'+' · '.join(link(prefix+item['path'],item['label']) for item in audit.get('evidence_links',[]))+'
'
- return section('调用链与日志检查',''+hero('Web Fetch 评测报告',f'{len(cases)} 个案例 · {len(names)} 种方法 · {track_label} 类分榜与一个综合通过率总榜。')
- if dataset.get('comparison',{}).get('call_chain_audit'):
- source_note='本轮未重新核验来源。' if dataset.get('comparison',{}).get('source_reverified') is False else ''
- home+=f'
{len(cases)} 个案例 · {len(records)} 次调用，机器判分。'+source_note+link('guide/index.html#call-chain-audit','查看调用链问题和日志留存范围')+'
'
from .report_explainer import article_title,render_explainer,supports_authored_explainer
TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else article_title(records) if authored_interpretation and supports_authored_explainer(cases) and not all(row.get('provider_type') == 'fixture' for row in records) else '本报告的任务范围与验收规则'
reading_note='从分数差异、失败案例和实现机制，理解如何选择抓取方法。' if authored_interpretation else '查看本报告的任务范围、结果数量与验收规则。'
- home+='
'
+ if dataset.get('comparison',{}).get('call_chain_audit'):
+ reading_url='guide/index.html'
+ home+='
报告解读 '+link(reading_url,'网页抓取成功之后，内容离可用还有多远')+'
从分数差异、失败案例和实现机制，理解如何选择抓取方法。
'+link(reading_url,'阅读全文 →')+''
+ if not dataset.get('comparison',{}).get('call_chain_audit'):home+='
'
if dataset.get('comparison',{}).get('call_chain_audit'):
TITLE='本轮调用链检查与日志范围'
- guide=local_call_chain_section(dataset)+section('如何读取本轮结果','
本轮按保存的实际返回进行机器判分，比例只对应本报告列出的案例与配置。日志可用于复查调用和判分；未经过独立人工审核，不能将失败一概归因于产品内部。
'+link('../analysis/index.html','逐次结果与筛选')+' · '+link('../methodology/index.html','题目与计时规则')+' · '+link('../reproduce/index.html','配置、版本和复现说明')+'
')
+ guide=section('如何读取本轮结果','
比例只对应本报告列出的案例与配置。先看总榜和分榜，再打开具体案例，对照验收要求与实际返回。请求成功只表示方法正常返回；任务通过表示返回内容满足该案例的全部要求。
评分规则与计时范围见“评测方法”，检查和修正记录见“核验记录”，方法版本、固定配置与日志范围见“环境与复现”。
')
else:
guide=render_explainer(out,cases,records,table=table,cell=cell,link=link,authored=authored_interpretation)
+ from .report_reading import render_archived_reading
+ guide=render_archived_reading(out) or guide
write(out,'guide/index.html','报告解读',guide)
panels=[]
for track,label in tracks:
@@ -2026,8 +2066,8 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
body=hero(esc(c['name']),hero_meta)+section('结论',summary,'overview')
reviewed=audited_attempts.get(r['attempt_id'])
if reviewed:
- independent='通过' if reviewed.get('agent_verdict')=='pass' else '未通过'
- final='通过' if reviewed.get('final_verdict')=='pass' else '未通过'
+ independent={'pass':'通过','fail':'未通过','insufficient':'证据不足'}.get(reviewed.get('agent_verdict'),'未记录')
+ final={'pass':'通过','fail':'未通过','unavailable':'无法判定'}.get(reviewed.get('final_verdict'),'未裁定；保留现有评分')
review_rows=[
('独立核验结论',independent),
('直接证据',reviewed.get('agent_evidence') or '—'),
@@ -2046,6 +2086,15 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
if not (c.get('ground_truth_main_content') or c.get('reference_answer')) and c.get('task_kind','body')=='body':
reference_heading='验收规则（无全文参考）'
source_meta=f'{link(reference_download,"下载验收规则")}；全文完整度、纯净度和 F1 不适用。'
+ if (c.get('track','web')=='web' and c.get('task_kind','body')=='body'
+ and c.get('required_facts') and not c.get('acceptance_contract')):
+ rules={key:c[key] for key in ('min_content_chars','expected_any','required_facts',
+ 'fact_coverage_threshold','required_patterns','body_requirements') if key in c}
+ source_body=required_text_answer_panel(c)
+ if c.get('ground_truth_main_content') or c.get('reference_answer'):
+ source_body+='
完整参考正文 '+f'
{esc(ref)} '
+ source_body+=('
查看原始验收规则 '
+ f'{esc(json.dumps(rules,ensure_ascii=False,indent=2))} ')
delivered=r.get('provider_metadata',{}).get('file_evaluation',{}).get('files',[])
if 'stable_task_pass' in m and c.get('acceptance_contract'):
reference_download='../contracts/'+c['id']+'.json'
@@ -2143,7 +2192,10 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
if records and all(row.get('provider_type') == 'fixture' for row in records):
write(out,'changes/index.html','示例说明',hero('合成示例','本报告由本地测试适配器生成。没有正式评测发布记录。'))
elif not authored_interpretation:
- write(out,'changes/index.html','本轮结果说明',hero('本轮结果说明','本报告从保存的调用、评分与证据构建，不属于仓库历史报告的发布版本。')+section('查看本轮证据','
'+link('../analysis/index.html','逐任务结果与证据')+' · '+link('../reproduce/index.html','运行参数与版本')+'
'))
+ review_path=out/'result-review.json'
+ review=json.loads(review_path.read_text()) if review_path.is_file() else {}
+ updates=''.join('
'+esc(row.get('date',''))+'
'+esc(row.get('title',''))+' '+esc(row.get('fix',''))+'
'+esc(row.get('scope',''))+'
' for row in review.get('review_rounds',[]))
+ write(out,'changes/index.html','变更记录',hero('变更记录','列出本轮核验后的修正；修正读取已保存的返回，不代表重新调用产品。')+section('本轮结果修正','
'+updates+'
')+section('详细依据','
'+link('../review/index.html','核验范围与修正说明')+' · '+link('../analysis/index.html','逐任务结果与证据')+' · '+link('../reproduce/index.html','运行参数与版本')+'
'))
else:
render_changes(out,cases,records,dataset)
if include_english:
@@ -2165,7 +2217,7 @@ def write_review_page(out,records):
raise ValueError('A reviewed report requires research/report-changes.json')
changes=json.loads(metadata_path.read_text())
from .report_changes import require_review_rounds
- rounds=require_review_rounds(changes)
+ rounds=(audit['review_rounds'] if audit.get('kind')=='local_independent_review' else require_review_rounds(changes))
rounds_html=''.join(
f'
第 {i} 轮{esc(row.get("date",""))}
{esc(row.get("title",""))} 范围： {esc(row.get("scope",""))}
发现： {esc(row.get("finding",""))}
纠正： {esc(row.get("fix",""))}
未完成： {esc(row.get("pending",""))}
'
for i,row in enumerate(rounds,1))
@@ -2173,7 +2225,7 @@ def write_review_page(out,records):
reviewed_attempts=audit.get('attempts',[])
disagreements=[row for row in reviewed_attempts
if row.get('agent_verdict') != row.get('machine_verdict')]
- adjudicated=[row for row in disagreements if row.get('final_verdict') in {'pass','fail'}]
+ adjudicated=[row for row in disagreements if row.get('final_verdict') in {'pass','fail','unavailable'}]
unresolved=audit.get('unresolved',0)
def coverage(label,item):
scope=item.get('review_protocol',{}).get('scope','all')
@@ -2188,6 +2240,11 @@ def coverage(label,item):
method=(f'
核验协议与执行者 执行者：{esc(reviewer.get("agent","独立代理"))}；模型：{esc(reviewer.get("model","—"))}；推理强度：{esc(reviewer.get("effort","—"))}。'
'核验时隐藏机器结论和分数，只提供任务契约、标准答案与完整实际输出；分歧回到原始证据裁定，未决非零则禁止发布。任务和输出未变化时沿用原核验；发生变化的调用必须重新盲审。
')
page+=section('核验口径',method)
+ dataset_path=out/'dataset.json'
+ if dataset_path.is_file():
+ findings=json.loads(dataset_path.read_text()).get('comparison',{}).get('call_chain_audit',{}).get('findings',[])
+ if findings:
+ page+=section('本轮修正与检查限制',''.join('
'+esc(item['text'])+'
' for item in findings))
write(out,'review/index.html','核验记录',page)
def write_support(out,records,manifests,*,authored_interpretation=True):
@@ -2247,10 +2304,12 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
for name in sorted({r['provider'] for r in records}):
subset=[r for r in records if r['provider']==name]
versions=sorted({str(value) for r in subset for value in
- [r.get('provider_metadata',{}).get('browser_version') or
+ [r.get('provider_metadata',{}).get('reported_version') or
+ r.get('provider_metadata',{}).get('browser_version') or
r.get('provider_metadata',{}).get('library_version')]
if value})
identities=[p.get('implementation_identity',{}) for m in manifests for p in m.get('providers',[]) if p.get('name')==name]
+ versions=versions or sorted({str(identity['reported_version']) for identity in identities if identity.get('reported_version')})
versions=versions or sorted({str(identity['repository']['head']) for identity in identities if identity.get('repository',{}).get('head')})
hashes=sorted({str(r.get('provider_metadata',{}).get('binary_sha256')) for r in subset if r.get('provider_metadata',{}).get('binary_sha256')})
versions=versions or sorted({provenance[h].get('version') or provenance[h].get('identity') for h in hashes if h in provenance})
@@ -2264,7 +2323,13 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
basis='运行时版本与 SHA-256'
else:
basis='运行时 SHA-256；nightly 源修订不可得'
- provider_rows.append(('',cell(link('../providers/'+name+'.html',provider_label(name)))+cell(esc('；'.join(versions) or '版本未记录'))+cell(esc(basis))+cell(esc('；'.join(deployments)))+cell('
'.join('
'+esc(x)+'' for x in hashes) or '—')+cell(str(len(subset)))))
+ if local_matrix:
+ version='；'.join(versions) or ('云端服务未公开后端版本' if not hashes and not any(identities) else '未取得可确认的版本信息')
+ if versions and all(re.fullmatch(r'[0-9a-f]{40}', value) for value in versions):
+ version='源码提交：'+version
+ provider_rows.append(('',cell(link('../providers/'+name+'.html',provider_label(name)))+cell('
'+esc(version)+' ')))
+ else:
+ provider_rows.append(('',cell(link('../providers/'+name+'.html',provider_label(name)))+cell(esc('；'.join(versions) or '版本未记录'))+cell(esc(basis))+cell(esc('；'.join(deployments)))+cell('
'.join('
'+esc(x)+'' for x in hashes) or '—')+cell(str(len(subset)))))
write_review_page(out,records)
repo='https://github.com/lexmount/browser-eval'
support=hero('环境与复现','代码、评测集和复现说明以 browser-eval 仓库为准；本页只列本轮必要配置。')
@@ -2275,8 +2340,22 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
support+=section('隔离资源证据','
'+link('../resource-cohort/resource-comparison.json','CPU、内存、网络汇总与共同成功样本')+' · '+link('../resource-cohort/results.jsonl','资源子集逐次记录')+' · '+link('../resource-cohort/execution.json','容器环境与镜像身份')+'
')
conditions=(f'最终报告纳入 {len(records):,} 次调用。任务与参考冻结后执行；每个方法内部串行，不复用个人登录状态。' if authored_interpretation else f'本报告纳入 {len(records):,} 个逻辑结果。运行参数见下表；保留的物理调用与输出见各结果详情。')
support+=section('公共条件','
'+conditions+'
'+table(['参数','值'],common,metrics=[]))
- support+=section('方法版本','
“事后核验”只说明留存二进制与官方归档完全一致，不改写成当时已经记录了版本。移动 nightly 无法追到不可得的源修订时，只用 SHA-256 标识精确可执行文件。
'+table(['方法','版本','身份依据','部署路径','二进制 SHA-256','调用数'],provider_rows,metrics=[]))
- support+=section('发布与加载','
本地构建为每个任务、方法和证据生成可独立检查的逻辑路由；这不等于线上部署同样数量的独立 HTML。发布器会把整份报告编译成一个通用 HTML 外壳、共享样式与脚本，以及按内容寻址的压缩数据块。页面切换由前端路由完成，证据只在打开对应页面时读取，超过 256 KiB 的完整文本还需用户主动加载。
发布清单记录逻辑路由数、源文件数、压缩块数和逐文件 SHA-256；发布门禁会重建每个文件并验证内部链接与原始下载字节。这样保留逐项可审计性，同时避免重复页面骨架和一次性加载全部证据。
')
+ if local_matrix:
+ support+=section('方法版本','
显示本轮使用的软件版本或源码提交。云端服务未公开可确认的后端版本号时，直接说明；这不代表服务没有版本。
'+table(['方法','本轮使用版本／源码提交'],provider_rows,metrics=[]))
+ else:
+ support+=section('方法版本','
版本优先使用调用时记录的版本号；源码实现使用冻结的提交号。云端服务没有公开后端版本时如实标注。二进制哈希用于确认具体执行文件；事后核验的版本会说明依据。
'+table(['方法','版本','身份依据','部署路径','二进制 SHA-256','调用数'],provider_rows,metrics=[]))
+ if local_matrix:
+ options_path=out/'frozen-method-options.json'
+ if options_path.is_file():
+ options=json.loads(options_path.read_text())['providers']
+ labels={'wait_until':'页面完成条件','block_private_networks':'阻止访问私网','max_bytes':'返回内容上限（字节）','max_age':'缓存最长年龄','proxy':'代理选择','formats':'请求输出格式','only_main_content':'只取主要正文','extract_depth':'提取深度','include_images':'包含图片','text_verbosity':'正文详细程度','max_age_hours':'缓存最长年龄（小时）','include_resource_links':'包含资源链接','rich_links_limit':'具名链接数量上限','document_max_bytes':'文档大小上限（字节）','min_interval_seconds':'调用间隔（秒）','log_level':'日志级别','retain_complete_diagnostic_log':'完整保存诊断日志','max_diagnostic_bytes':'日志保存上限（字节）','uniform_native_parameters':'使用统一固定原生选项','dns_mode':'域名解析方式'}
+ def option_text(value):
+ return '开启' if value is True else '关闭' if value is False else '、'.join(map(str,value)) if isinstance(value,list) else str(value)
+ config_rows=[('',cell(link('../providers/'+name+'.html',provider_label(name)))+cell(''.join('
'+esc(labels.get(k,k))+'：'+esc(option_text(v))+'
' for k,v in value.items()))) for name,value in options.items()]
+ support+=section('本轮实际固定配置','
以下选项来自调用前冻结的配置。每题调用一次，外层截止时间45秒；产品内部超时保留默认值。调用间隔不计入单次耗时。这里只展示影响抓取与日志的选项，具体请求和按任务选择的输出格式见各案例的调用日志。
'+table(['方法','固定选项'],config_rows,metrics=[]))
+ support+=local_call_chain_section(json.loads((out/'dataset.json').read_text()))
+ if not local_matrix:
+ support+=section('发布与加载','
本地构建为每个任务、方法和证据生成可独立检查的逻辑路由；这不等于线上部署同样数量的独立 HTML。发布器会把整份报告编译成一个通用 HTML 外壳、共享样式与脚本，以及按内容寻址的压缩数据块。页面切换由前端路由完成，证据只在打开对应页面时读取，超过 256 KiB 的完整文本还需用户主动加载。
发布清单记录逻辑路由数、源文件数、压缩块数和逐文件 SHA-256；发布门禁会重建每个文件并验证内部链接与原始下载字节。这样保留逐项可审计性，同时避免重复页面骨架和一次性加载全部证据。
')
if not authored_interpretation:
reproduce='
使用本轮任务配置与方法配置运行，或从保存的结果重新构建报告：
browser-harness-eval run --dataset DATASET --provider-config CONFIG --providers METHODS --output NEW_EVIDENCE\nbrowser-harness-eval report EVIDENCE 重新运行会调用配置中的方法；重建报告只读取已有结果。本轮参数、版本与可执行文件哈希见上表。
'
elif headless:
@@ -2284,10 +2363,10 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
else:
reproduce+='
方法配置、凭据变量和本轮实际版本分别见“方法介绍”“账号”和上方方法版本表；不存在的 headless 专用配置不属于本 cohort。
'
if local_matrix:
- reproduce+='
重建历史成绩时，先使用本轮绑定的评分提交；当前分支的评分规则可能已经更新。提交号和评分文件哈希见仓库 evaluation/dynamic-content-100-moli-webfetch/current-local-results.json 的 measured_scoring_git_revision 与 measured_scoring_source_sha256。先在该提交的代码目录执行上面的构建命令。只更新已有报告说明时，可在当前代码目录执行以下命令；它保留原评分和实测文件。
uv run --frozen python scripts/build_report.py --refresh-presentation evaluation/dynamic-content-100-moli-webfetch/current-local-results.json --output /absolute/path/to/report '
+ reproduce='
以下命令从本轮保存的证据构建本地报告，不重新调用产品：
uv run --frozen python scripts/build_report.py --local-audit --language '+('bilingual' if _ENGLISH_GENERATED else 'zh')+' --hide-local-status-banners --evidence /absolute/path/to/report-evidence --output /absolute/path/to/report\nuv run --frozen python scripts/check_report.py /absolute/path/to/report 使用与本轮结果对应的证据目录。调用时的版本、配置和评分器哈希见 '+link('../local-run-contract.json','调用前冻结协议')+' 与 '+link('../execution.json','执行批次')+'。上面的构建命令读取保存的评分；修改评分规则后，应先按声明范围复评分，不能仅重建页面就称成绩已经更新。
'
support+=section('核心复现指令',reproduce)
write(out,'reproduce/index.html','环境与复现',support)
- if not authored_interpretation:
+ if not authored_interpretation and not local_matrix:
rows=[]
for name in sorted({r['provider'] for r in records}):
selected=[r for r in records if r['provider']==name]
@@ -2315,8 +2394,12 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
{r['provider'] for r in records},
fixture_providers={r['provider'] for r in records if r.get('provider_type')=='fixture'},
)
- accountrows=[('',cell(esc(a['provider']))+cell(link(a['registration_url'],'注册 / 登录')+'
'+link(a['key_url'],'获取凭据'))+cell('
'.join('
'+esc(x)+'' for x in a['env_vars']))+cell(esc(a['steps']))+cell(esc(a['free_note']))) for a in accounts]
- write(out,'accounts/index.html','账号',hero('账号与凭据','只列官方入口和环境变量名称，不保存实际凭据。')+section('注册清单',table(['提供方','官方入口','环境变量','取得方式','免费 / 准入说明'],accountrows,cls='accounts-table')))
+ accountrows=[]
+ for a in accounts:
+ official=link(a['registration_url'],'官方获取入口' if local_matrix else '注册 / 登录')
+ if a['key_url']!=a['registration_url']:official+='
'+link(a['key_url'],'获取凭据')
+ accountrows.append(('',cell(esc(a['provider']))+cell(official)+cell('
'.join('
'+esc(x)+'' for x in a['env_vars']))+cell(esc(a['steps']))+cell(esc(a['free_note']))))
+ write(out,'accounts/index.html','账号与安装',hero('账号与安装','只列官方入口和环境变量名称，不保存实际凭据。')+section('安装与账号准备',table(['提供方','官方入口','环境变量','取得方式','免费 / 准入说明'],accountrows,cls='accounts-table')))
from .report_methods import render_methods
write(out,'adoption/index.html','方法介绍',render_methods(out,records,hero=hero,section=section,table=table,cell=cell,link=link,esc=esc,authored=authored_interpretation))
diff --git a/harness/web_fetch/report_article_cases.py b/harness/web_fetch/report_article_cases.py
index 16b6d2289..3331b459f 100644
--- a/harness/web_fetch/report_article_cases.py
+++ b/harness/web_fetch/report_article_cases.py
@@ -121,11 +121,6 @@ def remaining_fact_lines(text, excerpt):
f'至少 {required}/{len(regions)} 区达到 {threshold:.0%}；每区不低于 {floor:.0%}',
' / '.join(f'{r["coverage"]:.0%}' for r in regions),
metrics['body_coverage_pass']))
- if 'forbidden_hits' in metrics:
- forbidden = metrics['forbidden_hits']
- checks.append(_row('排除错误页', '排除：' + ' / '.join(case.get('forbidden_any', [])),
- ' / '.join(forbidden) if forbidden else '未命中排除项',
- not forbidden))
if 'text_encoding_corruption' in metrics:
checks.append(_row('文字解码', '替换字符数 ≥20 且占比 ≥2% 时拒绝',
f'{metrics.get("replacement_character_count", 0)} 个替换字符',
@@ -168,12 +163,6 @@ def _list_example(case, record, output):
f'{metrics["stable_complete_record_count"]} 条', metrics['stable_structure_pass']),
_row('唯一记录', f'去重后至少 {contract["minimum_records"]} 条',
f'{metrics["stable_unique_record_count"]} 条', metrics['stable_structure_pass'])]
- forbidden = metrics['forbidden_hits']
- blocked = metrics['stable_blocked_page']
- checks.append(_row('排除错误页',
- '排除：' + ' / '.join(case.get('forbidden_any', [])) + '；不得返回安全验证或拒绝访问页',
- ' / '.join(forbidden) if forbidden else ('命中拦截页特征' if blocked else '未命中排除项或拦截页特征'),
- not forbidden and not blocked))
checks.append(_row('文字解码', '替换字符数 ≥20 且占比 ≥2% 时拒绝',
f'{metrics["replacement_character_count"]} 个替换字符',
not metrics['text_encoding_corruption']))
@@ -204,10 +193,6 @@ def _dynamic_example(case, record, output):
metrics['stable_structure_pass']),
_row('全部固定名言', '10 组原句、作者与标签均须匹配独立参考',
f'{metrics["stable_anchor_hits"]}/{metrics["stable_anchor_total"]} 组命中', metrics['stable_content_pass']),
- _row('排除错误页', '排除：' + ' / '.join(case['forbidden_any']) + '；不得返回安全验证或拒绝访问页',
- ' / '.join(metrics['forbidden_hits']) if metrics['forbidden_hits'] else
- ('命中拦截页特征' if metrics['stable_blocked_page'] else '未命中排除项或拦截页特征'),
- not metrics['forbidden_hits'] and not metrics['stable_blocked_page']),
_row('文字解码', '替换字符数 ≥20 且占比 ≥2% 时拒绝',
f'{metrics["replacement_character_count"]} 个替换字符', not metrics['text_encoding_corruption']),
]
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index bb543e4da..4b0debb0b 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -203,7 +203,8 @@ def _install_report_tree(staging: Path, output: Path):
def build_report(source: Path, output: Path, *, include_english: bool = False,
local_audit: bool = False, authored_interpretation: bool = False,
- diagnostic_preview: bool = False, language: str | None = None) -> Path:
+ diagnostic_preview: bool = False, language: str | None = None,
+ show_local_status_banners: bool = True) -> Path:
from harness.report_languages import includes_english, select_report_language
include_english = includes_english(language, include_english)
source, output = source.resolve(), output.resolve()
@@ -493,6 +494,12 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
shutil.copytree(source/'history/file-baseline', staging/'history/file-baseline')
if dataset.get("comparison", {}).get("kind") == "uniform_native_local_measurement":
shutil.copy2(source / "local-run-contract.json", staging / "local-run-contract.json")
+ if (source/"providers.json").is_file():
+ # Publish only an explicit allowlist from the already validated frozen config.
+ config=json.loads((source/'providers.json').read_text())['providers']
+ allowed={'wait_until','block_private_networks','max_bytes','max_age','proxy','formats','only_main_content','extract_depth','include_images','text_verbosity','max_age_hours','include_resource_links','rich_links_limit','document_max_bytes','min_interval_seconds','log_level','retain_complete_diagnostic_log','max_diagnostic_bytes','uniform_native_parameters','dns_mode'}
+ public_options={name:{k:v for k,v in options.items() if k in allowed} for name,options in config.items()}
+ (staging/'frozen-method-options.json').write_text(json.dumps({'schema_version':1,'provider_config_sha256':hashlib.sha256((source/'providers.json').read_bytes()).hexdigest(),'providers':public_options},ensure_ascii=False,indent=2))
from harness.reporting.templates import resolve_template
(staging / "report-manifest.json").write_text(json.dumps({
"schema_version": 1, "template": resolve_template("webfetch"),
@@ -500,9 +507,13 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
"languages": ["zh", "en"] if include_english else ["zh"], "provider_calls_added": 0,
"dataset_sha256": hashlib.sha256((source / "dataset.json").read_bytes()).hexdigest(),
"results_sha256": hashlib.sha256((source / "results.jsonl").read_bytes()).hexdigest(),
+ "presentation": {"show_local_status_banners": show_local_status_banners},
}, ensure_ascii=False, indent=2))
+ from .report_reading import copy_archived_reading
+ copy_archived_reading(source,staging)
generate(staging,dataset,records,manifests,include_english=include_english,
- authored_interpretation=authored_interpretation)
+ authored_interpretation=authored_interpretation,
+ show_local_status_banners=show_local_status_banners)
if language is not None:
select_report_language(staging, language)
(staging/marker).write_text('Generated from persisted evidence. Rebuild; do not edit.\n')
@@ -1172,6 +1183,7 @@ def validate_local_presentation(output: Path, binding_path: Path):
not a fresh build or certification. All measured files remain untouched.
"""
from .safety import contained_path
+ from harness.report_languages import generated_languages
output = output.resolve()
binding = json.loads(binding_path.read_text())
manifest_path = contained_path(output, 'report-manifest.json')
@@ -1179,15 +1191,19 @@ def validate_local_presentation(output: Path, binding_path: Path):
if (binding.get('comparison_kind') != 'uniform_native_local_measurement'
or manifest.get('evidence_state') != 'local_measurement'
or manifest.get('certified') is not False
- or manifest.get('languages') != ['zh']
+ or manifest.get('languages') != list(generated_languages(output))
or not (output / '.generated-report').is_file()):
- raise ValueError('Presentation refresh requires a bound Chinese diagnostic snapshot')
+ raise ValueError('Presentation refresh requires a bound diagnostic snapshot')
for name, key in [('dataset.json', 'frozen_dataset_sha256'), ('results.jsonl', 'results_sha256')]:
expected = binding.get(key)
manifest_key = 'dataset_sha256' if name == 'dataset.json' else key
if (not expected or manifest.get(manifest_key) != expected
or hashlib.sha256(contained_path(output, name).read_bytes()).hexdigest() != expected):
raise ValueError('Presentation input binding changed: ' + name)
+ for item in binding.get('evidence_bindings', []):
+ path = contained_path(output, item['path'], reject_symlinks=True)
+ if hashlib.sha256(path.read_bytes()).hexdigest() != item['sha256']:
+ raise ValueError('Presentation evidence binding changed: ' + item['path'])
dataset = json.loads((output / 'dataset.json').read_text())
records = load_jsonl(contained_path(output, 'results.jsonl'))
if (len(records) != binding.get('result_count')
@@ -1214,6 +1230,8 @@ def refresh_local_presentation(output: Path, binding_path: Path) -> Path:
output = unlinked_path(output)
before_tree = evidence_tree_identity(output)
binding, dataset, records, manifest = validate_local_presentation(output, binding_path)
+ if manifest['languages'] != ['zh']:
+ raise ValueError('Presentation refresh supports Chinese snapshots only; rebuild both editions for bilingual changes')
manifest_path = output / "report-manifest.json"
# The existing owner renders from saved metrics. Only these explanatory
# routes may be installed; result pages, logs and every measured byte stay.
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index 5eec4fb54..a7ec1fd8f 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -176,7 +176,7 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
if presentation_binding and not local_measurement:
raise ValueError("Presentation binding requires a frozen uniform-native diagnostic snapshot")
if local_measurement:
- from .local_audit import require_local_measurement
+ from .local_audit import require_local_measurement, local_measurement_cases
if presentation_binding:
if profile:
raise ValueError("Presentation binding cannot certify a release profile")
@@ -185,7 +185,10 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
else:
require_local_measurement(root, d, rs)
inventory = {track: {"case_ids": [case["id"] for case in d["cases"] if case.get("track", "web") == track], "providers": providers} for track, providers in d["coverage_plan"].items()}
- canonical_dataset = repository / json.loads((root / "local-run-contract.json").read_text())["dataset_path"]
+ local_contract = json.loads((root / "local-run-contract.json").read_text())
+ canonical_cases = {case["id"]: case for case in local_measurement_cases(local_contract)}
+ canonical_dataset = (root / "dataset.json" if local_contract.get("dataset_sources")
+ else repository / local_contract["dataset_path"])
elif profile_path and (profile_path.parent / "dataset.json").is_file():
inventory = {
track: {
@@ -254,7 +257,11 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
for case in d["cases"]:
safe_identifier(case["id"])
validate_resource_source_evidence(case, canonical_dataset)
- reliability = require_reliable(d, rs)
+ if local_measurement:
+ from .local_audit import require_complete_local_inventory
+ reliability = require_complete_local_inventory(d, rs)
+ else:
+ reliability = require_reliable(d, rs)
saved_reliability = json.loads((root / "reliability.json").read_text())
if not saved_reliability == reliability:
raise ValueError("Saved reliability status differs from report evidence")
@@ -283,14 +290,15 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
)
if local_measurement:
from .stable_tasks import validate_acceptance_contract
- validate_acceptance_contract(c["acceptance_contract"])
+ if c.get("acceptance_contract"):
+ validate_acceptance_contract(c["acceptance_contract"])
+ elif not (stable_checks or c.get("task_kind") == "file"):
+ raise ValueError(c["id"])
elif not (unavailable and stable_checks):
raise ValueError(c["id"])
if c.get("ground_truth_main_content"):
expected = c["ground_truth_main_content"].encode()
if local_measurement:
- from .reference_inputs import read_dataset_bytes
- canonical_cases = {case["id"]: case for case in json.loads(read_dataset_bytes(canonical_dataset))["cases"]}
valid_reference = expected == canonical_cases[c["id"]]["ground_truth_main_content"].encode()
else:
valid_reference = hashlib.sha256(expected).hexdigest() == c["source_verification"]["reference_sha256"]
@@ -388,6 +396,12 @@ def canonical_link(page, value):
allowed_english_source_text = {"中文"} | {
case["name"] for case in d["cases"] if case.get("name")
}
+ archived_manifest = root / "archived-reading.json"
+ if archived_manifest.is_file():
+ from .report_reading import render_archived_reading
+ # Validate the archived source before allowing its original case names.
+ render_archived_reading(root, "en" if has_english else "zh")
+ allowed_english_source_text.update(json.loads(archived_manifest.read_text()).get("preserved_source_names", []))
if has_english:
for relative in english_primary:
if relative not in en_pages:
@@ -442,6 +456,8 @@ def canonical_link(page, value):
if case.get("task_kind") == "file"
else "标准答案与验收规则"
if case.get("task_kind") == "structure"
+ else "验收规则（无全文参考）"
+ if case.get("task_kind", "body") == "body" and not (case.get("ground_truth_main_content") or case.get("reference_answer"))
else "完整标准答案"
)
actual_heading = (
@@ -449,6 +465,7 @@ def canonical_link(page, value):
"目标文件": "Target file",
"标准答案与验收规则": "Reference and acceptance contract",
"完整标准答案": "Complete reference answer",
+ "验收规则（无全文参考）": "Acceptance rules (no full-text reference)",
}[heading]
if p.relative_to(root).parts[0] == "en"
else heading
diff --git a/harness/web_fetch/report_components.py b/harness/web_fetch/report_components.py
index 3db213b2b..e4e3b0487 100644
--- a/harness/web_fetch/report_components.py
+++ b/harness/web_fetch/report_components.py
@@ -11,7 +11,7 @@ def reference_text(case):
return '本例按资源或结构规则评分，不以页面短摘要计算全文 F1。'
rules=[('最少字符数',case.get('min_content_chars',1))]
for key,label in [('expected_any','至少命中一项文字'),('required_facts','必需文字组'),
- ('forbidden_any','不得出现的文字'),('required_patterns','格式与数量要求'),
+ ('required_patterns','格式与数量要求'),
('body_requirements','正文范围要求'),('acceptance_contract','验收契约')]:
if case.get(key):rules.append((label,case[key]))
if case.get('required_facts'):
@@ -35,7 +35,7 @@ def cards(items):
def metric_guide():
return '
静态网站正文、动态网站正文、热门网站正文和文档正文使用同一评分流程。页面来源与加载方式不同，改变的是案例要求，不改变指标定义。
'+cards([
- ('统一验收流程','接口返回 → 排除空内容与错误页 → 核对题目要求的文字和字段 → 检查列表数量、类型与唯一性 → 汇总任务结果。只执行该例适用的检查；失败详情保留具体未满足项。图片与视频沿用请求和交付检查，资源地址或文件字节另按格式核验。'),
+ ('统一验收流程','接口返回 → 检查目标内容是否为空 → 核对题目要求的文字和字段 → 检查列表数量、类型与唯一性 → 汇总任务结果。只执行该例适用的检查；失败详情保留具体未满足项。图片与视频沿用请求和交付检查，资源地址或文件字节另按格式核验。'),
('任务通过率','满足该例全部验收规则的次数 ÷ 已有验收证据的任务。段落、列表、字段都属于内容要求；
没有 F1 最低分门槛 。请求失败计入分母；任何计划调用缺失或无法判分时，整份报告标为不可靠并禁止发布。'),
('文字核对覆盖','每例命中的必需文字组数 ÷ 登记组数，再按案例取均值。固定标题、说明、字段标签或事实句均可作为核对组；易变名单和价格不固定为答案。没有文字依据的案例不记作满分，覆盖案例数另列。'),
('请求成功率','接口正常返回的次数占全部调用的比例。接口返回成功，不代表抓到了目标内容。'),
@@ -150,8 +150,8 @@ def metric_legend(headers):
def task_pass_guide(track, base='', cases=None):
rules = {
'dynamic': '核验初次加载范围：稳定内容锚点（稳定标题、栏目、说明）与条目的必需字段、数量、唯一性和类型共同决定任务通过。频繁变化的名单、条目标题和价格按声明的结构与类型核验，不固定旧快照取值。HTTP 页面壳不通过；渲染、内嵌数据和公开 API 均可作为实现路径。全文质量仅在有完整、可比的参考时另算。',
- 'web': '请求成功，并同时满足本例最低字符数、全部必需核对点、显著标记及结构要求；不得命中非参考正文自身包含的错误页标记。完整度、纯净度和 F1 另算，不作为通过门槛。',
- 'document': '文档正文采用与网页相同的文字验收规则：请求、最低长度、全部核对点及错误页检查满足；不是只拿到下载链接。F1 另算，不作为通过门槛。',
+ 'web': '请求成功，并同时满足本例最低字符数、全部必需核对点、显著标记及结构要求。验证码或错误提示文字不单独决定任务是否通过。完整度、纯净度和 F1 另算，不作为通过门槛。',
+ 'document': '文档正文按请求状态、最低长度与全部内容核对点验收；只拿到下载链接不能通过。验证或错误提示只作诊断。F1另算，不作为通过门槛。',
'image': '含图页面须在实际返回值中找齐指定图片地址；直接 PNG 须交付文件字节并通过格式、长度与目标 SHA-256 验收。',
'video': '按案例区分：找齐播放入口、核对标题与发布者，或交付通过格式、长度与目标 SHA-256 验收的原文件。链接不等于文件，均不测画面理解。',
'challenge': '请求成功，取得至少 3 条具名详情链接，字段完整、类型正确、条目不重复，链接符合目标站点规则；有独立内容证据的稳定标题、栏目、说明还须命中。频繁变化的名单和价格不固定旧值。通过表示满足上述取数要求，不证明变化值真实或最新。',
@@ -234,10 +234,10 @@ def examples():
('结构怎么测','至少 3 个有名称的本站详情链接，按地址去重。仅网址、导航按钮和登录提示不计入；频繁变化的条目名称、名次、价格或 ID 不固定为旧快照取值，不免除登记的稳定内容核对。'),
('怎样算通过','稳定内容锚点与必需字段、类型、唯一性和数量全部通过。通过不证明变化值的事实准确性或实时性；缺少完整、可比的参考时完整度、纯净度和 F1 为 —。
查看参考与实测 → '),
]),
- ('risk_control','历史访问受限网址内容',[
+ ('risk_control','风控网站',[
('任务','打开指定网址，取得首次自动加载出的目标正文、列表和真实链接，不进行点击、输入、滚动、翻页或登录。'),
('核对依据','按独立保存的目标页面证据登记稳定文字、完整正文分区，以及名称—链接、路径、字段、去重和数量要求。'),
- ('怎么测','先排除请求失败、空返回、验证或拦截页面和页面外壳，再逐项检查本例声明的稳定文字、正文覆盖与组合规则。'),
+ ('怎么测','核对请求是否成功，再逐项检查本例要求的稳定文字、正文覆盖与完整条目。验证或拦截提示只作诊断，出现提示不单独导致失败；只有页面外壳而没有合格目标内容仍不通过。'),
('怎样算通过','本例登记的全部规则均通过；文章只返回标题或开头、列表数量不足、链接不符合真实路径，均不通过。'),
]),
]
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index 63ccbef7c..fcdb4808a 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -60,6 +60,24 @@
"下载最终物理调用记录": "Download the final physical-call record",
"。运行记录保留请求状态、耗时和证据路径；原始输出保持调用后保存的字节。": ". The record retains request status, duration and evidence paths. Raw output preserves the saved response bytes.",
"请求成功，并同时满足本例最低字符数、全部必需核对点、显著标记及结构要求；不得命中非参考正文自身包含的错误页标记。完整度、纯净度和 F1 另算，不作为通过门槛。": "A successful request must satisfy the task's minimum length, all required text checks, salient markers and structure requirements. Error-page markers absent from the reference must not appear. Recall, precision and F1 are calculated separately and do not determine acceptance.",
+ "云端服务未公开后端版本": "The cloud service does not disclose its backend version",
+ "未取得可确认的版本信息": "No verifiable version information is available",
+ "本轮使用版本／源码提交": "Version / source commit used in this run",
+ "显示本轮使用的软件版本或源码提交。云端服务未公开可确认的后端版本号时，直接说明；这不代表服务没有版本。": "Software versions or source commits used for this run. Undisclosed cloud backend versions are stated explicitly; this does not mean the service has no version.",
+ "本轮启用自身 DEBUG 并捕获网络库日志。": "This run enabled native DEBUG output and captured network-library logs.",
+ "列出本轮核验后的修正；修正读取已保存的返回，不代表重新调用产品。": "Corrections after verification of this run use saved responses; they do not represent new product calls.",
+ "本轮结果修正": "Corrections to this run",
+ "详细依据": "Supporting evidence",
+ "核验范围与修正说明": "Review scope and corrections",
+ "产品原生返回；保留实际输出与日志": "Native product output; actual responses and logs are retained",
+ "缓存与等待选项见本轮固定配置": "See the frozen configuration for cache and wait settings",
+ "Firecrawl、Tavily、Exa、Cloudflare；通过各自 API 调用。": "Firecrawl, Tavily, Exa and Cloudflare, called through their respective APIs.",
+ "本轮实际固定配置": "Frozen settings used in this run",
+ "本轮保存了哪些日志": "Logs retained in this run",
+ "产品与依赖库日志范围": "Product and dependency log coverage",
+
+ "比例只对应本报告列出的案例与配置。先看总榜和分榜，再打开具体案例，对照验收要求与实际返回。请求成功只表示方法正常返回；任务通过表示返回内容满足该案例的全部要求。": "Rates apply only to the cases and configurations in this report. Start with the overall and category rankings, then open individual cases to compare requirements with actual output. Request success means the method returned normally; task success means the returned content met every requirement for that case.",
+ "评分规则与计时范围见“评测方法”，检查和修正记录见“核验记录”，方法版本、固定配置与日志范围见“环境与复现”。": "See Methodology for scoring and timing, Review history for checks and corrections, and Reproduce for method versions, fixed options and log coverage.",
"已保存的诊断日志": "Saved diagnostic logs",
"已保存文件截断；原生日志范围见说明": "Saved file truncated; see native log coverage notes",
"已保存文件未截断；原生日志范围见说明": "Saved file not truncated; see native log coverage notes",
@@ -79,6 +97,7 @@
"原生日志范围": "Native log coverage",
"检查结论": "Audit findings",
"如何读取本轮结果": "How to read these results",
+ "本轮按保存的实际返回进行机器判分，比例只对应本报告列出的案例与配置。日志可用于复查调用和判分；失败原因须结合原始响应、调用记录及核验记录判断，不能一概归因于产品内部。": "Saved outputs are machine-scored. Rates apply only to the cases and configurations listed here. Assess failure causes using the native response, call log and review record; failures cannot all be attributed to product internals.",
"本轮按保存的实际返回进行机器判分，比例只对应本报告列出的案例与配置。日志可用于复查调用和判分；未经过独立人工审核，不能将失败一概归因于产品内部。": "Saved outputs are machine-scored. Rates apply only to the cases and configurations listed here. Logs support review of calls and scoring; without independent human review, failures cannot all be attributed to product internals.",
"逐次结果与筛选": "Per-call results and filters",
"题目与计时规则": "Tasks and timing rules",
@@ -150,7 +169,7 @@
"从官方 v1.1.7 release 获取二进制，配置 MOLI_1_1_7_BINARY。": "Obtain the binary from the official v1.1.7 release and configure MOLI_1_1_7_BINARY.",
"Moli 1.1.7 正式发布": "Moli 1.1.7 official release",
"本方法只使用官方 1.1.7 发行二进制，与其他 Moli 版本分开计分。普通网页使用原生 Markdown；直接文档与媒体的裸 fetch 路线在正式二进制上实测确认。正文任务若取得受支持的原始文档，会在同一计时预算内解析文字；文件任务直接验收字节。不在失败后切换引擎或输出模式。": "This method uses only the official 1.1.7 release binary and is scored separately from other Moli versions. Ordinary web pages use native Markdown; bare-fetch routes for direct documents and media were verified on the release binary. If a body-text task retrieves a supported original document, text is parsed within the same timing budget; file tasks validate bytes directly. The method does not switch engines or output modes after a failure.",
- "历史访问受限网址正文": "Historically access-restricted URL body text",
+ "风控网站": "Risk-control websites",
"历史访问受限网址内容": "Historically access-restricted URL content",
"请求指定网址": "Request the specified URL",
"识别目标页或拦截页": "Identify the target page or a blocked page",
@@ -3620,16 +3639,433 @@ def localize_html(page: str, source: PurePosixPath, *, translations=None) -> str
english = PurePosixPath("en") / source
page = page.replace('', '', 1)
page = re.sub(r'
', _header(source, english), page, count=1, flags=re.S)
+ # Raw evidence and executable/style blocks are preserved, never translated.
+ # Mask them before parsing so large responses do not incur entity-by-entity work.
+ preserved = []
+ def preserve(match):
+ preserved.append(match.group(0))
+ return f""
+ page = re.sub(r"<(pre|script|style)\b[^>]*>.*?", preserve, page, flags=re.S | re.I)
parser = Localizer(source, translations)
parser.feed(page)
parser.close()
- return _rewrite_links("".join(parser.parts), source, english)
+ localized = "".join(parser.parts)
+ for index, block in enumerate(preserved):
+ localized = localized.replace(f"", block)
+ return _rewrite_links(localized, source, english)
+TEXT.update({
+ "安装与账号准备": "Installation and account setup",
+ "账号与安装 · Web Fetch": "Access & setup · Web Fetch",
+})
+
+TEXT.update({'内容选择': 'Content selection',
+ '典型处理流程': 'Typical processing flow',
+ '变更记录 · Web Fetch': 'Changelog · Web Fetch',
+ '已知 URL → 网页正文、页面资源地址或直接文档文字': 'Known URL → page body, resource URLs, or direct document text',
+ '已知 URL → 可读文本/文档文字': 'Known URL → readable text or document text',
+ '本地浏览器渲染 + Markdown 抽取': 'Local browser rendering + Markdown extraction',
+ '匿名 HTTP provider + Turndown': 'Anonymous HTTP provider + Turndown',
+ '已知 URL → 正文文本': 'Known URL → body text',
+ 'HTTP + BeautifulSoup 网页文本加载器': 'HTTP + BeautifulSoup page-text loader',
+ '已知 URL → 渲染正文、结构化资源 URL 或原文件字节': 'Known URL → rendered body, structured resource URLs, or original file '
+ 'bytes',
+ 'HTTP GET + Readability 正文抽取（关闭 Firecrawl 回退）': 'HTTP GET + Readability body extraction (Firecrawl fallback '
+ 'disabled)',
+ '已知 URL → raw_content': 'Known URL → raw_content',
+ '逐任务结果与证据': 'Per-task results and evidence',
+ '运行参数与版本': 'Run parameters and versions',
+ '本报告纳入 4,300 个逻辑结果。运行参数见下表；保留的物理调用与输出见各结果详情。': 'This report includes 4,300 logical results. Run parameters '
+ 'are listed below; retained physical calls and outputs are '
+ 'available in each result detail.',
+ '以下选项来自调用前冻结的配置。每题调用一次，外层截止时间45秒；产品内部超时保留默认值。调用间隔不计入单次耗时。这里只展示影响抓取与日志的选项，具体请求和按任务选择的输出格式见各案例的调用日志。': 'These '
+ 'options '
+ 'come '
+ 'from '
+ 'configuration '
+ 'frozen '
+ 'before '
+ 'calls. '
+ 'Each '
+ 'task '
+ 'is '
+ 'called '
+ 'once '
+ 'with '
+ 'an '
+ 'outer '
+ 'deadline '
+ 'of 45 '
+ 'seconds; '
+ 'product-internal '
+ 'timeouts '
+ 'keep '
+ 'their '
+ 'defaults. '
+ 'Inter-call '
+ 'intervals '
+ 'are '
+ 'excluded '
+ 'from '
+ 'call '
+ 'latency. '
+ 'Only '
+ 'retrieval '
+ 'and '
+ 'logging '
+ 'options '
+ 'are '
+ 'shown '
+ 'here; '
+ 'see '
+ 'each '
+ 'call '
+ 'log '
+ 'for '
+ 'actual '
+ 'requests '
+ 'and '
+ 'task-dependent '
+ 'output '
+ 'formats.',
+ '固定选项': 'Fixed options',
+ '页面完成条件：': 'Page completion condition:',
+ '阻止访问私网：': 'Block private-network access:',
+ '返回内容上限（字节）：': 'Maximum returned bytes:',
+ '调用间隔（秒）：': 'Inter-call interval (seconds):',
+ '日志级别：': 'Log level:',
+ '完整保存诊断日志：': 'Retain full diagnostic logs:',
+ '日志保存上限（字节）：': 'Maximum retained log bytes:',
+ '使用统一固定原生选项：': 'Use uniform fixed native options:',
+ '域名解析方式：': 'DNS resolution mode:',
+ '缓存最长年龄：': 'Maximum cache age:',
+ '代理选择：': 'Proxy selection:',
+ '请求输出格式：': 'Requested output formats:',
+ '只取主要正文：': 'Main content only:',
+ '提取深度：': 'Extraction depth:',
+ '包含图片：': 'Include images:',
+ '正文详细程度：': 'Text verbosity:',
+ '缓存最长年龄（小时）：': 'Maximum cache age (hours):',
+ '包含资源链接：': 'Include resource links:',
+ '具名链接数量上限：': 'Maximum named links:',
+ '文档大小上限（字节）：': 'Maximum document bytes:',
+ '每次调用都保存了测试程序的调用记录：何时开始、传入什么参数、何时返回，以及是否发生错误。各案例详情中的“调用方事件”展示这些记录。': 'Every call retains harness events: '
+ 'start time, input parameters, return '
+ 'time, and errors. These appear under '
+ 'Caller events in each case detail.',
+ '“产品与依赖库日志”展示产品或底层库实际输出的运行信息。云端服务没有开放的内部日志，不会用测试程序的记录代替。下表说明各方法留下了哪些日志。': 'Product and dependency logs '
+ 'contain runtime information '
+ 'actually emitted by the product '
+ 'or underlying libraries. Harness '
+ 'events do not substitute for '
+ 'unavailable cloud-internal logs. '
+ 'The table lists what was '
+ 'retained for each method.',
+ '已启用产品 debug；日志文件另含调用方记录。': 'Product debug enabled; the log file also contains caller events.',
+ '保留真实请求、返回错误及可用请求标识；未取得云服务内部 debug，后台能力尚未确认。': 'Actual requests, returned errors, and available request IDs '
+ 'retained; cloud-internal debug was not obtained, and '
+ 'backend capabilities remain unconfirmed.',
+ '已启用 debug；实际原生日志可能很少，不能以日志文件存在推断内部过程完整。': 'Debug enabled; native logs may be sparse, and the presence of a '
+ 'log file does not prove complete internal-process coverage.',
+ '保留真实调用、返回和错误；此抓取路径未确认可开启内部 debug。': 'Actual calls, responses, and errors retained; internal debug '
+ 'availability for this retrieval path is unconfirmed.',
+ '保留底层 HTTP 库日志；不等同于完整产品内部执行日志。': 'Underlying HTTP-library logs retained; these are not complete '
+ 'product-internal execution logs.',
+ '启用原生 verbose 运行日志，并捕获进程输出。': 'Native verbose logging enabled, with process output captured.',
+ '完整日志在各案例详情中查看。日志可以帮助定位问题，但没有记录的内部过程不能据此推断；日志文件存在，也不等于产品的全部内部过程都有记录。': 'Full retained logs are available in '
+ 'each case detail. They help locate '
+ 'problems, but unrecorded internal '
+ 'processes cannot be inferred; a log '
+ 'file does not mean every internal '
+ 'process was recorded.',
+ '以下命令从本轮保存的证据构建本地报告，不重新调用产品：': 'These commands build the local report from saved evidence without '
+ 'additional product calls:',
+ '使用与本轮结果对应的证据目录。调用时的版本、配置和评分器哈希见': 'Use the evidence directory corresponding to these results. Call-time '
+ 'versions, configuration, and scorer hashes are in',
+ '调用前冻结协议': 'Pre-call frozen protocol',
+ '。上面的构建命令读取保存的评分；修改评分规则后，应先按声明范围复评分，不能仅重建页面就称成绩已经更新。': '. The build commands read saved scores. After '
+ 'changing scoring rules, rescore the declared scope '
+ 'before rebuilding; rebuilding pages alone does not '
+ 'update scores.',
+ '430 个计划任务中的通过数 / 可判定数；每题等权，无法判定结果不参与分母。': 'Passed / decidable results among 430 planned tasks; each task '
+ 'has equal weight, and undecidable results are excluded from the '
+ 'denominator.',
+ '每种方法均使用同一组 430 个任务，包含通过、未通过、错误和超时。耗时是单次调用从输入到返回的时间，不包含调用之间的统一冷却间隔。每种方法内部按案例依次执行；方法之间的并发安排和限制见调用链检查。耗时仍受本地与远程环境、网络和产品内部限制影响。': 'Each '
+ 'method '
+ 'uses '
+ 'the '
+ 'same '
+ '430 '
+ 'tasks, '
+ 'including '
+ 'passes, '
+ 'failures, '
+ 'errors, '
+ 'and '
+ 'timeouts. '
+ 'Latency '
+ 'runs '
+ 'from '
+ 'input '
+ 'to '
+ 'return '
+ 'and '
+ 'excludes '
+ 'the '
+ 'common '
+ 'cooldown '
+ 'between '
+ 'calls. '
+ 'Cases '
+ 'execute '
+ 'sequentially '
+ 'within '
+ 'each '
+ 'method; '
+ 'inter-method '
+ 'scheduling '
+ 'and '
+ 'limits '
+ 'are '
+ 'documented '
+ 'in '
+ 'the '
+ 'call-chain '
+ 'audit. '
+ 'Local '
+ 'and '
+ 'remote '
+ 'environments, '
+ 'networks, '
+ 'and '
+ 'product '
+ 'limits '
+ 'still '
+ 'affect '
+ 'latency.',
+ '全部调用均值': 'All-call mean',
+ '全部调用 P50': 'All-call P50',
+ '全部调用 P95': 'All-call P95',
+ '请求成功，并同时满足本例最低字符数、全部必需核对点、显著标记及结构要求。验证码或错误提示文字不单独决定任务是否通过。完整度、纯净度和 F1 另算，不作为通过门槛。 已登记正文覆盖规则的案例，还须在最多 5 个参考区域中逐区达到 80% 词语计数覆盖，不能只返回标题或开头。': 'The '
+ 'request '
+ 'must '
+ 'succeed '
+ 'and '
+ 'satisfy '
+ 'the '
+ 'case '
+ 'minimum '
+ 'length, '
+ 'every '
+ 'required '
+ 'checkpoint, '
+ 'significant '
+ 'marker, '
+ 'and '
+ 'structure '
+ 'requirement. '
+ 'Verification '
+ 'or '
+ 'error '
+ 'text '
+ 'does '
+ 'not '
+ 'independently '
+ 'determine '
+ 'acceptance. '
+ 'Completeness, '
+ 'purity, '
+ 'and '
+ 'F1 '
+ 'are '
+ 'separate '
+ 'metrics, '
+ 'not '
+ 'pass '
+ 'thresholds. '
+ 'Cases '
+ 'with '
+ 'body-coverage '
+ 'rules '
+ 'must '
+ 'reach '
+ '80% '
+ 'token-count '
+ 'coverage '
+ 'in '
+ 'each '
+ 'of '
+ 'up '
+ 'to '
+ '5 '
+ 'reference '
+ 'regions; '
+ 'a '
+ 'title '
+ 'or '
+ 'opening '
+ 'alone '
+ 'is '
+ 'insufficient.',
+ '文档正文按请求状态、最低长度与全部内容核对点验收；只拿到下载链接不能通过。验证或错误提示只作诊断。F1另算，不作为通过门槛。 已登记正文覆盖规则的案例，还须在最多 5 个参考区域中逐区达到 80% 词语计数覆盖，不能只返回标题或开头。': 'Document '
+ 'text '
+ 'is '
+ 'accepted '
+ 'by '
+ 'request '
+ 'status, '
+ 'minimum '
+ 'length, '
+ 'and '
+ 'all '
+ 'content '
+ 'checkpoints; '
+ 'a '
+ 'download '
+ 'link '
+ 'alone '
+ 'is '
+ 'insufficient. '
+ 'Verification '
+ 'and '
+ 'error '
+ 'notices '
+ 'are '
+ 'diagnostic '
+ 'only. '
+ 'F1 '
+ 'is '
+ 'separate, '
+ 'not '
+ 'a '
+ 'pass '
+ 'threshold. '
+ 'Cases '
+ 'with '
+ 'body-coverage '
+ 'rules '
+ 'must '
+ 'reach '
+ '80% '
+ 'token-count '
+ 'coverage '
+ 'in '
+ 'each '
+ 'of '
+ 'up '
+ 'to '
+ '5 '
+ 'reference '
+ 'regions; '
+ 'a '
+ 'title '
+ 'or '
+ 'opening '
+ 'alone '
+ 'is '
+ 'insufficient.',
+ '不支持输入类型': 'Unsupported input type',
+ '任务规则未满足': 'Task requirements not met',
+ '取页返回 404': 'Retrieval returned 404',
+ '取页连接失败': 'Retrieval connection failed',
+ '方法安全限制': 'Method safety restriction',
+ '方法禁止跨站跳转': 'Method blocks cross-site redirects',
+ '方法空页规则拦截': 'Method empty-page rule blocked retrieval',
+ '正文提取为空': 'Body extraction empty',
+ '源站要求验证码': 'Source site requires verification',
+ '搜索': 'Search',
+ '搜索案例、方法或原因': 'Search cases, methods, or reasons',
+ '参考标注': 'Reference annotations',
+ '任务验收': 'Task acceptance',
+ '评分口径': 'Scoring scope',
+ '核对请求是否成功，再逐项检查本例要求的稳定文字、正文覆盖与完整条目。验证或拦截提示只作诊断，出现提示不单独导致失败；只有页面外壳而没有合格目标内容仍不通过。': 'Check request success, '
+ 'then validate the '
+ 'required stable text, '
+ 'body coverage, and '
+ 'complete records. '
+ 'Verification or blocking '
+ 'notices are diagnostic '
+ 'only and do not '
+ 'independently cause '
+ 'failure; a page shell '
+ 'without qualifying '
+ 'target content still '
+ 'fails.',
+ '接口返回 → 检查目标内容是否为空 → 核对题目要求的文字和字段 → 检查列表数量、类型与唯一性 → 汇总任务结果。只执行该例适用的检查；失败详情保留具体未满足项。图片与视频沿用请求和交付检查，资源地址或文件字节另按格式核验。': 'Response '
+ '→ '
+ 'check '
+ 'for '
+ 'empty '
+ 'target '
+ 'content '
+ '→ '
+ 'verify '
+ 'required '
+ 'text '
+ 'and '
+ 'fields '
+ '→ '
+ 'check '
+ 'list '
+ 'count, '
+ 'types, '
+ 'and '
+ 'uniqueness '
+ '→ '
+ 'aggregate '
+ 'the '
+ 'task '
+ 'verdict. '
+ 'Only '
+ 'checks '
+ 'applicable '
+ 'to '
+ 'that '
+ 'case '
+ 'run; '
+ 'failure '
+ 'details '
+ 'retain '
+ 'unmet '
+ 'requirements. '
+ 'Images '
+ 'and '
+ 'videos '
+ 'use '
+ 'request '
+ 'and '
+ 'delivery '
+ 'checks, '
+ 'with '
+ 'resource '
+ 'URLs '
+ 'or '
+ 'file '
+ 'bytes '
+ 'validated '
+ 'by '
+ 'format.'})
+REGEX.insert(0, (re.compile(r"^源码提交：(.+)$"), r"Source commit: \1"))
+
+TEXT.update({'本轮修正与检查限制': 'Corrections and review limits'})
+PHRASES.update({"Codex 独立复核": "Codex independent review", "见原始复核材料绑定记录": "see the binding in the original review evidence", "见原始复核材料": "see original review evidence"})
+
+TEXT.update({
+ "请求成功，并同时满足本例最低字符数、全部必需核对点、显著标记及结构要求。验证码或错误提示文字不单独决定任务是否通过。完整度、纯净度和 F1 另算，不作为通过门槛。": "The request must succeed and satisfy minimum length, every required checkpoint, significant marker, and structure requirement. Verification or error notices do not independently determine acceptance. Completeness, purity, and F1 are separate metrics, not pass thresholds.",
+})
+REGEX.insert(0, (re.compile(r"^(\d+) 个计划任务中的通过数 / 可判定数；每题等权，无法判定结果不参与分母。$"), r"Passed / decidable results among \1 planned tasks; every task has equal weight, and undecidable results are excluded from the denominator."))
+
+TEXT.update({
+ "版本优先使用调用时记录的版本号；源码实现使用冻结的提交号。云端服务没有公开后端版本时如实标注。二进制哈希用于确认具体执行文件；事后核验的版本会说明依据。": "Versions use call-time records where available; source implementations use frozen commits. Undisclosed cloud backend versions are stated explicitly. Binary hashes identify the executable, and retrospectively verified versions state their evidence.",
+})
+
def write_english_report(out: Path) -> int:
"""Mirror every Chinese HTML route and return the mirrored route count."""
history_path = out / "report-changes.json"
translations = release_translations(json.loads(history_path.read_text())) if history_path.is_file() else {}
+ from .report_reading import archived_translations
+ translations.update(archived_translations(out))
english_root = out / "en"
if english_root.exists():
shutil.rmtree(english_root)
@@ -3640,12 +4076,21 @@ def write_english_report(out: Path) -> int:
allowed_source_text.update(
case.get("name") for case in dataset.get("cases", []) if case.get("name")
)
+ archived_manifest = out / "archived-reading.json"
+ if archived_manifest.is_file():
+ allowed_source_text.update(json.loads(archived_manifest.read_text()).get("preserved_source_names", []))
sources = sorted(path for path in out.rglob("*.html") if "en" not in path.relative_to(out).parts)
for source_path in sources:
source = PurePosixPath(source_path.relative_to(out).as_posix())
target = out / "en" / Path(str(source))
target.parent.mkdir(parents=True, exist_ok=True)
- localized = localize_html(source_path.read_text(encoding="utf-8"), source, translations=translations)
+ raw=source_path.read_text(encoding="utf-8")
+ if str(source)=='guide/index.html' and (out/'archived-reading.json').is_file():
+ from .report_reading import render_archived_reading
+ shell=re.sub(r'
.*? ', '
',raw,flags=re.S)
+ localized=localize_html(shell,source,translations=translations).replace('',render_archived_reading(out,'en'))
+ else:
+ localized=localize_html(raw,source,translations=translations)
if source in {PurePosixPath("guide/index.html"), PurePosixPath("guide/article.html")}:
untranslated = untranslated_english_narrative(localized, allowed_source_text)
if untranslated:
diff --git a/harness/web_fetch/report_interfaces.py b/harness/web_fetch/report_interfaces.py
index d11938dd9..a5d53c22d 100644
--- a/harness/web_fetch/report_interfaces.py
+++ b/harness/web_fetch/report_interfaces.py
@@ -124,7 +124,9 @@ def fields(rows):return '
'+''.join('
if not bindings:
raise ValueError("Measured native interface requires a saved call log: "+name)
request=json.loads(verify_public_call_log(out,bindings[-1]).read_text())["request"]
- s={**s,"params":[],"request":request,"notes":"此处展示该次公开调用记录保存的请求字段；完整配置和日志见环境与调用详情。产品内部超时以冻结配置和实际请求为准。"}
+ from .methods import method_page_model
+ model=method_page_model(name,name,[x for x in records if x["provider"]==name])
+ s={**s,"entry":model["endpoint"],"params":[],"request":request,"notes":"此处展示该次公开调用记录保存的请求字段；完整配置和日志见环境与调用详情。产品内部超时以冻结配置和实际请求为准。"}
access=HEADLESS_ACCESS.get(name)
if r:
content=contained_path(out,r['artifact_path']).read_text();sample={k:r.get(k) for k in ['ok','status_code','title','final_url']};sample['content_excerpt']=content[:450];sample['content_characters']=len(content)
diff --git a/harness/web_fetch/report_methods.py b/harness/web_fetch/report_methods.py
index 0e7bb7af3..dc4185658 100644
--- a/harness/web_fetch/report_methods.py
+++ b/harness/web_fetch/report_methods.py
@@ -63,7 +63,8 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True)
'输入为任务定义与超时预算，输出为预设的内容或错误。完整输出见该方法的结果详情。
',
'interface-' + name)
return body
- if not authored:
+ native=(out/'dataset.json').is_file() and json.loads((out/'dataset.json').read_text()).get('comparison',{}).get('kind')=='uniform_native_local_measurement'
+ if not authored and not native:
body=hero('本轮方法与接口','以下内容来自本轮结果记录与适配器契约。')
anchors={'method-guide-pricing'}
for name in sorted({row['provider'] for row in records}):
@@ -94,6 +95,11 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True)
intro=[];rows=[];details=''
for name in names:
kind,desc,url=PROFILES.get(name,('测试适配器','本地可重复测试实现。','https://example.com'));r=next(r for r in records if r['provider']==name);info=method_info(r['provider_type'])
+ if native:
+ model=method_page_model(name,name,[row for row in records if row['provider']==name])
+ desc=model['contract']
+ if name=='moli-webfetch':url='https://github.com/lexmount/moli'
+ info={**info,'output':'产品原生返回；保留实际输出与日志','freshness':'缓存与等待选项见本轮固定配置'}
intro.append(('',cell(link('../providers/'+name+'.html',display_name(name)))+cell(kind)+cell(desc)+cell(esc(info['output']))+cell(esc(info['freshness']))+cell(link(url,'文档 / 源码 ↗'))))
for item in counts:
name=item['provider'];n=item['repository_count'];known=n is not None
@@ -118,7 +124,7 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True)
body+=section('参评方法',cards([('远程正文接口','Kitesurf 使用 Cloudflare 账户级 Browser Run /markdown API；耗时包含公网往返、远端排队与执行。'),('本地原生产物','Lightpanda、Obscura 与 Moli 1.1.1 的普通网页使用 CLI 正文输出。Moli 的直接文档和媒体 URL 使用裸 fetch：正文任务在预算内解析，文件任务校验字节。'),('本地 DOM 转换','Chromium 152 由 Playwright 取完整渲染 DOM，再用 node-html-markdown 2.0.0 转换。五种方法都不调用模型。')]))
body+=section('Kitesurf 能否本地部署',cards([('当前结论','截至 2026-09-07，Cloudflare 尚未发布 Kitesurf 源码或自行部署包；本轮只能通过 Cloudflare 账号版 Browser Run 远程测试，不能把它当作本地开源浏览器。'),('后续意向','Cloudflare 官方博客表示计划在准备好后开源，并希望允许客户部署到自己的账户。这是未来计划，不是当前已有能力。')])+'当前官方接入说明 · 官方开源计划说明
')
else:
- body+=section('参评方法',cards([('托管 API','Lexmount 两个区域、Firecrawl、Tavily、Exa、Cloudflare；通过各自 API 调用。'),('项目内置','CowAgent、OpenClaw、DeepSeek Harness；按本轮保留的实现与配置执行。'),('本地运行','LangChain WebBaseLoader、Crawl4AI、Moli WebFetch；使用本机网络与运行环境。')]))
+ body+=section('参评方法',cards([('托管 API','Firecrawl、Tavily、Exa、Cloudflare；通过各自 API 调用。'),('项目内置','CowAgent、OpenClaw、DeepSeek Harness；按本轮保留的实现与配置执行。'),('本地运行','LangChain WebBaseLoader、Crawl4AI、Moli WebFetch；使用本机网络与运行环境。')]))
body+=section('能力、价格与接口',tabs('method-guide',[
('capabilities','官方能力',render_capabilities(out,names,table=table,cell=cell,link=link)),
('pricing','价格与计费',render_pricing(out,names,table=table,cell=cell,link=link,esc=esc)),
diff --git a/harness/web_fetch/report_reading.py b/harness/web_fetch/report_reading.py
new file mode 100644
index 000000000..011ef057d
--- /dev/null
+++ b/harness/web_fetch/report_reading.py
@@ -0,0 +1,64 @@
+"""An explicitly supplied historical article, separate from current measurements."""
+import hashlib
+import json
+import re
+import shutil
+from pathlib import Path
+
+from .safety import contained_path
+
+
+def copy_archived_reading(source: Path, out: Path) -> None:
+ manifest = source / "archived-reading.json"
+ if not manifest.is_file():
+ return
+ data = json.loads(manifest.read_text())
+ for item in [*data["languages"].values(), *([data["translations"]] if data.get("translations") else [])]:
+ origin = contained_path(source, item["path"])
+ if origin.is_symlink() or hashlib.sha256(origin.read_bytes()).hexdigest() != item["sha256"]:
+ raise ValueError("Archived article content differs from its manifest")
+ target = contained_path(out, item["path"])
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(origin, target)
+ shutil.copy2(manifest, out / "archived-reading.json")
+
+
+def render_archived_reading(out: Path, language: str = "zh") -> str | None:
+ manifest = out / "archived-reading.json"
+ if not manifest.is_file():
+ return None
+ data = json.loads(manifest.read_text())
+ item = data["languages"][language]
+ raw = contained_path(out, item["path"]).read_bytes()
+ if hashlib.sha256(raw).hexdigest() != item["sha256"]:
+ raise ValueError("Archived article hash mismatch")
+ text = raw.decode()
+ body = re.search(r"]*>(.*?)", text, re.S)
+ if not body:
+ raise ValueError("Archived article has no main content")
+ styles = "".join(re.findall(r"]*>.*?", text, re.S))
+ date = data.get("source_data_date")
+ if language == "zh":
+ date_note = f"（截至{date}）" if date else ""
+ note = f"本文沿用旧报告文章与原有数据{date_note}，不代表本轮评测结果。"
+ else:
+ date_note = f" through {date}" if date else ""
+ note = f"This article retains the older report and its data{date_note}; it does not describe the current benchmark results."
+ return (styles + '' + body[1] + '
')
+
+
+def archived_translations(out: Path) -> dict[str, str]:
+ manifest = out / "archived-reading.json"
+ if not manifest.is_file():
+ return {}
+ item = json.loads(manifest.read_text()).get("translations")
+ if not item:
+ return {}
+ raw = contained_path(out, item["path"], reject_symlinks=True).read_bytes()
+ if hashlib.sha256(raw).hexdigest() != item["sha256"]:
+ raise ValueError("Archived translations hash mismatch")
+ text = json.loads(raw)["text"]
+ if not isinstance(text, dict) or any(not isinstance(k, str) or not isinstance(v, str) for k, v in text.items()):
+ raise ValueError("Report translations require text pairs")
+ return text
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index 5a9509359..74718a834 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -460,6 +460,8 @@ def run_evaluation(
provider_concurrency: int = 1,
max_failure_retries: int = 0,
resource_sample_interval_ms: int | None = None,
+ reference_policy: str = "fresh",
+ case_admission: Any | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
output_dir = output_dir.resolve()
for name, value, minimum in (
@@ -496,7 +498,17 @@ def run_evaluation(
for filename in ['artifacts', PUBLIC_CALL_LOG_ROOT, 'manifest.json', 'dataset.json',
'results.jsonl', 'summary.json']:
contained_path(output_dir, filename)
- reference_checks = preflight_reference_checks(cases, dataset_path)
+ if reference_policy not in {"fresh", "frozen_existing"}:
+ raise ValueError("unknown reference policy")
+ if reference_policy == "frozen_existing":
+ # Explicit local measurement of existing rules; no live-source claim.
+ from .reference_lifecycle import validate_resource_source_evidence
+ for case in cases:
+ validate_resource_source_evidence(case, dataset_path)
+ frozen_dataset_sha256 = hashlib.sha256(read_dataset_bytes(dataset_path)).hexdigest()
+ reference_checks = None
+ else:
+ reference_checks = preflight_reference_checks(cases, dataset_path)
for provider in providers:
missing = provider.validate()
if missing:
@@ -524,6 +536,7 @@ def run_evaluation(
"network_preflight": ({"dns_mode": network_audit["dns_mode"],
"sha256": hashlib.sha256((output_dir / 'network-preflight.json').read_bytes()).hexdigest()}
if network_audit is not None else None),
+ "reference_policy": reference_policy,
"unexecuted": [],
"deadline_policy": "end_to_end_answer; non_cancellable_adapter_quarantined",
"public_call_log_schema_version": 1,
@@ -581,6 +594,11 @@ def run_evaluation(
json.dumps(dataset_meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
+ if case_admission is not None:
+ manifest["scheduling"] = case_admission.describe()
+ (output_dir / "manifest.json").write_text(
+ json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
results_path = output_dir / "results.jsonl"
result_lock = threading.Lock()
with results_path.open("w", encoding="utf-8") as results_file:
@@ -598,7 +616,14 @@ def run_provider(provider: Provider) -> None:
binding = reference_checks["cases"].get(case.id) if reference_checks else None
# Outside provider exception handling: expired references
# stop measurement, never create a supplier failure record.
- check_case_reference_fresh(case, dataset_path, binding)
+ if reference_policy == "frozen_existing":
+ if hashlib.sha256(read_dataset_bytes(dataset_path)).hexdigest() != frozen_dataset_sha256:
+ raise ReferenceLifecycleError("Frozen rules changed during the run", "reference_changed")
+ else:
+ check_case_reference_fresh(case, dataset_path, binding)
+ # Admission waits before the answer timer and 45-second budget.
+ admission_token = (case_admission.acquire(provider, case)
+ if case_admission is not None else None)
started = utc_now()
physical_attempts = []
stopped_provider = False
@@ -783,6 +808,8 @@ def run_provider(provider: Provider) -> None:
)
if delay:
time.sleep(delay)
+ if case_admission is not None:
+ case_admission.release(admission_token, pending=pending, record=record)
if stopped_provider:
remaining_attempts = ((runs - run_number) * len(cases)
+ len(cases) - case_index)
diff --git a/harness/web_fetch/scoring_regressions.py b/harness/web_fetch/scoring_regressions.py
index 16f71c9af..0da2f00c3 100644
--- a/harness/web_fetch/scoring_regressions.py
+++ b/harness/web_fetch/scoring_regressions.py
@@ -30,7 +30,14 @@ def load_scoring_regressions(path: Path | None = None) -> dict[str, dict[str, An
if document.get("schema_version") != 1 or not isinstance(document.get("cases"), list):
raise ValueError("Unsupported scoring regression registry")
rows: dict[str, dict[str, Any]] = {}
+ revisions = document.get("control_revisions", {}).get("overrides", {})
+ if not isinstance(revisions, dict) or set(revisions) - {row.get("id") for row in document["cases"]}:
+ raise ValueError("Control revisions must name existing frozen controls")
for index, row in enumerate(document["cases"], 1):
+ revision = revisions.get(row.get("id"), {})
+ if not isinstance(revision, dict) or set(revision) - {"case", "expected_task_pass", "expected_metrics"}:
+ raise ValueError("Control revisions may only update synthetic inputs and expectations")
+ row = {**row, **revision}
identifier = row.get("id")
if not isinstance(identifier, str) or not identifier.strip() or identifier in rows:
raise ValueError(f"Scoring regression {index} requires a unique nonempty id")
diff --git a/harness/web_fetch/stable_tasks.py b/harness/web_fetch/stable_tasks.py
index be0ea4661..edf97ed06 100644
--- a/harness/web_fetch/stable_tasks.py
+++ b/harness/web_fetch/stable_tasks.py
@@ -133,11 +133,30 @@ def _unwrap_links(text, schema):
def _visible(text, schema=None):
+ # Structured text records use the same visible HTML body as body anchors.
+ # Link and table schemas retain their existing DOM ownership parsers.
+ projected_html = (schema in SCHEMA_FIELDS and schema not in {"named_link", "linked_product", "table_row", "body_text", "page_identity"}
+ and re.match(r'\s*(?:',
+ lambda m: m.group(0) if m.group(1) else " ", text,
+ flags=re.I | re.S)
# Decode one character-reference layer, as a browser does for both HTML
# and Markdown. In particular, ``&`` is the visible ``&`` while an
# intentionally escaped ``&`` remains the literal text ``&``.
- text = unescape(text)
+ if not projected_html:
+ text = unescape(text)
# Extractors represent Markdown hard breaks using one or two backslashes.
text = re.sub(r"\\{1,2}[ \t]*(?=\n)", "", text)
text = _unwrap_links(text, schema)
@@ -423,6 +442,22 @@ def _operations(lines):
return records
+def _closed_title_descriptions(text):
+ """A repeated visible title closes its intervening description span."""
+ paragraphs = [block.strip() for block in re.split(r"\n\s*\n", text) if block.strip()]
+ records = []
+ for first, description, repeated in zip(paragraphs, paragraphs[1:], paragraphs[2:]):
+ if any("\n" in value for value in (first, description, repeated)):
+ continue
+ name = _line(first)
+ if _key(name) != _key(_line(repeated)):
+ continue
+ record = _record({"name": name, "description": _line(description)}, "name_description")
+ if record["valid"]:
+ records.append(record)
+ return records
+
+
def _descriptions(text):
records = []
if _CARD_BOUNDARY in text:
@@ -485,6 +520,16 @@ def _earthquakes(text):
# line. Split only a complete trailing timestamp, within this card.
separated = []
for line in lines:
+ compact = re.fullmatch(
+ r"(.+?)(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}"
+ r"(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}|\s*\(UTC(?:[+-]\d{2}:\d{2})?\))?)"
+ r"\s+(-?\d+(?:\.\d+)?\s*km)", line, re.I,
+ )
+ if (compact and _text(compact[1])
+ and _valid_field("time", compact[2])
+ and _valid_field("depth", compact[3])):
+ separated.extend((compact[1].strip(), compact[2], compact[3]))
+ continue
joined = re.fullmatch(
r"(.+?)(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}"
r"(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}|\s*\(UTC(?:[+-]\d{2}:\d{2})?\))?)",
@@ -529,7 +574,19 @@ def _parse_records(visible, schema):
for section in sections:
records.extend(_operations([_line(line) for line in section.splitlines() if line.strip()]))
elif schema == "name_description":
- records.extend(_descriptions(remainder))
+ closed = _closed_title_descriptions(remainder)
+ weak = _descriptions(remainder)
+ for record in weak:
+ description = _key(record["fields"].get("description", ""))
+ name = _key(record["fields"].get("name", ""))
+ if any(name != _key(owner["fields"]["name"])
+ and _key(owner["fields"]["name"]) in description
+ and _key(owner["fields"]["description"]) in description
+ for owner in closed):
+ record["valid"] = False
+ record["errors"].append("description_contains_separate_owned_record")
+ records.extend(weak)
+ records.extend(closed)
elif schema == "taxonomy_family":
records.extend(_record({"family": _field(line, "family")}, schema) for line in lines
if re.fullmatch(r"(?:family\s*:\s*)?[A-Za-z]+idae", line, re.I))
@@ -572,14 +629,54 @@ def is_record(url):
return None
-def _anchor_matches(anchor, records, visible, schema, non_record_visible=None,
- anchor_visible=None):
+def _stable_literal_match(surface, value, registered_words=()):
+ """An atomic Latin label is a word, not part of another word.
+
+ Native camel boundaries and fully covered chains of registered labels
+ preserve adjacent UI text without accepting arbitrary word fragments.
+ Longer declared text retains the existing exact-phrase containment rule.
+ """
from .metrics import fact_in_text, normalize_for_match
+ expected = normalize_for_match(_visible(value))
+ folded = normalize_for_match(surface)
+ if not fact_in_text(folded, expected):
+ return False
+ if not re.fullmatch(r"[A-Za-z]+", value):
+ return True
+ boundary = lambda needle, text: bool(re.search(
+ r"(? None:
@@ -869,8 +968,6 @@ def evaluate_stable_task(content: str, contract: dict, base_url: str = "", eligi
content_errors = (["page_identity_mismatch"] if not all(r["pass"] for r in page_results) else [])
from .reference_quality import blocked_page_hits
blocked_page = bool(blocked_page_hits(text_surface))
- if blocked_page:
- content_errors.append("blocked_page")
errors = []
if schema in {"named_link", "linked_product"}:
link_diagnostics = []
@@ -929,9 +1026,12 @@ def evaluate_stable_task(content: str, contract: dict, base_url: str = "", eligi
if schema == 'named_link' and any(a.get('match_scope') == 'non_record_lines'
for a in contract.get('stable_text_anchors', [])):
non_record_visible = _non_record_visible(content, contract, records, base_url)
+ registered_words = tuple(value for anchor in contract.get("stable_text_anchors", [])
+ for value in anchor["alternatives"]
+ if re.fullmatch(r"[A-Za-z]+", value))
for anchor in contract.get("stable_text_anchors", []):
matched = _anchor_matches(anchor, records, visible, schema,
- non_record_visible, anchor_visible)
+ non_record_visible, anchor_visible, registered_words)
anchors.append({**anchor, "matched": matched, "pass": bool(matched)})
content_errors += ["missing_stable_anchor:" + r["id"] for r in anchors if not r["pass"]]
complete = [r for r in records if r["valid"]]
diff --git a/harness/web_fetch/structured_tables.py b/harness/web_fetch/structured_tables.py
index 0d78c3a07..2563525fc 100644
--- a/harness/web_fetch/structured_tables.py
+++ b/harness/web_fetch/structured_tables.py
@@ -178,6 +178,8 @@ def parse_table_records(content, visible, contract):
columns, scope = contract['table_columns'], contract['table_scope']
names = [c['name'] for c in columns]
candidates = []
+ aliases = {c['label'].casefold(): c['name'] for c in columns}
+ aliases.update({n.casefold(): n for n in names})
document_pattern = r'\s*(?:]*>\s*)?<(?:html|body|main|div|table|section|article|ul|ol)\b'
if not re.match(document_pattern, content, re.I):
content = _without_example_fences(content)
@@ -247,7 +249,7 @@ def parse_table_records(content, visible, contract):
if ('|' in line and len(cells) == len(separators)
and all(re.fullmatch(r'\s*:?-{3,}:?\s*', cell)
for cell in separators)
- and any(re.fullmatch(scope['start_pattern'], cell.strip())
+ and any(re.fullmatch(scope['start_pattern'], _header_label(cell, aliases))
for cell in cells)):
break
offset += len(line)
@@ -265,14 +267,23 @@ def capture(match):
return '\n' * match.group().count('\n')
text = re.sub(scope['plain_multiline_row_pattern'], capture, text)
lines = text.splitlines()
- aliases = {c['label'].casefold(): c['name'] for c in columns}
- aliases.update({n.casefold(): n for n in names})
occupied = set()
table_count = 0
for i in range(len(lines) - 1):
if i in occupied or '|' not in lines[i]:
continue
- labels = [s.strip().casefold() for s in _pipe_cells(lines[i])]
+ labels = [_header_label(s, aliases).casefold() for s in _pipe_cells(lines[i])]
+ first_row = i + 2
+ # Some native conversions split a sticky header from its body:
+ # an empty header/separator is followed by the complete labels.
+ # Require that explicit label row; never inherit prior columns.
+ if not any(labels) and first_row < len(lines):
+ recovered = [_header_label(s, aliases).casefold()
+ for s in _pipe_cells(lines[first_row])]
+ mapped = [aliases.get(label) for label in recovered]
+ if len(mapped) == len(names) and set(mapped) == set(names):
+ labels = recovered
+ first_row += 1
ignored = {v.casefold() for v in scope.get('ignored_column_labels', [])}
selected = [j for j, label in enumerate(labels) if label not in ignored]
headings = [aliases.get(labels[j]) for j in selected]
@@ -281,7 +292,7 @@ def capture(match):
or len(separators) != len(labels)
or not all(re.fullmatch(r'\s*:?-{3,}:?\s*', s) for s in separators)):
continue
- j = i + 2
+ j = first_row
while j < len(lines) and '|' in lines[j]:
values = [v.strip() for v in _pipe_cells(lines[j])]
if len(values) == len(labels):
@@ -308,6 +319,12 @@ def capture(match):
return records
+def _header_label(value, aliases):
+ value = value.strip()
+ # Sorting icons decorate a declared column; they are not part of its name.
+ undecorated = re.sub(r'[\s↑↓↕⇅▲▼▴▾]+$', '', value)
+ return undecorated if undecorated.casefold() in aliases else value
+
def _pipe_cells(line):
# Remove only the table's outer delimiters; adjacent pipes are empty cells.
return line.strip().removeprefix('|').removesuffix('|').split('|')
diff --git a/harness/web_fetch/workers/cowagent_worker.py b/harness/web_fetch/workers/cowagent_worker.py
index 803ab7a46..132212746 100644
--- a/harness/web_fetch/workers/cowagent_worker.py
+++ b/harness/web_fetch/workers/cowagent_worker.py
@@ -17,6 +17,12 @@ def main():
sys.path.insert(0, str(root))
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode
+ from harness.web_fetch.diagnostics import diagnostic_context, redact
+
+ class NativeLogSink:
+ def emit(self, source, event, **fields):
+ print(json.dumps(redact({"source": source, "event": event, **fields}),
+ ensure_ascii=False, default=str), file=sys.stderr, flush=True)
module = None
for line in sys.stdin:
if not line.strip():
@@ -34,7 +40,10 @@ def main():
module, 'DEFAULT_TIMEOUT', None)
tool = module.WebFetch({'cwd': str(workspace)})
dns_mode = validate_dns_mode(request.get('dns_mode', 'system'))
- with public_dns_context(dns_mode, request['timeout_seconds']) as dns_trace:
+ with diagnostic_context(
+ NativeLogSink() if request.get('log_level') == 'debug' else None,
+ names=('log', 'urllib3', 'requests', 'langchain_community'),
+ ), public_dns_context(dns_mode, request['timeout_seconds']) as dns_trace:
result = tool.execute({'url': request['url']})
metadata.update(dns_mode=dns_mode, dns_resolutions=dns_trace,
effective_timeout_seconds=metadata['cowagent_default_timeout_seconds'],
diff --git a/harness/web_fetch/workers/crawl4ai_worker.py b/harness/web_fetch/workers/crawl4ai_worker.py
index 1184089cc..5887b7485 100755
--- a/harness/web_fetch/workers/crawl4ai_worker.py
+++ b/harness/web_fetch/workers/crawl4ai_worker.py
@@ -166,7 +166,7 @@ async def fetch_file(request, directory):
) as crawler:
result = await crawler.crawl(
request["url"],
- config=CrawlerRunConfig(page_timeout=int(request["timeout_seconds"] * 1000)),
+ config=CrawlerRunConfig(**({} if request.get("uniform_native_parameters") else {"page_timeout": int(request["timeout_seconds"] * 1000)})),
)
return {
"ok": 200 <= result.status_code < 300,
@@ -192,9 +192,9 @@ async def _run_pdf_document(request):
url=request["url"],
config=CrawlerRunConfig(
cache_mode=CacheMode.BYPASS,
- page_timeout=int(timeout_seconds * 1000),
+ **({} if request.get("uniform_native_parameters") else {"page_timeout": int(timeout_seconds * 1000)}),
scraping_strategy=scraping_strategy,
- verbose=False,
+ verbose=request.get("log_level") == "debug",
),
)
markdown = result.markdown.raw_markdown if result.markdown else ""
@@ -408,7 +408,7 @@ async def main() -> None:
"ok": bool(result.success),
# Preserve HTML, canonical Markdown, headers and native errors
# privately; answer selection remains raw_markdown.
- "raw_tool_response": result.model_dump(mode="json"),
+ "raw_tool_response": result.model_dump(mode="json", fallback=str),
"content": markdown,
"files": files,
"status_code": result.redirected_status_code if result.redirected_status_code is not None else result.status_code,
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
new file mode 100644
index 000000000..1b60b5573
--- /dev/null
+++ b/research/native-430-local-report.json
@@ -0,0 +1,38 @@
+{
+ "schema_version": 1,
+ "comparison_kind": "uniform_native_local_measurement",
+ "case_count": 430,
+ "result_count": 4300,
+ "providers": [
+ "cloudflare-markdown",
+ "cowagent-built-in",
+ "crawl4ai",
+ "deepseek-harness-built-in",
+ "exa",
+ "firecrawl",
+ "langchain-webbase",
+ "moli-webfetch",
+ "openclaw-built-in",
+ "tavily"
+ ],
+ "frozen_dataset_sha256": "2745cb80a1941fe1931e46e23de174437194984ea838d9cdd1f02b9b081df030",
+ "results_sha256": "8feaeac89b7edc52961103d16c6be4f42310a5de715ec2dd29aa2f7c7cb1a0cc",
+ "measured_scoring_source_sha256": "4ce40cdf2ecb1af4088915a2b869ffbdcb65dce5222d1429578e97d175f13bfc",
+ "evidence_state": "local_measurement",
+ "certified": false,
+ "evidence_storage": "External data-home snapshot; provider calls, review payloads and generated pages are not stored in Git.",
+ "evidence_bindings": [
+ {
+ "path": "local-run-contract.json",
+ "sha256": "350b59d55e872b9b8405a8718b80b83bbe95891a9517d10ef1b89a1c90ccad17"
+ },
+ {
+ "path": "result-review.json",
+ "sha256": "733996bb2b05650e832b99b8d28b2489c8a72eba2fe60a645f94c5b2d90cdb96"
+ },
+ {
+ "path": "archived-reading.json",
+ "sha256": "85da5e52c295eb7fffbcd3e10e41d33294eeadf233ec7a6958de68ed0c9e30fb"
+ }
+ ]
+}
diff --git a/scripts/build_report.py b/scripts/build_report.py
index 484c365b7..bc3cbede8 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -26,6 +26,8 @@
help='refresh an existing local report using a hash-bound current-local-results.json; preserve measured evidence')
parser.add_argument('--local-audit',action='store_true',
help='build an explicitly local, non-publishable audit report from bound evidence')
+ parser.add_argument('--hide-local-status-banners', action='store_true',
+ help='omit repeated top status banners from local audit pages; retain evidence and disclosures')
parser.add_argument('--diagnostic-cohorts',action='store_true',
help='render an explicitly diagnostic cohort model with the maintained WebFetch template')
parser.add_argument('--wcxb-source',type=Path)
@@ -33,6 +35,8 @@
parser.add_argument('--wcxb',action='store_true',help='Build the independently bound WCXB report')
parser.add_argument('--webmainbench-full',action='store_true',help='Build the separately certified complete 545-case report')
args=parser.parse_args()
+ if args.hide_local_status_banners and not args.local_audit:
+ parser.error('--hide-local-status-banners requires --local-audit')
from harness.reporting.templates import resolve_template
resolve_template('webfetch')
if args.refresh_presentation:
@@ -68,7 +72,8 @@ def requested_language(output, supported=SUPPORTED_REPORT_LANGUAGES):
evidence=require_evidence(args.evidence)
print(build_report(evidence,args.output,
include_english=args.include_english, language=requested_language(args.output),
- local_audit=True))
+ local_audit=True,
+ show_local_status_banners=not args.hide_local_status_banners))
raise SystemExit(0)
if sum(bool(value) for value in (args.profile, args.wcxb, args.webmainbench_full)) > 1:
parser.error('--profile, --wcxb and --webmainbench-full are mutually exclusive')
diff --git a/tests/test_case_admission.py b/tests/test_case_admission.py
new file mode 100644
index 000000000..7edbce459
--- /dev/null
+++ b/tests/test_case_admission.py
@@ -0,0 +1,50 @@
+import json, threading, time
+from pathlib import Path
+from types import SimpleNamespace
+from harness.web_fetch.admission import CaseAdmission
+from harness.web_fetch.models import EvalCase, ProviderResult
+from harness.web_fetch.providers.base import Provider
+from harness.web_fetch.runner import run_evaluation
+
+def method(name):return SimpleNamespace(name=name)
+def case(host):return SimpleNamespace(id=host,url='https://'+host+'/')
+
+def test_same_site_waits_and_distinct_cloud_local_can_overlap(tmp_path):
+ local=CaseAdmission(tmp_path,group='local',site_cooldown=0)
+ cloud=CaseAdmission(tmp_path,group='cloud',site_cooldown=0)
+ a=local.acquire(method('a'),case('www.example.com'));seen=[]
+ t=threading.Thread(target=lambda:seen.append(cloud.acquire(method('b'),case('api.example.com'))));t.start();time.sleep(.15);assert not seen
+ local.release(a);t.join(1);assert seen;cloud.release(seen[0])
+ a=local.acquire(method('a'),case('one.test'));b=cloud.acquire(method('b'),case('two.test'));state=json.loads((tmp_path/'state.json').read_text());assert len(state['active'])==2;local.release(a);cloud.release(b)
+
+def test_second_local_waits_and_parallel_timeout_reduces_to_serial(tmp_path):
+ local=CaseAdmission(tmp_path,group='local',site_cooldown=0);cloud=CaseAdmission(tmp_path,group='cloud',site_cooldown=0)
+ a=local.acquire(method('a'),case('one.test'));seen=[];t=threading.Thread(target=lambda:seen.append(local.acquire(method('b'),case('two.test'))));t.start();time.sleep(.15);assert not seen
+ c=cloud.acquire(method('c'),case('three.test'));cloud.release(c,record={'error_type':'timeout','attempt_id':'c'});assert not json.loads((tmp_path/'state.json').read_text())['parallel_enabled'];local.release(a);t.join(1);assert seen;local.release(seen[0])
+
+def test_pending_call_keeps_site_and_resource_reserved(tmp_path):
+ local=CaseAdmission(tmp_path,group='local',site_cooldown=0);a=local.acquire(method('a'),case('one.test'));local.release(a,pending=SimpleNamespace(is_alive=lambda:True));state=json.loads((tmp_path/'state.json').read_text());assert state['active'][a['key']]['quarantined'];assert a['key'] in state['active']
+
+def test_admission_wait_excluded_from_answer_timeout_and_latency(tmp_path):
+ class Fixture(Provider):
+ def fetch(self,case,timeout):return ProviderResult(True,'Expected')
+ class Gate:
+ def describe(self):return {'fixture':True}
+ def acquire(self,provider,case):time.sleep(.3);return 'token'
+ def release(self,token,**kwargs):assert token=='token'
+ p=Fixture('fixture',{'type':'fixture'});c=EvalCase('case','Case','https://example.com','static',['Expected'])
+ start=time.monotonic();records,_=run_evaluation(output_dir=tmp_path/'run',dataset_meta={'id':'test'},dataset_path=Path('dataset.json'),cases=[c],providers=[p],provider_config_path=Path('providers.json'),runs=1,timeout=.1,delay=0,case_admission=Gate());assert time.monotonic()-start>=.3;assert records[0]['ok'];assert records[0]['elapsed_ms']<100;assert records[0].get('error_type') is None
+
+
+def test_same_method_interval_is_not_replaced_by_site_cooldown(tmp_path):
+ gate = CaseAdmission(tmp_path, group='local', site_cooldown=.3)
+ a = gate.acquire(method('a'), case('one.test'))
+ gate.release(a)
+ begin = time.monotonic()
+ a = gate.acquire(method('a'), case('one.test'))
+ assert time.monotonic()-begin < .1
+ gate.release(a)
+ begin = time.monotonic()
+ b = gate.acquire(method('b'), case('one.test'))
+ assert time.monotonic()-begin >= .2
+ gate.release(b)
diff --git a/tests/test_dynamic_100_moli_119.py b/tests/test_dynamic_100_moli_119.py
index 0c3963382..fc1df4a33 100644
--- a/tests/test_dynamic_100_moli_119.py
+++ b/tests/test_dynamic_100_moli_119.py
@@ -7,7 +7,7 @@
from harness.web_fetch import local_audit
from harness.web_fetch.reference_inputs import input_tree
from harness.web_fetch.report import local_audit_banner, provider_configuration
-from harness.web_fetch.scoring import SCORING_SOURCE_SHA256
+from harness.web_fetch.storage import data_home
from harness.web_fetch.scoring_regressions import load_scoring_regressions
from harness.web_fetch.stable_tasks import validate_acceptance_contract
from tests.conftest import read_frozen_dataset_bytes
@@ -23,7 +23,7 @@ def _dataset():
return json.loads(read_frozen_dataset_bytes(DATASET_PATH).decode("utf-8"))
-def test_dynamic_100_dataset_matches_the_frozen_local_audit():
+def test_dynamic_100_dataset_matches_the_frozen_local_audit(pytestconfig):
dataset = _dataset()
cases = dataset["cases"]
receipt = json.loads(RECEIPT_PATH.read_text(encoding="utf-8"))
@@ -71,9 +71,27 @@ def test_dynamic_100_dataset_matches_the_frozen_local_audit():
assert set(scorer_revision['regression_ids']) <= set(followup['regression_ids'])
scorer_revision = {**scorer_revision, **followup}
assert scorer_revision["main_baseline_sha256"] == compatibility["bindings"]["scoring_source_sha256"]
- assert scorer_revision["revised_sha256"] == SCORING_SOURCE_SHA256
+ # Validate the frozen audit against its actual source bytes, not a later
+ # implementation. Keep historical snapshots external and read-only.
+ scorer_files = ("scoring.py", "metrics.py", "stable_tasks.py", "dynamic_lists.py",
+ "list_verification.py", "reference_quality.py", "dynamic_entry_matching.py",
+ "response_input.py", "body_coverage.py", "structured_tables.py",
+ "ordered_tree.py", "composite_tasks.py")
+ snapshot = data_home() / "scoring-snapshots" / scorer_revision["revised_sha256"]
+ required = ["harness/web_fetch/" + name for name in scorer_files]
+ required += ["evaluation/scoring-regressions.json", "tests/test_composite_tasks.py", "tests/test_stable_tasks.py"]
+ missing = [name for name in required if not (snapshot / name).is_file()]
+ if missing:
+ message = "Requires frozen historical scorer inputs: " + str(snapshot)
+ if pytestconfig.getoption("--require-reference-inputs"):
+ pytest.fail(message)
+ pytest.skip(message)
+ assert not snapshot.is_symlink()
+ assert not any((snapshot / name).is_symlink() for name in required)
+ assert scorer_revision["revised_sha256"] == hashlib.sha256(b"\0".join(
+ (snapshot / "harness/web_fetch" / name).read_bytes() for name in scorer_files)).hexdigest()
assert scorer_revision["scoring_regressions_sha256"] == hashlib.sha256(
- (ROOT / "evaluation/scoring-regressions.json").read_bytes()).hexdigest()
+ (snapshot / "evaluation/scoring-regressions.json").read_bytes()).hexdigest()
assert set(scorer_revision["regression_ids"]) <= set(controls)
modes = {}
for identifier in scorer_revision["regression_ids"]:
@@ -109,10 +127,10 @@ def test_dynamic_100_dataset_matches_the_frozen_local_audit():
assert bindings["scoring_source_sha256"] == scorer_revision["main_baseline_sha256"]
assert bindings["scoring_regressions_sha256"] != scorer_revision["scoring_regressions_sha256"]
assert bindings["composite_tests_sha256"] == hashlib.sha256(
- (ROOT / "tests/test_composite_tasks.py").read_bytes()
+ (snapshot / "tests/test_composite_tasks.py").read_bytes()
).hexdigest()
assert bindings["stable_tests_sha256"] == hashlib.sha256(
- (ROOT / "tests/test_stable_tasks.py").read_bytes()
+ (snapshot / "tests/test_stable_tasks.py").read_bytes()
).hexdigest()
assert "Moli 1.1.9 local audit complete" in dataset["approval_status"]
assert dataset["comparison"]["local_audit_receipt"] == RECEIPT_PATH.relative_to(ROOT).as_posix()
diff --git a/tests/test_failure_diagnosis.py b/tests/test_failure_diagnosis.py
index 038b1852e..cd5f4cfc0 100644
--- a/tests/test_failure_diagnosis.py
+++ b/tests/test_failure_diagnosis.py
@@ -99,56 +99,33 @@ def assert_category(self, metrics, expected, **case_fields):
self.assertEqual((record, case), before)
return result
- def test_blocked_page_with_enough_links_is_not_record_shortfall(self):
+ def test_prompt_diagnostics_do_not_replace_missing_target_reason(self):
contract = self.contract(page_patterns=['Catalog'], stable_text_anchors=[
{'id': 'heading', 'alternatives': ['Browse products']}])
metrics = evaluate_stable_task('Access denied\n' + self.links, contract, self.base_url)
self.assertTrue(metrics['stable_structure_pass'])
- self.assertEqual(metrics['stable_unique_record_count'], 3)
self.assertTrue(metrics['stable_blocked_page'])
- self.assertIn('blocked_page', metrics['stable_task_errors'])
- self.assertNotIn('too_few_complete_records', metrics['stable_task_errors'])
- result = self.assert_category(metrics, '返回非目标内容', acceptance_contract=contract)
- self.assertIn('blocked_page', result['evidence'])
- self.assertNotIn('条目不足', result['evidence'])
- # Both persisted representations are understood independently.
- without_flag = deepcopy(metrics)
- without_flag.pop('stable_blocked_page')
- self.assert_category(without_flag, '返回非目标内容')
- without_code = deepcopy(metrics)
- without_code['stable_task_errors'].remove('blocked_page')
- self.assert_category(without_code, '返回非目标内容')
+ self.assertNotIn('blocked_page', metrics['stable_task_errors'])
+ result = diagnose(self.record(metrics), {'task_kind': 'body', 'acceptance_contract': contract}, '')
+ self.assertIsNotNone(result)
+ self.assertNotIn('blocked_page', result['evidence'])
- def test_forbidden_and_blocked_page_have_same_semantics(self):
- metrics = evaluate_content('Access denied', [], forbidden_any=['Access denied'])
- self.assert_category(metrics, '返回非目标内容')
- stable = evaluate_content('Access denied\n' + self.links, [],
- forbidden_any=['Access denied'], acceptance_contract=self.contract(), base_url=self.base_url)
- self.assert_category(stable, '返回非目标内容')
- self.assert_category({'stable_task_pass': False, 'stable_task_errors': ['forbidden_content']},
- '返回非目标内容')
+ def test_qualified_content_with_forbidden_hint_has_no_failure(self):
+ facts = [{'id': 'target', 'alternatives': ['Required target content']}]
+ metrics = evaluate_content('Required target content\nAccess denied', [], required_facts=facts,
+ forbidden_any=['Access denied'])
+ self.assertEqual(metrics['forbidden_hits'], ['Access denied'])
+ self.assertIsNone(diagnose(self.record(metrics), {'task_kind': 'body'}, ''))
+ missing = evaluate_content('Access denied', [], required_facts=facts, forbidden_any=['Access denied'])
+ self.assertFalse(missing['content_pass'])
+ self.assertIsNotNone(diagnose(self.record(missing), {'task_kind': 'body'}, ''))
- def test_generic_blocked_page_precedes_resource_success(self):
- blocked = evaluate_content('# Making sure you\'re not a bot!', [])
- blocked.update(resource_pass=True, resource_results=[{'id': 'hero', 'pass': True}])
- result = diagnose(self.record(blocked), {'task_kind': 'resource'}, '')
- self.assertEqual(result['category'], '返回非目标内容')
- self.assertIn("Making sure you're not a bot", result['evidence'])
-
- prose = evaluate_content("The article explains why sites say you're not a bot.", [])
- prose.update(resource_pass=True, resource_results=[{'id': 'hero', 'pass': True}])
- self.assertIsNone(diagnose(self.record(prose), {'task_kind': 'resource'}, ''))
-
- def test_chinese_gate_heading_precedes_resource_success_without_prose_fp(self):
- blocked = evaluate_content('# 验证中心', [])
- blocked.update(resource_pass=True, resource_results=[{'id': 'hero', 'pass': True}])
- result = diagnose(self.record(blocked), {'task_kind': 'resource'}, '')
- self.assertEqual(result['category'], '返回非目标内容')
- self.assertIn('验证中心', result['evidence'])
-
- prose = evaluate_content('本文介绍验证中心的设计与用户体验。', [])
- prose.update(resource_pass=True, resource_results=[{'id': 'hero', 'pass': True}])
- self.assertIsNone(diagnose(self.record(prose), {'task_kind': 'resource'}, ''))
+ def test_prompt_diagnostics_do_not_veto_delivered_resource(self):
+ for heading in ("# Making sure you're not a bot!", '# 验证中心'):
+ metrics = evaluate_content(heading, [])
+ metrics.update(resource_pass=True, resource_results=[{'id': 'hero', 'pass': True}])
+ self.assertTrue(metrics['blocked_page'])
+ self.assertIsNone(diagnose(self.record(metrics), {'task_kind': 'resource'}, ''))
def test_too_many_records_reports_upper_bound_not_too_few(self):
contract = self.contract(maximum_records=3)
diff --git a/tests/test_fairness.py b/tests/test_fairness.py
index df84ffc00..2f249a78b 100644
--- a/tests/test_fairness.py
+++ b/tests/test_fairness.py
@@ -290,7 +290,7 @@ def test_product_prefix_fields_follow_explicit_card_boundaries(self):
self.assertNotIn('shipping', missing['stable_records'][0]['fields'])
self.assertEqual(missing['stable_records'][1]['fields']['shipping'], 'Free shipping')
- def test_dynamic_declared_exclusions_are_executable_not_prose_only(self):
+ def test_dynamic_prompt_hits_are_diagnostic_when_target_is_complete(self):
cases = {case['id']: case for case in self.historical_dataset()['cases']}
supplemental = {
'dynamic-react-cart': [
@@ -317,7 +317,7 @@ def test_dynamic_declared_exclusions_are_executable_not_prose_only(self):
self.assertTrue(extended['stable_structure_pass'])
if identifier == 'dynamic-react-cart' or token == 'Similar Products':
self.assertEqual(extended['forbidden_hits'], [token])
- self.assertFalse(task_passed(True, extended))
+ self.assertTrue(task_passed(True, extended))
else:
self.assertEqual(extended['forbidden_hits'], [])
self.assertTrue(task_passed(True, extended))
@@ -347,7 +347,7 @@ def test_historical_file_missing_is_unmeasured_not_failed(self):
m = score_attempt(c,b'link only',ok=True)
r = {'provider':'p','ok':True,'metrics':m,'elapsed_ms':20,'case_id':'x'}
self.assertFalse(task_measured(m))
- self.assertEqual(result_label(r,c),'未测')
+ self.assertEqual(result_label(r,c),'无法判定')
self.assertIn('data-result="unmeasured"',case_attrs(r,c))
self.assertIn('0/1', ranking([r],{'x':c},'image'))
diff --git a/tests/test_file_report_reason.py b/tests/test_file_report_reason.py
index b0a667098..dd2fb5415 100644
--- a/tests/test_file_report_reason.py
+++ b/tests/test_file_report_reason.py
@@ -17,3 +17,35 @@ def test_failed_and_unmeasured_remain_distinct(self):
def test_late_bytes_do_not_get_success_explanation(self):
row={'ok':True,'metrics':{'task_kind':'file','raw_file_pass':True,'deadline_overrun':True}}
self.assertIn('超过',reason(row,{'task_kind':'file'}))
+
+ def test_missing_page_identity_is_not_a_file_or_uncalled_result(self):
+ from harness.web_fetch.report import failure_category, result_label
+ row={'ok':True,'metrics':{'request_ok':True,'page_identity_required':True,
+ 'page_identity_status':'unavailable'}}
+ self.assertEqual(result_label(row,{}),'无法判定')
+ self.assertIn('最终网页身份',reason(row,{}))
+ self.assertNotIn('原文件',reason(row,{}))
+ self.assertIn('无法判定',failure_category(row,{}))
+
+ def test_shared_evidence_lines_preserve_complete_exact_matches(self):
+ from harness.web_fetch.report import _exact_evidence_lines, _normalized_evidence_lines
+ output=' A & B \nUnrelated\nA & B\nA & B\n'
+ cached=_normalized_evidence_lines(output)
+ for expected in ['A & B','Unrelated','missing','']:
+ self.assertEqual(_exact_evidence_lines(expected,output),
+ _exact_evidence_lines(expected,output,normalized_lines=cached))
+ self.assertEqual(_exact_evidence_lines('Unrelated',output,normalized_lines=cached),'Unrelated')
+ self.assertEqual(_exact_evidence_lines('missing',output,normalized_lines=cached),'')
+
+ def test_unknown_result_is_visible_but_not_in_acceptance_denominator(self):
+ from harness.web_fetch.report import overall_ranking
+ cases={'a':{'id':'a','track':'web'},'b':{'id':'b','track':'web'}}
+ rows=[{'provider':'p','case_id':'a','ok':True,'elapsed_ms':1,
+ 'metrics':{'content_pass':True}},
+ {'provider':'p','case_id':'b','ok':True,'elapsed_ms':1,
+ 'metrics':{'page_identity_required':True,'page_identity_status':'unavailable'}}]
+ rendered=overall_ranking(rows,cases)
+ self.assertIn('无法判定',rendered)
+ self.assertIn('1/1',rendered)
+ self.assertIn('可判定数',rendered)
+ self.assertNotIn('每题计一次通过或失败',rendered)
diff --git a/tests/test_fixed_native_policy.py b/tests/test_fixed_native_policy.py
index 09cb8ba60..3cdc27b47 100644
--- a/tests/test_fixed_native_policy.py
+++ b/tests/test_fixed_native_policy.py
@@ -1,7 +1,9 @@
import json
+from pathlib import Path
from dataclasses import replace
from unittest.mock import patch
import pytest
+from harness.web_fetch.config import load_json
from harness.web_fetch.models import EvalCase
from harness.web_fetch.providers.base import build_provider
from harness.web_fetch.providers.http_client import HttpResponse
@@ -39,14 +41,14 @@ def test_fixed_native_requests_preserve_capabilities_without_product_timers(kind
@pytest.mark.parametrize("kind", ["moli_webfetch", "firecrawl", "tavily_extract", "exa_contents",
"cloudflare_markdown", "cowagent_builtin", "subprocess_jsonl"])
-def test_fixed_native_rejects_hints_and_auxiliary_routes_before_call(kind):
+def test_fixed_native_rejects_hints_and_unknown_outputs_before_call(kind):
provider = build_provider(kind, {"type": kind, "uniform_native_parameters": True})
case = EvalCase("one", "one", "https://example.com/", "control", [],
retrieval_hints={"completion": {"selector": ".answer"}})
with pytest.raises(ValueError, match="per-case"):
provider.fetch(case, 45)
- case = replace(case, retrieval_hints={}, track="document")
- with pytest.raises(ValueError, match="auxiliary"):
+ case = replace(case, retrieval_hints={}, task_kind="undeclared")
+ with pytest.raises(ValueError, match="declared native output"):
provider.fetch(case, 45)
@@ -59,3 +61,75 @@ def test_runner_rejects_extra_attempts_before_preflight(tmp_path, runs, retries)
cases=[EvalCase("one", "one", "https://example.com/", "control", [])], providers=[provider], provider_config_path=tmp_path / "absent-providers.json",
runs=runs, timeout=45, delay=0, max_failure_retries=retries)
assert not (tmp_path / "run").exists()
+
+
+@pytest.mark.parametrize("kind,module", [("firecrawl", "firecrawl"), ("exa_contents", "exa"),
+ ("tavily_extract", "tavily"), ("cloudflare_markdown", "cloudflare")])
+def test_web_family_does_not_change_native_cloud_request(kind, module):
+ provider = build_provider(module, {"type": kind, "uniform_native_parameters": True,
+ "api_key": "fixture", "api_token": "fixture", "account_id": "fixture"})
+ bodies = []
+ for track in ("web", "dynamic", "challenge", "risk_control"):
+ case = EvalCase("one", "one", "https://example.com/", "control", [], track=track,
+ acceptance_contract=load_json(Path(__file__).resolve().parents[1] /
+ "evaluation/dynamic-content-100-moli-webfetch/dataset.json")["cases"][0]["acceptance_contract"])
+ with patch(f"harness.web_fetch.providers.{module}.request") as call:
+ call.return_value = HttpResponse(200, case.url, {}, b'{}')
+ provider.fetch(case, 45)
+ assert call.call_count == 1
+ bodies.append(call.call_args)
+ assert all(body == bodies[0] for body in bodies)
+
+
+def test_native_cloudflare_document_never_downloads_in_caller():
+ provider = build_provider("cf", {"type": "cloudflare_markdown", "uniform_native_parameters": True})
+ case = EvalCase("pdf", "pdf", "https://example.com/a.pdf", "control", [], track="document")
+ with patch("harness.web_fetch.providers.cloudflare._get_document", side_effect=AssertionError("extra download")), \
+ patch("harness.web_fetch.providers.cloudflare.request") as call:
+ call.return_value = HttpResponse(200, case.url, {}, b'{"success":true,"result":"native text"}')
+ assert provider.fetch(case, 45).content == "native text"
+ assert call.call_count == 1
+ assert call.call_args.args[0].endswith("/markdown")
+ assert call.call_args.kwargs["json_body"]["url"] == case.url
+
+
+def test_native_cloudflare_media_has_no_answer_dependent_selectors_or_timers():
+ provider = build_provider("cf", {"type": "cloudflare_markdown", "uniform_native_parameters": True})
+ bodies = []
+ for expected in ([], [{"kind": "video-link", "urls": ["https://secret-answer.invalid/"]}]):
+ case = EvalCase("media", "media", "https://example.com/", "control", [],
+ track="video", task_kind="resource", expected_resources=expected)
+ with patch("harness.web_fetch.providers.cloudflare.request") as call:
+ call.return_value = HttpResponse(200, case.url, {}, b'{"success":true,"result":[]}')
+ provider.fetch(case, 45)
+ assert call.call_count == 1
+ bodies.append(call.call_args.kwargs["json_body"])
+ assert bodies[0] == bodies[1]
+ assert "actionTimeout" not in bodies[0]
+ assert "timeout" not in bodies[0]["gotoOptions"]
+
+
+def test_all_430_frozen_cases_pass_native_input_guard():
+ from pathlib import Path
+ from harness.web_fetch.config import load_dataset
+ provider = build_provider("fixture", {"type": "fixture", "uniform_native_parameters": True})
+ count = 0
+ for filename in ("dataset.json", "dynamic-content-100-moli-webfetch/dataset.json",
+ "popular-sites/dataset.json", "risk-control/dataset.json"):
+ _, cases = load_dataset(Path(__file__).resolve().parents[1] / "evaluation" / filename)
+ for case in cases:
+ if filename == "dataset.json" and case.track not in {"web", "document", "image", "video"}:
+ continue
+ provider.check_native_call(case)
+ count += 1
+ assert count == 430
+
+
+def test_native_media_projection_keeps_returned_sources_and_video_links():
+ from harness.web_fetch.providers.cloudflare import _resource_urls
+ payload = {"result": [
+ {"selector": "video", "results": [{"attributes": [{"name": "src", "value": "/a.mp4"}]}]},
+ {"selector": "a[href]", "results": [{"attributes": [{"name": "href", "value": "/video/abc"}]}]}]}
+ assert _resource_urls(payload, base_url="https://example.com/", track="video",
+ include_video_page_links=True, only_video_page_links=False) == [
+ "https://example.com/a.mp4", "https://example.com/video/abc"]
diff --git a/tests/test_fixed_native_workers.py b/tests/test_fixed_native_workers.py
index e6f39b506..5f9c0ab26 100644
--- a/tests/test_fixed_native_workers.py
+++ b/tests/test_fixed_native_workers.py
@@ -7,6 +7,10 @@
from types import SimpleNamespace
from unittest.mock import patch
+import pytest
+from pydantic import BaseModel
+from typing import Any
+
ROOT = Path(__file__).resolve().parents[1]
REQUEST = {"url": "https://example.invalid/", "timeout_seconds": 45,
"uniform_native_parameters": True, "track": "web", "task_kind": "body",
@@ -28,7 +32,18 @@ def load(self): return [SimpleNamespace(page_content="all native text", metadata
assert json.loads(output.getvalue())["content"] == "all native text"
-def test_crawl4ai_retains_native_capabilities_and_page_timeout_default():
+@pytest.mark.parametrize("native_success", [True, False])
+def test_crawl4ai_retains_native_capabilities_and_page_timeout_default(native_success):
+ class NativeEnvelope(BaseModel):
+ success: bool
+ markdown: str
+ error_message: str | None
+ # Crawl4AI 0.9.4 exposes a deprecated descriptor in failure results.
+ fit_html: Any = property(lambda _: None)
+
+ native_text = "all native text" if native_success else ""
+ native_error = None if native_success else "native navigation failed"
+ envelope = NativeEnvelope(success=native_success, markdown=native_text, error_message=native_error)
seen = []
def run_config(**kwargs):
seen.append(kwargs)
@@ -39,9 +54,9 @@ async def __aenter__(self): return self
async def __aexit__(self, *args): pass
async def arun(self, **kwargs):
assert kwargs["config"].page_timeout == 60000
- return SimpleNamespace(success=True, markdown=SimpleNamespace(raw_markdown="all native text"),
+ return SimpleNamespace(success=native_success, markdown=SimpleNamespace(raw_markdown=native_text),
downloaded_files=[], status_code=200, url=REQUEST["url"], redirected_url=None, redirected_status_code=None,
- error_message=None, media={}, links={}, model_dump=lambda **kw: {"native": "complete"})
+ error_message=native_error, media={}, links={}, model_dump=envelope.model_dump)
fake = {"crawl4ai": SimpleNamespace(AsyncWebCrawler=Crawler, BrowserConfig=lambda **kw: kw,
CacheMode=SimpleNamespace(BYPASS="bypass"), CrawlerRunConfig=run_config),
"crawl4ai.async_configs": SimpleNamespace(HTTPCrawlerConfig=object),
@@ -58,8 +73,12 @@ async def arun(self, **kwargs):
assert seen[0]["remove_overlay_elements"] is True
assert not {"js_code", "wait_for"} & seen[0].keys()
result = json.loads(output.getvalue())
- assert result["ok"], result
- assert result["content"] == "all native text"
+ assert result["ok"] is native_success, result
+ assert result["content"] == native_text
+ assert result["error_message"] == native_error
+ assert result["raw_tool_response"]["success"] is native_success
+ assert result["raw_tool_response"]["markdown"] == native_text
+ assert result["raw_tool_response"]["error_message"] == native_error
def test_typescript_workers_keep_native_timers(tmp_path):
@@ -124,3 +143,31 @@ def test_typescript_workers_keep_native_timers(tmp_path):
answer = json.loads(result.stdout)
assert 'raw_tool_response' in answer, answer
assert answer['raw_tool_response']['seen']['web-fetch-http']['timeoutMs'] == 30000
+
+
+def test_cowagent_worker_keeps_native_url_only_and_real_debug_logs(tmp_path):
+ import logging
+ calls = []
+ class Tool:
+ def __init__(self, config): pass
+ def execute(self, arguments):
+ calls.append(arguments)
+ logging.getLogger("log").debug("native fixture diagnostic")
+ return SimpleNamespace(status="success", result="native answer", ext_data={}, display="native")
+ request = {**REQUEST, "log_level": "debug"}
+ module = SimpleNamespace(WebFetch=Tool, DEFAULT_TIMEOUT=30)
+ import importlib
+ real_import = importlib.import_module
+ def import_native(name, *args, **kwargs):
+ return module if name == "agent.tools.web_fetch.web_fetch" else real_import(name, *args, **kwargs)
+ with patch("importlib.import_module", side_effect=import_native), \
+ patch("sys.argv", ["worker", str(tmp_path)]), \
+ patch("sys.stdin", io.StringIO(json.dumps(request) + "\n")), \
+ patch("sys.stdout", new_callable=io.StringIO) as output, \
+ patch("sys.stderr", new_callable=io.StringIO) as logs:
+ worker = runpy.run_path(str(ROOT / "harness/web_fetch/workers/cowagent_worker.py"))
+ worker["main"]()
+ assert calls == [{"url": request["url"]}]
+ assert json.loads(output.getvalue())["content"] == "native answer"
+ assert "native fixture diagnostic" in logs.getvalue()
+ assert "native answer" not in logs.getvalue()
diff --git a/tests/test_local_audit.py b/tests/test_local_audit.py
index f636f33e5..2fe0b2c78 100644
--- a/tests/test_local_audit.py
+++ b/tests/test_local_audit.py
@@ -233,8 +233,8 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp
build_report(evidence, output, local_audit=True)
home = (output / "index.html").read_text()
assert "本地实测版" in home
- assert "1 个案例 · 1 次调用，机器判分。" in home
- assert "本轮未重新核验来源。" in home
+ assert "查看调用链问题和日志留存范围" not in home
+ assert 'href="guide/index.html"' in home
assert "5案例小范围" not in home
guide = (output / "guide/index.html").read_text()
assert "比例只对应本报告列出的案例与配置" in guide
@@ -385,13 +385,14 @@ def test_local_call_chain_section_discloses_real_log_gaps_and_escapes_text():
dataset = {"comparison": {"call_chain_audit": {
"summary": "原判定保留；评分问题未更正",
"methods": [{"provider": "moli-webfetch", "calls": 5, "request_ok": 0,
- "task_pass": 0, "logs": 5, "native_log_note": "2次原生日志缺失",
+ "task_pass": 0, "logs": 5, "native_log_note": "2次原生日志缺失；<内部过程未知>",
"assessment": "<原生内部原因未知>"}],
"findings": [{"text": "无新增调用"}]}}}
html = local_call_chain_section(dataset)
- assert "2次原生日志缺失" in html and "无新增调用" in html
- assert "<原生内部原因未知>" in html
- assert "原判定保留；评分问题未更正" in html
+ assert "2次原生日志缺失" in html
+ assert "<内部过程未知>" in html
+ assert "调用数" not in html and "题目通过" not in html
+ assert "调用方事件" in html and "产品与依赖库日志" in html
def test_report_real_diagnostics_do_not_claim_missing_debug(tmp_path):
@@ -544,3 +545,129 @@ def test_worker_protocol_response_is_not_a_runtime_log():
assert "native_tool_returned" in caller
assert "raw_tool_response" not in native+caller
assert runtime_log_text(response+progress+worker)==progress+worker
+
+
+def test_local_offline_projection_keeps_native_url_and_call_facts(tmp_path):
+ import copy
+ original = {"attempt_id": "a", "ok": True, "elapsed_ms": 17,
+ "final_url": "https://example.test/input", "metrics": {"content_sha256": "body"}}
+ corrected = copy.deepcopy(original)
+ corrected.update(final_url="https://example.test/actual", scoring={"mode": "offline_rescore"})
+ corrected["metrics"]["score"] = 1
+ native = tmp_path / "native.json"
+ _write(native, {"data": {"url": corrected["final_url"]}})
+ receipt = {"original_call_identities": {"a": local_audit.local_measurement_call_sha256(original)},
+ "call_identities": {"a": local_audit.local_measurement_call_sha256(corrected)},
+ "field_projections": {"a": {"final_url": {"path": "native.json", "sha256": _digest(native),
+ "json_pointer": ["data", "url"], "original_value": original["final_url"]}}}}
+ path = tmp_path / "local-evidence/scoring-correction.json"
+ _write(path, receipt)
+ local_audit._require_local_record_correction(tmp_path, original, corrected)
+ changed = copy.deepcopy(corrected)
+ changed["elapsed_ms"] = 1
+ receipt["call_identities"]["a"] = local_audit.local_measurement_call_sha256(changed)
+ _write(path, receipt)
+ with pytest.raises(ValueError, match="measured call facts"):
+ local_audit._require_local_record_correction(tmp_path, original, changed)
+ _write(path, {**receipt, "call_identities": {"a": local_audit.local_measurement_call_sha256(corrected)}})
+ _write(native, {"data": {"url": "https://example.test/wrong"}})
+ with pytest.raises(ValueError, match="native response"):
+ local_audit._require_local_record_correction(tmp_path, original, corrected)
+
+
+def test_local_review_retains_deferred_failure_and_rejects_wrong_coverage(tmp_path):
+ import copy
+ (tmp_path / "results.jsonl").write_text("saved results")
+ record = {"attempt_id": "a", "ok": False, "metrics": {"content_sha256": "body"}}
+ review = {"kind": "local_independent_review", "review_protocol": {"scope": "all"},
+ "reviewed_attempts": 1, "expected_attempts": 1,
+ "results_sha256": _digest(tmp_path / "results.jsonl"), "unresolved": 1,
+ "attempts": [{"attempt_id": "a", "content_sha256": "body", "agent_verdict": "pass",
+ "current_verdict": "fail", "final_verdict": None,
+ "status": "deferred_unadjudicated"}]}
+ path = tmp_path / "review.json"
+ def bind(value):
+ _write(path, value)
+ return {"comparison": {"independent_review": {"path": "review.json", "sha256": _digest(path)}}}
+ assert local_audit._require_local_review(tmp_path, bind(review), [record])["unresolved"] == 1
+ changed = copy.deepcopy(review)
+ changed["attempts"][0]["final_verdict"] = "pass"
+ with pytest.raises(ValueError, match="scored evidence"):
+ local_audit._require_local_review(tmp_path, bind(changed), [record])
+ changed = copy.deepcopy(review)
+ changed["attempts"].append(changed["attempts"][0])
+ with pytest.raises(ValueError, match="scope differs"):
+ local_audit._require_local_review(tmp_path, bind(changed), [record])
+ changed = {**review, "unresolved": 0}
+ with pytest.raises(ValueError, match="unresolved count"):
+ local_audit._require_local_review(tmp_path, bind(changed), [record])
+
+
+def test_local_review_banner_distinguishes_reviewed_and_original_measurements():
+ from harness.web_fetch.report import local_audit_banner
+ dataset = {"comparison": {"kind": "uniform_native_local_measurement",
+ "publication_state": "local_audit_only_not_for_release"}}
+ assert "未进行独立人工审核" in local_audit_banner(dataset)
+ dataset["comparison"]["independent_review"] = {"path": "review.json"}
+ text = local_audit_banner(dataset)
+ assert "未裁定项保留现有评分" in text
+ assert "未进行独立人工审核" not in text
+
+
+def test_retrospective_page_review_preserves_call_and_checks_other_requirements(tmp_path):
+ import copy
+ from harness.web_fetch.scoring import score_attempt
+ from harness.web_fetch.metrics import task_passed
+ body = 'Expected complete body. A second distinctive sentence describing this page.'
+ case = {'id': 'synthetic', 'url': 'https://example.test/start', 'track': 'risk_control',
+ 'task_kind': 'body', 'source_benchmark': {'risk_control': {'navigation_contract': {
+ 'requested_url': 'https://example.test/start',
+ 'observed_final_url': 'https://example.test/document'}}},
+ 'acceptance_contract': {'version': 1, 'schema': 'body_text', 'minimum_records': 0,
+ 'stable_text_anchors': [{'id': 'body', 'alternatives': ['Expected complete body']}]}}
+ (tmp_path / 'answer.txt').write_text(body)
+ record = {'attempt_id': 'method__synthetic__r001', 'case_id': 'synthetic',
+ 'artifact_path': 'answer.txt', 'ok': True, 'final_url': None,
+ 'metrics': score_attempt(case, body.encode(), ok=True)}
+ captures = [{'url': 'https://example.test/document', 'text': body}]
+ annotation = {'attempt_id': record['attempt_id'],
+ 'case_sha256': local_audit._canonical_sha256(case),
+ 'call_sha256': local_audit.local_measurement_call_sha256(record),
+ 'content_sha256': record['metrics']['content_sha256'],
+ 'basis': 'retrospective_content_correspondence', 'reviewed_at': '2026-10-13',
+ 'capture_index': 0, 'capture_sha256': local_audit._canonical_sha256(captures[0]),
+ 'corresponding_url': captures[0]['url'],
+ 'matching_samples': ['Expected complete body.',
+ 'A second distinctive sentence describing this page.']}
+ before = copy.deepcopy(record)
+ scored = local_audit.score_reviewed_page_identity(tmp_path, case, record, annotation, captures)
+ assert record == before and record['final_url'] is None
+ assert task_passed(True, scored)
+ assert scored['page_identity_basis'] == 'retrospective_content_correspondence'
+ for field, value, message in (
+ ('content_sha256', 'wrong', 'binding'),
+ ('capture_sha256', 'wrong', 'capture'),
+ ('matching_samples', ['Missing substantive text', 'Other missing substantive text'], 'matching'),
+ ):
+ bad = {**annotation, field: value}
+ with pytest.raises(ValueError, match=message):
+ local_audit.score_reviewed_page_identity(tmp_path, case, record, bad, captures)
+ wrong = [{'url': 'https://example.test/other', 'text': body}]
+ bad = {**annotation, 'corresponding_url': wrong[0]['url'],
+ 'capture_sha256': local_audit._canonical_sha256(wrong[0])}
+ with pytest.raises(ValueError, match='URL requirement'):
+ local_audit.score_reviewed_page_identity(tmp_path, case, record, bad, wrong)
+ # A reviewed page identity must never excuse missing required content.
+ missing = copy.deepcopy(case)
+ missing['acceptance_contract']['stable_text_anchors'].append(
+ {'id': 'missing', 'alternatives': ['A required fact that was not returned']})
+ bad = {**annotation, 'case_sha256': local_audit._canonical_sha256(missing)}
+ with pytest.raises(ValueError, match='unavailable page identity'):
+ local_audit.score_reviewed_page_identity(tmp_path, missing, record, bad, captures)
+
+
+def test_retrospective_page_review_requires_bound_receipt(tmp_path):
+ with pytest.raises(ValueError, match='lack a receipt'):
+ local_audit._require_page_identity_review(tmp_path, {}, [
+ {'attempt_id': 'synthetic', 'metrics': {
+ 'page_identity_basis': 'retrospective_content_correspondence'}}])
diff --git a/tests/test_local_measurement_lineage.py b/tests/test_local_measurement_lineage.py
new file mode 100644
index 000000000..203067952
--- /dev/null
+++ b/tests/test_local_measurement_lineage.py
@@ -0,0 +1,141 @@
+"""Frozen inputs and physical calls must survive report assembly unchanged."""
+import copy
+import hashlib
+import json
+
+import pytest
+
+from harness.web_fetch import local_audit
+from harness.web_fetch.reliability import require_reliable
+
+
+def bind(root, name, value, *, lines=False):
+ path = root / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ text = ''.join(json.dumps(row) + '\n' for row in value) if lines else json.dumps(value)
+ path.write_text(text)
+ return {'path': name, 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
+
+
+def test_unknown_verdict_can_be_shown_locally_without_becoming_publishable():
+ dataset = {'comparison': {'kind': 'uniform_native_local_measurement',
+ 'publication_state': 'local_audit_only_not_for_release'},
+ 'coverage_plan': {'web': ['p']}, 'cases': [{'id': 'a', 'track': 'web'}]}
+ row = {'provider': 'p', 'case_id': 'a', 'metrics': {'page_identity_required': True,
+ 'page_identity_status': 'unavailable', 'request_ok': True}}
+ reliability = local_audit.require_complete_local_inventory(dataset, [row])
+ assert reliability['present_calls'] == 1 and reliability['judged_calls'] == 0
+ assert not reliability['complete']
+ with pytest.raises(ValueError):
+ require_reliable(dataset, [row])
+ for records in [[], [row, row], [{**row, 'case_id': 'other'}]]:
+ with pytest.raises(ValueError):
+ local_audit.require_complete_local_inventory(dataset, records)
+ dataset['comparison']['publication_state'] = 'certified'
+ with pytest.raises(ValueError):
+ local_audit.require_complete_local_inventory(dataset, [row])
+
+
+def test_multiple_frozen_sources_preserve_order_and_reject_changes(tmp_path, monkeypatch):
+ monkeypatch.setattr(local_audit, 'ROOT', tmp_path)
+ a = bind(tmp_path, 'first.json', {'cases': [{'id': 'a'}, {'id': 'b'}]})
+ b = bind(tmp_path, 'second.json', {'cases': [{'id': 'c'}]})
+ contract = {'dataset_sources': [a, b]}
+ assert local_audit.local_measurement_cases(contract) == [{'id': 'a'}, {'id': 'b'}, {'id': 'c'}]
+ for selection in [['b', 'a'], ['a', 'a'], ['missing']]:
+ changed = copy.deepcopy(contract)
+ changed['dataset_sources'][0]['case_ids'] = selection
+ with pytest.raises(ValueError):
+ local_audit.local_measurement_cases(changed)
+ changed = copy.deepcopy(contract)
+ changed['dataset_sources'].append(a)
+ with pytest.raises(ValueError):
+ local_audit.local_measurement_cases(changed)
+ bind(tmp_path, 'second.json', {'cases': [{'id': 'changed'}]})
+ with pytest.raises(ValueError):
+ local_audit.local_measurement_cases(contract)
+
+
+@pytest.fixture
+def execution(tmp_path, monkeypatch):
+ monkeypatch.setattr(local_audit, 'local_measurement_cases', lambda _: [{'id': c} for c in 'abc'])
+ original_config = {'providers': {'p': {'command': ['{harness_root}/scripts/oss/p.py'], 'max_bytes': 20_000_000}, 'q': {'version': 'fixed'}}}
+ migrated_config = copy.deepcopy(original_config)
+ migrated_config['providers']['p']['command'] = ['{harness_root}/harness/web_fetch/workers/p.py']
+ old_config = bind(tmp_path, 'original-config.json', original_config)
+ new_config = bind(tmp_path, 'migrated-config.json', migrated_config)
+ policy = {'runs': 1, 'timeout': 45, 'delay': .5, 'provider_concurrency': 1, 'max_failure_retries': 0}
+ original = {'kind': 'uniform_native_local_measurement', 'providers': ['p', 'q'], 'dataset_sources': [], 'scoring_source_sha256': 'd' * 64, 'runs': 1, 'timeout_seconds': 45, 'delay': .5, 'max_failure_retries': 0, 'runtime_source_sha256': 'a' * 64, 'provider_config_sha256': old_config['sha256']}
+ resumed = {'kind': 'post_merge_resume_and_runtime_correction', 'parameters': policy, 'case_ids_by_provider': {'p': ['b', 'c'], 'q': ['a', 'b', 'c']}, 'runtime_source_sha256': 'b' * 64, 'provider_config_sha256': new_config['sha256']}
+ plans = {'original': bind(tmp_path, 'original-plan.json', original), 'resumed': bind(tmp_path, 'resumed-plan.json', resumed)}
+ scheduling = {'max_parallel': 2, 'max_local': 1, 'max_cloud': 1}
+ admitted = {'kind': 'bounded_admission_continuation', 'predecessor_plan_sha256': plans['resumed']['sha256'], 'parameters': policy, 'providers': ['p', 'q'], 'case_ids_by_provider': {'p': ['c'], 'q': ['c']}, 'runtime_source_sha256': 'c' * 64, 'provider_config_sha256': new_config['sha256'], 'scheduling': scheduling}
+ plans['admitted'] = bind(tmp_path, 'admitted-plan.json', admitted)
+ for key in plans:
+ plans[key]['configuration'] = old_config if key == 'original' else new_config
+ contract = {**original, 'execution_plans': plans}
+ values = {'original': original, 'resumed': resumed, 'admitted': admitted}
+ batches, records = [], []
+ for name in ['p', 'q']:
+ segments, method_rows = [], []
+ chunks = [('original', 'a'), ('resumed', 'b'), ('admitted', 'c')] if name == 'p' else [('resumed', 'a'), ('resumed', 'b'), ('admitted', 'c')]
+ for index, (key, case) in enumerate(chunks):
+ row = {'attempt_id': f'{name}__{case}__r001', 'provider': name, 'case_id': case, 'run': 1, 'ok': True, 'elapsed_ms': 7, 'metrics': {}}
+ timeout = name == 'q' and case == 'a'
+ if timeout:
+ row.update(ok=False, error_type='timeout', elapsed_ms=45_000)
+ method_rows.append(row)
+ providers = [{'name': name, 'type': 'fixture'}]
+ manifest = {'providers': providers, 'parameters': policy, 'runtime_source_sha256': values[key]['runtime_source_sha256'], 'provider_config_sha256': values[key]['provider_config_sha256'], 'completed_at': 'complete', 'attempt_ids': [row['attempt_id']], 'attempt_count': 1, 'unexecuted': []}
+ if timeout:
+ manifest['unexecuted'] = [{'after_attempt_id': row['attempt_id'], 'reason': 'non_cancellable_call_still_running; remaining provider cases not executed'}]
+ if key == 'admitted':
+ manifest['scheduling'] = scheduling
+ stem = f'{name}-{index}'
+ segment = {'plan_id': key, 'manifest': bind(tmp_path, stem + '-manifest.json', manifest), 'records': bind(tmp_path, stem + '-rows.jsonl', [row], lines=True), 'providers': providers, 'parameters': policy, 'attempt_ids': [row['attempt_id']], 'attempt_count': 1}
+ if name == 'q' and case == 'b':
+ continuation = {'kind': 'resume_unexecuted_after_process_exit', 'method': name, 'resumed_plan_sha256': plans[key]['sha256'], 'parameters': policy, 'provider_config_sha256': values[key]['provider_config_sha256'], 'runtime_source_sha256': values[key]['runtime_source_sha256'], 'case_ids': ['b', 'c']}
+ segment['continuation_plan'] = bind(tmp_path, 'continuation.json', continuation)
+ segments.append(segment)
+ records.extend(method_rows)
+ batches.append({'providers': [{'name': name}], 'segments': segments, 'attempt_ids': [row['attempt_id'] for row in method_rows], 'attempt_count': 3})
+ return tmp_path, contract, batches, records
+
+
+def test_retained_calls_full_method_correction_and_three_runtime_phases(execution):
+ local_audit._require_local_execution_segments(*execution)
+
+
+@pytest.mark.parametrize('change', ['omit_timeout', 'alter_latency', 'repeat_method', 'alter_parameters', 'wrong_runtime', 'alter_scheduling', 'alter_product_config'])
+def test_segment_assembly_rejects_protocol_or_evidence_changes(execution, change):
+ root, contract, batches, records = copy.deepcopy(execution)
+ if change == 'omit_timeout':
+ batches[1]['segments'].pop(0)
+ elif change == 'alter_latency':
+ records[0]['elapsed_ms'] = 1
+ elif change == 'repeat_method':
+ batches.append(copy.deepcopy(batches[0]))
+ elif change == 'alter_parameters':
+ batches[0]['segments'][0]['parameters']['timeout'] = 90
+ elif change in {'wrong_runtime', 'alter_scheduling'}:
+ segment = batches[0]['segments'][-1]
+ manifest = json.loads((root / segment['manifest']['path']).read_text())
+ if change == 'wrong_runtime':
+ manifest['runtime_source_sha256'] = 'wrong'
+ else:
+ manifest['scheduling'] = {'max_parallel': 10}
+ segment['manifest'] = bind(root, 'changed-manifest.json', manifest)
+ else:
+ config = json.loads((root / contract['execution_plans']['resumed']['configuration']['path']).read_text())
+ config['providers']['p']['max_bytes'] = 1
+ binding = bind(root, 'changed-config.json', config)
+ for key in ['resumed', 'admitted']:
+ plan = json.loads((root / contract['execution_plans'][key]['path']).read_text())
+ plan['provider_config_sha256'] = binding['sha256']
+ if key == 'admitted':
+ plan['predecessor_plan_sha256'] = contract['execution_plans']['resumed']['sha256']
+ replacement = bind(root, f'changed-{key}-plan.json', plan)
+ replacement['configuration'] = binding
+ contract['execution_plans'][key] = replacement
+ with pytest.raises(ValueError, match='beyond worker paths' if change == 'alter_product_config' else None):
+ local_audit._require_local_execution_segments(root, contract, batches, records)
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index e96d8a154..420a21eec 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -123,7 +123,7 @@ def test_content_metrics_are_deterministic(self) -> None:
self.assertEqual(result["markdown_headings"], 1)
self.assertEqual(result["markdown_links"], 1)
- def test_ground_truth_requires_every_fact_and_rejects_error_pages(self) -> None:
+ def test_ground_truth_requires_every_fact_and_keeps_prompt_diagnostics(self) -> None:
facts = [
{"id": "title", "alternatives": ["Example Domain"]},
{"id": "purpose", "alternatives": ["documentation examples"]},
@@ -135,7 +135,7 @@ def test_ground_truth_requires_every_fact_and_rejects_error_pages(self) -> None:
"Example Domain documentation examples Access denied",
["Example Domain"], required_facts=facts, forbidden_any=["Access denied"],
)
- self.assertFalse(blocked["content_pass"])
+ self.assertTrue(blocked["content_pass"])
self.assertEqual(blocked["forbidden_hits"], ["Access denied"])
def test_summary_separates_transport_and_content(self) -> None:
diff --git a/tests/test_moli_provider.py b/tests/test_moli_provider.py
index 20ad08893..d1f16ee85 100644
--- a/tests/test_moli_provider.py
+++ b/tests/test_moli_provider.py
@@ -201,6 +201,33 @@ def case_with(self, **changes):
values.update(changes)
return SimpleNamespace(**values)
+ def test_native_families_never_use_caller_scripts_or_conversion(self):
+ provider = self.provider(
+ "assert '--eval' not in sys.argv and '--wait-script' not in sys.argv\n"
+ "assert '--trace-network' not in sys.argv\n"
+ "if sys.argv[-1].endswith(('.pdf', '.docx')):\n"
+ " assert '--dump' not in sys.argv\n"
+ "else:\n"
+ " assert sys.argv[sys.argv.index('--dump') + 1] in ('markdown', 'html')\n"
+ "print('native output')\n", uniform_native_parameters=True)
+ with patch("harness.web_fetch.providers.moli.extract_document_text", side_effect=AssertionError("caller conversion")):
+ for track, kind, url in [("challenge", "structure", "https://example.com/"),
+ ("risk_control", "body", "https://example.com/"),
+ ("document", "body", "https://example.com/a.docx"),
+ ("document", "body", "https://example.com/a.pdf"),
+ ("document", "body", "https://example.com/viewer.html"),
+ ("image", "resource", "https://example.com/"),
+ ("video", "resource", "https://example.com/"),
+ ("video", "body", "https://example.com/")]:
+ result = provider.fetch(self.case_with(track=track, task_kind=kind, url=url), 5)
+ self.assertTrue(result.ok, result.error_message)
+ if url.endswith((".pdf", ".docx")):
+ self.assertEqual(result.content, "")
+ self.assertEqual(result.files[0]["data"].strip(), b"native output")
+ else:
+ self.assertEqual(result.content.strip(), "native output")
+ self.assertEqual(result.metadata["dump"], "html" if kind == "resource" else "markdown")
+
def test_fixed_native_cli_omits_internal_timers(self):
provider = self.provider(
"import sys\n"
diff --git a/tests/test_network_preflight.py b/tests/test_network_preflight.py
index c1411b6d7..95f3e3bea 100644
--- a/tests/test_network_preflight.py
+++ b/tests/test_network_preflight.py
@@ -4,6 +4,19 @@
class NetworkPreflightTests(unittest.TestCase):
+ def test_non_public_dns_is_not_a_fair_public_comparison(self):
+ for addresses,accepted in ((['43.162.217.19'],True),(['2606:4700:4700::1111'],True),
+ (['10.2.0.11'],False),(['127.0.0.1'],False),(['169.254.1.1'],False),
+ (['fd00::1'],False),(['43.162.217.19','10.2.0.11'],False)):
+ with self.subTest(addresses=addresses):
+ def resolve(host,port,**kwargs):
+ return [(socket.AF_INET6 if ':' in a else socket.AF_INET,socket.SOCK_STREAM,6,'',(a,port)) for a in addresses]
+ audit=inspect_system_dns(['https://public.example/page'],resolver=resolve)
+ self.assertEqual(audit['usable_for_guarded_local_comparison'],accepted)
+ if accepted:require_usable_public_dns(audit)
+ else:
+ with self.assertRaisesRegex(ValueError,'Do not disable'):
+ require_usable_public_dns(audit)
def test_synthetic_dns_fails_before_calls_but_public_dns_passes(self):
for address, accepted in [('198.19.93.254', False), ('185.199.108.153', True)]:
with self.subTest(address=address):
@@ -53,6 +66,31 @@ def fetch(self, case, timeout):
self.assertFalse(output.exists())
+ def test_runner_rejects_non_public_dns_before_calls_and_artifacts(self):
+ from pathlib import Path
+ from tempfile import TemporaryDirectory
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.providers.base import Provider
+ from harness.web_fetch.runner import run_evaluation
+ calls = []
+ class GuardedFixture(Provider):
+ def fetch(self, case, timeout):
+ calls.append(case.id)
+ raise AssertionError("No provider calls allowed in invalid DNS environment")
+ provider = GuardedFixture('guarded', {'type': 'openclaw_builtin'})
+ fake = [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.2.0.11', 443))]
+ with TemporaryDirectory() as directory, patch('socket.getaddrinfo', return_value=fake):
+ output = Path(directory) / 'measurement'
+ with self.assertRaisesRegex(ValueError, 'non-public'):
+ run_evaluation(output_dir=output, dataset_meta={}, dataset_path=Path('fixture.json'),
+ cases=[EvalCase('control', 'control', 'https://example.com/', 'control', [])],
+ providers=[provider], provider_config_path=Path('fixture-providers.json'),
+ runs=1, timeout=45, delay=0)
+ self.assertEqual(calls, [])
+ self.assertFalse(output.exists())
+
+
def test_public_dns_mode_uses_real_resolver_and_records_mode(self):
from types import SimpleNamespace
from unittest.mock import patch
diff --git a/tests/test_providers.py b/tests/test_providers.py
index 1356d6107..e2ce37ec9 100644
--- a/tests/test_providers.py
+++ b/tests/test_providers.py
@@ -7,7 +7,7 @@
from pathlib import Path
from unittest.mock import patch
-from harness.web_fetch.metrics import evaluate_resources
+from harness.web_fetch.metrics import evaluate_resources, task_passed, task_measured
from harness.web_fetch.models import EvalCase
from harness.web_fetch.providers.cloudflare import (
CloudflareContentProvider,
@@ -23,9 +23,36 @@
from harness.web_fetch.providers.tavily import TavilyExtractProvider
from harness.web_fetch.publication_privacy import project_file
from harness.web_fetch.runner import run_evaluation
+from harness.web_fetch.scoring import score_attempt
class ProviderTest(unittest.TestCase):
+ @patch('harness.web_fetch.providers.firecrawl.request')
+ def test_native_final_url_controls_identity(self, request):
+ case=EvalCase(id='redirect-control',name='Redirect control',url='https://example.test/start',
+ category='static',expected_any=['Expected body'],track='risk_control',
+ acceptance_contract={'version':1,'schema':'body_text','minimum_records':0,
+ 'stable_text_anchors':[{'id':'body','alternatives':['Expected body']}]},
+ source_benchmark={'risk_control':{'navigation_contract':{
+ 'requested_url':'https://example.test/start','observed_final_url':'https://example.test/final'}}})
+ for native_url, passed, measured, identity in (
+ ('https://example.test/final',True,True,'verified'),
+ ('https://example.test/other',False,True,'mismatch'),
+ (None,False,False,'unavailable'),
+ ):
+ with self.subTest(native_url=native_url):
+ metadata={'sourceURL':case.url,'statusCode':200}
+ if native_url is not None: metadata['url']=native_url
+ request.return_value=HttpResponse(200,'https://api.firecrawl.dev/v2/scrape',{},
+ json.dumps({'success':True,'data':{'markdown':'Expected body','metadata':metadata}}).encode())
+ result=FirecrawlProvider('firecrawl',{'api_key':'synthetic','uniform_native_parameters':True}).fetch(case,45)
+ self.assertEqual(result.final_url,native_url)
+ m=score_attempt(case,result.content.encode(),ok=result.ok,metadata=result.metadata,
+ status_code=result.status_code,final_url=result.final_url)
+ self.assertEqual(m['page_identity_status'],identity)
+ self.assertEqual(task_passed(result.ok,m),passed)
+ self.assertEqual(task_measured(m),measured)
+
@patch("harness.web_fetch.providers.firecrawl.request")
def test_firecrawl_native_page_scope_is_explicit_and_preserves_answer(self, mock_request):
returned = "Header\n\nMain text\n\nFooter"
diff --git a/tests/test_report_article_cases.py b/tests/test_report_article_cases.py
index 1a908611f..cdd9c83ea 100644
--- a/tests/test_report_article_cases.py
+++ b/tests/test_report_article_cases.py
@@ -45,8 +45,7 @@ def test_text_evidence_is_escaped_without_rewriting_its_content(self):
text_encoding_corruption=True,
replacement_character_count=12)
_, _, checks, _ = _text_example(case, record, text)
- self.assertIn('Access denied', checks[-2])
- self.assertIn('未通过', checks[-2])
+ self.assertNotIn('排除错误页', ''.join(checks))
self.assertIn('12 个替换字符', checks[-1])
self.assertIn('未通过', checks[-1])
@@ -72,7 +71,7 @@ def test_pdf_exposes_matching_lines_outside_abstract_without_fabricating_them(se
with self.assertRaisesRegex(ValueError, 'fact excerpt is missing'):
_text_example(case, record, text.replace('Scaled Dot-Product Attention', 'absent'), pdf=True)
- def test_list_rejects_blocked_or_corrupt_content_despite_sufficient_records(self):
+ def test_list_keeps_decode_failure_without_prompt_veto(self):
case = {'acceptance_contract': {'minimum_records': 3, 'hosts': ['example.com'],
'entry_path': '/shop/[^/]+',
'stable_text_anchors': [{'id': 'title', 'alternatives': ['Catalog']}]},
@@ -83,8 +82,7 @@ def test_list_rejects_blocked_or_corrupt_content_despite_sufficient_records(self
stable_structure_pass=True, forbidden_hits=[], stable_blocked_page=True,
replacement_character_count=20, text_encoding_corruption=True)
_, _, checks, _ = _list_example(case, {'metrics': metrics}, output)
- self.assertIn('命中拦截页特征', checks[-2])
- self.assertIn('未通过', checks[-2])
+ self.assertNotIn('排除错误页', ''.join(checks))
self.assertIn('20 个替换字符', checks[-1])
self.assertIn('未通过', checks[-1])
diff --git a/tests/test_report_body_scope.py b/tests/test_report_body_scope.py
index 845c6eae2..c15049bf2 100644
--- a/tests/test_report_body_scope.py
+++ b/tests/test_report_body_scope.py
@@ -187,6 +187,11 @@ def failed(**metrics):
failure_category(failed(blocked_page=True, composite_acceptance_pass=False), c),
"返回验证或拦截页面",
)
+ for track in ("risk_control", "web", "challenge", "popular"):
+ qualified_case = case("qualified", "body", track)
+ qualified = record(qualified_case, passed=True)
+ qualified["metrics"].update(blocked_page=True, stable_blocked_page=True)
+ self.assertEqual(failure_category(qualified, qualified_case), "—")
self.assertEqual(failure_category(failed(content_chars=0), c), "完全空返回")
self.assertEqual(
failure_category(failed(content_chars=37, markdown_headings=0, markdown_links=0,
@@ -937,3 +942,21 @@ def test_build_rejects_stale_scoring_implementation_before_render(self):
if __name__=='__main__':
unittest.main()
+
+
+def test_body_display_shares_native_markup_and_entity_handling_with_scorer():
+ from harness.web_fetch.scoring import score_attempt
+ contract = {"version": 3, "unit": "region_compact_hexagram_recall",
+ "minimum_coverage": .8, "minimum_floor": .6, "minimum_region_passes": 1,
+ "regions": [{"id": "body", "text": "abcdefghijkl"}]}
+ definition = {"id": "markup", "url": "https://example.test", "body_requirements": contract}
+ for output in ("abcdefabcdef "):
+ record = {"ok": True, "title": "abcdefghijkl"}
+ metrics = score_attempt(definition, output.encode(), ok=True, title=record["title"])
+ display = _body_scoring_text(record, output)
+ rows, cards = body_region_rows(definition, metrics, display)
+ assert len(rows) == 1
+ assert "'
+ for body,passed in ((cards,True),(hidden,False),(cards.replace('2026-09-02',''),False),(cards.replace('2026-09-02','2026-09-02'),False),(cards.replace('','
',1),False)):
+ with self.subTest(body=body):
+ self.assertEqual(task_passed(True,score_attempt(case,(''+body+'').encode(),ok=True)),passed)
+ def test_inline_html_break_preserves_visible_body_anchor(self):
+ case={'url':'https://example.test/','task_kind':'body','expected_any':[],
+ 'acceptance_contract':{'version':1,'schema':'body_text','minimum_records':0,
+ 'stable_text_anchors':[{'id':'statement','alternatives':['One platform. Endless ways to work together.']}]}}
+ for body,passed in (
+ ('## One platform.
Endless ways to work together.',True),
+ ('## One platform.
Endless ways to work together.',True),
+ ('## One platform.\nEndless ways to work together.',True),
+ ('## One platform. <br>Endless ways to work together.',False),
+ ('## One platform. \\
Endless ways to work together.',False),
+ ('`One platform.
Endless ways to work together.`',False),
+ ('## One platform.
Different ways to work together.',False),
+ ):
+ with self.subTest(body=body):
+ self.assertEqual(task_passed(True,score_attempt(case,body.encode(),ok=True)),passed)
+ def test_parallel_column_scopes_cannot_share_other_column_records(self):
+ case={'url':'https://example.test/','task_kind':'body','expected_any':[],
+ 'composite_acceptance':{'version':1,'groups':[
+ {'id':'first','kind':'named_link','minimum_records':2,'hosts':['example.test'],
+ 'entry_path':r'/item/\d+','start_pattern':'First Division','end_pattern':'Second Division'},
+ {'id':'second','kind':'named_link','minimum_records':2,'hosts':['example.test'],
+ 'entry_path':r'/item/\d+','start_pattern':'Second Division','end_pattern':'End'}]}}
+ html='
End '
+ markdown='| First Division | Second Division |\n| --- | --- |\n| [Alpha](/item/1) | [Gamma](/item/3) |\n| [Beta](/item/2) | [Delta](/item/4) |\n\n# End'
+ for content,passed in (
+ (html,True),(markdown,True),
+ (html.replace('
Beta ',''),False),
+ (markdown.replace('[Beta](/item/2)',''),False),
+ (html.replace('
End ',''),False),
+ (markdown.replace('# End',''),False),
+ ):
+ with self.subTest(content=content):
+ m=score_attempt(case,content.encode(),ok=True)
+ self.assertEqual(task_passed(True,m),passed)
+ def test_thumbnail_card_owns_immediate_visible_caption(self):
+ case={'url':'https://example.test/catalog','task_kind':'body','expected_any':[],
+ 'acceptance_contract':{'version':1,'schema':'named_link','minimum_records':2,
+ 'hosts':['example.test'],'entry_path':r'/item/\d+',
+ 'entry_scope':{'html_selector':'.card','title_selector':'p.name','link_selector':'a[href]',
+ 'start_pattern':'^Catalog$','end_pattern':'^Footer$'}}}
+ first='[](/item/1)\n\nFirst delivered title\n\n'
+ second='[](/item/2)\n\nSecond delivered title\n\n'
+ for content,passed in (
+ ('Catalog\n'+first+second+'Footer',True),
+ ('Catalog\n'+first+'[](/item/2)\n\nFooter',False),
+ ('Catalog\n[](/item/1)\n\n'+second+'Footer',False),
+ ('Catalog\n'+first+'[](/item/2)\n\n## Footer\nSecond title outside scope',False),
+ ('Catalog\n'+first+'[](/item/2)\n\n[Navigation](/home)\n\nFooter',False),
+ ):
+ with self.subTest(content=content):
+ m=score_attempt(case,content.encode(),ok=True)
+ self.assertEqual(task_passed(True,m),passed)
+ def test_linked_headers_use_visible_labels_only(self):
+ case={'url':'https://example.test/','task_kind':'body','expected_any':[],
+ 'composite_acceptance':{'version':1,'groups':[{'id':'rows','kind':'linked_table','minimum_records':2,
+ 'hosts':['example.test'],'entry_path':r'/item/\d+','required_columns':[
+ {'id':'name','kind':'text','aliases':['Name']},{'id':'version','kind':'version','aliases':['Version']}],
+ 'identity_column':'name','link_column':'name'}]}}
+ rows='\n| --- | --- |\n| [Alpha](/item/1) | 1.2.3 |\n| [Beta](/item/2) | 2.3.4 |'
+ for header,body,passed in (
+ ('| Name | Version |',rows,True),
+ ('| [Name](/sort/name) | [Version](/sort/version "Sort packages") |',rows,True),
+ ('| [Other](/sort/Name) | Version |',rows,False),
+ ('| [Name](/sort/name) | Version |',rows.replace('2.3.4','unknown'),False),
+ ('| [Name](/sort/name) | Version |',rows.replace('/item/2','/item/1'),False),
+ ):
+ with self.subTest(header=header,body=body):
+ m=score_attempt(case,(header+body).encode(),ok=True)
+ self.assertEqual(task_passed(True,m),passed)
+ def test_visible_entity_equivalence_keeps_body_requirements(self):
+ case={'url':'https://example.test/body','task_kind':'body','expected_any':[],
+ 'body_requirements':{'version':3,'unit':'region_compact_hexagram_recall','minimum_coverage':1.0,'minimum_floor':1.0,
+ 'minimum_region_passes':1,'regions':[{'id':'body','text':'Alpha Beta Gamma Delta'}]}}
+ for content,passed in (
+ ('Alpha Beta Gamma Delta',True),
+ ('Alpha Beta Gamma Delta',True),
+ ('Alpha Beta Gamma Wrong',False),
+ ('Alpha Beta Gamma Delta',False),
+ ('Alpha Beta',False),
+ ):
+ with self.subTest(content=content):
+ m=score_attempt(case,content.encode(),ok=True)
+ self.assertEqual(task_passed(True,m),passed)
+
+ def test_composite_keeps_visible_html_links(self):
+ case={'url':'https://example.test/base/','task_kind':'body','expected_any':['Directory'],
+ 'composite_acceptance':{'version':1,'groups':[{'id':'entries','kind':'named_link','minimum_records':2,
+ 'hosts':['example.test'],'entry_path':r'/base/[^/]+'}]}}
+ for content,passed in (
+ ('
Directory Alpha Beta ',True),
+ ('
Directory Alpha Beta ',False),
+ ('
Directory Alpha Beta ',False),
+ ('
Directory Alpha ',False),
+ ):
+ with self.subTest(content=content):
+ m=score_attempt(case,content.encode(),ok=True)
+ self.assertEqual(task_passed(True,m),passed)
+ def test_inline_html_and_named_metadata_keep_text_scope(self):
+ case={'url':'https://example.test/catalog','task_kind':'body','expected_any':[],
+ 'acceptance_contract':{'version':1,'schema':'named_link','minimum_records':2,
+ 'hosts':['example.test'],'entry_path':r'/item/\d+',
+ 'entry_scope':{'html_selector':'main a[href]','start_pattern':'^Target list$','end_pattern':'^Footer$'}}}
+ labels=[{'text':'First entry','href':'https://example.test/item/1'},
+ {'text':'Second entry','href':'https://example.test/item/2'}]
+ for text,links,passed in (
+ ('## Target list\n[
First entry ](/item/1)\n[
Second entry ](/item/2)\n## Footer',[],True),
+ ('## Target list\nFirst entry\nSecond entry\n## Footer',labels,True),
+ ('## Target list\nFirst entry\n## Footer\nSecond entry',labels,False),
+ ('
Target list ',[],False),
+ ('## Target list\nFirst entry\nSecond entry\n## Footer\nFirst entry',labels,False),
+ ):
+ with self.subTest(text=text):
+ m=score_attempt(case,text.encode(),ok=True,metadata={'links':links})
+ self.assertEqual(task_passed(True,m),passed,m.get('stable_task_errors'))
+
+ def test_equivalent_headers_preserve_declared_scope_and_columns(self):
+ case={'url':'https://example.test/table','task_kind':'body','expected_any':[],
+ 'acceptance_contract':{'version':1,'schema':'table_row','minimum_records':2,
+ 'table_columns':[{'name':'name','label':'Name','type':'text'},
+ {'name':'amount','label':'Amount','type':'integer'}],
+ 'record_identity_fields':['name'],'table_scope':{'row_selector':'table tbody tr','cell_selector':'td',
+ 'start_pattern':'^Name$','end_pattern':'^Footer$',
+ 'plain_row_pattern':r'(?P
[^\t]+)\t(?P\d+)'}}}
+ for text,passed in (
+ ('| Name ↕ | Amount ↕ |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |',True),
+ ('| Name | Amount |\n| --- | --- |\n\n| | |\n| --- | --- |\n| Name | Amount |\n| Alpha | 7 |\n| Beta | 8 |',True),
+ ('| Name ↕ | Other ↕ |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |',False),
+ ('| Name | Amount |\n| --- | --- |\n\n| | |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |',False),
+ ):
+ with self.subTest(text=text):
+ m=score_attempt(case,text.encode(),ok=True)
+ self.assertEqual(task_passed(True,m),passed,m.get('stable_task_errors'))
+ case['acceptance_contract']['table_scope']['start_pattern']='^Target section$'
+ text='| Name ↕ | Amount ↕ |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |'
+ self.assertFalse(task_passed(True,score_attempt(case,text.encode(),ok=True)))
+
def test_untagged_source_requires_explicitly_observed_empty_language(self):
c = case(track="dynamic", evaluation_url="https://example.test/page",
expected_final_url="https://example.test/page",
@@ -337,7 +494,7 @@ def test_body_reference_can_exempt_its_standalone_captcha_heading_only(self):
challenge = score_attempt(
case(ground_truth_main_content="Accessibility guidance",
- required_facts=[]),
+ required_facts=[{"id":"guidance","alternatives":["Accessibility guidance"]}]),
b"CAPTCHA", ok=True,
)
self.assertEqual(challenge["blocked_page_hits"], ["CAPTCHA"])
@@ -351,7 +508,7 @@ def test_body_reference_can_exempt_its_standalone_captcha_heading_only(self):
)
self.assertEqual(stable["blocked_page_hits"], ["CAPTCHA"])
self.assertEqual(stable["blocked_page_reference_exemptions"], [])
- self.assertFalse(task_passed(True, stable))
+ self.assertTrue(task_passed(True, stable))
def test_quote_fields_reject_lost_tag_boundaries(self):
quote = "“The world as we have created it is a process of our thinking.”"
diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py
index e816ae3d7..42d9c846b 100644
--- a/tests/test_scoring_regressions.py
+++ b/tests/test_scoring_regressions.py
@@ -4,7 +4,7 @@
import unittest
from pathlib import Path
-from harness.web_fetch.metrics import task_passed
+from harness.web_fetch.metrics import task_measured, task_passed
from harness.web_fetch.reference_lifecycle import (
reference_revision_cases,
scoring_revision_history_sha256,
@@ -71,6 +71,8 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self)
row["ok"], metrics, row["case"].get("task_kind", "body")
)
self.assertEqual(verdict, row["expected_task_pass"])
+ if "expected_task_measured" in row:
+ self.assertEqual(task_measured(metrics), row["expected_task_measured"])
for key, expected in row.get("expected_metrics", {}).items():
self.assertEqual(metrics.get(key), expected, key)
root_modes.setdefault(row["root_cause"], set()).add(row["failure_mode"])
diff --git a/tests/test_stable_tasks.py b/tests/test_stable_tasks.py
index 071acbce6..d0072a23d 100644
--- a/tests/test_stable_tasks.py
+++ b/tests/test_stable_tasks.py
@@ -521,7 +521,7 @@ def test_anchor_validation_and_numeric_boundaries(self):
self.assertTrue(result["stable_structure_pass"])
self.assertFalse(result["stable_content_pass"])
- def test_block_pages_with_complete_cards_never_pass(self):
+ def test_prompt_headings_do_not_veto_complete_cards(self):
cards = "Alpha\n$10\n\nBeta\n$20"
for heading in ("# Access denied", "403 Forbidden", "Verify you are human", "Just a moment...",
"W3 Performing security verification", "Making sure you're not a bot",
@@ -529,21 +529,21 @@ def test_block_pages_with_complete_cards_never_pass(self):
"验证一下，购物无忧"):
result = self.check(heading + "\n" + cards)
self.assertTrue(result["stable_structure_pass"])
- self.assertFalse(result["stable_content_pass"])
- self.assertFalse(result["stable_task_pass"])
+ self.assertTrue(result["stable_content_pass"])
+ self.assertTrue(result["stable_task_pass"])
self.assertTrue(result["stable_blocked_page"])
- def test_explicit_forbidden_rule_survives_stable_override_and_reference_exemptions(self):
+ def test_forbidden_diagnostic_does_not_override_target_content(self):
text = "Custom access failure\nAlpha\n$10\n\nBeta\n$20"
result = evaluate_content(text, [], forbidden_any=["Custom access failure"],
ground_truth_main_content=text, acceptance_contract=contract("product"))
self.assertTrue(result["stable_structure_pass"])
- self.assertFalse(result["stable_content_pass"])
- self.assertFalse(result["content_pass"])
- self.assertFalse(task_passed(True, result))
+ self.assertTrue(result["stable_content_pass"])
+ self.assertTrue(result["content_pass"])
+ self.assertTrue(task_passed(True, result))
self.assertEqual(result["reference_term_exemptions"], [])
self.assertEqual(result["word_f1"], 1.0)
- self.assertFalse(task_passed(True, {**result, "stable_task_pass": True}))
+ self.assertTrue(task_passed(True, {**result, "stable_task_pass": True}))
self.assertFalse(task_passed(False, {"stable_task_pass": True}))
def test_block_page_words_in_explanatory_prose_are_not_automatically_block_pages(self):
diff --git a/tests/test_three_cohort_review_followup.py b/tests/test_three_cohort_review_followup.py
index eed455c97..5313eb2e8 100644
--- a/tests/test_three_cohort_review_followup.py
+++ b/tests/test_three_cohort_review_followup.py
@@ -59,12 +59,12 @@ def test_inline_challenge_phrase_is_not_a_standalone_challenge(tag):
'stable_text_anchors':[{'id':'body','alternatives':['Required body']} ]}
case={'id':'article','url':'https://example.com/article','track':'web','task_kind':'body','acceptance_contract':rules}
body=f'Required body. If you see <{tag}>Verify you are human, complete the verification before continuing.
'
- for html, expected in [(body, True), ('Required body.
Verify you are human Required body.
Verify you are human Verify you are human ', False)]:
html=''+html+''
assert evaluate_stable_task(html,rules)['stable_task_pass'] is expected
scored=score_attempt(case,html.encode(),ok=True)
assert scored['content_pass'] is expected
- assert scored['blocked_page'] is (not expected)
+ assert scored['blocked_page'] is ('' in html)
@pytest.mark.parametrize('markup', [' Required body.
Verify you are human
',
From a2ba2237456c6563a0aa9645ecb70613ea014cb3 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Tue, 13 Oct 2026 20:22:55 +0800
Subject: [PATCH 02/50] Fix frozen local report reproduction and admission
cleanup
---
harness/web_fetch/README.md | 10 +-
harness/web_fetch/admission.py | 9 +-
harness/web_fetch/report.py | 11 +-
harness/web_fetch/report_build.py | 58 +++-
harness/web_fetch/runner.py | 372 +++++++++++++-------------
research/native-430-local-report.json | 3 +-
scripts/build_report.py | 7 +-
tests/test_case_admission.py | 46 +++-
tests/test_fixed_native_policy.py | 11 +-
tests/test_local_audit.py | 38 +++
tests/test_report_body_scope.py | 15 ++
11 files changed, 380 insertions(+), 200 deletions(-)
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index 8fdf0ee98..e34ca52fa 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -836,7 +836,9 @@ their frozen evidence and are restored from their pinned release.
### Current local 430-task report
`research/native-430-local-report.json` binds the reviewed ten-method snapshot
-by dataset, result and scorer hashes. The 4,300 measured responses, execution
+by dataset, result and scorer hashes. Its `evidence_path` is relative to
+`BROWSER_EVAL_DATA_HOME` (default `~/.browser-eval`), so an installed snapshot
+can be located without inferring it from earlier work logs. The 4,300 measured responses, execution
segments, review payloads, archived interpretation article and its translations
remain in the external evidence directory; they are not distributed in Git.
The shared report owner reads those saved inputs and does not call providers
@@ -849,7 +851,11 @@ uv run --frozen python scripts/check_report.py /absolute/path/to/local-report \
```
For a fresh build, use `scripts/build_report.py --local-audit` with the matching
-external evidence, an explicit output directory, and `--language bilingual`.
+external evidence, an explicit output directory, and `--language zh --include-english`.
+When rebuilding the bound historical report with a newer scorer, also pass
+`--presentation-binding research/native-430-local-report.json`; this verifies
+the existing report snapshot and preserves its saved scores and evidence bytes.
+The matching binding is required for `check_report.py` as shown above.
Any scoring change must first follow the declared correction protocol; a new
render alone does not update scores. The public release profile and its
publication target are separate and are not changed by this local snapshot.
diff --git a/harness/web_fetch/admission.py b/harness/web_fetch/admission.py
index 96ecb6e18..f3ca803d8 100644
--- a/harness/web_fetch/admission.py
+++ b/harness/web_fetch/admission.py
@@ -4,6 +4,7 @@
import fcntl
import json
import os
+import threading
import time
from contextlib import contextmanager
from pathlib import Path
@@ -88,6 +89,10 @@ def acquire(self, provider, case):
return token
time.sleep(0.1)
+ def _release_when_finished(self, token, pending):
+ pending.join()
+ self.release(token)
+
def release(self, token, *, pending=None, record=None):
with self._state() as state:
if token['key'] not in state['active']:
@@ -103,7 +108,9 @@ def release(self, token, *, pending=None, record=None):
'reason': 'timeout_or_429_observed_during_overlap; causality_unproven'})
if pending is not None and pending.is_alive():
state['active'][token['key']]['quarantined'] = True
- self._event({'event': 'quarantined_until_process_exit', **token})
+ self._event({'event': 'quarantined_until_call_exit', **token})
+ threading.Thread(target=self._release_when_finished, args=(token, pending),
+ daemon=True, name='webfetch-admission-cleanup').start()
return
del state['active'][token['key']]
state['finished_sites'][token['site']] = {'at': time.time(), 'provider': token['provider']}
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index dd6606af8..0a8ec6a26 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -702,7 +702,6 @@ def table(headers,rows,id='',sortable=False,cls='',metrics=None,index_header='
body=''.join(f'{i+1} {values}' for i,(attrs,values) in enumerate(rows))
sort_attrs=(f' data-col="{default_sort[0]}" data-dir="{default_sort[1]}"' if default_sort else '')
return metric_legend(headers if metrics is None else metrics)+f''
-EXTRA+='\ntable.ranking th:nth-child(4),table.ranking th:nth-child(4) button,table.ranking td:nth-child(4){color:#117541;font-weight:800}[data-theme="dark"] table.ranking th:nth-child(4),[data-theme="dark"] table.ranking th:nth-child(4) button,[data-theme="dark"] table.ranking td:nth-child(4){color:#74e1a8}\n'
EXTRA+='\n.change-log section{padding:24px 0}.change-log h2{font-size:24px}.change-log p{margin:10px 0;line-height:1.75}.change-log summary{font-weight:650;color:var(--accent)}.change-log .table-wrap{overflow-x:auto}\n'
EXTRA+='''
.anchor-grid[data-scroll-after="5"],.answer-list[data-scroll-after="5"],.record-scroll{max-height:620px;overflow:auto;overscroll-behavior:contain;scrollbar-gutter:stable;padding-right:6px}
@@ -1238,19 +1237,20 @@ def body_region_rows(case, metrics, output):
from .body_coverage import evaluate_body_requirements
contract=case.get('body_requirements') or {}
if not contract:return [],''
- recalculated={x['id']:x for x in evaluate_body_requirements(output,contract)['body_region_results']}
saved_regions=metrics.get('body_region_results',[])
+ recalculated={}
if saved_regions and all('matched_output_passages' in item for item in saved_regions):
evidence={item['id']:[passage.get('text','') for passage in item.get('matched_output_passages',[]) if passage.get('text')]
for item in saved_regions}
evidence_counts={item['id']:item.get('matched_units',item.get('matched_tokens',0)) for item in saved_regions}
else:
+ recalculated={x['id']:x for x in evaluate_body_requirements(output,contract)['body_region_results']}
evidence,evidence_counts=_body_region_evidence(output,contract)
regions={x['id']:x for x in contract['regions']}
rows=[];cards=[];total=len(regions)
for index,item in enumerate(metrics.get('body_region_results',[]),1):
actual={**recalculated.get(item['id'],{}),**item}
- if evidence_counts.get(item['id'])!=recalculated.get(item['id'],{}).get('matched_units'):
+ if recalculated and evidence_counts.get(item['id'])!=recalculated.get(item['id'],{}).get('matched_units'):
raise ValueError('Body evidence allocation differs from scorer: '+item['id'])
expected=regions.get(item['id'],{}).get('text','')
expected_count=actual.get('expected_units',actual.get('expected_tokens',len(expected.split())))
@@ -2363,7 +2363,10 @@ def option_text(value):
else:
reproduce+='方法配置、凭据变量和本轮实际版本分别见“方法介绍”“账号”和上方方法版本表；不存在的 headless 专用配置不属于本 cohort。
'
if local_matrix:
- reproduce='以下命令从本轮保存的证据构建本地报告，不重新调用产品：
uv run --frozen python scripts/build_report.py --local-audit --language '+('bilingual' if _ENGLISH_GENERATED else 'zh')+' --hide-local-status-banners --evidence /absolute/path/to/report-evidence --output /absolute/path/to/report\nuv run --frozen python scripts/check_report.py /absolute/path/to/report 使用与本轮结果对应的证据目录。调用时的版本、配置和评分器哈希见 '+link('../local-run-contract.json','调用前冻结协议')+' 与 '+link('../execution.json','执行批次')+'。上面的构建命令读取保存的评分；修改评分规则后，应先按声明范围复评分，不能仅重建页面就称成绩已经更新。
'
+ frozen_binding=(out/'presentation-binding.json').is_file()
+ build_binding=' --presentation-binding /absolute/path/to/report-evidence/presentation-binding.json' if frozen_binding else ''
+ check_binding=' --presentation-binding /absolute/path/to/report/presentation-binding.json' if frozen_binding else ''
+ reproduce='以下命令从本轮保存的证据构建本地报告，不重新调用产品：
uv run --frozen python scripts/build_report.py --local-audit --language '+('zh --include-english' if _ENGLISH_GENERATED else 'zh')+' --hide-local-status-banners --evidence /absolute/path/to/report-evidence --output /absolute/path/to/report'+build_binding+'\nuv run --frozen python scripts/check_report.py /absolute/path/to/report'+check_binding+' 使用与本轮结果对应的证据目录。调用时的版本、配置和评分器哈希见 '+link('../local-run-contract.json','调用前冻结协议')+' 与 '+link('../execution.json','执行批次')+'。上面的构建命令读取保存的评分；修改评分规则后，应先按声明范围复评分，不能仅重建页面就称成绩已经更新。
'
support+=section('核心复现指令',reproduce)
write(out,'reproduce/index.html','环境与复现',support)
if not authored_interpretation and not local_matrix:
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 4b0debb0b..26b664c31 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -201,10 +201,59 @@ def _install_report_tree(staging: Path, output: Path):
staging.rename(output)
+def _build_frozen_presentation(source, output, presentation_binding, *,
+ include_english, language, authored_interpretation,
+ show_local_status_banners):
+ """Render a bound historical report using saved scores and public evidence."""
+ from harness.report_languages import select_report_language
+ from .safety import evidence_tree_identity
+ before = evidence_tree_identity(source)
+ _, dataset, records, manifest = validate_local_presentation(source, presentation_binding)
+ review_path = source / "result-review.json"
+ binding = json.loads(presentation_binding.read_text())
+ if review_path.is_file() and not any(
+ item.get("path") == "result-review.json"
+ and item.get("sha256") == hashlib.sha256(review_path.read_bytes()).hexdigest()
+ for item in binding.get("evidence_bindings", [])
+ ):
+ raise ValueError("Frozen presentation review lacks a hash binding")
+ output.parent.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix=".frozen-presentation-", dir=output.parent) as work:
+ staging = Path(work) / "report"
+ shutil.copytree(source, staging)
+ shutil.rmtree(staging / "en", ignore_errors=True)
+ shutil.copy2(presentation_binding, staging / "presentation-binding.json")
+ batches = json.loads((source / "execution.json").read_text())["batches"]
+ for record in records:
+ metadata = record.setdefault("provider_metadata", {})
+ metadata["public_log_note"] = dataset.get("comparison", {}).get(
+ "call_chain_audit", {}).get("call_log_notes", {}).get(record["attempt_id"], "")
+ for kind, suffix in (("debug_log", "debug"), ("response", "response")):
+ relative = "logs/" + record["attempt_id"] + "." + suffix + ".txt"
+ if relative in before:
+ metadata["public_" + kind + "_path"] = relative
+ metadata["public_" + kind + "_sha256"] = before[relative]
+ generate(staging, dataset, records, batches, include_english=include_english,
+ authored_interpretation=authored_interpretation,
+ show_local_status_banners=show_local_status_banners)
+ manifest["languages"] = ["zh", "en"] if include_english else ["zh"]
+ manifest["presentation"] = {"show_local_status_banners": show_local_status_banners,
+ "frozen_scores": True}
+ (staging / "report-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
+ if language is not None:
+ select_report_language(staging, language)
+ validate_local_presentation(staging, presentation_binding)
+ if evidence_tree_identity(source) != before:
+ raise ValueError("Presentation source changed during rendering")
+ _install_report_tree(staging, output)
+ return output / "index.html"
+
+
def build_report(source: Path, output: Path, *, include_english: bool = False,
local_audit: bool = False, authored_interpretation: bool = False,
diagnostic_preview: bool = False, language: str | None = None,
- show_local_status_banners: bool = True) -> Path:
+ show_local_status_banners: bool = True,
+ presentation_binding: Path | None = None) -> Path:
from harness.report_languages import includes_english, select_report_language
include_english = includes_english(language, include_english)
source, output = source.resolve(), output.resolve()
@@ -213,6 +262,13 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
marker = '.generated-report'
if output.exists() and not (output/marker).is_file():
raise ValueError(f'Refusing to replace an unmarked directory: {output}')
+ if presentation_binding is not None:
+ if not local_audit:
+ raise ValueError("A frozen presentation binding requires local audit mode")
+ return _build_frozen_presentation(
+ source, output, presentation_binding, include_english=include_english,
+ language=language, authored_interpretation=authored_interpretation,
+ show_local_status_banners=show_local_status_banners)
dataset_probe = json.loads(contained_path(source, 'dataset.json').read_text())
diagnostic_preview = (diagnostic_preview and
dataset_probe.get('comparison', {}).get('kind') == 'web_fetch_diagnostic')
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index 74718a834..60c69b696 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -624,192 +624,196 @@ def run_provider(provider: Provider) -> None:
# Admission waits before the answer timer and 45-second budget.
admission_token = (case_admission.acquire(provider, case)
if case_admission is not None else None)
- started = utc_now()
- physical_attempts = []
- stopped_provider = False
- for physical_attempt in range(1, max_failure_retries + 2):
- retry_throttle_wait_ms = 0.0
- if physical_attempt > 1 and provider.min_interval_seconds:
- remaining = provider.min_interval_seconds - (time.monotonic() - previous_request_completed)
- if remaining > 0:
- retry_throttle_wait_ms = round(remaining * 1000, 2)
- time.sleep(remaining)
- physical_started = utc_now()
- physical_begin = time.perf_counter()
- pending = None
- caught_exception = None
- diagnostics = None
- if provider.config.get("log_level") == "debug":
- diagnostics = CallDiagnostics(output_dir / "private_artifacts" / "live-logs" / provider.name / case.id / f"run-{run_number:03d}-physical-{physical_attempt:03d}.jsonl", attempt_id)
- diagnostics.emit("caller", "call_started", provider=provider.name, case_id=case.id, url=case.url, timeout_seconds=timeout)
- provider._call_diagnostics = diagnostics
- try:
- result, pending = fetch_with_deadline(provider, case, timeout)
- except Exception as error: # each physical failure remains inspectable
- caught_exception = error
- error_type, error_message = classify_exception(error)
- public_metadata = getattr(error, "public_metadata", {})
- public_metadata = public_metadata if isinstance(public_metadata, dict) else {}
- allowed_diagnostic = {
- key: public_metadata[key]
- for key in ("_answer_elapsed_ms", "exit_code", "stdout",
- "stdout_bytes", "stdout_truncated",
- "stderr", "stderr_bytes", "stderr_truncated")
- if key in public_metadata
- }
- result = ProviderResult(
- ok=False,
- # Adapter exceptions are diagnostics, never a
- # supplier answer for the public scorer.
- content="",
- error_type=error_type,
- error_message=_public_text(error_message),
- metadata=allowed_diagnostic,
+ pending = None
+ record = None
+ try:
+ started = utc_now()
+ physical_attempts = []
+ stopped_provider = False
+ for physical_attempt in range(1, max_failure_retries + 2):
+ retry_throttle_wait_ms = 0.0
+ if physical_attempt > 1 and provider.min_interval_seconds:
+ remaining = provider.min_interval_seconds - (time.monotonic() - previous_request_completed)
+ if remaining > 0:
+ retry_throttle_wait_ms = round(remaining * 1000, 2)
+ time.sleep(remaining)
+ physical_started = utc_now()
+ physical_begin = time.perf_counter()
+ pending = None
+ caught_exception = None
+ diagnostics = None
+ if provider.config.get("log_level") == "debug":
+ diagnostics = CallDiagnostics(output_dir / "private_artifacts" / "live-logs" / provider.name / case.id / f"run-{run_number:03d}-physical-{physical_attempt:03d}.jsonl", attempt_id)
+ diagnostics.emit("caller", "call_started", provider=provider.name, case_id=case.id, url=case.url, timeout_seconds=timeout)
+ provider._call_diagnostics = diagnostics
+ try:
+ result, pending = fetch_with_deadline(provider, case, timeout)
+ except Exception as error: # each physical failure remains inspectable
+ caught_exception = error
+ error_type, error_message = classify_exception(error)
+ public_metadata = getattr(error, "public_metadata", {})
+ public_metadata = public_metadata if isinstance(public_metadata, dict) else {}
+ allowed_diagnostic = {
+ key: public_metadata[key]
+ for key in ("_answer_elapsed_ms", "exit_code", "stdout",
+ "stdout_bytes", "stdout_truncated",
+ "stderr", "stderr_bytes", "stderr_truncated")
+ if key in public_metadata
+ }
+ result = ProviderResult(
+ ok=False,
+ # Adapter exceptions are diagnostics, never a
+ # supplier answer for the public scorer.
+ content="",
+ error_type=error_type,
+ error_message=_public_text(error_message),
+ metadata=allowed_diagnostic,
+ )
+ wall_elapsed_ms = round((time.perf_counter() - physical_begin) * 1000, 2)
+ if diagnostics is not None:
+ diagnostics.finish(result)
+ # Cancellable adapters may need a bounded post-deadline
+ # unwind. That cleanup determines quarantine safety but
+ # is not part of the answer latency.
+ physical_elapsed_ms = result.metadata.pop(
+ '_answer_elapsed_ms', wall_elapsed_ms
)
- wall_elapsed_ms = round((time.perf_counter() - physical_begin) * 1000, 2)
- if diagnostics is not None:
- diagnostics.finish(result)
- # Cancellable adapters may need a bounded post-deadline
- # unwind. That cleanup determines quarantine safety but
- # is not part of the answer latency.
- physical_elapsed_ms = result.metadata.pop(
- '_answer_elapsed_ms', wall_elapsed_ms
- )
- previous_request_completed = time.monotonic()
- if case.task_kind == 'file':
- result.metadata.setdefault('file_evaluation', {'files': [], 'pass': False,
- 'scope': 'returned_bytes_signature_length_sha256'})
- suffix = (f'run-{run_number:03d}.txt' if max_failure_retries == 0 else
- f'run-{run_number:03d}-physical-{physical_attempt:03d}.txt')
- artifact_rel = Path('artifacts') / provider.name / case.id / suffix
- artifact = persist_result_artifacts(output_dir, artifact_rel, result)
- artifact_bytes = result.content.encode('utf-8', errors='replace')
- measured = score_attempt(
- case, artifact_bytes, ok=result.ok, metadata=result.metadata,
- title=result.title, final_url=result.final_url,
- list_reference=load_list_reference(case, dataset_path.parent),
- status_code=result.status_code,
- prepared_answer=result.prepared_answer,
- )
- measured['deadline_overrun'] = physical_elapsed_ms > timeout * 1000
- measured['deadline_budget_ms'] = timeout * 1000
- if binding is not None:
- measured['reference_sha256'] = binding['reference_sha256']
- physical_passed = task_passed(result.ok, measured, case.task_kind)
- physical_completed = utc_now()
- call_log = persist_public_call_log(
- output_dir, provider=provider, case=case, attempt_id=attempt_id,
- run_number=run_number, physical_attempt=physical_attempt,
- started_at=physical_started, completed_at=physical_completed,
- elapsed_ms=physical_elapsed_ms,
- retry_throttle_wait_ms=retry_throttle_wait_ms, timeout=timeout,
- artifact_rel=artifact_rel,
- content_sha256=measured['content_sha256'], result=result,
- exception=caught_exception,
- )
- artifact_paths = [artifact_rel.as_posix()]
- for key in ('rendered_html_path', 'raw_stdout_path'):
- if result.metadata.get(key):
- artifact_paths.append(result.metadata[key])
- artifact_paths.extend(
- item['artifact_path']
- for item in result.metadata.get('file_evaluation', {}).get('files', [])
- if item.get('artifact_path')
- )
- artifact_hashes = {
- relative: hashlib.sha256(contained_path(
- output_dir, relative, reject_symlinks=True).read_bytes()).hexdigest()
- for relative in artifact_paths
- }
- if artifact_hashes[artifact_rel.as_posix()] != measured['content_sha256']:
+ previous_request_completed = time.monotonic()
+ if case.task_kind == 'file':
+ result.metadata.setdefault('file_evaluation', {'files': [], 'pass': False,
+ 'scope': 'returned_bytes_signature_length_sha256'})
+ suffix = (f'run-{run_number:03d}.txt' if max_failure_retries == 0 else
+ f'run-{run_number:03d}-physical-{physical_attempt:03d}.txt')
+ artifact_rel = Path('artifacts') / provider.name / case.id / suffix
+ artifact = persist_result_artifacts(output_dir, artifact_rel, result)
+ artifact_bytes = result.content.encode('utf-8', errors='replace')
+ measured = score_attempt(
+ case, artifact_bytes, ok=result.ok, metadata=result.metadata,
+ title=result.title, final_url=result.final_url,
+ list_reference=load_list_reference(case, dataset_path.parent),
+ status_code=result.status_code,
+ prepared_answer=result.prepared_answer,
+ )
+ measured['deadline_overrun'] = physical_elapsed_ms > timeout * 1000
+ measured['deadline_budget_ms'] = timeout * 1000
+ if binding is not None:
+ measured['reference_sha256'] = binding['reference_sha256']
+ physical_passed = task_passed(result.ok, measured, case.task_kind)
+ physical_completed = utc_now()
+ call_log = persist_public_call_log(
+ output_dir, provider=provider, case=case, attempt_id=attempt_id,
+ run_number=run_number, physical_attempt=physical_attempt,
+ started_at=physical_started, completed_at=physical_completed,
+ elapsed_ms=physical_elapsed_ms,
+ retry_throttle_wait_ms=retry_throttle_wait_ms, timeout=timeout,
+ artifact_rel=artifact_rel,
+ content_sha256=measured['content_sha256'], result=result,
+ exception=caught_exception,
+ )
+ artifact_paths = [artifact_rel.as_posix()]
+ for key in ('rendered_html_path', 'raw_stdout_path'):
+ if result.metadata.get(key):
+ artifact_paths.append(result.metadata[key])
+ artifact_paths.extend(
+ item['artifact_path']
+ for item in result.metadata.get('file_evaluation', {}).get('files', [])
+ if item.get('artifact_path')
+ )
+ artifact_hashes = {
+ relative: hashlib.sha256(contained_path(
+ output_dir, relative, reject_symlinks=True).read_bytes()).hexdigest()
+ for relative in artifact_paths
+ }
+ if artifact_hashes[artifact_rel.as_posix()] != measured['content_sha256']:
+ raise ValueError('Persisted output differs from scored bytes')
+ private_artifacts = []
+ private_binding = raw_response_binding(result.metadata)
+ if private_binding:
+ private_artifacts.append(private_binding)
+ diagnostic_binding = diagnostic_log_binding(result.metadata)
+ if diagnostic_binding:
+ private_artifacts.append(diagnostic_binding)
+ physical_attempts.append({
+ 'physical_attempt': physical_attempt,
+ 'started_at': physical_started,
+ 'completed_at': physical_completed,
+ 'elapsed_ms': physical_elapsed_ms,
+ 'retry_throttle_wait_ms': retry_throttle_wait_ms,
+ 'ok': result.ok,
+ 'status_code': result.status_code,
+ 'error_type': result.error_type,
+ 'error_message': result.error_message,
+ 'task_pass': physical_passed,
+ 'content_sha256': measured['content_sha256'],
+ 'artifact_path': artifact_rel.as_posix(),
+ 'artifact_paths': artifact_paths,
+ 'artifact_sha256': artifact_hashes,
+ 'call_log': call_log,
+ **({'private_artifacts': private_artifacts} if private_artifacts else {}),
+ })
+ stopped_provider = (result.error_type == 'timeout' and pending is not None
+ and pending.is_alive())
+ if physical_passed or stopped_provider or physical_attempt > max_failure_retries:
+ break
+ elapsed_ms = round(sum(item['elapsed_ms'] + item['retry_throttle_wait_ms']
+ for item in physical_attempts), 2)
+ result.metadata['call_log'] = physical_attempts[-1]['call_log']
+ if len(physical_attempts) > 1:
+ result.metadata['retry'] = {
+ 'physical_attempts': len(physical_attempts),
+ 'retries_used': len(physical_attempts) - 1,
+ 'trigger': 'previous physical attempt failed the task predicate',
+ 'selection': 'final physical attempt is authoritative; no best-of selection',
+ 'attempts': physical_attempts,
+ }
+ finished = utc_now()
+ if hashlib.sha256(artifact.read_bytes()).hexdigest() != measured['content_sha256']:
raise ValueError('Persisted output differs from scored bytes')
- private_artifacts = []
- private_binding = raw_response_binding(result.metadata)
- if private_binding:
- private_artifacts.append(private_binding)
- diagnostic_binding = diagnostic_log_binding(result.metadata)
- if diagnostic_binding:
- private_artifacts.append(diagnostic_binding)
- physical_attempts.append({
- 'physical_attempt': physical_attempt,
- 'started_at': physical_started,
- 'completed_at': physical_completed,
- 'elapsed_ms': physical_elapsed_ms,
- 'retry_throttle_wait_ms': retry_throttle_wait_ms,
- 'ok': result.ok,
- 'status_code': result.status_code,
- 'error_type': result.error_type,
- 'error_message': result.error_message,
- 'task_pass': physical_passed,
- 'content_sha256': measured['content_sha256'],
- 'artifact_path': artifact_rel.as_posix(),
- 'artifact_paths': artifact_paths,
- 'artifact_sha256': artifact_hashes,
- 'call_log': call_log,
- **({'private_artifacts': private_artifacts} if private_artifacts else {}),
- })
- stopped_provider = (result.error_type == 'timeout' and pending is not None
- and pending.is_alive())
- if physical_passed or stopped_provider or physical_attempt > max_failure_retries:
- break
- elapsed_ms = round(sum(item['elapsed_ms'] + item['retry_throttle_wait_ms']
- for item in physical_attempts), 2)
- result.metadata['call_log'] = physical_attempts[-1]['call_log']
- if len(physical_attempts) > 1:
- result.metadata['retry'] = {
- 'physical_attempts': len(physical_attempts),
- 'retries_used': len(physical_attempts) - 1,
- 'trigger': 'previous physical attempt failed the task predicate',
- 'selection': 'final physical attempt is authoritative; no best-of selection',
- 'attempts': physical_attempts,
- }
- finished = utc_now()
- if hashlib.sha256(artifact.read_bytes()).hexdigest() != measured['content_sha256']:
- raise ValueError('Persisted output differs from scored bytes')
- record = AttemptRecord(
- attempt_id=attempt_id,
- provider=provider.name,
- provider_type=provider.type,
- case_id=case.id,
- case_name=case.name,
- category=case.category,
- url=case.url,
- run=run_number,
- started_at=started,
- elapsed_ms=elapsed_ms,
- ok=result.ok,
- status_code=result.status_code,
- final_url=result.final_url,
- title=result.title,
- error_type=result.error_type,
- error_message=result.error_message,
- artifact_path=artifact_rel.as_posix(),
- metrics=measured,
- provider_metadata={
- **result.metadata,
- "configured_min_interval_seconds": provider.min_interval_seconds,
- "client_throttle_wait_ms": throttle_wait_ms,
- },
- ).to_dict()
- apply_request_status(record)
- record["completed_at"] = finished
- from .models import execution_id
- record['run_id'] = execution_id(record)
- passed = task_passed(record['ok'],record['metrics'],case.task_kind)
- state = "PASS" if passed else "FAIL"
- with result_lock:
- records.append(record)
- results_file.write(json.dumps(record, ensure_ascii=True) + "\n")
- results_file.flush()
- print(
- f"{state:4} {provider.name:14} {case.id:24} "
- f"{elapsed_ms:9.2f} ms {result.error_type or ''}",
- flush=True,
- )
- if delay:
- time.sleep(delay)
- if case_admission is not None:
- case_admission.release(admission_token, pending=pending, record=record)
+ record = AttemptRecord(
+ attempt_id=attempt_id,
+ provider=provider.name,
+ provider_type=provider.type,
+ case_id=case.id,
+ case_name=case.name,
+ category=case.category,
+ url=case.url,
+ run=run_number,
+ started_at=started,
+ elapsed_ms=elapsed_ms,
+ ok=result.ok,
+ status_code=result.status_code,
+ final_url=result.final_url,
+ title=result.title,
+ error_type=result.error_type,
+ error_message=result.error_message,
+ artifact_path=artifact_rel.as_posix(),
+ metrics=measured,
+ provider_metadata={
+ **result.metadata,
+ "configured_min_interval_seconds": provider.min_interval_seconds,
+ "client_throttle_wait_ms": throttle_wait_ms,
+ },
+ ).to_dict()
+ apply_request_status(record)
+ record["completed_at"] = finished
+ from .models import execution_id
+ record['run_id'] = execution_id(record)
+ passed = task_passed(record['ok'],record['metrics'],case.task_kind)
+ state = "PASS" if passed else "FAIL"
+ with result_lock:
+ records.append(record)
+ results_file.write(json.dumps(record, ensure_ascii=True) + "\n")
+ results_file.flush()
+ print(
+ f"{state:4} {provider.name:14} {case.id:24} "
+ f"{elapsed_ms:9.2f} ms {result.error_type or ''}",
+ flush=True,
+ )
+ if delay:
+ time.sleep(delay)
+ finally:
+ if case_admission is not None:
+ case_admission.release(admission_token, pending=pending, record=record)
if stopped_provider:
remaining_attempts = ((runs - run_number) * len(cases)
+ len(cases) - case_index)
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
index 1b60b5573..eed10466b 100644
--- a/research/native-430-local-report.json
+++ b/research/native-430-local-report.json
@@ -34,5 +34,6 @@
"path": "archived-reading.json",
"sha256": "85da5e52c295eb7fffbcd3e10e41d33294eeadf233ec7a6958de68ed0c9e30fb"
}
- ]
+ ],
+ "evidence_path": "runs/native-430-20261010T071648Z/local-report"
}
diff --git a/scripts/build_report.py b/scripts/build_report.py
index bc3cbede8..1b71a0f2e 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -24,6 +24,8 @@
help='generate both editions; --language selects the homepage')
parser.add_argument('--refresh-presentation', type=Path,
help='refresh an existing local report using a hash-bound current-local-results.json; preserve measured evidence')
+ parser.add_argument('--presentation-binding', type=Path,
+ help='rebuild a hash-bound historical local report without rescoring')
parser.add_argument('--local-audit',action='store_true',
help='build an explicitly local, non-publishable audit report from bound evidence')
parser.add_argument('--hide-local-status-banners', action='store_true',
@@ -35,6 +37,8 @@
parser.add_argument('--wcxb',action='store_true',help='Build the independently bound WCXB report')
parser.add_argument('--webmainbench-full',action='store_true',help='Build the separately certified complete 545-case report')
args=parser.parse_args()
+ if args.presentation_binding and not args.local_audit:
+ parser.error('--presentation-binding requires --local-audit')
if args.hide_local_status_banners and not args.local_audit:
parser.error('--hide-local-status-banners requires --local-audit')
from harness.reporting.templates import resolve_template
@@ -73,7 +77,8 @@ def requested_language(output, supported=SUPPORTED_REPORT_LANGUAGES):
print(build_report(evidence,args.output,
include_english=args.include_english, language=requested_language(args.output),
local_audit=True,
- show_local_status_banners=not args.hide_local_status_banners))
+ show_local_status_banners=not args.hide_local_status_banners,
+ presentation_binding=args.presentation_binding))
raise SystemExit(0)
if sum(bool(value) for value in (args.profile, args.wcxb, args.webmainbench_full)) > 1:
parser.error('--profile, --wcxb and --webmainbench-full are mutually exclusive')
diff --git a/tests/test_case_admission.py b/tests/test_case_admission.py
index 7edbce459..0ebe82e64 100644
--- a/tests/test_case_admission.py
+++ b/tests/test_case_admission.py
@@ -23,7 +23,29 @@ def test_second_local_waits_and_parallel_timeout_reduces_to_serial(tmp_path):
c=cloud.acquire(method('c'),case('three.test'));cloud.release(c,record={'error_type':'timeout','attempt_id':'c'});assert not json.loads((tmp_path/'state.json').read_text())['parallel_enabled'];local.release(a);t.join(1);assert seen;local.release(seen[0])
def test_pending_call_keeps_site_and_resource_reserved(tmp_path):
- local=CaseAdmission(tmp_path,group='local',site_cooldown=0);a=local.acquire(method('a'),case('one.test'));local.release(a,pending=SimpleNamespace(is_alive=lambda:True));state=json.loads((tmp_path/'state.json').read_text());assert state['active'][a['key']]['quarantined'];assert a['key'] in state['active']
+ local = CaseAdmission(tmp_path, group='local', site_cooldown=0)
+ cloud = CaseAdmission(tmp_path, group='cloud', site_cooldown=0)
+ token = local.acquire(method('a'), case('one.test'))
+ finish = threading.Event()
+ pending = threading.Thread(target=finish.wait, daemon=True)
+ pending.start()
+ local.release(token, pending=pending)
+ state = json.loads((tmp_path / 'state.json').read_text())
+ assert state['active'][token['key']]['quarantined']
+ admitted = []
+ waiter = threading.Thread(target=lambda: admitted.append(
+ cloud.acquire(method('b'), case('one.test'))), daemon=True)
+ waiter.start()
+ try:
+ time.sleep(.15)
+ assert not admitted
+ finally:
+ finish.set()
+ pending.join(1)
+ waiter.join(2)
+ assert admitted, 'Finished background calls must release their reservation'
+ cloud.release(admitted[0])
+ assert not json.loads((tmp_path / 'state.json').read_text())['active']
def test_admission_wait_excluded_from_answer_timeout_and_latency(tmp_path):
class Fixture(Provider):
@@ -48,3 +70,25 @@ def test_same_method_interval_is_not_replaced_by_site_cooldown(tmp_path):
b = gate.acquire(method('b'), case('one.test'))
assert time.monotonic()-begin >= .2
gate.release(b)
+
+
+def test_failed_scoring_releases_admission_without_hiding_error(tmp_path, monkeypatch):
+ import pytest
+ from harness.web_fetch import runner
+
+ class Fixture(Provider):
+ def fetch(self, case, timeout):
+ return ProviderResult(True, 'Expected')
+
+ gate = CaseAdmission(tmp_path / 'admission', group='local', site_cooldown=0)
+ def broken_score(*args, **kwargs):
+ raise ValueError('scoring failed')
+ monkeypatch.setattr(runner, 'score_attempt', broken_score)
+ with pytest.raises(ValueError, match='scoring failed'):
+ run_evaluation(output_dir=tmp_path / 'run', dataset_meta={'id': 'test'},
+ dataset_path=Path('dataset.json'),
+ cases=[EvalCase('case', 'Case', 'https://example.com', 'static', ['Expected'])],
+ providers=[Fixture('fixture', {'type': 'fixture'})],
+ provider_config_path=Path('providers.json'), runs=1, timeout=1, delay=0,
+ case_admission=gate)
+ assert not json.loads((gate.root / 'state.json').read_text())['active']
diff --git a/tests/test_fixed_native_policy.py b/tests/test_fixed_native_policy.py
index 3cdc27b47..e8cff000d 100644
--- a/tests/test_fixed_native_policy.py
+++ b/tests/test_fixed_native_policy.py
@@ -3,7 +3,7 @@
from dataclasses import replace
from unittest.mock import patch
import pytest
-from harness.web_fetch.config import load_json
+from tests.conftest import read_frozen_dataset_bytes
from harness.web_fetch.models import EvalCase
from harness.web_fetch.providers.base import build_provider
from harness.web_fetch.providers.http_client import HttpResponse
@@ -71,8 +71,8 @@ def test_web_family_does_not_change_native_cloud_request(kind, module):
bodies = []
for track in ("web", "dynamic", "challenge", "risk_control"):
case = EvalCase("one", "one", "https://example.com/", "control", [], track=track,
- acceptance_contract=load_json(Path(__file__).resolve().parents[1] /
- "evaluation/dynamic-content-100-moli-webfetch/dataset.json")["cases"][0]["acceptance_contract"])
+ acceptance_contract={"version": 1, "schema": "body_text", "minimum_records": 0,
+ "stable_text_anchors": [{"id": "answer", "alternatives": ["Expected answer"]}]})
with patch(f"harness.web_fetch.providers.{module}.request") as call:
call.return_value = HttpResponse(200, case.url, {}, b'{}')
provider.fetch(case, 45)
@@ -110,13 +110,14 @@ def test_native_cloudflare_media_has_no_answer_dependent_selectors_or_timers():
def test_all_430_frozen_cases_pass_native_input_guard():
- from pathlib import Path
from harness.web_fetch.config import load_dataset
provider = build_provider("fixture", {"type": "fixture", "uniform_native_parameters": True})
count = 0
for filename in ("dataset.json", "dynamic-content-100-moli-webfetch/dataset.json",
"popular-sites/dataset.json", "risk-control/dataset.json"):
- _, cases = load_dataset(Path(__file__).resolve().parents[1] / "evaluation" / filename)
+ path = Path(__file__).resolve().parents[1] / "evaluation" / filename
+ read_frozen_dataset_bytes(path)
+ _, cases = load_dataset(path)
for case in cases:
if filename == "dataset.json" and case.track not in {"web", "document", "image", "video"}:
continue
diff --git a/tests/test_local_audit.py b/tests/test_local_audit.py
index 2fe0b2c78..e8b5e02dc 100644
--- a/tests/test_local_audit.py
+++ b/tests/test_local_audit.py
@@ -286,6 +286,44 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp
"measured_scoring_source_sha256": historical_scorer,
"measured_scoring_git_revision": "historical-revision"}
_write(binding_path, binding)
+ public_logs = output / "logs"
+ public_logs.mkdir(exist_ok=True)
+ attempt_id = frozen_rows[0]["attempt_id"]
+ debug = public_logs / (attempt_id + ".debug.txt")
+ response = public_logs / (attempt_id + ".response.txt")
+ debug.write_text("actual library event\n")
+ response.write_text('{"observed": "actual API response"}\n')
+ rebuilt = tmp_path / "frozen-bilingual-report"
+ before_results = (output / "results.jsonl").read_bytes()
+ with pytest.raises(ValueError, match="stale scoring"):
+ build_report(output, rebuilt, local_audit=True, include_english=True)
+ build_report(output, rebuilt, local_audit=True, include_english=True,
+ presentation_binding=binding_path)
+ assert (rebuilt / "results.jsonl").read_bytes() == before_results
+ assert (rebuilt / "dataset.json").read_bytes() == (output / "dataset.json").read_bytes()
+ assert (rebuilt / "en/index.html").is_file()
+ for prefix in ("", "en/"):
+ detail = (rebuilt / prefix / "attempts" / (attempt_id + ".html")).read_text()
+ assert attempt_id + ".debug.txt" in detail
+ assert attempt_id + ".response.html" in detail
+ assert (rebuilt / "logs" / debug.name).read_bytes() == debug.read_bytes()
+ assert (rebuilt / "logs" / response.name).read_bytes() == response.read_bytes()
+ assert json.loads((rebuilt / "report-manifest.json").read_text())["presentation"]["frozen_scores"]
+ reproduce = (rebuilt / "reproduce/index.html").read_text()
+ assert "--language zh --include-english" in reproduce
+ assert "--language bilingual" not in reproduce
+ assert "--presentation-binding" in reproduce
+ assert all(row["metrics"]["scoring_source_sha256"] == historical_scorer
+ for row in map(json.loads, (rebuilt / "results.jsonl").read_text().splitlines()))
+ chinese = tmp_path / "frozen-chinese-report"
+ build_report(rebuilt, chinese, local_audit=True, presentation_binding=binding_path)
+ assert list((chinese / "en").rglob("*.html")) == [chinese / "en/index.html"]
+ assert "not been generated" in (chinese / "en/index.html").read_text()
+ _write(binding_path, {**binding, "results_sha256": "0" * 64})
+ with pytest.raises(ValueError, match="input binding changed"):
+ build_report(output, tmp_path / "invalid-rebuild", local_audit=True,
+ presentation_binding=binding_path)
+ _write(binding_path, binding)
(output / "changes/index.html").write_text("Old release history")
before = {path.relative_to(output).as_posix(): _digest(path)
for path in output.rglob("*") if path.is_file()}
diff --git a/tests/test_report_body_scope.py b/tests/test_report_body_scope.py
index c15049bf2..1118e2c69 100644
--- a/tests/test_report_body_scope.py
+++ b/tests/test_report_body_scope.py
@@ -78,6 +78,21 @@ def test_body_display_uses_visible_body_without_title_or_scripts(self):
self.assertEqual(len(rows),1)
self.assertNotIn('ghijkl',cards)
+ def test_bound_body_evidence_is_displayed_without_rescoring(self):
+ from harness.web_fetch.scoring import score_attempt
+ contract={'version':3,'unit':'region_compact_hexagram_recall',
+ 'minimum_coverage':.8,'minimum_floor':.6,'minimum_region_passes':1,
+ 'regions':[{'id':'body','text':'abcdefghijkl'}]}
+ c=case('body',kind='body');c['body_requirements']=contract
+ metrics=score_attempt(c,b'abcdefghijkl',ok=True)
+ saved=copy.deepcopy(metrics)
+ with patch('harness.web_fetch.body_coverage.evaluate_body_requirements',
+ side_effect=AssertionError('presentation must not rescore saved passages')):
+ rows,cards=body_region_rows(c,metrics,'abcdefghijkl')
+ self.assertEqual(metrics,saved)
+ self.assertEqual(len(rows),1)
+ self.assertIn('abcdefghijkl',''.join(cards))
+
def test_body_evidence_uses_the_canonical_global_allocation(self):
shared='abcdefghijklmnopqrstuvwx'*3
contract={'version':3,'unit':'region_compact_hexagram_recall',
From 47b6cf2f4e0ea0d09645424e87592197b365024e Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Tue, 13 Oct 2026 20:54:45 +0800
Subject: [PATCH 03/50] Bind frozen report inputs and localize evidence
interfaces safely
---
harness/web_fetch/report.py | 12 ++++-----
harness/web_fetch/report_build.py | 16 ++++++-----
harness/web_fetch/report_i18n.py | 21 ++++++++++++++-
research/native-430-local-report.json | 4 +++
tests/test_local_audit.py | 38 ++++++++++++++++++++++++++-
5 files changed, 77 insertions(+), 14 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 0a8ec6a26..260c75496 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -1272,8 +1272,8 @@ def stable_answer_panel(contract):
items=[]
for index,anchor in enumerate(anchors,1):
alternatives=anchor.get('alternatives',[])
- text='\n或\n'.join(alternatives)
- items.append(f'必须出现的稳定文字 {index}（任一写法即可） {esc(text)} 或 \n'.join(esc(value) for value in alternatives)
+ items.append(f'必须出现的稳定文字 {index}（任一写法即可） {text} 正文长度要求 至少 {case.get("min_content_chars",1):,} 字符 必需文字组 {index}（任一写法即可） {esc(text)} 或 \n'.join(esc(value) for value in fact.get('alternatives',[]))
+ items.append(f'必需文字组 {index}（任一写法即可） {text} 全部文字组都须命中。
' if threshold==1 else
f'文字组覆盖至少达到 {pct(threshold)}。
')
@@ -1381,7 +1381,7 @@ def _quote_field_issue(metrics):
def stable_anchor_evidence(metrics,output):
cards=[];normalized_lines=None
for index,item in enumerate(metrics.get('stable_anchor_results',[]),1):
- alternatives=item.get('alternatives',[]);expected='\n或\n'.join(alternatives)
+ alternatives=item.get('alternatives',[]);expected='\n或 \n'.join(esc(value) for value in alternatives)
excerpt=''
for alternative in alternatives:
if normalized_lines is None:normalized_lines=_normalized_evidence_lines(output)
@@ -1405,7 +1405,7 @@ def stable_anchor_evidence(metrics,output):
field_detail='整条记录按字段判定；文字出现不等于列表边界完整。
'+table(['字段','标准答案','实际解析','判定'],rows,metrics=[])
badge='字段边界不合格'
actual=(f'{esc(excerpt)}
' if excerpt else '实际返回中未找到可供定位的连续原文。
')
- cards.append(f'
稳定文字 {index} {badge} {field_detail}
稳定文字 {index} {badge} {field_detail}共 {len(cards)} 项稳定文字判定，全部参与评分；当前区域显示约 5 项高度，向下滚动查看全部。
'
if len(cards)>5 else '')
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 26b664c31..420dfdfac 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -207,16 +207,20 @@ def _build_frozen_presentation(source, output, presentation_binding, *,
"""Render a bound historical report using saved scores and public evidence."""
from harness.report_languages import select_report_language
from .safety import evidence_tree_identity
+ if source in output.parents:
+ raise ValueError("Frozen presentation output must be outside its source tree")
before = evidence_tree_identity(source)
_, dataset, records, manifest = validate_local_presentation(source, presentation_binding)
review_path = source / "result-review.json"
binding = json.loads(presentation_binding.read_text())
- if review_path.is_file() and not any(
- item.get("path") == "result-review.json"
- and item.get("sha256") == hashlib.sha256(review_path.read_bytes()).hexdigest()
- for item in binding.get("evidence_bindings", [])
- ):
- raise ValueError("Frozen presentation review lacks a hash binding")
+ required = ["local-run-contract.json", "execution.json"]
+ if review_path.is_file():
+ required.append("result-review.json")
+ for relative in required:
+ if not any(item.get("path") == relative
+ and item.get("sha256") == before.get(relative)
+ for item in binding.get("evidence_bindings", [])):
+ raise ValueError("Frozen presentation input lacks a hash binding: " + relative)
output.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".frozen-presentation-", dir=output.parent) as work:
staging = Path(work) / "report"
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index fcdb4808a..66fbfbaf4 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -79,6 +79,23 @@
"比例只对应本报告列出的案例与配置。先看总榜和分榜，再打开具体案例，对照验收要求与实际返回。请求成功只表示方法正常返回；任务通过表示返回内容满足该案例的全部要求。": "Rates apply only to the cases and configurations in this report. Start with the overall and category rankings, then open individual cases to compare requirements with actual output. Request success means the method returned normally; task success means the returned content met every requirement for that case.",
"评分规则与计时范围见“评测方法”，检查和修正记录见“核验记录”，方法版本、固定配置与日志范围见“环境与复现”。": "See Methodology for scoring and timing, Review history for checks and corrections, and Reproduce for method versions, fixed options and log coverage.",
"已保存的诊断日志": "Saved diagnostic logs",
+ "已保存的诊断日志 · Web Fetch": "Saved diagnostic logs · Web Fetch",
+ "完整原始响应": "Complete original response",
+ "完整原始响应 · Web Fetch": "Complete original response · Web Fetch",
+ "查看完整 API 响应（已隐藏凭据与本机路径）": "View complete API response (credentials and local paths hidden)",
+ "查看已保存的诊断日志（已隐藏凭据与本机路径）": "View saved diagnostic logs (credentials and local paths hidden)",
+ "已隐藏凭据与本机路径": "Credentials and local paths hidden",
+ "真实保存记录；仅为浏览显示，未增加产品调用。": "Actual saved records, formatted for browsing without additional product calls.",
+ "记录来源": "Record source",
+ "下载原始公开文件": "Download original public file",
+ "仅包含实际观测的日志。": "Contains only observed logs.",
+ "产品实际返回；公开副本已隐藏凭据与本机路径。": "Actual product response; credentials and local paths are hidden in this public copy.",
+ "或": "or",
+ "全部文字组都须命中。": "Every text group must match.",
+ "完整参考正文": "Complete reference body",
+ "查看原始验收规则": "View raw acceptance rules",
+ "正文长度要求": "Body length requirement",
+ "· 公开 SHA-256：": "· Public SHA-256:",
"已保存文件截断；原生日志范围见说明": "Saved file truncated; see native log coverage notes",
"已保存文件未截断；原生日志范围见说明": "Saved file not truncated; see native log coverage notes",
"本次没有取得产品或依赖库的原生日志。已保存的调用方记录见下方。": "No native product or dependency logs were obtained for this call. Saved caller records appear below.",
@@ -2708,7 +2725,7 @@
(re.compile(r"^实际解析出 (\d+) 条完整且不重复的记录，要求至少 (\d+) 条；结构检查未通过，稳定文字检查未通过。$"), r"Parsed \1 complete, unique records against a minimum of \2; structure and stable-text checks failed."),
(re.compile(r"^实际解析出 (\d+) 条完整且不重复的记录，要求至少 (\d+) 条；结构检查(通过|未通过)，稳定文字检查(通过|未通过)。$"), lambda m: f"Parsed {m.group(1)} complete, unique records against a minimum of {m.group(2)}; structure {'passed' if m.group(3) == '通过' else 'failed'} and stable-text checks {'passed' if m.group(4) == '通过' else 'failed'}."),
(re.compile(r"^完整度 / 纯净度 / F1 为 ([^。]+)。$"), r"Recall / precision / F1: \1."),
- (re.compile(r"^正文没有抓全；低于目标的位置为：(.+)。下方会把完整标准答案与实际返回并排展示。$"), r"Body coverage is incomplete. Regions below target: \1. The complete reference and actual output are shown side by side below."),
+ (re.compile(r"^正文没有抓全；低于目标的位置为：(.+)。下方会把完整标准答案与实际返回并排展示。$"), lambda m: "Body coverage is incomplete. Regions below target: " + m[1].replace("位置 ", "region ") + ". The complete reference and actual output are shown side by side below."),
(re.compile(r"^这一段标准答案产生 ([\d,]+) 个连续六字符片段；在上方完整实际返回（([\d,]+) 字符）中命中 ([\d,]+) 个，所以覆盖率为 ([\d,]+) ÷ ([\d,]+) = ([\d.]+%)。目标 ([\d.]+%)，硬下限 ([\d.]+%)。$"), r"This reference region produces \1 contiguous six-character shingles. \3 are found in the complete output above (\2 characters), so coverage is \4 ÷ \5 = \6. Target: \7; hard floor: \8."),
(re.compile(r"^· 采集时间 (.+)$"), r"· captured at \1"),
]
@@ -3431,6 +3448,8 @@ def release_translations(history: dict) -> dict[str, str]:
]
REGEX.extend([
+ (re.compile(r"^必需文字组 (\d+)（任一写法即可）$"), r"Required text group \1 (any alternative may match)"),
+ (re.compile(r"^· 原始采集记录 ([\d,]+) 字节（可含协议返回） · 原始 SHA-256：$"), r"· Original capture: \1 bytes (may include protocol responses) · Original SHA-256:"),
(re.compile(r"^本报告纳入 ([\d,]+) 个逻辑结果。运行参数见下表；保留的物理调用与输出见各结果详情。$"),
r"This report includes \1 logical results. Run parameters are listed below; each result detail contains its retained physical calls and outputs."),
(re.compile(r"^· 请求：(成功|失败) · 耗时：([\d,.]+) ms · 实际输出：([\d,]+) 字符$"),
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
index eed10466b..264021924 100644
--- a/research/native-430-local-report.json
+++ b/research/native-430-local-report.json
@@ -33,6 +33,10 @@
{
"path": "archived-reading.json",
"sha256": "85da5e52c295eb7fffbcd3e10e41d33294eeadf233ec7a6958de68ed0c9e30fb"
+ },
+ {
+ "path": "execution.json",
+ "sha256": "42a7adef621fd925308c6c82b82aa08eca0f42dcb41affcc76947c912323b4ac"
}
],
"evidence_path": "runs/native-430-20261010T071648Z/local-report"
diff --git a/tests/test_local_audit.py b/tests/test_local_audit.py
index e8b5e02dc..2c86de9ec 100644
--- a/tests/test_local_audit.py
+++ b/tests/test_local_audit.py
@@ -284,8 +284,20 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp
"frozen_dataset_sha256": _digest(output / "dataset.json"),
"results_sha256": manifest["results_sha256"],
"measured_scoring_source_sha256": historical_scorer,
- "measured_scoring_git_revision": "historical-revision"}
+ "measured_scoring_git_revision": "historical-revision",
+ "evidence_bindings": [{"path": name, "sha256": _digest(output / name)}
+ for name in ("local-run-contract.json", "execution.json")]}
_write(binding_path, binding)
+ for missing in ("local-run-contract.json", "execution.json"):
+ _write(binding_path, {**binding, "evidence_bindings": [
+ item for item in binding["evidence_bindings"] if item["path"] != missing]})
+ with pytest.raises(ValueError, match="input lacks a hash binding: " + missing):
+ build_report(output, tmp_path / (missing + "-unbound"), local_audit=True,
+ presentation_binding=binding_path)
+ _write(binding_path, binding)
+ with pytest.raises(ValueError, match="outside its source tree"):
+ build_report(output, output / "nested-report", local_audit=True,
+ presentation_binding=binding_path)
public_logs = output / "logs"
public_logs.mkdir(exist_ok=True)
attempt_id = frozen_rows[0]["attempt_id"]
@@ -306,6 +318,30 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp
detail = (rebuilt / prefix / "attempts" / (attempt_id + ".html")).read_text()
assert attempt_id + ".debug.txt" in detail
assert attempt_id + ".response.html" in detail
+ from harness.web_fetch.report_i18n import untranslated_english_narrative, localize_html
+ from pathlib import PurePosixPath
+ english_detail = (rebuilt / "en/attempts" / (attempt_id + ".html")).read_text()
+ assert "View complete API response (credentials and local paths hidden)" in english_detail
+ for suffix in ("debug", "response"):
+ view = (rebuilt / "en/logs" / (attempt_id + "." + suffix + ".html")).read_text()
+ assert not untranslated_english_narrative(view, {"中文"})
+ sample = ('必需文字组 3（任一写法即可）
first\n或 \nsecond 引文 first\n或\nsecond '
+ '正文没有抓全；低于目标的位置为：位置 1 77.5%。下方会把完整标准答案与实际返回并排展示。
'
+ '原始返回：位置 1\n或\n真实正文 ')
+ localized = localize_html(sample, PurePosixPath("attempts/example.html"))
+ assert "Required text group 3" in localized
+ assert "first\nor \nsecond" in localized
+ assert "引文 first\n或\nsecond" in localized
+ from harness.web_fetch.report import required_text_answer_panel, stable_answer_panel
+ alternatives = ["\n或\noriginal", "second"]
+ panels = [required_text_answer_panel({"required_facts": [{"alternatives": alternatives}]}),
+ stable_answer_panel({"stable_text_anchors": [{"alternatives": alternatives}], "minimum_records": 1})]
+ for panel in panels:
+ assert "<source>\n或\noriginal\n或 \nsecond" in panel
+ english_panel = localize_html(panel, PurePosixPath("attempts/example.html"))
+ assert "<source>\n或\noriginal\nor \nsecond" in english_panel
+ assert "region 1 77.5%" in localized
+ assert '原始返回：位置 1\n或\n真实正文 ' in localized
assert (rebuilt / "logs" / debug.name).read_bytes() == debug.read_bytes()
assert (rebuilt / "logs" / response.name).read_bytes() == response.read_bytes()
assert json.loads((rebuilt / "report-manifest.json").read_text())["presentation"]["frozen_scores"]
From 4fda8ab3a26933a18e1911baf8549027bfc80ce8 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Tue, 13 Oct 2026 21:09:06 +0800
Subject: [PATCH 04/50] Preserve source evidence during report localization
---
harness/web_fetch/report.py | 12 ++++++------
harness/web_fetch/report_i18n.py | 15 +++++++++++++--
tests/test_local_audit.py | 4 ++--
tests/test_report_languages.py | 19 +++++++++++++++++++
4 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 260c75496..43214544d 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -1272,7 +1272,7 @@ def stable_answer_panel(contract):
items=[]
for index,anchor in enumerate(anchors,1):
alternatives=anchor.get('alternatives',[])
- text='\n或 \n'.join(esc(value) for value in alternatives)
+ text='\n或 \n'.join(f'{esc(value)} ' for value in alternatives)
items.append(f'必须出现的稳定文字 {index}（任一写法即可） {text} 正文长度要求 至少 {case.get("min_content_chars",1):,} 字符 或 \n'.join(esc(value) for value in fact.get('alternatives',[]))
+ text='\n或 \n'.join(f'{esc(value)} ' for value in fact.get('alternatives',[]))
items.append(f'必需文字组 {index}（任一写法即可） {text} 全部文字组都须命中。
' if threshold==1 else
@@ -1381,7 +1381,7 @@ def _quote_field_issue(metrics):
def stable_anchor_evidence(metrics,output):
cards=[];normalized_lines=None
for index,item in enumerate(metrics.get('stable_anchor_results',[]),1):
- alternatives=item.get('alternatives',[]);expected='\n或 \n'.join(esc(value) for value in alternatives)
+ alternatives=item.get('alternatives',[]);expected='\n或 \n'.join(f'{esc(value)} ' for value in alternatives)
excerpt=''
for alternative in alternatives:
if normalized_lines is None:normalized_lines=_normalized_evidence_lines(output)
@@ -1401,10 +1401,10 @@ def stable_anchor_evidence(metrics,output):
wanted=expected_fields.get(field,'');got=returned_fields.get(field,'')
compact=(normalize_for_match(wanted).replace(' ','')==normalize_for_match(got).replace(' ',''))
verdict='命中' if exact[field] else '文字齐全，但分隔丢失' if field=='tags' and compact else '不一致'
- rows.append(('',cell(esc(labels[field]))+cell(esc(wanted))+cell(esc(got))+cell(esc(verdict))))
+ rows.append(('',cell(esc(labels[field]))+cell(f'{esc(wanted)} ')+cell(f'{esc(got)} ')+cell(esc(verdict))))
field_detail='整条记录按字段判定；文字出现不等于列表边界完整。
'+table(['字段','标准答案','实际解析','判定'],rows,metrics=[])
badge='字段边界不合格'
- actual=(f'{esc(excerpt)}
' if excerpt else '实际返回中未找到可供定位的连续原文。
')
+ actual=(f'{esc(excerpt)}
' if excerpt else '实际返回中未找到可供定位的连续原文。
')
cards.append(f'
稳定文字 {index} {badge} {field_detail}共 {len(cards)} 项稳定文字判定，全部参与评分；当前区域显示约 5 项高度，向下滚动查看全部。
'
@@ -2117,7 +2117,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
reference_heading='目标文件'
expected_hash=c.get('source_verification',{}).get('file_sha256') or '未登记'
source_meta='本任务验收原始文件字节，不要求 Markdown。'
- source_body=(f''
+ source_body=(f'
'
f'
目标 SHA-256 {esc(expected_hash)}
')
if delivered:
actual_rows=[('',cell(esc(f.get('mime') or '未知格式'))+cell(f'{f.get("bytes",0):,}')+
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index 66fbfbaf4..e3b219306 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -3498,10 +3498,18 @@ def __init__(self, source: PurePosixPath | None = None, translations=None) -> No
self.stack: list[str] = []
self.source = source
self.translations = translations
+ self.translate_stack = []
+
+ def _allows_translation(self, attrs):
+ mode = (dict(attrs).get("translate") or "").lower()
+ if mode in {"yes", "no"}:
+ return mode == "yes"
+ return self.translate_stack[-1] if self.translate_stack else True
def _start_tag(self, attrs) -> str:
original = self.get_starttag_text()
- if any(tag in self.stack for tag in ("pre", "code", "script", "style")):
+ if (not self._allows_translation(attrs)
+ or any(tag in self.stack for tag in ("pre", "code", "script", "style"))):
return original
values = dict(attrs)
article = self.source in {PurePosixPath("guide/index.html"), PurePosixPath("guide/article.html")}
@@ -3533,6 +3541,7 @@ def handle_decl(self, decl: str) -> None:
self.parts.append(f"")
def handle_starttag(self, tag: str, attrs) -> None:
+ self.translate_stack.append(self._allows_translation(attrs))
self.stack.append(tag)
self.parts.append(self._start_tag(attrs))
@@ -3542,10 +3551,12 @@ def handle_startendtag(self, tag: str, attrs) -> None:
def handle_endtag(self, tag: str) -> None:
if tag in self.stack:
self.stack = self.stack[:len(self.stack) - 1 - self.stack[::-1].index(tag)]
+ self.translate_stack = self.translate_stack[:len(self.stack)]
self.parts.append(f"")
def handle_data(self, data: str) -> None:
- if any(tag in self.stack for tag in ("pre", "code", "script", "style")):
+ if ((self.translate_stack and not self.translate_stack[-1])
+ or any(tag in self.stack for tag in ("pre", "code", "script", "style"))):
self.parts.append(data)
else:
self.parts.append(translate_text(data, self.translations))
diff --git a/tests/test_local_audit.py b/tests/test_local_audit.py
index 2c86de9ec..e30fbcf37 100644
--- a/tests/test_local_audit.py
+++ b/tests/test_local_audit.py
@@ -337,9 +337,9 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp
panels = [required_text_answer_panel({"required_facts": [{"alternatives": alternatives}]}),
stable_answer_panel({"stable_text_anchors": [{"alternatives": alternatives}], "minimum_records": 1})]
for panel in panels:
- assert "<source>\n或\noriginal\n
或 \nsecond" in panel
+ assert '
<source>\n或\noriginal \n
或 \n
second ' in panel
english_panel = localize_html(panel, PurePosixPath("attempts/example.html"))
- assert "<source>\n或\noriginal\n
or \nsecond" in english_panel
+ assert '
<source>\n或\noriginal \n
or \n
second ' in english_panel
assert "region 1 77.5%" in localized
assert '
原始返回：位置 1\n或\n真实正文 ' in localized
assert (rebuilt / "logs" / debug.name).read_bytes() == debug.read_bytes()
diff --git a/tests/test_report_languages.py b/tests/test_report_languages.py
index c870fd889..06268ae7b 100644
--- a/tests/test_report_languages.py
+++ b/tests/test_report_languages.py
@@ -135,3 +135,22 @@ def test_release_rejects_a_wrong_primary_language(tmp_path, damage):
path.write_text(json.dumps(contract))
with pytest.raises(ValueError, match="language|entrypoint"):
check_language_contract(tmp_path)
+
+
+def test_evidence_translate_no_preserves_nested_text_and_attributes(monkeypatch):
+ from pathlib import PurePosixPath
+ from harness.web_fetch import report_i18n
+ raw = "记录来源" * 1000
+ original = report_i18n.translate_text
+
+ def translate(value, translations=None):
+ assert raw not in value, "Raw evidence reached narrative translation"
+ return original(value, translations)
+
+ monkeypatch.setattr(report_i18n, "translate_text", translate)
+ page = ('
' + raw + ' 记录来源
记录来源
')
+ actual = report_i18n.localize_html(page, PurePosixPath("attempts/example.html"))
+ assert '
' + raw + ' Record source ' in actual
+ assert actual.endswith('
Record source
')
From 6c7738cc8bfcc7e16bdc33b3c89105f8f873f081 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Tue, 13 Oct 2026 21:19:44 +0800
Subject: [PATCH 05/50] Localize body contract conclusions
---
harness/web_fetch/report_i18n.py | 3 +++
tests/test_report_languages.py | 15 +++++++++++++++
2 files changed, 18 insertions(+)
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index e3b219306..c440a06df 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -91,6 +91,9 @@
"仅包含实际观测的日志。": "Contains only observed logs.",
"产品实际返回；公开副本已隐藏凭据与本机路径。": "Actual product response; credentials and local paths are hidden in this public copy.",
"或": "or",
+ "稳定内容与结构检查通过。": "Stable content and structure checks passed.",
+ "条目结构检查通过；本例未设置稳定文字核对点。": "Record structure passed; this task has no stable-text checks.",
+ "稳定文字或记录结构没有满足任务要求。": "Stable text or record structure did not meet the task requirements.",
"全部文字组都须命中。": "Every text group must match.",
"完整参考正文": "Complete reference body",
"查看原始验收规则": "View raw acceptance rules",
diff --git a/tests/test_report_languages.py b/tests/test_report_languages.py
index 06268ae7b..af53f4cf9 100644
--- a/tests/test_report_languages.py
+++ b/tests/test_report_languages.py
@@ -154,3 +154,18 @@ def translate(value, translations=None):
assert '
' + raw + ' Record source ' in actual
assert actual.endswith('
Record source
')
+
+
+@pytest.mark.parametrize('passed,anchors,expected', [
+ (True, [{'pass': True}], 'Stable content and structure checks passed.'),
+ (True, [], 'Record structure passed; this task has no stable-text checks.'),
+ (False, [], 'Stable text or record structure did not meet the task requirements.'),
+])
+def test_body_contract_conclusion_is_localized(passed, anchors, expected):
+ from pathlib import PurePosixPath
+ from harness.web_fetch.report import reason
+ from harness.web_fetch.report_i18n import localize_html
+ record = {'ok': True, 'metrics': {'stable_task_pass': passed,
+ 'stable_task_schema': 'body_text', 'stable_anchor_results': anchors}}
+ conclusion = reason(record, {'task_kind': 'body'})
+ assert localize_html('
' + conclusion + '
', PurePosixPath('attempts/example.html')) == '
' + expected + '
'
From 96a76d65f69ba10753beffcb42822d910688da46 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Wed, 14 Oct 2026 14:08:55 +0800
Subject: [PATCH 06/50] Version scoring rules and enforce diagnostic report
evidence
---
evaluation/scoring-regressions.json | 902 ++++++++++++--------
harness/web_fetch/README.md | 32 +
harness/web_fetch/adjudication_evidence.py | 75 ++
harness/web_fetch/cohort_exclusions.py | 38 +
harness/web_fetch/local_audit.py | 14 +-
harness/web_fetch/local_revision.py | 166 ++++
harness/web_fetch/metrics.py | 6 +
harness/web_fetch/models.py | 3 +
harness/web_fetch/report.py | 116 ++-
harness/web_fetch/report_build.py | 13 +-
harness/web_fetch/report_check.py | 24 +
harness/web_fetch/report_components.py | 4 +-
harness/web_fetch/report_i18n.py | 61 +-
harness/web_fetch/report_quality.py | 31 +
harness/web_fetch/report_reading.py | 61 +-
harness/web_fetch/result_ledger.py | 58 ++
harness/web_fetch/scoring.py | 17 +
harness/web_fetch/scoring_regressions.py | 16 +-
harness/web_fetch/stable_tasks.py | 38 +-
research/native-430-local-report.json | 20 +-
scripts/build_report.py | 6 +-
tests/test_adjudication_evidence.py | 26 +
tests/test_cohort_exclusions.py | 35 +
tests/test_local_audit.py | 10 +-
tests/test_local_diagnostic_presentation.py | 64 ++
tests/test_local_revision.py | 83 ++
tests/test_report_presentation.py | 5 +-
tests/test_report_quality.py | 17 +
tests/test_report_reading.py | 57 +-
tests/test_report_ui.mjs | 6 +
tests/test_result_ledger.py | 46 +
tests/test_scoring_policy_versions.py | 29 +
tests/test_stable_text_roles.py | 62 ++
33 files changed, 1716 insertions(+), 425 deletions(-)
create mode 100644 harness/web_fetch/adjudication_evidence.py
create mode 100644 harness/web_fetch/cohort_exclusions.py
create mode 100644 harness/web_fetch/local_revision.py
create mode 100644 harness/web_fetch/report_quality.py
create mode 100644 harness/web_fetch/result_ledger.py
create mode 100644 tests/test_adjudication_evidence.py
create mode 100644 tests/test_cohort_exclusions.py
create mode 100644 tests/test_local_diagnostic_presentation.py
create mode 100644 tests/test_local_revision.py
create mode 100644 tests/test_report_quality.py
create mode 100644 tests/test_result_ledger.py
create mode 100644 tests/test_scoring_policy_versions.py
create mode 100644 tests/test_stable_text_roles.py
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index b183749c9..d96a32d91 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -30761,414 +30761,596 @@
"artifact": "Unrelated content sufficiently long\nCAPTCHA",
"ok": true,
"expected_task_pass": false
- }
- ]
- ,"control_revisions": {
- "reason": "User-approved scoring policy: prompt matches are diagnostic only. Frozen historical controls remain intact; current expectations and hint-only target requirements are declared here.",
- "overrides": {
- "standalone-captcha-without-reference-remains-blocked": {
- "case": {
- "url": "https://example.test/accessibility",
- "ground_truth_main_content": "Accessibility guidance",
- "required_facts": [
- {
- "id": "target-body",
- "alternatives": [
- "Accessibility guidance"
- ]
- }
- ]
- },
- "expected_task_pass": false,
- "expected_metrics": {
- "blocked_page": true,
- "blocked_page_hits": [
- "CAPTCHA"
- ],
- "blocked_page_reference_exemptions": []
+ },
+ {
+ "id": "ordinary-text-player-control-only",
+ "category": "acceptance-contract",
+ "root_cause": "excluded_control_used_as_ordinary_text",
+ "failure_mode": "false-positive",
+ "case": {
+ "id": "ordinary-text-control", "url": "https://example.test/", "task_kind": "structure",
+ "content_score_applicable": false,
+ "acceptance_contract": {
+ "version": 1, "schema": "body_text", "minimum_records": 0,
+ "stable_text_anchors": [{"id": "live", "alternatives": ["Live"]}],
+ "text_exclusions": {"html_selectors": ["button", "[role=button]"], "text_patterns": ["^\\s*\\d+:\\d{2}\\s*/\\s*Live\\s*[•·]?\\s*Watch full video\\b"]}
}
},
- "complete-records-with-declared-excluded-ui-fail": {
- "case": {
- "url": "https://example.test/products",
- "forbidden_any": [
- "Similar Products"
- ],
- "acceptance_contract": {
- "version": 1,
- "schema": "product",
- "minimum_records": 2,
- "stable_text_anchors": []
- }
- },
- "expected_task_pass": true,
- "expected_metrics": {
- "forbidden_hits": [
- "Similar Products"
- ],
- "stable_structure_pass": true
+ "artifact": "0:00\n\n/\n\nLive\n\n•Watch full video",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "ordinary-text-with-player-control",
+ "category": "acceptance-contract",
+ "root_cause": "excluded_control_used_as_ordinary_text",
+ "failure_mode": "false-negative",
+ "case": {
+ "id": "ordinary-text-control", "url": "https://example.test/", "task_kind": "structure",
+ "content_score_applicable": false,
+ "acceptance_contract": {
+ "version": 1, "schema": "body_text", "minimum_records": 0,
+ "stable_text_anchors": [{"id": "live", "alternatives": ["Live"]}],
+ "text_exclusions": {"html_selectors": ["button", "[role=button]"], "text_patterns": ["^\\s*\\d+:\\d{2}\\s*/\\s*Live\\s*[•·]?\\s*Watch full video\\b"]}
}
},
- "known-challenge-headings-block-complete-records": {
- "case": {
- "url": "https://example.test/gallery",
- "task_kind": "resource",
- "content_score_applicable": false,
- "expected_resources": [
- {
- "id": "hero",
- "urls": [
- "/hero.jpg"
- ]
- }
- ]
- },
- "expected_task_pass": true,
- "expected_metrics": {
- "resource_pass": true,
- "blocked_page": true
- }
+ "artifact": "[Live](https://example.test/live)\n\n0:00\n/\nLive\n•Watch full video",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "full-page-stable-body-and-navigation",
+ "category": "acceptance-contract",
+ "root_cause": "whole_page_scope_requires_target_content",
+ "failure_mode": "false-negative",
+ "case": {
+ "id": "full-page-control", "url": "https://example.test/", "task_kind": "body",
+ "task_prompt": "获取目标正文及导航、页脚中登记的稳定文字。",
+ "body_requirements": {"version": 3, "unit": "region_compact_hexagram_recall", "minimum_coverage": 0.8, "minimum_floor": 0.6, "minimum_region_passes": 1, "regions": [{"id": "body", "text": "this specific article explains the observed retrieval contract"}]},
+ "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "content_scope": "full_page_stable_coverage", "stable_text_anchors": [{"id": "home", "alternatives": ["Home"], "source_location": "nav > a"}]}
},
- "unsplash-bot-heading-blocks-complete-records": {
- "case": {
- "url": "https://example.test/gallery",
- "task_kind": "resource",
- "content_score_applicable": false,
- "expected_resources": [
- {
- "id": "hero",
- "urls": [
- "/hero.jpg"
- ]
- }
- ]
+ "artifact": "Home\nThis specific article explains the observed retrieval contract.",
+ "ok": true, "expected_task_pass": true
+ },
+ {
+ "id": "full-page-stable-navigation-only",
+ "category": "acceptance-contract",
+ "root_cause": "whole_page_scope_requires_target_content",
+ "failure_mode": "false-positive",
+ "case": {
+ "id": "full-page-control", "url": "https://example.test/", "task_kind": "body",
+ "task_prompt": "获取目标正文及导航、页脚中登记的稳定文字。",
+ "body_requirements": {"version": 3, "unit": "region_compact_hexagram_recall", "minimum_coverage": 0.8, "minimum_floor": 0.6, "minimum_region_passes": 1, "regions": [{"id": "body", "text": "this specific article explains the observed retrieval contract"}]},
+ "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "content_scope": "full_page_stable_coverage", "stable_text_anchors": [{"id": "home", "alternatives": ["Home"], "source_location": "nav > a"}]}
+ },
+ "artifact": "Home",
+ "ok": true, "expected_task_pass": false
+ },
+ {
+ "id": "standalone-captcha-without-reference-remains-blocked-policy-v2-diagnostic",
+ "category": "task-predicate",
+ "root_cause": "blocked_page_detection",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/accessibility",
+ "ground_truth_main_content": "Accessibility guidance",
+ "required_facts": [
+ {
+ "id": "target-body",
+ "alternatives": [
+ "Accessibility guidance"
+ ]
+ }
+ ],
+ "acceptance_policy_version": 2
+ },
+ "artifact": "CAPTCHA",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "blocked_page": true,
+ "blocked_page_hits": [
+ "CAPTCHA"
+ ],
+ "blocked_page_reference_exemptions": []
+ }
+ },
+ {
+ "id": "complete-records-with-declared-excluded-ui-fail-policy-v2-diagnostic",
+ "category": "acceptance-contract",
+ "root_cause": "declared_scope_exclusions",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/products",
+ "forbidden_any": [
+ "Similar Products"
+ ],
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "product",
+ "minimum_records": 2,
+ "stable_text_anchors": []
},
- "expected_task_pass": true,
- "expected_metrics": {
- "resource_pass": true,
- "blocked_page": true
- }
+ "acceptance_policy_version": 2
},
- "chinese-gate-heading-blocks-complete-records": {
- "case": {
- "url": "https://example.test/catalog",
- "task_kind": "structure",
- "acceptance_contract": {
- "version": 1,
- "schema": "product",
- "minimum_records": 2
+ "artifact": "Alpha\n$10.00\n\nBeta\n$20.00\n\nSimilar Products\nGamma\n$30.00",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "forbidden_hits": [
+ "Similar Products"
+ ],
+ "stable_structure_pass": true
+ }
+ },
+ {
+ "id": "known-challenge-headings-block-complete-records-policy-v2-diagnostic",
+ "category": "task-predicate",
+ "root_cause": "blocked_page_detection",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/gallery",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "hero",
+ "urls": [
+ "/hero.jpg"
+ ]
}
+ ],
+ "acceptance_policy_version": 2
+ },
+ "artifact": "W3 Performing security verification\n",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true,
+ "blocked_page": true
+ }
+ },
+ {
+ "id": "unsplash-bot-heading-blocks-complete-records-policy-v2-diagnostic",
+ "category": "task-predicate",
+ "root_cause": "blocked_page_detection",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/gallery",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "hero",
+ "urls": [
+ "/hero.jpg"
+ ]
+ }
+ ],
+ "acceptance_policy_version": 2
+ },
+ "artifact": "Making sure you're not a bot\n",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true,
+ "blocked_page": true
+ }
+ },
+ {
+ "id": "chinese-gate-heading-blocks-complete-records-policy-v2-diagnostic",
+ "category": "task-predicate",
+ "root_cause": "blocked_page_detection",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/catalog",
+ "task_kind": "structure",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "product",
+ "minimum_records": 2
},
- "expected_task_pass": true,
- "expected_metrics": {
- "blocked_page": true,
- "stable_blocked_page": true,
- "stable_structure_pass": true,
- "stable_content_pass": true
- }
+ "acceptance_policy_version": 2
},
- "markdown-emphasized-forbidden-status-heading-is-blocked": {
- "case": {
- "url": "https://example.test/platform",
- "required_facts": [
+ "artifact": "验证一下，购物无忧\nAlpha shirt\n$10\n\nBeta shirt\n$20",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "blocked_page": true,
+ "stable_blocked_page": true,
+ "stable_structure_pass": true,
+ "stable_content_pass": true
+ }
+ },
+ {
+ "id": "markdown-emphasized-forbidden-status-heading-is-blocked-policy-v2-diagnostic",
+ "category": "task-predicate",
+ "root_cause": "blocked_page_detection",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/platform",
+ "required_facts": [
+ {
+ "id": "target-body",
+ "alternatives": [
+ "Accessibility guidance"
+ ]
+ }
+ ],
+ "acceptance_policy_version": 2
+ },
+ "artifact": "**Error: Forbidden / 403**",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "blocked_page": true
+ }
+ },
+ {
+ "id": "dynamic-target-placeholder-cannot-pass-policy-v2-diagnostic",
+ "category": "body-scope",
+ "root_cause": "unrendered_dynamic_target_placeholder",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.com/table",
+ "task_kind": "body",
+ "required_facts": [
+ {
+ "id": "title",
+ "alternatives": [
+ "Virtual DOM"
+ ]
+ }
+ ],
+ "forbidden_any": [
+ "Loading Example..."
+ ],
+ "acceptance_policy_version": 2
+ },
+ "artifact": "Virtual DOM\nLoading Example...\nView Source",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "forbidden_hits": [
+ "Loading Example..."
+ ]
+ }
+ },
+ {
+ "id": "removing-normal-theme-control-does-not-hide-real-block-page-policy-v2-diagnostic",
+ "category": "acceptance-contract",
+ "root_cause": "normal_page_control_misclassified_as_forbidden_content",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/docs",
+ "task_kind": "body",
+ "forbidden_any": [
+ "CAPTCHA",
+ "Access denied"
+ ],
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "body_text",
+ "minimum_records": 0,
+ "stable_text_anchors": [
{
- "id": "target-body",
+ "id": "identity",
"alternatives": [
- "Accessibility guidance"
+ "Documentation"
]
- }
- ]
- },
- "expected_task_pass": false,
- "expected_metrics": {
- "blocked_page": true
- }
- },
- "dynamic-target-placeholder-cannot-pass": {
- "case": {
- "url": "https://example.com/table",
- "task_kind": "body",
- "required_facts": [
+ },
{
- "id": "title",
+ "id": "install",
"alternatives": [
- "Virtual DOM"
+ "Install"
+ ]
+ },
+ {
+ "id": "import",
+ "alternatives": [
+ "Import"
]
}
- ],
- "forbidden_any": [
- "Loading Example..."
]
},
- "expected_task_pass": true,
- "expected_metrics": {
- "forbidden_hits": [
- "Loading Example..."
- ]
- }
+ "acceptance_policy_version": 2
},
- "removing-normal-theme-control-does-not-hide-real-block-page": {
- "case": {
- "url": "https://example.test/docs",
- "task_kind": "body",
- "forbidden_any": [
- "CAPTCHA",
- "Access denied"
- ],
- "acceptance_contract": {
- "version": 1,
- "schema": "body_text",
- "minimum_records": 0,
- "stable_text_anchors": [
- {
- "id": "identity",
- "alternatives": [
- "Documentation"
- ]
- },
- {
- "id": "install",
- "alternatives": [
- "Install"
- ]
- },
- {
- "id": "import",
- "alternatives": [
- "Import"
- ]
- }
- ]
- }
- },
- "expected_task_pass": true,
- "expected_metrics": {
- "forbidden_hits": [
- "CAPTCHA"
- ]
- }
+ "artifact": "Documentation\nInstall\nImport\nCAPTCHA",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "forbidden_hits": [
+ "CAPTCHA"
+ ]
+ }
+ },
+ {
+ "id": "forbidden-html-surface-visible-challenge-policy-v2-diagnostic",
+ "category": "answer-surface",
+ "root_cause": "dormant_html_templates_not_blocking_notices",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/catalog",
+ "task_kind": "body",
+ "expected": [
+ "Game catalog"
+ ],
+ "forbidden_any": [
+ "CAPTCHA",
+ "Access denied"
+ ],
+ "min_content_chars": 1,
+ "acceptance_policy_version": 2
},
- "forbidden-html-surface-visible-challenge": {
- "case": {
- "url": "https://example.test/catalog",
- "task_kind": "body",
- "expected": [
- "Game catalog"
- ],
- "forbidden_any": [
- "CAPTCHA",
- "Access denied"
- ],
- "min_content_chars": 1
- },
- "expected_task_pass": true
+ "artifact": "
Game catalog CAPTCHA required
",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "forbidden-html-surface-actual-denial-policy-v2-diagnostic",
+ "category": "answer-surface",
+ "root_cause": "dormant_html_templates_not_blocking_notices",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/catalog",
+ "task_kind": "body",
+ "expected": [
+ "Game catalog"
+ ],
+ "forbidden_any": [
+ "CAPTCHA",
+ "Access denied"
+ ],
+ "min_content_chars": 1,
+ "acceptance_policy_version": 2
},
- "forbidden-html-surface-actual-denial": {
- "case": {
- "url": "https://example.test/catalog",
- "task_kind": "body",
- "expected": [
- "Game catalog"
- ],
- "forbidden_any": [
- "CAPTCHA",
- "Access denied"
- ],
- "min_content_chars": 1
- },
- "expected_task_pass": true
+ "artifact": "
Game catalogAccess denied
",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "forbidden-html-surface-plain-challenge-policy-v2-diagnostic",
+ "category": "answer-surface",
+ "root_cause": "dormant_html_templates_not_blocking_notices",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/catalog",
+ "task_kind": "body",
+ "expected": [
+ "Game catalog"
+ ],
+ "forbidden_any": [
+ "CAPTCHA",
+ "Access denied"
+ ],
+ "min_content_chars": 1,
+ "acceptance_policy_version": 2
},
- "forbidden-html-surface-plain-challenge": {
- "case": {
- "url": "https://example.test/catalog",
- "task_kind": "body",
- "expected": [
- "Game catalog"
+ "artifact": "Game catalog\nCAPTCHA required",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "consent-does-not-hide-access-denied-policy-v2-diagnostic",
+ "category": "structured-extraction",
+ "root_cause": "cookie_preferences_not_blocked_documentation",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://mui.com/x/react-data-grid/",
+ "task_kind": "body",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "table_row",
+ "minimum_records": 5,
+ "stable_text_anchors": [],
+ "record_identity_fields": [
+ "id"
],
- "forbidden_any": [
- "CAPTCHA",
- "Access denied"
+ "table_columns": [
+ {
+ "name": "id",
+ "label": "ID",
+ "type": "integer",
+ "minimum": 1
+ },
+ {
+ "name": "first",
+ "label": "First name",
+ "type": "text"
+ },
+ {
+ "name": "last",
+ "label": "Last name",
+ "type": "text"
+ },
+ {
+ "name": "age",
+ "label": "Age",
+ "type": "integer",
+ "minimum": 0,
+ "allow_empty": true
+ }
],
- "min_content_chars": 1
- },
- "expected_task_pass": true
- },
- "consent-does-not-hide-access-denied": {
- "case": {
- "url": "https://mui.com/x/react-data-grid/",
- "task_kind": "body",
- "acceptance_contract": {
- "version": 1,
- "schema": "table_row",
- "minimum_records": 5,
- "stable_text_anchors": [],
- "record_identity_fields": [
- "id"
- ],
- "table_columns": [
+ "table_scope": {
+ "root_selector": "[role=grid]",
+ "root_index": 0,
+ "row_selector": "[role=row][data-id]",
+ "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])",
+ "start_pattern": "Community version \\(free forever\\)",
+ "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$",
+ "plain_row_pattern": "(?P
\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))",
+ "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$",
+ "reference_kind": "fixed_demo",
+ "reference_rows": [
{
- "name": "id",
- "label": "ID",
- "type": "integer",
- "minimum": 1
+ "id": "1",
+ "first": "Jon",
+ "last": "Snow",
+ "age": "14"
},
{
- "name": "first",
- "label": "First name",
- "type": "text"
+ "id": "2",
+ "first": "Cersei",
+ "last": "Lannister",
+ "age": "31"
},
{
- "name": "last",
- "label": "Last name",
- "type": "text"
+ "id": "3",
+ "first": "Jaime",
+ "last": "Lannister",
+ "age": "31"
},
{
- "name": "age",
- "label": "Age",
- "type": "integer",
- "minimum": 0,
- "allow_empty": true
- }
- ],
- "table_scope": {
- "root_selector": "[role=grid]",
- "root_index": 0,
- "row_selector": "[role=row][data-id]",
- "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])",
- "start_pattern": "Community version \\(free forever\\)",
- "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$",
- "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))",
- "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$",
- "reference_kind": "fixed_demo",
- "reference_rows": [
- {
- "id": "1",
- "first": "Jon",
- "last": "Snow",
- "age": "14"
- },
- {
- "id": "2",
- "first": "Cersei",
- "last": "Lannister",
- "age": "31"
- },
- {
- "id": "3",
- "first": "Jaime",
- "last": "Lannister",
- "age": "31"
- },
- {
- "id": "4",
- "first": "Arya",
- "last": "Stark",
- "age": "11"
- },
- {
- "id": "5",
- "first": "Daenerys",
- "last": "Targaryen",
- "age": ""
- }
- ]
- }
- },
- "body_requirements": {
- "version": 3,
- "unit": "region_compact_hexagram_recall",
- "minimum_coverage": 0.8,
- "minimum_floor": 0.6,
- "minimum_region_passes": 1,
- "segmentation_method": "sentences",
- "regions": [
+ "id": "4",
+ "first": "Arya",
+ "last": "Stark",
+ "age": "11"
+ },
{
- "id": "body-region-1",
- "text": "the grid displays structured rows and columns each row preserves the fields of one record",
- "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.",
- "segmentation_method": "sentences"
+ "id": "5",
+ "first": "Daenerys",
+ "last": "Targaryen",
+ "age": ""
}
]
- },
- "forbidden_any": [
- "Access denied",
- "CAPTCHA"
+ }
+ },
+ "body_requirements": {
+ "version": 3,
+ "unit": "region_compact_hexagram_recall",
+ "minimum_coverage": 0.8,
+ "minimum_floor": 0.6,
+ "minimum_region_passes": 1,
+ "segmentation_method": "sentences",
+ "regions": [
+ {
+ "id": "body-region-1",
+ "text": "the grid displays structured rows and columns each row preserves the fields of one record",
+ "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.",
+ "segmentation_method": "sentences"
+ }
]
},
- "expected_task_pass": true
+ "forbidden_any": [
+ "Access denied",
+ "CAPTCHA"
+ ],
+ "acceptance_policy_version": 2
},
- "html-doctype-visible-challenge-still-blocks": {
- "case": {
- "url": "https://example.test/products",
- "task_kind": "body",
- "required_facts": [
+ "artifact": "Access denied\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "html-doctype-visible-challenge-policy-v2-diagnostic",
+ "category": "format-normalization",
+ "root_cause": "html_visible_forbidden_surface",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/products",
+ "task_kind": "body",
+ "required_facts": [
+ {
+ "id": "products",
+ "alternatives": [
+ "Available products"
+ ]
+ }
+ ],
+ "forbidden_any": [
+ "CAPTCHA"
+ ],
+ "acceptance_policy_version": 2
+ },
+ "artifact": "Available products CAPTCHA
CAPTCHA Welcome
Required body.
Verify you are human {esc(label)} '
def cell(value,sort=None):return '{value}'
+def model_text(value,english=None):
+ return esc(value) if english is None else ''+esc(value)+' '
def rate(a,b):return pct(a/b) if b else '—'
def section(title,body,id=''):return ''
def hero(title,desc):return f'Web Fetch · 评测报告
{title} {desc}
{esc(index_header)} '+''.join(f'{h} ↕ {h} ' for i,h in enumerate(headers))
- body=''.join(f'{i+1} {values}' for i,(attrs,values) in enumerate(rows))
+def table(headers,rows,id='',sortable=False,cls='',metrics=None,index_header='序号',default_sort=None,include_index=True):
+ head=(f'{esc(index_header)} ' if include_index else '')+''.join(f'{h} ↕ {h} ' for i,h in enumerate(headers))
+ body=''.join(f''+ (f'{i+1} ' if include_index else '')+f'{values}' for i,(attrs,values) in enumerate(rows))
sort_attrs=(f' data-col="{default_sort[0]}" data-dir="{default_sort[1]}"' if default_sort else '')
+ if not include_index:sort_attrs+=' data-row-index="false"'
return metric_legend(headers if metrics is None else metrics)+f''
EXTRA+='\n.change-log section{padding:24px 0}.change-log h2{font-size:24px}.change-log p{margin:10px 0;line-height:1.75}.change-log summary{font-weight:650;color:var(--accent)}.change-log .table-wrap{overflow-x:auto}\n'
EXTRA+='''
@@ -1054,6 +1057,7 @@ def measured(r,c):
return task_measured(r['metrics'],c.get('task_kind','body'))
def result_label(r,c):
+ if r['metrics'].get('evaluation_excluded'):return '排除比较'
if not measured(r,c):return '无法判定'
state='通过' if succeeded(r,c) else '未通过'
return '列表任务：'+state if c.get('task_kind')=='structure' else state
@@ -1063,11 +1067,13 @@ def case_rate(rs,c):
value=rate(sum(succeeded(r,c) for r in rs),len(rs))
return value
-def overview_card(records,cases,track,label,names):
+def overview_card(records,cases,track,label,names,*,diagnostic=False):
selected=[c for c in cases.values() if c.get('track','web')==track]
if not selected:return ''
rs=[r for r in records if cases[r['case_id']].get('track','web')==track]
eligible=[r for r in rs if measured(r,cases[r['case_id']])]
+ if diagnostic:
+ return f'{len(selected)} 个任务
{link("#home-rankings-"+track,label)} 本地诊断结果，不提供名次。
'+model_text(
+ f'保留历史复核记录 {audit.get("reviewed_attempts",0)}/{audit.get("expected_attempts",0)}；本次补充逐案裁定 {count} 条。未新增全量盲审，也不构成正式认证。',
+ f'Historical review records: {audit.get("reviewed_attempts",0)}/{audit.get("expected_attempts",0)}; {count} case-specific adjudications supplemented. No new full blind review or certification is claimed.')+' '+link(prefix+'review/index.html','查看核验轮次 →')+'
')
scope=audit.get('review_protocol',{}).get('scope','all')
if scope=='supplement-25-local-audit':
return (f'新增25例结果审核 '
@@ -1096,7 +1107,7 @@ def review_cell(audit, rows, base=''):
if not audit:return ''
indexed=review_index(audit);items=[indexed[r['attempt_id']] for r in rows if r['attempt_id'] in indexed]
unresolved=sum(not x.get('final_verdict') for x in items)
- label=f'{len(items)}/{len(rows)} · Codex 已核验'
+ label=f'{len(items)}/{len(rows)} · 复核记录'
if unresolved:label+=f' · 未决 {unresolved}'
return cell(link(base+'review/index.html',label),len(items)/len(rows) if rows else -1)
@@ -1413,6 +1424,7 @@ def stable_anchor_evidence(metrics,output):
return note+f'
'+''.join(cards)+'
'
def failure_category(r,c):
+ if r['metrics'].get('evaluation_excluded'):return '评测环境异常：统一排除比较'
if not measured(r,c):return '无法判定：最终网页身份缺少证据' if r['metrics'].get('page_identity_required') and r['metrics'].get('page_identity_status')=='unavailable' else '未测：缺少验收证据'
if succeeded(r,c):return '—'
# Classification describes an already failed result; hints never veto success.
@@ -1469,6 +1481,8 @@ def stable_failure_summary(metrics):
return '；'.join(parts) or '稳定文字或记录结构没有满足任务要求'
def reason(r,c):
+ if r['metrics'].get('evaluation_excluded'):
+ return r['metrics']['evaluation_exclusion_reason']
if not measured(r,c):
if r['metrics'].get('page_identity_required') and r['metrics'].get('page_identity_status')=='unavailable':
return '无法判定：已保存实际返回，但缺少可核实的最终网页身份，不计入通过数、失败数或任务通过率的分母。'
@@ -1531,16 +1545,19 @@ def stable_checks(m):
if len(records)>5:record_table=f'{record_table}
'
return content+record_table+('未通过原因：'+esc(stable_failure_summary(m))+'。
' if m.get('stable_task_errors') else '')
-def ranking(records,cases,track,base='',id=None,compact=False,audit=None):
+def ranking(records,cases,track,base='',id=None,compact=False,audit=None,*,diagnostic=False):
+ excluded={r['case_id'] for r in records if r['metrics'].get('evaluation_excluded')}
+ records=[r for r in records if not r['metrics'].get('evaluation_excluded')]
from .report_components import comparable_body_reference
title_scope=track=='challenge' and any('title_quality' in r for r in records if cases[r['case_id']].get('track')==track)
- selected={k:c for k,c in cases.items() if c.get('track','web')==track}
+ selected={k:c for k,c in cases.items() if c.get('track','web')==track and k not in excluded}
text_track=track in {'web','dynamic','document','challenge'} and any(c.get('task_kind','body') in {'body','structure'} for c in selected.values())
groups=defaultdict(list)
for r in records:
if cases[r['case_id']].get('track','web')==track:groups[r['provider']].append(r)
def sort_key(item):
name,rs=item
+ if diagnostic:return name
n=sum(measured(r,cases[r['case_id']]) for r in rs)
return (-sum(succeeded(r,cases[r['case_id']]) for r in rs)/n if n else 1,name)
rows=[];text_scopes={}
@@ -1603,7 +1620,7 @@ def avg(key):
missing=sorted({r['provider'] for r in records}-set(groups))
note+=' 各类型按各自任务范围比较，不合并为跨类型总分。'
if missing:note+=' 未测方法（不计为失败）：'+esc('、'.join(missing))+'。'
- result=table(heads,rows,id or 'ranking-'+track,True,'ranking',metrics=[] if compact else None)
+ result=table(heads,rows,id or 'ranking-'+track,True,'ranking',metrics=[] if compact else None,include_index=not diagnostic)
if text_scopes and len(set(text_scopes.values()))>1:
result='文字核对覆盖范围 '+''.join(''+esc(name)+f'：{len(scope)}/{len(selected)} 例；'+esc('、'.join(selected[k]['name'] for k in sorted(scope)) or '无文字核对记录')+' ' for name,scope in sorted(text_scopes.items()))+' '+note+'
'+result.replace(''+note+''+(result if diagnostic else result.replace('同题耗时 · 全部方法共同通过子集 '+note+'
'+ (table(['方法','同题均值','同题 P50'],rows,metrics=[])+ ''+links+'
' if rows else '')+''
def overview_all_call_latency(records,cases):
+ excluded={r['case_id'] for r in records if r['metrics'].get('evaluation_excluded')}
+ records=[r for r in records if r['case_id'] not in excluded]
+ cases={key:case for key,case in cases.items() if key not in excluded}
names=sorted({r['provider'] for r in records})
keys={(r['provider'],r['case_id']) for r in records}
if len(keys)!=len(records) or any((name,cid) not in keys for name in names for cid in cases):
@@ -1641,6 +1661,17 @@ def overview_all_call_latency(records,cases):
return (f'每种方法均使用同一组 {len(cases)} 个任务，包含通过、未通过、错误和超时。耗时是单次调用从输入到返回的时间，不包含调用之间的统一冷却间隔。每种方法内部按案例依次执行；方法之间的并发安排和限制见调用链检查。耗时仍受本地与远程环境、网络和产品内部限制影响。
'+
table(['方法','调用数','全部调用均值','全部调用 P50','全部调用 P95'],rows,'all-call-latency-table',metrics=[]))
+
+def result_ledger_section(out, records, prefix=''):
+ path=out/'local-evidence/result-ledger.json'
+ if not path.is_file():return ''
+ from .result_ledger import validate_ledger
+ ledger=validate_ledger(json.loads(path.read_text()),records)
+ rows=[('',cell(esc(row['id']))+cell(str(row['before_passes']))+cell(str(row['added_passes']))+
+ cell(str(row['removed_passes']))+cell(str(row['totals']['pass']))+
+ cell(str(row['totals']['excluded']))) for row in ledger['rounds']]
+ return section('评分修正账目','每轮增加、减少与最终通过数均从逐条状态记录计算；排除结果保留原始调用，不计入比较分母。
'+table(['阶段','修正前通过','新增通过','减少通过','修正后通过','排除比较'],rows,metrics=[])+link(prefix+'local-evidence/result-ledger.json','下载逐条增减账目'),'result-ledger')
+
def overview_matched_latency(records,cases):
from .report_explainer import _article_speed_population
names=sorted({r['provider'] for r in records})
@@ -1704,7 +1735,10 @@ def title_diagnostics(groups,selected,base,identifier):
'该诊断回答“已经返回的条目标题是否完整、干净”，不衡量榜单召回率、实时性或整页正文质量。
'+steps
+table(['方法','标题完整度','标题纯净度','标题 F1','标题参考覆盖（可评分 / 返回）'],rows,identifier+'-titles',True,metrics=[])+ '')
-def overall_ranking(records,cases,audit=None):
+def overall_ranking(records,cases,audit=None,*,diagnostic=False):
+ excluded={r['case_id'] for r in records if r['metrics'].get('evaluation_excluded')}
+ records=[r for r in records if not r['metrics'].get('evaluation_excluded')]
+ cases={k:c for k,c in cases.items() if k not in excluded}
has_unknown=any(not measured(r,cases[r["case_id"]]) for r in records)
groups=defaultdict(list)
for r in records:groups[r['provider']].append(r)
@@ -1723,7 +1757,7 @@ def overall_ranking(records,cases,audit=None):
values+=cell(ms(percentile(times,.95)),percentile(times,.95))
if audit:values+=review_cell(audit,rs)
rows.append((passed,name,('',values)))
- rows=[row for _,_,row in sorted(rows,key=lambda x:(-x[0],x[1]))]
+ rows=[row for _,_,row in sorted(rows,key=lambda x:x[1] if diagnostic else (-x[0],x[1]))]
tracks=active_tracks(cases)
heads=['方法','请求成功率','综合任务通过率']+(['无法判定'] if has_unknown else [])+[label for _,label in tracks]+['均值','P50','P95']
if audit:heads+=['审查结果']
@@ -1740,7 +1774,11 @@ def overall_ranking(records,cases,audit=None):
f'
全调用均值 / P50 / P95 每种方法在同一组 {case_count} 个计划任务上的端到端耗时，包含通过与失败；三项不参与通过率排名。主矩阵调用跨日期、跨批次，不等同于同窗受控速度比较。 '
'
审查结果 独立盲审覆盖数、审查方法和未决数。 '
'')
- return intro+help+table(heads,rows,'ranking-overall',True,'ranking',metrics=[]).replace('{state}')+cell(esc(failure_category(r,c)))+cell(ms(r['elapsed_ms']))
+ values=cell(link(base+'attempts/'+r['attempt_id']+'.html',c['name'] if label is None else label))+cell(case_track_label(c))+cell(f'{state} ')+cell(esc(failure_category(r,c)))+cell(ms(r['elapsed_ms']))
values+=cell(f'{esc(r.get("run_id") or execution_id(r))}')
return case_attrs(r,c),values
@@ -1819,14 +1857,16 @@ def local_audit_banner(dataset):
if dataset.get('comparison',{}).get('publication_state')!='local_audit_only_not_for_release':
return ''
if dataset.get('comparison',{}).get('kind')=='uniform_native_local_measurement':
- reviewed=dataset.get('comparison',{}).get('independent_review')
- note='已完成全量独立复核；未裁定项保留现有评分，具体范围见核验记录。' if reviewed else '结果为机器判定，未进行独立人工审核；'
- return '本地实测版 全部方法使用同一组已提交案例和判分规则，每例一次调用，统一45秒外部时限，不设逐例等待或提取参数，不重试。'+note+'本报告仅供本地查看，不代表正式发布。
本地未认证诊断快照 · 不可用于排名 来源未在调用前后完成全量核验。保存的评分、质量指标和复核记录仅供诊断，不代表正式发布或认证结果。
'+note+'
本地审核版 本页合并两个不同日期的 Moli 调用批次，保留每例的真实调用时间。本次仅更换报告展示代码，没有新增 Moli 调用，也不代表正式发布。
调用已完整，部分结果无法判定 '+esc(f'{reliability["present_calls"]} 条调用均有记录；其中 {len(reliability["unjudged"])} 条缺少足够验收证据。通过率按可判定结果计算，分母和无法判定数量分别显示。本地报告不代表完整验收或正式发布。')+'
'
+ excluded=[r for r in records if r['metrics'].get('evaluation_excluded')]
+ if len(excluded)==len(reliability['unjudged']):
+ _RELIABILITY_BANNER=''+model_text(
+ f'{len(excluded)} 条结果因已确认的评测环境异常统一排除比较；原始调用和输出保留。',
+ f'{len(excluded)} results are excluded across all methods for a verified evaluation-environment defect; original calls and outputs are retained.')+'
'
+ else:
+ _RELIABILITY_BANNER='调用已完整，部分结果无法判定 '+esc(f'{reliability["present_calls"]} 条调用均有记录；其中 {len(reliability["unjudged"])} 条缺少足够验收证据。通过率按可判定结果计算，分母和无法判定数量分别显示。本地报告不代表完整验收或正式发布。')+'
'
else:
_RELIABILITY_BANNER='报告不可靠：评测未完整 '+esc('；'.join(reliability['errors']))+'。本页排名不得用于结论或发布。
'
if not show_local_status_banners and dataset.get('comparison',{}).get('publication_state')=='local_audit_only_not_for_release' and not any(reliability[key] for key in ('missing','extra','duplicates')):
@@ -1863,8 +1909,9 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
for r in records:bycase[r['case_id']].append(r);byprovider[r['provider']].append(r)
cards=''
for track,label in tracks:
- cards+=overview_card(records,cases,track,label,names)
- home=''+hero('Web Fetch 评测报告',f'{len(cases)} 个案例 · {len(names)} 种方法 · {track_label} 类分榜与一个综合通过率总榜。')
+ cards+=overview_card(records,cases,track,label,names,diagnostic=diagnostic)
+ description=(f'{len(cases)} 个案例 · {len(names)} 种方法 · 本地未认证诊断快照。' if diagnostic else f'{len(cases)} 个案例 · {len(names)} 种方法 · {track_label} 类分榜与一个综合通过率总榜。')
+ home='
'+hero('Web Fetch 评测报告',description)
from .report_explainer import article_title,render_explainer,supports_authored_explainer
TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else article_title(records) if authored_interpretation and supports_authored_explainer(cases) and not all(row.get('provider_type') == 'fixture' for row in records) else '本报告的任务范围与验收规则'
reading_note='从分数差异、失败案例和实现机制，理解如何选择抓取方法。' if authored_interpretation else '查看本报告的任务范围、结果数量与验收规则。'
@@ -1884,15 +1931,19 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
for track,label in tracks:
selected=[c for c in cases.values() if c.get('track','web')==track]
coverage=f'{len(selected)} 个案例 · {len({r["provider"] for r in records if cases[r["case_id"]].get("track","web")==track})} 种已测方法'
- panels.append((track,label,f'
{label} {coverage} '+ranking(records,cases,track,compact=True,audit=audit)+f'
{link("datasets/index.html#track-"+track,"案例与参考 →")}
'))
+ panels.append((track,label,f'
{label} {coverage} '+ranking(records,cases,track,compact=True,audit=audit,diagnostic=diagnostic)+f'
{link("datasets/index.html#track-"+track,"案例与参考 →")}
'))
home+=section(f'{track_label} 类任务概览',review_summary(out)+'
调用完整： '+str(reliability['expected_calls'])+'/'+str(reliability['expected_calls'])+' 次均有结果。
只在同一任务类别内比较。
'+cards+'
','overview')
- home+=section('综合通过率总榜',overall_ranking(records,cases,audit),'overall-ranking')
+ home+=section('任务判定汇总（非排名）' if diagnostic else '综合通过率总榜',overall_ranking(records,cases,audit,diagnostic=diagnostic),'overall-ranking')
+ if diagnostic:
+ home+=result_ledger_section(out,records)
+ low=[r for r in records if not r['metrics'].get('evaluation_excluded') and succeeded(r,cases[r['case_id']]) and cases[r['case_id']].get('ground_truth_main_content') and r['metrics'].get('word_precision') is not None and r['metrics']['word_precision']<.3]
+ home+=section('通过结果的内容诊断',f'
{len(low)} 条通过结果的参考词精确率低于 30%。此比例用于定位检查对象，不是统一失败线；通过不保证干净正文，正文参考的范围和时间也会影响该指标。
','coverage-quality')
if dataset.get('comparison',{}).get('kind')=='uniform_native_local_measurement':
home+=section('同一任务集合的全部调用耗时',overview_all_call_latency(records,cases),'all-call-latency')
home+=section('全部方法共同成功任务的耗时',overview_matched_latency(records,cases),'overall-matched-latency')
if (out/'resource-cohort/resource-comparison.json').is_file():
home+=section('CPU、内存与网络',overview_resources(out),'overall-resources')
- home+=section(f'{track_label} 类分榜','
分榜用于判断具体能力。默认按任务通过率排序，点击列名可切换；质量分数与通过率分别计算。
'+tabs('home-rankings',panels),'results')
+ home+=section('按任务类别查看诊断结果' if diagnostic else f'{track_label} 类分榜',('
方法按名称排列，不提供名次。任务判定、内容质量和耗时分别展示。
' if diagnostic else '
分榜用于判断具体能力。默认按任务通过率排序，点击列名可切换；质量分数与通过率分别计算。
')+tabs('home-rankings',panels),'results')
write(out,'index.html','评测报告',home+'
')
controls=filter_controls([('track','类型',tracks),('category','类别',[(x,x) for x in dict.fromkeys(c['category'] for c in cases.values())])])
rows=[]
@@ -1963,7 +2014,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
for r in ([] if presentation_only else records):
c=cases[r['case_id']];m=r['metrics'];ref=reference_text(c)
output=contained_path(out,r['artifact_path']).read_text(encoding='utf-8',errors='replace')
- rows=[('',cell(k)+cell(v)) for k,v in [('任务 ID',f'
{esc(c["id"])}'),('方法',link('../providers/'+r['provider']+'.html',provider_label(r['provider']))),('请求','成功' if r['ok'] else '失败'),('任务结果','通过' if succeeded(r,c) else '未通过'),('HTTP 状态',esc(r.get('status_code'))),('耗时',ms(r['elapsed_ms'])),('全文完整度 / 纯净度 / F1',f'{pct(m.get("word_recall"))} / {pct(m.get("word_precision"))} / {pct(m.get("word_f1"))}'),('输出字符',f'{len(output):,}'),('输出校验值',f'
{esc(m.get("content_sha256"))}')]]
+ rows=[('',cell(k)+cell(v)) for k,v in [('任务 ID',f'
{esc(c["id"])}'),('方法',link('../providers/'+r['provider']+'.html',provider_label(r['provider']))),('请求','成功' if r['ok'] else '失败'),('任务结果',result_label(r,c)),('HTTP 状态',esc(r.get('status_code'))),('耗时',ms(r['elapsed_ms'])),('全文完整度 / 纯净度 / F1',f'{pct(m.get("word_recall"))} / {pct(m.get("word_precision"))} / {pct(m.get("word_f1"))}'),('输出字符',f'{len(output):,}'),('输出校验值',f'
{esc(m.get("content_sha256"))}')]]
if not c.get('ground_truth_main_content'):
rows[6]=('',cell('全文完整度 / 纯净度 / F1')+cell('— / — / —（全文指标无完整参考，未评测）'))
else:
@@ -2046,7 +2097,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
hero_meta=(f'
任务 ID：{esc(c["id"])} {esc(task_instruction(c))}'
f'
方法：{esc(provider_label(r["provider"]))} '
f'运行 ID：{esc(r.get("run_id") or execution_id(r))} ')
- summary=(f'
'
+ summary=(f'
'
f'
{esc(state)} {esc(reason(r,c))}
'
f'
方法：{link("../providers/"+r["provider"]+".html",provider_label(r["provider"]))} · '
f'请求：{"成功" if r["ok"] else "失败"} · 耗时：{ms(r["elapsed_ms"])} · {actual_summary}
')
@@ -2055,7 +2106,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
'
查看本次完整实际返回 '
'
查看逐项判定 '
'
查看运行日志与证据 ')
- if r.get('diagnosis') and not passed:
+ if r.get('diagnosis') and not passed and measured(r,c):
d=r['diagnosis'];s=d.get('independent_source_check') or {}
quote_issue=_quote_field_issue(m)
facts=[('失败分类','字段边界丢失' if quote_issue else d['category']),
@@ -2064,6 +2115,8 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
if s:facts.extend([('独立源页复查',f"{s.get('checked_at','')} · HTTP {s.get('status','未取得')} · {s.get('title') or s.get('error','')}"),('复查边界',d['source_check_boundary'])])
summary+='
查看失败分类与归因边界 '+table(['项目','依据'],[('',cell(esc(k))+cell(esc(v))) for k,v in facts],metrics=[])+""
body=hero(esc(c['name']),hero_meta)+section('结论',summary,'overview')
+ from .report_quality import render_quality_observations
+ body+=section('目标覆盖与附加内容',render_quality_observations(r,c,output,esc=esc),'quality-diagnostics')
reviewed=audited_attempts.get(r['attempt_id'])
if reviewed:
independent={'pass':'通过','fail':'未通过','insufficient':'证据不足'}.get(reviewed.get('agent_verdict'),'未记录')
@@ -2073,11 +2126,13 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
('直接证据',reviewed.get('agent_evidence') or '—'),
('最终结论',final),
]
- if reviewed.get('adjudication'):
+ if reviewed.get('adjudication_evidence'):
review_rows.append(('分歧裁定',reviewed['adjudication']))
body+=section('独立结果核验',table(
['项目','核验记录'],
- [('',cell(esc(label))+cell(esc(value))) for label,value in review_rows],
+ [('',cell(esc(label))+cell(
+ model_text(value,reviewed.get('adjudication_evidence',{}).get('reason_en')) if label=='分歧裁定'
+ else '
'+esc(value)+' ' if label=='直接证据' else esc(value))) for label,value in review_rows],
metrics=[]),'independent-review')
reference_heading='完整标准答案' if c.get('task_kind')!='structure' else '标准答案与验收规则'
reference_download='../references/'+c['id']+'.txt'
@@ -2235,7 +2290,10 @@ def coverage(label,item):
return f'{label}全量 {item.get("reviewed_attempts",0):,}/{item.get("total_attempts",item.get("expected_attempts",0)):,}'
conclusion=(f'
{coverage("主结果",audit)}；'
f'代理分歧 {len(disagreements)}；已裁定 {len(adjudicated)}；未决 {unresolved}。
')
+ if audit.get('adjudication_revision'):
+ conclusion=review_summary(out,'../')
page+=section('当前状态',conclusion)
+ page+=result_ledger_section(out,records,'../')
page+=section('核验轮次','
'+rounds_html+'
')
method=(f'
核验协议与执行者 执行者：{esc(reviewer.get("agent","独立代理"))}；模型：{esc(reviewer.get("model","—"))}；推理强度：{esc(reviewer.get("effort","—"))}。'
'核验时隐藏机器结论和分数，只提供任务契约、标准答案与完整实际输出；分歧回到原始证据裁定，未决非零则禁止发布。任务和输出未变化时沿用原核验；发生变化的调用必须重新盲审。
')
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 420dfdfac..612b239bc 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -1266,15 +1266,24 @@ def validate_local_presentation(output: Path, binding_path: Path):
raise ValueError('Presentation evidence binding changed: ' + item['path'])
dataset = json.loads((output / 'dataset.json').read_text())
records = load_jsonl(contained_path(output, 'results.jsonl'))
+ scoring_sources = {row['attempt_id']: binding.get('measured_scoring_source_sha256') for row in records}
+ if binding.get('offline_revision'):
+ from .local_revision import require_local_revision
+ scoring_sources = require_local_revision(output, binding['offline_revision'], dataset, records)
if (len(records) != binding.get('result_count')
or len(dataset['cases']) != binding.get('case_count')
or sorted({row['provider'] for row in records}) != sorted(binding['providers'])
- or any(row['metrics'].get('scoring_source_sha256') != binding.get('measured_scoring_source_sha256') for row in records)):
+ or any(row['metrics'].get('scoring_source_sha256') != scoring_sources[row['attempt_id']] for row in records)):
raise ValueError('Presentation refresh cannot change the measured matrix or scoring revision')
if any(path.is_symlink() for path in output.rglob("*")):
raise ValueError("Presentation snapshot must not contain symlinks")
validate_report_ids(dataset, records)
- if not assess_reliability(dataset, records)['complete']:
+ reliability = assess_reliability(dataset, records)
+ from .cohort_exclusions import require_cohort_exclusions
+ excluded = require_cohort_exclusions(output, dataset, records)
+ if (not reliability['complete'] and
+ (any(reliability[key] for key in ('missing', 'extra', 'duplicates'))
+ or any(item['case_id'] not in excluded for item in reliability['unjudged']))):
raise ValueError('Presentation refresh requires the complete declared matrix')
for row in records:
artifact = contained_path(output, row['artifact_path'])
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index a7ec1fd8f..adb1d328c 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -133,6 +133,19 @@ def handle_data(self, text):
self.numbers.extend(visible_numbers(text))
+def check_local_diagnostic_presentation(root: Path):
+ """Completeness must never turn uncertified evidence into a displayed ranking."""
+ for path in root.rglob('*.html'):
+ page = path.read_text(encoding='utf-8')
+ if '
' not in page:
+ continue # Original response/source HTML is evidence, not report prose.
+ if 'data-evidence-state="local_measurement"' not in page:
+ raise ValueError('Local diagnostic disclosure missing: ' + str(path.relative_to(root)))
+ for table in re.findall(r'
]*\bid="ranking-[^"]+".*?
', page, re.S):
+ if 'data-dir="desc"' in table or re.search(r'\s*(?:序号|Rank|No\.)\s* ', table):
+ raise ValueError('Uncertified diagnostic comparison assigns ranks: ' + str(path.relative_to(root)))
+
+
def check(root: Path, *, authority_root: Path, profile: Path | None = None, presentation_binding: Path | None = None):
reject_evidence_symlinks(root)
root = root.resolve()
@@ -176,6 +189,17 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
if presentation_binding and not local_measurement:
raise ValueError("Presentation binding requires a frozen uniform-native diagnostic snapshot")
if local_measurement:
+ check_local_diagnostic_presentation(root)
+ if (root / 'result-review.json').is_file():
+ from .adjudication_evidence import require_adjudication_evidence, require_current_review_state
+ review=json.loads((root / 'result-review.json').read_text())
+ require_current_review_state(rs,review)
+ require_adjudication_evidence(root,d,rs,review)
+ if (root / 'local-evidence/result-ledger.json').is_file():
+ from .result_ledger import validate_ledger
+ ledger = validate_ledger(json.loads((root / 'local-evidence/result-ledger.json').read_text()), rs)
+ if ledger['states'][-1]['source_results_sha256'] != hashlib.sha256((root / 'results.jsonl').read_bytes()).hexdigest():
+ raise ValueError('Result ledger source hash differs from current results')
from .local_audit import require_local_measurement, local_measurement_cases
if presentation_binding:
if profile:
diff --git a/harness/web_fetch/report_components.py b/harness/web_fetch/report_components.py
index e4e3b0487..afb72b995 100644
--- a/harness/web_fetch/report_components.py
+++ b/harness/web_fetch/report_components.py
@@ -91,9 +91,9 @@ def metric_legend(headers):
TRACK_CONTEXT = {
'web': {
- 'flow': ['请求网页', '定位正文区域', '去除导航等杂文', '输出正文'],
+ 'flow': ['请求网页', '定位正文区域', '按题目范围保留文字', '输出正文'],
'flow_note': '目标是读出页面内容，而不是保存整份网页源码。正文若已在响应中，普通 HTTP 加解析器就可能完成。',
- 'meaning': '常见网页的正文提取：从文章、文档站、百科或项目主页中取回要读的内容，剔除导航、广告等干扰。难点是找对正文边界，既不漏段落，也不混入大量杂文。',
+ 'meaning': '从文章、文档站、百科或项目主页中取回题目要求的内容。导航和页脚是否需要保留，由每道题登记的范围决定；它们不能替代要求的正文、列表或表格。',
'example': '项目仓库页以 README 为正文：只返回仓库名称和简介不够；把文件目录、导航一起带回来，也可能降低纯净度。',
'boundary': '这组按正文提取任务选样，没有逐页按渲染方式分组，不能等同于“纯静态网页”。请求成功、任务通过、全文质量分别衡量；不测站内搜索、点击或多页浏览。',
},
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index c440a06df..6a628c100 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -110,6 +110,35 @@
"已保存真实诊断（原生日志范围见说明）": "Observed diagnostics saved (see native log coverage notes)",
"本轮未重新核验来源。": "Sources were not reverified for this run.",
"本地实测版": "Local measured comparison",
+ "本地未认证诊断快照 · 不可用于排名": "Local uncertified diagnostic snapshot · not for ranking",
+ "来源未在调用前后完成全量核验。保存的评分、质量指标和复核记录仅供诊断，不代表正式发布或认证结果。": "Sources were not fully verified before and after calls. Saved scores, quality metrics and review records are diagnostic evidence, not a certified release.",
+ "本地诊断结果，不提供名次。": "Local diagnostic results; no ranks are assigned.",
+ "任务判定汇总（非排名）": "Task judgments (not a ranking)",
+ "按任务类别查看诊断结果": "Diagnostic results by task family",
+ "方法按名称排列，不提供名次。任务判定、内容质量和耗时分别展示。": "Methods are listed by name without ranks. Task judgments, content quality and latency are shown separately.",
+ "本地未认证诊断结果，仅展示保存的任务判定与耗时，不用于产品排名。通过不代表返回内容干净；请同时查看全文指标和挑战提示。": "Local uncertified diagnostic results show saved task judgments and latency, not product rankings. Passing does not establish clean content; also inspect full-text metrics and challenge indicators.",
+ "诊断指标说明": "Diagnostic metric definitions",
+ "无命中": "No matches",
+ "评分修正账目": "Scoring revision ledger",
+ "每轮增加、减少与最终通过数均从逐条状态记录计算；排除结果保留原始调用，不计入比较分母。": "Additions, removals and final passes are derived from individual result states. Excluded results retain their original calls and stay outside comparison denominators.",
+ "阶段": "Stage",
+ "修正前通过": "Passes before",
+ "新增通过": "Passes added",
+ "减少通过": "Passes removed",
+ "修正后通过": "Passes after",
+ "下载逐条增减账目": "Download per-result revision ledger",
+ "通过结果的内容诊断": "Content diagnostics for passing results",
+ "目标覆盖与附加内容": "Target coverage and additional content",
+ "目标要求：": "Target requirements:",
+ "附加内容诊断：": "Additional-content diagnostics:",
+ "参考词精确率：": "Reference word precision:",
+ "包含验证码组件文字": "Challenge-widget text is present",
+ "未检测到该组件文字特征": "No matching widget-text signature detected",
+ "排除比较": "Excluded from comparison",
+ "通过只表示满足登记的目标要求，不保证干净正文。提示词命中不证明整页被拦截；词精确率也受参考范围和时间影响。低于 30% 仅用于定位检查对象，不是统一失败线。": "Passing means the registered target requirements were met, not that the body is clean. Hint words do not establish that the whole page was blocked; word precision also depends on reference scope and time. Below 30% identifies inspection candidates, not a universal failure threshold.",
+ "评测环境异常：统一排除比较": "Evaluation environment defect: excluded across all methods",
+ "已有复核记录；覆盖和裁定依据见核验页面。": "Review records are available; see the review page for coverage and adjudication evidence.",
+ "结果为机器判定，未进行独立人工审核。": "Results are machine judgments without independent human review.",
"全部方法使用同一组已提交案例和判分规则，每例一次调用，统一45秒外部时限，不设逐例等待或提取参数，不重试。结果为机器判定，未进行独立人工审核；本报告仅供本地查看，不代表正式发布。": "All methods use the same committed cases and scoring rules: one call per case, a shared 45-second caller deadline, no per-case wait or extraction parameters, and no retries. Results are machine-scored without independent human review. This report is for local inspection and is not a formal release.",
"调用链与日志检查": "Call-chain and log audit",
"题目通过": "Task passes",
@@ -1707,6 +1736,8 @@
"请求网页": "Request page",
"定位正文区域": "Locate body region",
"去除导航等杂文": "Remove navigation and noise",
+ "按题目范围保留文字": "Retain text within the task scope",
+ "从文章、文档站、百科或项目主页中取回题目要求的内容。导航和页脚是否需要保留，由每道题登记的范围决定；它们不能替代要求的正文、列表或表格。": "Retrieve the required content from articles, documentation, encyclopedias or project pages. Each task declares whether navigation and footer text must be retained; they cannot substitute for the required body, lists or tables.",
"输出正文": "Return body text",
"目标是读出页面内容，而不是保存整份网页源码。正文若已在响应中，普通 HTTP 加解析器就可能完成。": "The goal is readable page content, not a copy of the complete page source. When the body is already present in the response, ordinary HTTP plus a parser may be sufficient.",
"项目仓库页以 README 为正文：只返回仓库名称和简介不够；把文件目录、导航一起带回来，也可能降低纯净度。": "For a repository page, the README is the body target. A repository name and short description are insufficient, while including the file tree and navigation may reduce precision.",
@@ -2618,6 +2649,7 @@
lambda match: f"The evaluation does not cover its plan; {match.group(1)} unplanned calls are present"
+ (f"; {match.group(2)} calls have no pass/fail judgement" if match.group(2) else "")
+ ". Rankings on this page must not be used for conclusions or publication."),
+ (re.compile(r"^(\d+) 个案例 · (\d+) 种方法 · 本地未认证诊断快照。$"), r"\1 tasks · \2 methods · local uncertified diagnostic snapshot."),
(re.compile(r"^(\d+) 个案例 · (\d+) 种方法 · (\d+)\s*类分榜与一个综合通过率总榜。$"), r"\1 tasks · \2 methods · \3 family rankings and one overall coverage ranking."),
(re.compile(r"^(\d+)\s*类任务概览$"), r"\1 task families"),
(re.compile(r"^(\d+)\s*类分榜$"), r"Rankings for \1 task families"),
@@ -3502,6 +3534,7 @@ def __init__(self, source: PurePosixPath | None = None, translations=None) -> No
self.source = source
self.translations = translations
self.translate_stack = []
+ self.model_text_stack = []
def _allows_translation(self, attrs):
mode = (dict(attrs).get("translate") or "").lower()
@@ -3544,9 +3577,14 @@ def handle_decl(self, decl: str) -> None:
self.parts.append(f"")
def handle_starttag(self, tag: str, attrs) -> None:
+ localized = dict(attrs).get('data-report-en') if tag == 'span' else None
+ parent_model = any(self.model_text_stack)
+ self.model_text_stack.append(localized is not None or parent_model)
self.translate_stack.append(self._allows_translation(attrs))
self.stack.append(tag)
self.parts.append(self._start_tag(attrs))
+ if localized is not None and not parent_model:
+ self.parts.append(html.escape(localized))
def handle_startendtag(self, tag: str, attrs) -> None:
self.parts.append(self._start_tag(attrs))
@@ -3555,9 +3593,12 @@ def handle_endtag(self, tag: str) -> None:
if tag in self.stack:
self.stack = self.stack[:len(self.stack) - 1 - self.stack[::-1].index(tag)]
self.translate_stack = self.translate_stack[:len(self.stack)]
+ self.model_text_stack = self.model_text_stack[:len(self.stack)]
self.parts.append(f"")
def handle_data(self, data: str) -> None:
+ if any(self.model_text_stack):
+ return
if ((self.translate_stack and not self.translate_stack[-1])
or any(tag in self.stack for tag in ("pre", "code", "script", "style"))):
self.parts.append(data)
@@ -3565,9 +3606,13 @@ def handle_data(self, data: str) -> None:
self.parts.append(translate_text(data, self.translations))
def handle_entityref(self, name: str) -> None:
+ if any(self.model_text_stack):
+ return
self.parts.append(f"&{name};")
def handle_charref(self, name: str) -> None:
+ if any(self.model_text_stack):
+ return
self.parts.append(f"&#{name};")
def handle_comment(self, data: str) -> None:
@@ -4088,6 +4133,8 @@ def preserve(match):
"请求成功，并同时满足本例最低字符数、全部必需核对点、显著标记及结构要求。验证码或错误提示文字不单独决定任务是否通过。完整度、纯净度和 F1 另算，不作为通过门槛。": "The request must succeed and satisfy minimum length, every required checkpoint, significant marker, and structure requirement. Verification or error notices do not independently determine acceptance. Completeness, purity, and F1 are separate metrics, not pass thresholds.",
})
REGEX.insert(0, (re.compile(r"^(\d+) 个计划任务中的通过数 / 可判定数；每题等权，无法判定结果不参与分母。$"), r"Passed / decidable results among \1 planned tasks; every task has equal weight, and undecidable results are excluded from the denominator."))
+REGEX.insert(0, (re.compile(r"^(\d+)/(\d+) · 复核记录$"), r"\1/\2 · review records"))
+REGEX.insert(0, (re.compile(r"^(\d+) 条通过结果的参考词精确率低于 30%。此比例用于定位检查对象，不是统一失败线；通过不保证干净正文，正文参考的范围和时间也会影响该指标。$"), r"\1 passing results have reference word precision below 30%. This locates inspection candidates, not a universal failure threshold. Passing does not establish clean body text; reference scope and time also affect this metric."))
TEXT.update({
"版本优先使用调用时记录的版本号；源码实现使用冻结的提交号。云端服务没有公开后端版本时如实标注。二进制哈希用于确认具体执行文件；事后核验的版本会说明依据。": "Versions use call-time records where available; source implementations use frozen commits. Undisclosed cloud backend versions are stated explicitly. Binary hashes identify the executable, and retrospectively verified versions state their evidence.",
@@ -4098,7 +4145,7 @@ def write_english_report(out: Path) -> int:
history_path = out / "report-changes.json"
translations = release_translations(json.loads(history_path.read_text())) if history_path.is_file() else {}
from .report_reading import archived_translations
- translations.update(archived_translations(out))
+ article_translations = archived_translations(out)
english_root = out / "en"
if english_root.exists():
shutil.rmtree(english_root)
@@ -4120,8 +4167,16 @@ def write_english_report(out: Path) -> int:
raw=source_path.read_text(encoding="utf-8")
if str(source)=='guide/index.html' and (out/'archived-reading.json').is_file():
from .report_reading import render_archived_reading
- shell=re.sub(r'.*? ', ''+esc(hints)+'' if info['hint_words'] else '无命中 '
+ return ('目标要求： '+verdict+'
'
+ '
附加内容诊断： '+component+' ；'+hint_html+'
'
+ '
参考词精确率： '+precision+'
'
+ '
通过只表示满足登记的目标要求，不保证干净正文。提示词命中不证明整页被拦截；词精确率也受参考范围和时间影响。低于 30% 仅用于定位检查对象，不是统一失败线。
]*>(.*?)", text, re.S)
if not body:
raise ValueError("Archived article has no main content")
- styles = "".join(re.findall(r"]*>.*?", text, re.S))
date = data.get("source_data_date")
if language == "zh":
date_note = f"（截至{date}）" if date else ""
@@ -44,8 +99,8 @@ def render_archived_reading(out: Path, language: str = "zh") -> str | None:
else:
date_note = f" through {date}" if date else ""
note = f"This article retains the older report and its data{date_note}; it does not describe the current benchmark results."
- return (styles + '' + body[1] + '
')
+ return ('' + note
+ + '
' + sanitize_article_html(body[1]) + '
Complete results
')
+ with pytest.raises(ValueError, match='disclosure missing'):
+ check_local_diagnostic_presentation(tmp_path)
+ banner = 'Uncertified
'
+ page.write_text(prefix + banner + '')
+ with pytest.raises(ValueError, match='assigns ranks'):
+ check_local_diagnostic_presentation(tmp_path)
+ page.write_text(prefix + banner + '')
+ check_local_diagnostic_presentation(tmp_path)
diff --git a/tests/test_local_revision.py b/tests/test_local_revision.py
new file mode 100644
index 000000000..642e588cf
--- /dev/null
+++ b/tests/test_local_revision.py
@@ -0,0 +1,83 @@
+import hashlib
+import json
+
+import pytest
+
+from harness.web_fetch.local_revision import require_local_revision, revise_local_snapshot
+from harness.web_fetch.scoring import score_attempt
+from harness.web_fetch.metrics import task_passed
+
+
+def test_offline_revision_preserves_calls_and_rejects_tampering(tmp_path):
+ raw = b'0:00\n/\nLive\nWatch full video'
+ (tmp_path / 'answer.txt').write_bytes(raw)
+ case = {'id': 'a', 'url': 'https://example.test/', 'task_kind': 'structure',
+ 'content_score_applicable': False,
+ 'acceptance_contract': {'version': 1, 'schema': 'body_text', 'minimum_records': 0,
+ 'stable_text_anchors': [{'id': 'live', 'alternatives': ['Live']}]}}
+ original = {'attempt_id': 'p__a', 'provider': 'p', 'case_id': 'a', 'ok': True,
+ 'artifact_path': 'answer.txt', 'elapsed_ms': 7,
+ 'metrics': score_attempt(case, raw, ok=True)}
+ assert task_passed(True, original['metrics'])
+ (tmp_path / 'dataset.json').write_text(json.dumps({'cases': [case], 'comparison': {}}))
+ (tmp_path / 'results.jsonl').write_text(json.dumps(original) + '\n')
+ changed = {**case, 'acceptance_contract': {**case['acceptance_contract'],
+ 'text_exclusions': {'text_patterns': [r'^\s*\d+:\d{2}\s*/\s*Live\s*Watch full video\b']}}}
+ config = tmp_path / 'config.json'
+ config.write_text(json.dumps({'cases': [changed], 'rule_revision': True, 'authority': {'description': 'Retain ordinary page text requirement'},
+ 'regression_ids': ['ordinary-text-player-control-only', 'ordinary-text-with-player-control']}))
+ invalid = json.loads(config.read_text())
+ invalid['rule_revision'] = False
+ config.write_text(json.dumps(invalid))
+ with pytest.raises(ValueError, match='explicit rule revision'):
+ revise_local_snapshot(tmp_path, config)
+ invalid['rule_revision'] = True
+ config.write_text(json.dumps(invalid))
+ receipt = revise_local_snapshot(tmp_path, config)
+ assert receipt['provider_calls_added'] == 0
+ assert receipt['rule_revision'] is True
+ assert receipt['task_verdict_changes'] == [{'attempt_id': 'p__a', 'before': True, 'after': False}]
+ assert (tmp_path / 'answer.txt').read_bytes() == raw
+ rows = [json.loads((tmp_path / 'results.jsonl').read_text())]
+ assert rows[0]['scoring']['scored_at']
+ dataset = json.loads((tmp_path / 'dataset.json').read_text())
+ path = next((tmp_path / 'local-evidence/offline-revisions').glob('*/receipt.json'))
+ binding = {'path': str(path.relative_to(tmp_path)), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
+ require_local_revision(tmp_path, binding, dataset, rows)
+ # A self-consistent changed result hash does not authorize changing a call.
+ rows[0]['elapsed_ms'] = 8
+ (tmp_path / 'results.jsonl').write_text(json.dumps(rows[0]) + '\n')
+ receipt['output_results_sha256'] = hashlib.sha256((tmp_path / 'results.jsonl').read_bytes()).hexdigest()
+ path.write_text(json.dumps(receipt))
+ binding['sha256'] = hashlib.sha256(path.read_bytes()).hexdigest()
+ with pytest.raises(ValueError, match='measured call facts'):
+ require_local_revision(tmp_path, binding, dataset, rows)
+
+
+def test_policy_revision_records_version_and_rejects_parser_only_label(tmp_path):
+ raw = b'Required body\nCAPTCHA'
+ (tmp_path / 'answer.txt').write_bytes(raw)
+ case = {'id': 'a', 'url': 'https://example.test/', 'task_kind': 'body', 'acceptance_policy_version': 1,
+ 'forbidden_any': ['CAPTCHA'], 'acceptance_contract': {'version': 1, 'schema': 'body_text',
+ 'minimum_records': 0, 'stable_text_anchors': [{'id': 'body', 'alternatives': ['Required body']}]}}
+ original = {'attempt_id': 'p__a', 'provider': 'p', 'case_id': 'a', 'ok': True,
+ 'artifact_path': 'answer.txt', 'metrics': score_attempt(case, raw, ok=True)}
+ assert not task_passed(True, original['metrics'])
+ (tmp_path / 'dataset.json').write_text(json.dumps({'cases': [case], 'comparison': {}}))
+ (tmp_path / 'results.jsonl').write_text(json.dumps(original)+'\n')
+ config = tmp_path / 'config.json'
+ config.write_text(json.dumps({'cases': [{**case, 'acceptance_policy_version': 2}], 'rule_revision': True,
+ 'authority': {'description': 'Explicitly adopt policy 2; verification words are diagnostics'},
+ 'regression_ids': ['html-doctype-visible-challenge-policy-v2-diagnostic', 'prompt-diagnostic-only-verification']}))
+ receipt = revise_local_snapshot(tmp_path, config)
+ assert receipt['rule_revision'] is True and receipt['acceptance_policy_version'] == 2
+ rows = [json.loads((tmp_path / 'results.jsonl').read_text())]
+ assert task_passed(True, rows[0]['metrics'])
+ rows[0]['scoring']['rule_revision'] = False
+ (tmp_path / 'results.jsonl').write_text(json.dumps(rows[0])+'\n')
+ path = next((tmp_path / 'local-evidence/offline-revisions').glob('*/receipt.json'))
+ receipt['output_results_sha256'] = hashlib.sha256((tmp_path / 'results.jsonl').read_bytes()).hexdigest()
+ path.write_text(json.dumps(receipt))
+ binding = {'path': str(path.relative_to(tmp_path)), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
+ with pytest.raises(ValueError, match='parser-only'):
+ require_local_revision(tmp_path, binding, json.loads((tmp_path / 'dataset.json').read_text()), rows)
diff --git a/tests/test_report_presentation.py b/tests/test_report_presentation.py
index da7737ff7..fbcd63686 100644
--- a/tests/test_report_presentation.py
+++ b/tests/test_report_presentation.py
@@ -114,10 +114,11 @@ def test_current_interface_and_capability_copy_is_localized(self):
self.assertNotRegex(translate_text(source),r'[\u4e00-\u9fff]',source)
def test_regional_endpoint_distinct(self):
self.assertNotEqual(SPECS['lexmount-cn']['entry'],SPECS['lexmount-global']['entry'])
- def test_review_label_names_codex(self):
+ def test_review_label_links_records_without_claiming_fresh_verification(self):
audit={'attempts':[{'attempt_id':'one','final_verdict':'pass'}]}
row=review_cell(audit,[{'attempt_id':'one'}])
- self.assertIn('1/1 · Codex 已核验',row)
+ self.assertIn('1/1 · 复核记录',row)
+ self.assertNotIn('已核验',row)
def test_headless_access_covers_every_method_and_lists_setup_variables(self):
self.assertEqual(set(HEADLESS_ACCESS),{'kitesurf','lightpanda','obscura','moli-1.0.6','moli','moli-1.1.3','moli-1.1.7','moli-1.1.11','moli-pr643','chromium'})
for access in HEADLESS_ACCESS.values():
diff --git a/tests/test_report_quality.py b/tests/test_report_quality.py
new file mode 100644
index 000000000..96cc31140
--- /dev/null
+++ b/tests/test_report_quality.py
@@ -0,0 +1,17 @@
+from harness.web_fetch.report_quality import quality_observations
+
+
+def test_target_coverage_and_challenge_content_are_distinct():
+ record={'ok':True,'metrics':{'content_pass':True,'word_precision':.2,'forbidden_hits':['CAPTCHA']}}
+ case={'ground_truth_main_content':'Reference body','task_kind':'body'}
+ actual=quality_observations(record,case,'hCaptcha\nPlease try again.\nVerify\nhttps://hcaptcha.com/privacy')
+ assert actual['task_pass'] and actual['low_precision'] and actual['challenge_component_text']
+ normal=quality_observations(record,case,'This documentation describes CAPTCHA integration.')
+ assert normal['task_pass'] and not normal['challenge_component_text']
+ assert normal['hint_words']==['CAPTCHA']
+
+
+def test_missing_reference_is_unavailable_not_zero_precision():
+ record={'ok':True,'metrics':{'content_pass':True,'word_precision':0}}
+ actual=quality_observations(record,{'task_kind':'body'})
+ assert actual['word_precision'] is None and not actual['low_precision']
diff --git a/tests/test_report_reading.py b/tests/test_report_reading.py
index 97dc51925..3a5928e0b 100644
--- a/tests/test_report_reading.py
+++ b/tests/test_report_reading.py
@@ -23,7 +23,7 @@ def test_historical_article_preserves_source_data_and_language(tmp_path):
assert heading in rendered
assert "94/100" in rendered
assert "current benchmark results" in rendered if language == "en" else "本轮评测结果" in rendered
- assert "Read Saved content
'
+ safe = sanitize_article_html(raw)
+ soup = BeautifulSoup(safe, 'html.parser')
+ assert not soup.find(['style', 'script', 'iframe', 'form', 'input'])
+ assert not soup.img.get('src') and not soup.img.get('onerror')
+ assert not soup.a.get('href') and not soup.a.get('onclick')
+ assert soup.p['id'] == 'archived-rows'
+ assert 'Saved content' in safe
+ import base64
+ png='data:image/png;base64,'+base64.b64encode(b'\x89PNG\r\n\x1a\n').decode()
+ preserved=BeautifulSoup(sanitize_article_html('核验记录
本地未认证诊断快照 · 不可用于排名
'
+ article = render_archived_reading(tmp_path)
+ (tmp_path / 'guide/index.html').write_text(''+banner+article+' ')
+ root = '核验记录
核验记录
'+model_text('该结果的说明','Specific explanation ')
+ english=localize_html(raw,PurePosixPath('attempts/example.html'))
+ assert 'Specific explanation <script>bad</script>' in english
+ assert '该结果的说明' not in english
+ assert 'Available products Available products CAPTCHA
Available products Available products CAPTCHA
Set search: true to enable the search plugin:
Search examples
To enable sorting, simply add sort: true to your config:
Sorting examples
Visible product catalog is ready.
Hello ","title":"Home","ok":true,"expected_task_pass":false,"expected_metrics":{"length_pass":false}},
- {"id":"html-visible-challenge-heading-blocks-body","category":"format-normalization","root_cause":"html_visible_answer_surface","failure_mode":"false-positive","case":{"url":"https://example.test/","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Welcome"]}]}},"artifact":"CAPTCHA Welcome
Required body
","title":"Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title ","ok":true,"expected_task_pass":true,"expected_metrics":{"stable_anchor_hits":1}},
{"id":"long-metadata-title-cannot-expose-script-anchor","category":"format-normalization","root_cause":"html_title_prefix_detection","failure_mode":"false-positive","case":{"url":"https://example.test/article","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"Other text
","title":"Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title ","ok":true,"expected_task_pass":false,"expected_metrics":{"stable_anchor_hits":0}},
{"id":"html-inline-challenge-explanation-is-prose","category":"format-normalization","root_cause":"html_block_line_boundaries","failure_mode":"false-negative","case":{"url":"https://example.test/article","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"Required body. If you see Verify you are human , complete the verification before continuing.
","ok":true,"expected_task_pass":true,"expected_metrics":{"blocked_page":false,"stable_blocked_page":false}},
- {"id":"html-independent-challenge-heading-still-blocks","category":"format-normalization","root_cause":"html_block_line_boundaries","failure_mode":"false-positive","case":{"url":"https://example.test/article","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"Required body.
Verify you are human [A-Z]?[0-9]+)\\s+(?P.+)$"}]}]}},"artifact":"Section 1. First\nProblem\tTried\tAC\tDifficulty\nP1 Hello World\t1\t1\t7\nP2 Output Practice\t2\t1\t5\nSection 2. Next\nProblem\tTried\tAC\tDifficulty\nB003 Addition Exercise\t3\t2\t4\nSection 6. End","ok":true,"expected_task_pass":true},
{"id":"combined-problem-cells-cannot-omit-name","category":"structured-extraction","root_cause":"composite_table_equivalent_fields","failure_mode":"false-positive","case":{"url":"https://example.test/training","task_kind":"body","composite_acceptance":{"version":1,"groups":[{"id":"problems","kind":"table","minimum_records":3,"start_pattern":"(?m)^Section\\s+1\\.","end_pattern":"(?m)^Section\\s+6\\.","required_columns":[{"id":"number","kind":"text","pattern":"^[A-Z]?[0-9]+$","aliases":["Number"]},{"id":"name","kind":"text","aliases":["Name"]},{"id":"attempts","kind":"integer","aliases":["Tried"]},{"id":"accepted","kind":"integer","aliases":["AC"]},{"id":"difficulty","kind":"integer","aliases":["Difficulty"]}],"identity_column":"number","combined_columns":[{"aliases":["Problem"],"pattern":"^(?P[A-Z]?[0-9]+)\\s+(?P.+)$"}]}]}},"artifact":"Section 1. First\nProblem\tTried\tAC\tDifficulty\nP1 Hello World\t1\t1\t7\nP2 Output Practice\t2\t1\t5\nSection 2. Next\nProblem\tTried\tAC\tDifficulty\nB003\t3\t2\t4\nSection 6. End","ok":true,"expected_task_pass":false},
{"id":"html-head-title-cannot-supply-body","category":"format-normalization","root_cause":"html_head_is_not_body","failure_mode":"false-positive","case":{"url":"https://example.test/article","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"Required body Other text
","ok":true,"expected_task_pass":false,"expected_metrics":{"stable_anchor_hits":0}},
@@ -30646,35 +30560,6 @@
"composite_acceptance_pass": false
}
},
- {
- "id": "prompt-diagnostic-qualified-with-forbidden",
- "category": "task-predicate",
- "root_cause": "prompt_diagnostic_used_as_acceptance_veto",
- "failure_mode": "false-negative",
- "case": {
- "id": "prompt-diagnostic-control",
- "url": "https://events.example.test/help",
- "track": "web",
- "task_kind": "body",
- "expected_any": [],
- "required_facts": [
- {
- "id": "required",
- "alternatives": [
- "Return purchased items within thirty days"
- ]
- }
- ],
- "min_content_chars": 30,
- "forbidden_any": [
- "CAPTCHA",
- "Access denied"
- ]
- },
- "artifact": "Return purchased items within thirty days\nCAPTCHA",
- "ok": true,
- "expected_task_pass": true
- },
{
"id": "prompt-diagnostic-qualified-with-block-heading",
"category": "task-predicate",
@@ -30827,7 +30712,36 @@
"ok": true, "expected_task_pass": false
},
{
- "id": "standalone-captcha-without-reference-remains-blocked-policy-v2-diagnostic",
+ "id": "prompt-qualified-with-diagnostic-hints",
+ "category": "task-predicate",
+ "root_cause": "prompt_diagnostic_used_as_acceptance_veto",
+ "failure_mode": "false-negative",
+ "case": {
+ "id": "prompt-diagnostic-control",
+ "url": "https://events.example.test/help",
+ "track": "web",
+ "task_kind": "body",
+ "expected_any": [],
+ "required_facts": [
+ {
+ "id": "required",
+ "alternatives": [
+ "Return purchased items within thirty days"
+ ]
+ }
+ ],
+ "min_content_chars": 30,
+ "diagnostic_any": [
+ "CAPTCHA",
+ "Access denied"
+ ]
+ },
+ "artifact": "Return purchased items within thirty days\nCAPTCHA",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "standalone-captcha-without-reference-remains-blocked-diagnostic-hints",
"category": "task-predicate",
"root_cause": "blocked_page_detection",
"failure_mode": "false-positive",
@@ -30836,13 +30750,13 @@
"ground_truth_main_content": "Accessibility guidance",
"required_facts": [
{
- "id": "target-body",
+ "id": "target",
"alternatives": [
"Accessibility guidance"
]
}
],
- "acceptance_policy_version": 2
+ "diagnostic_any": []
},
"artifact": "CAPTCHA",
"ok": true,
@@ -30856,35 +30770,34 @@
}
},
{
- "id": "complete-records-with-declared-excluded-ui-fail-policy-v2-diagnostic",
+ "id": "complete-records-with-declared-excluded-ui-fail-diagnostic-hints",
"category": "acceptance-contract",
"root_cause": "declared_scope_exclusions",
"failure_mode": "false-positive",
"case": {
"url": "https://example.test/products",
- "forbidden_any": [
- "Similar Products"
- ],
"acceptance_contract": {
"version": 1,
"schema": "product",
"minimum_records": 2,
"stable_text_anchors": []
},
- "acceptance_policy_version": 2
+ "diagnostic_any": [
+ "Similar Products"
+ ]
},
"artifact": "Alpha\n$10.00\n\nBeta\n$20.00\n\nSimilar Products\nGamma\n$30.00",
"ok": true,
"expected_task_pass": true,
"expected_metrics": {
- "forbidden_hits": [
+ "stable_structure_pass": true,
+ "diagnostic_hits": [
"Similar Products"
- ],
- "stable_structure_pass": true
+ ]
}
},
{
- "id": "known-challenge-headings-block-complete-records-policy-v2-diagnostic",
+ "id": "known-challenge-headings-block-complete-records-diagnostic-hints",
"category": "task-predicate",
"root_cause": "blocked_page_detection",
"failure_mode": "false-negative",
@@ -30900,7 +30813,7 @@
]
}
],
- "acceptance_policy_version": 2
+ "diagnostic_any": []
},
"artifact": "W3 Performing security verification\n",
"ok": true,
@@ -30911,7 +30824,7 @@
}
},
{
- "id": "unsplash-bot-heading-blocks-complete-records-policy-v2-diagnostic",
+ "id": "unsplash-bot-heading-blocks-complete-records-diagnostic-hints",
"category": "task-predicate",
"root_cause": "blocked_page_detection",
"failure_mode": "false-negative",
@@ -30927,7 +30840,7 @@
]
}
],
- "acceptance_policy_version": 2
+ "diagnostic_any": []
},
"artifact": "Making sure you're not a bot\n",
"ok": true,
@@ -30938,7 +30851,7 @@
}
},
{
- "id": "chinese-gate-heading-blocks-complete-records-policy-v2-diagnostic",
+ "id": "chinese-gate-heading-blocks-complete-records-diagnostic-hints",
"category": "task-predicate",
"root_cause": "blocked_page_detection",
"failure_mode": "false-negative",
@@ -30950,7 +30863,7 @@
"schema": "product",
"minimum_records": 2
},
- "acceptance_policy_version": 2
+ "diagnostic_any": []
},
"artifact": "验证一下，购物无忧\nAlpha shirt\n$10\n\nBeta shirt\n$20",
"ok": true,
@@ -30963,7 +30876,7 @@
}
},
{
- "id": "markdown-emphasized-forbidden-status-heading-is-blocked-policy-v2-diagnostic",
+ "id": "markdown-emphasized-forbidden-status-heading-is-blocked-diagnostic-hints",
"category": "task-predicate",
"root_cause": "blocked_page_detection",
"failure_mode": "false-negative",
@@ -30971,13 +30884,13 @@
"url": "https://example.test/platform",
"required_facts": [
{
- "id": "target-body",
+ "id": "target",
"alternatives": [
- "Accessibility guidance"
+ "Required platform text"
]
}
],
- "acceptance_policy_version": 2
+ "diagnostic_any": []
},
"artifact": "**Error: Forbidden / 403**",
"ok": true,
@@ -30987,7 +30900,7 @@
}
},
{
- "id": "dynamic-target-placeholder-cannot-pass-policy-v2-diagnostic",
+ "id": "dynamic-target-placeholder-cannot-pass-diagnostic-hints",
"category": "body-scope",
"root_cause": "unrendered_dynamic_target_placeholder",
"failure_mode": "false-positive",
@@ -31002,32 +30915,27 @@
]
}
],
- "forbidden_any": [
+ "diagnostic_any": [
"Loading Example..."
- ],
- "acceptance_policy_version": 2
+ ]
},
"artifact": "Virtual DOM\nLoading Example...\nView Source",
"ok": true,
"expected_task_pass": true,
"expected_metrics": {
- "forbidden_hits": [
+ "diagnostic_hits": [
"Loading Example..."
]
}
},
{
- "id": "removing-normal-theme-control-does-not-hide-real-block-page-policy-v2-diagnostic",
+ "id": "removing-normal-theme-control-does-not-hide-real-block-page-diagnostic-hints",
"category": "acceptance-contract",
"root_cause": "normal_page_control_misclassified_as_forbidden_content",
"failure_mode": "false-positive",
"case": {
"url": "https://example.test/docs",
"task_kind": "body",
- "forbidden_any": [
- "CAPTCHA",
- "Access denied"
- ],
"acceptance_contract": {
"version": 1,
"schema": "body_text",
@@ -31053,19 +30961,22 @@
}
]
},
- "acceptance_policy_version": 2
+ "diagnostic_any": [
+ "CAPTCHA",
+ "Access denied"
+ ]
},
"artifact": "Documentation\nInstall\nImport\nCAPTCHA",
"ok": true,
"expected_task_pass": true,
"expected_metrics": {
- "forbidden_hits": [
+ "diagnostic_hits": [
"CAPTCHA"
]
}
},
{
- "id": "forbidden-html-surface-visible-challenge-policy-v2-diagnostic",
+ "id": "forbidden-html-surface-visible-challenge-diagnostic-hints",
"category": "answer-surface",
"root_cause": "dormant_html_templates_not_blocking_notices",
"failure_mode": "false-positive",
@@ -31075,19 +30986,18 @@
"expected": [
"Game catalog"
],
- "forbidden_any": [
+ "min_content_chars": 1,
+ "diagnostic_any": [
"CAPTCHA",
"Access denied"
- ],
- "min_content_chars": 1,
- "acceptance_policy_version": 2
+ ]
},
"artifact": "Game catalog CAPTCHA required
",
"ok": true,
"expected_task_pass": true
},
{
- "id": "forbidden-html-surface-actual-denial-policy-v2-diagnostic",
+ "id": "forbidden-html-surface-actual-denial-diagnostic-hints",
"category": "answer-surface",
"root_cause": "dormant_html_templates_not_blocking_notices",
"failure_mode": "false-positive",
@@ -31097,19 +31007,18 @@
"expected": [
"Game catalog"
],
- "forbidden_any": [
+ "min_content_chars": 1,
+ "diagnostic_any": [
"CAPTCHA",
"Access denied"
- ],
- "min_content_chars": 1,
- "acceptance_policy_version": 2
+ ]
},
"artifact": "Game catalogAccess denied
",
"ok": true,
"expected_task_pass": true
},
{
- "id": "forbidden-html-surface-plain-challenge-policy-v2-diagnostic",
+ "id": "forbidden-html-surface-plain-challenge-diagnostic-hints",
"category": "answer-surface",
"root_cause": "dormant_html_templates_not_blocking_notices",
"failure_mode": "false-positive",
@@ -31119,19 +31028,18 @@
"expected": [
"Game catalog"
],
- "forbidden_any": [
+ "min_content_chars": 1,
+ "diagnostic_any": [
"CAPTCHA",
"Access denied"
- ],
- "min_content_chars": 1,
- "acceptance_policy_version": 2
+ ]
},
"artifact": "Game catalog\nCAPTCHA required",
"ok": true,
"expected_task_pass": true
},
{
- "id": "consent-does-not-hide-access-denied-policy-v2-diagnostic",
+ "id": "consent-does-not-hide-access-denied-diagnostic-hints",
"category": "structured-extraction",
"root_cause": "cookie_preferences_not_blocked_documentation",
"failure_mode": "false-positive",
@@ -31231,18 +31139,17 @@
}
]
},
- "forbidden_any": [
+ "diagnostic_any": [
"Access denied",
"CAPTCHA"
- ],
- "acceptance_policy_version": 2
+ ]
},
"artifact": "Access denied\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version",
"ok": true,
"expected_task_pass": true
},
{
- "id": "html-doctype-visible-challenge-policy-v2-diagnostic",
+ "id": "html-doctype-visible-challenge-diagnostic-hints",
"category": "format-normalization",
"root_cause": "html_visible_forbidden_surface",
"failure_mode": "false-positive",
@@ -31257,23 +31164,22 @@
]
}
],
- "forbidden_any": [
+ "diagnostic_any": [
"CAPTCHA"
- ],
- "acceptance_policy_version": 2
+ ]
},
"artifact": "Available products CAPTCHA
CAPTCHA Welcome
Required body.
Verify you are human '+model_text('按本次所列任务的通过率排序；未完成仓库认证流程。测试日期和核验范围见详情。','Ordered by pass rate on the listed tasks; repository certification is incomplete. See details for test dates and verification scope.')+'
'
note='已有复核记录；覆盖和裁定依据见核验页面。' if dataset['comparison'].get('independent_review') else '结果为机器判定，未进行独立人工审核。'
return '本地未认证诊断快照 · 不可用于排名 来源未在调用前后完成全量核验。保存的评分、质量指标和复核记录仅供诊断，不代表正式发布或认证结果。
'+note+'
本地审核版 本页合并两个不同日期的 Moli 调用批次，保留每例的真实调用时间。本次仅更换报告展示代码，没有新增 Moli 调用，也不代表正式发布。
'+hero('Web Fetch 评测报告',description)
from .report_explainer import article_title,render_explainer,supports_authored_explainer
TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else article_title(records) if authored_interpretation and supports_authored_explainer(cases) and not all(row.get('provider_type') == 'fixture' for row in records) else '本报告的任务范围与验收规则'
@@ -1940,7 +1944,8 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
coverage=f'{len(selected)} 个案例 · {len({r["provider"] for r in records if cases[r["case_id"]].get("track","web")==track})} 种已测方法'
panels.append((track,label,f'
{label} {coverage} '+ranking(records,cases,track,compact=True,audit=audit,diagnostic=diagnostic)+f'
{link("datasets/index.html#track-"+track,"案例与参考 →")}
'))
home+=section(f'{track_label} 类任务概览',('' if local_pass_rate_ranking(dataset) else review_summary(out))+'
调用完整： '+str(reliability['expected_calls'])+'/'+str(reliability['expected_calls'])+' 次均有结果。
只在同一任务类别内比较。
'+cards+'
','overview')
- home+=section('任务判定汇总（非排名）' if diagnostic else '综合通过率总榜',overall_ranking(records,cases,audit,diagnostic=diagnostic),'overall-ranking')
+ overall_title=('任务判定汇总（非排名）' if diagnostic else model_text('本轮任务通过率排名','Task pass-rate ranking for this sample') if local_pass_rate_ranking(dataset) else '综合通过率总榜')
+ home+=section(overall_title,overall_ranking(records,cases,audit,diagnostic=diagnostic),'overall-ranking')
if diagnostic:
home+=result_ledger_section(out,records)
low=[r for r in records if not r['metrics'].get('evaluation_excluded') and succeeded(r,cases[r['case_id']]) and cases[r['case_id']].get('ground_truth_main_content') and r['metrics'].get('word_precision') is not None and r['metrics']['word_precision']<.3]
@@ -2288,8 +2293,8 @@ def current_local_review_findings(out,dataset,records,audit):
f"Currently {passed} pass among {len(eligible)} eligible results; {len(records)-len(eligible)} are excluded. Pages and downloadable summaries use the same saved results."),
(f"保留独立复核记录 {audit.get('reviewed_attempts',0)}/{len(records)}；另有 {unreviewed} 条未独立复核，其中机器通过 {audit.get('accepted_unreviewed_passes',0)} 条。补充裁定 {adjudicated} 条，不称为新一轮全量盲审。",
f"Retained independent review covers {audit.get('reviewed_attempts',0)}/{len(records)} results. Another {unreviewed} are not independently reviewed, including {audit.get('accepted_unreviewed_passes',0)} machine passes. There are {adjudicated} supplemental adjudications; this is not a new full blind review."),
- (f"评分规则版本为 {dataset['comparison'].get('scoring_policy_version',1)}。目标内容要求、正文质量和验证提示分别展示；提示文字不单独决定任务是否通过。",
- f"Scoring policy version: {dataset['comparison'].get('scoring_policy_version',1)}. Target requirements, body quality and verification hints are reported separately; hint words alone do not determine acceptance."),
+ ("目标内容要求、正文质量和诊断文字分别展示；具体评分实现哈希见各结果详情，诊断词不单独决定任务是否通过。",
+ "Target requirements, body quality and diagnostic text are reported separately. Each result binds its scorer hash; diagnostic words alone do not determine acceptance."),
]
if local_pass_rate_ranking(dataset):
rows.append(('名次按当前任务通过率计算，耗时单独展示，不参与名次。历史调用未完成调用前后的全量来源核验，保留实际检查日期和范围，不声称已完成来源认证。',
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 011ccb520..05e1a87aa 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -229,6 +229,10 @@ def _build_frozen_presentation(source, output, presentation_binding, *,
shutil.copy2(presentation_binding, staging / "presentation-binding.json")
batches = json.loads((source / "execution.json").read_text())["batches"]
for record in records:
+ case = next(item for item in dataset['cases'] if item['id'] == record['case_id'])
+ if not record.get('diagnosis'):
+ text = contained_path(source, record['artifact_path']).read_text(errors='replace')
+ record['diagnosis'] = diagnose(record, case, text)
metadata = record.setdefault("provider_metadata", {})
metadata["public_log_note"] = dataset.get("comparison", {}).get(
"call_chain_audit", {}).get("call_log_notes", {}).get(record["attempt_id"], "")
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index eaff454eb..be4d71a68 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -55,12 +55,14 @@ def require_table_index_headers(parser, *, english=False, diagnostic=False):
raise ValueError('Unexpected table index header: ' + header)
-def require_home_comparison_layout(home, track_count, *, diagnostic=False):
- overall = '任务判定汇总（非排名）' if diagnostic else '综合通过率总榜'
+def require_home_comparison_layout(home, track_count, *, diagnostic=False, sample_ranking=False):
+ overall = ('任务判定汇总（非排名）' if diagnostic else
+ '本轮任务通过率排名' if sample_ranking else '综合通过率总榜')
details = '按任务类别查看诊断结果' if diagnostic else f'{track_count} 类分榜'
- markers = [f'
{track_count} 类任务概览 ', f'
{overall} ', f'
{details} ']
- if any(marker not in home for marker in markers) or not (
- home.index(markers[0]) < home.index(markers[1]) < home.index(markers[2])):
+ markers = [f'{track_count} 类任务概览', overall, details]
+ headings = [re.sub(r'<[^>]+>', '', value) for value in re.findall(r'
]*>(.*?)', home, re.S)]
+ if any(marker not in headings for marker in markers) or not (
+ headings.index(markers[0]) < headings.index(markers[1]) < headings.index(markers[2])):
raise ValueError('Task overview and comparison must precede detailed results')
if '判分完整性' not in home or ('本类最高：' in home) == diagnostic:
raise ValueError('Comparison winner disclosure differs from evidence state')
@@ -188,15 +190,48 @@ def require_replacement_execution(root, binding, physical, logical):
raise ValueError('Task replacement physical/logical call inventory differs')
+class VisibleLocalScope(HTMLParser):
+ """Disclosure attributes count only when attached to visible explanatory text."""
+ def __init__(self):
+ super().__init__()
+ self.stack = []
+ self.visible = set()
+
+ def handle_starttag(self, tag, attrs):
+ attrs = dict(attrs)
+ inherited = self.stack[-1][1:] if self.stack else (False, set())
+ style = re.sub(r'\s+', '', attrs.get('style', '')).lower()
+ hidden = (inherited[0] or tag in {'script', 'style', 'template'}
+ or 'hidden' in attrs or attrs.get('aria-hidden') == 'true'
+ or 'display:none' in style or 'visibility:hidden' in style)
+ kinds = set(inherited[1])
+ if attrs.get('data-evidence-state') == 'local_measurement': kinds.add('evidence')
+ if attrs.get('data-ranking-scope') == 'saved-task-pass-rate': kinds.add('ranking')
+ if tag not in {'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}:
+ self.stack.append((tag, hidden, kinds))
+
+ def handle_endtag(self, tag):
+ for index in range(len(self.stack) - 1, -1, -1):
+ if self.stack[index][0] == tag:
+ del self.stack[index:]
+ break
+
+ def handle_data(self, data):
+ if data.strip() and self.stack and not self.stack[-1][1]:
+ self.visible.update(self.stack[-1][2])
+
+
def check_local_diagnostic_presentation(root: Path, *, ranked=False):
"""Track evidence state and scope any explicitly requested sample ranking."""
for path in root.rglob('*.html'):
page = path.read_text(encoding='utf-8')
if ']*\bid="ranking-[^"]+".*?
', page, re.S):
if not ranked and ('data-dir="desc"' in table or re.search(r'\s*(?:序号|Rank|No\.)\s* ', table)):
@@ -590,7 +625,7 @@ def canonical_link(page, value):
raise ValueError("Report acceptance check failed")
home = (root / "index.html").read_text()
track_count = len({case.get("track", "web") for case in d["cases"]})
- require_home_comparison_layout(home, track_count, diagnostic=local_measurement and not sample_ranking)
+ require_home_comparison_layout(home, track_count, diagnostic=local_measurement and not sample_ranking, sample_ranking=sample_ranking)
if (root / "resource-cohort/resource-comparison.json").is_file():
guide = (root / "guide/index.html").read_text()
for page_name, text in [("home", home), ("guide", guide)]:
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index 6a628c100..5141553c0 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -2458,6 +2458,7 @@
# are deliberately narrow report phrases; they never include task names,
# reference answers, or provider output.
PHRASES = {
+ "处理路线：": "Retrieval path: ",
"的 Moli 已保存输出仍包含": "'s saved Moli output still contains ",
"，内容规则未通过。请求成功在这里记录了一个正常返回的结果，却没有等来任务要求的内容。这比方法是否具备浏览器功能更接近本次接入要解决的问题。": ", but failed the content rule. A normal response did not wait for the required content. This is closer to the integration problem than whether the method has browser features.",
"Quotes to Scrape 的": "Quotes to Scrape's ",
@@ -2594,6 +2595,17 @@
TEXT.update(adoption_text)
+def _body_region_summary(match):
+ regions = match[1]
+ for source, target in (("开头", "Opening"), ("前部", "Early section"),
+ ("中部", "Middle"), ("后部", "Late section"),
+ ("结尾", "Ending"), ("位置", "Region")):
+ regions = regions.replace(source + " ", target + " ")
+ return ("Body coverage is incomplete. Regions below target: "
+ + regions.replace("、", ", ")
+ + ". The complete reference and actual output are shown side by side below.")
+
+
REGEX = [
(re.compile(r"^本报告包含 (\d+) 个任务、(\d+) 个逻辑结果，任务通过 (\d+)/(\d+)。结论仅适用于已登记的任务和规则。$"), r"This report contains \1 tasks and \2 logical results, with \3/\4 task passes. Conclusions apply only to the registered tasks and rules."),
(re.compile(r"^的 F1 为 ([\d.]+%)，但没有满足该例任务规则。零通过表示没有一次满足全部任务门槛，不表示输出与参考毫无词语重合。这里的 F1 对照已保存的样本文本，只诊断文字重合，不承担实时事实核验。词频相似度允许部分文字得分，任务规则检查指定的内容或字段结构；二者应保留各自的含义。$"),
@@ -2760,7 +2772,7 @@
(re.compile(r"^实际解析出 (\d+) 条完整且不重复的记录，要求至少 (\d+) 条；结构检查未通过，稳定文字检查未通过。$"), r"Parsed \1 complete, unique records against a minimum of \2; structure and stable-text checks failed."),
(re.compile(r"^实际解析出 (\d+) 条完整且不重复的记录，要求至少 (\d+) 条；结构检查(通过|未通过)，稳定文字检查(通过|未通过)。$"), lambda m: f"Parsed {m.group(1)} complete, unique records against a minimum of {m.group(2)}; structure {'passed' if m.group(3) == '通过' else 'failed'} and stable-text checks {'passed' if m.group(4) == '通过' else 'failed'}."),
(re.compile(r"^完整度 / 纯净度 / F1 为 ([^。]+)。$"), r"Recall / precision / F1: \1."),
- (re.compile(r"^正文没有抓全；低于目标的位置为：(.+)。下方会把完整标准答案与实际返回并排展示。$"), lambda m: "Body coverage is incomplete. Regions below target: " + m[1].replace("位置 ", "region ") + ". The complete reference and actual output are shown side by side below."),
+ (re.compile(r"^正文没有抓全；低于目标的位置为：(.+)。下方会把完整标准答案与实际返回并排展示。$"), _body_region_summary),
(re.compile(r"^这一段标准答案产生 ([\d,]+) 个连续六字符片段；在上方完整实际返回（([\d,]+) 字符）中命中 ([\d,]+) 个，所以覆盖率为 ([\d,]+) ÷ ([\d,]+) = ([\d.]+%)。目标 ([\d.]+%)，硬下限 ([\d.]+%)。$"), r"This reference region produces \1 contiguous six-character shingles. \3 are found in the complete output above (\2 characters), so coverage is \4 ÷ \5 = \6. Target: \7; hard floor: \8."),
(re.compile(r"^· 采集时间 (.+)$"), r"· captured at \1"),
]
diff --git a/harness/web_fetch/report_quality.py b/harness/web_fetch/report_quality.py
index 6d2a9ee10..e9033564d 100644
--- a/harness/web_fetch/report_quality.py
+++ b/harness/web_fetch/report_quality.py
@@ -14,7 +14,7 @@ def quality_observations(record, case, output=''):
and re.search(r'hcaptcha\.com/(?:privacy|what-is-hcaptcha)', output, re.I))
return {'task_pass': task_passed(record['ok'], metrics, case.get('task_kind')),
'word_precision': precision, 'low_precision': precision is not None and precision < .3,
- 'hint_words': list(dict.fromkeys(metrics.get('forbidden_hits', []) + metrics.get('blocked_page_hits', []))),
+ 'hint_words': list(dict.fromkeys(metrics.get('diagnostic_hits', []) + metrics.get('blocked_page_hits', []))),
'challenge_component_text': component}
diff --git a/harness/web_fetch/report_reading.py b/harness/web_fetch/report_reading.py
index 9719c28c1..a4787fdad 100644
--- a/harness/web_fetch/report_reading.py
+++ b/harness/web_fetch/report_reading.py
@@ -61,6 +61,8 @@ def sanitize_article_html(fragment):
if element.name == 'a' and 'href' in attrs:
attrs['rel'] = 'noopener noreferrer'
element.attrs = attrs
+ for table in soup.find_all('table'):
+ table.wrap(soup.new_tag('div', attrs={'class': 'table-wrap'}))
return str(soup)
diff --git a/harness/web_fetch/report_style.py b/harness/web_fetch/report_style.py
index 0f505332c..3b05df79e 100644
--- a/harness/web_fetch/report_style.py
+++ b/harness/web_fetch/report_style.py
@@ -1,5 +1,6 @@
"""Shared visual styling for the current report and standalone editorial export."""
STYLE = r"""
+[data-archived-reading] img{max-width:100%;height:auto}
:root{color-scheme:light;--bg:#f6f8fb;--panel:#fff;--panel2:#eef3f8;--ink:#17212b;--muted:#5d6b78;--line:#d9e2ea;--accent:#1268d8;--accent-soft:#e9f2ff;--violet:#6756d8;--lime:#3f7b26;--coral:#bd402f;--good:#117a47;--bad:#c22e43;--warn:#8a5a00;--callout:#fff6d8;--failure-bg:#fff0f2;--success-bg:#eaf8f0;--code-bg:#101820;--code-ink:#e9f1f5;--shadow:0 8px 28px rgba(30,55,80,.08);--max:1180px}
:root[data-theme="dark"]{color-scheme:dark;--bg:#071015;--panel:#0d1a21;--panel2:#122630;--ink:#edf6f7;--muted:#9aadb4;--line:#263c46;--accent:#49d6ed;--accent-soft:#10303a;--violet:#8b7cff;--lime:#c6ed3d;--coral:#ff8168;--good:#67df9f;--bad:#ff6b7a;--warn:#ffd166;--callout:#251f13;--failure-bg:#2a1419;--success-bg:#10271d;--code-bg:#050d11;--code-ink:#d7e8eb;--shadow:none}
*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:linear-gradient(180deg,var(--accent-soft) 0,transparent 240px),var(--bg);color:var(--ink);font:15px/1.65 Inter,system-ui,-apple-system,"Segoe UI","PingFang SC",sans-serif;transition:background .2s,color .2s}a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline}.wrap{width:min(var(--max),calc(100% - 40px));margin:auto}.top{border-bottom:1px solid var(--line);background:color-mix(in srgb,var(--panel) 92%,transparent);position:sticky;top:0;z-index:5;backdrop-filter:blur(14px)}nav{display:flex;align-items:center;gap:17px;overflow:auto;padding:10px 0;white-space:nowrap}.brand{font-weight:850;color:var(--ink);margin-right:auto}.theme-toggle{border:1px solid var(--line);background:var(--panel);color:var(--ink);border-radius:999px;padding:7px 12px;font:inherit;font-size:12px;cursor:pointer}.theme-toggle:hover{border-color:var(--accent)}.hero{padding:42px 0 30px}.eyebrow{font:750 12px/1.2 ui-monospace,monospace;letter-spacing:.12em;color:var(--accent)}h1{font-size:clamp(34px,5vw,58px);line-height:1.08;letter-spacing:-.04em;margin:13px 0;max-width:980px}h2{font-size:clamp(25px,3vw,36px);line-height:1.16;letter-spacing:-.025em;margin:0 0 12px}h3{font-size:19px;margin:24px 0 8px}.lead,.sub{color:var(--muted);max-width:900px}.meta{display:flex;gap:8px;flex-wrap:wrap}.pill,.tag{border:1px solid var(--line);background:var(--panel);border-radius:999px;padding:5px 10px;color:var(--muted);font-size:12px}.tag{padding:2px 7px}.grid{display:grid;grid-template-columns:repeat(12,1fr);gap:14px}.card{background:var(--panel);border:1px solid var(--line);border-radius:14px;padding:20px;box-shadow:var(--shadow)}.kpi{grid-column:span 3;border-top:3px solid var(--accent)}.kpi:nth-child(2){border-top-color:var(--violet)}.kpi:nth-child(3){border-top-color:var(--lime)}.kpi:nth-child(4){border-top-color:var(--coral)}.n{font-size:32px;font-weight:850;letter-spacing:-.04em;margin:7px 0}.section{padding:38px 0;border-top:1px solid var(--line)}.table-wrap{overflow:auto;border:1px solid var(--line);border-radius:13px;background:var(--panel);box-shadow:var(--shadow)}table{border-collapse:collapse;width:100%;min-width:760px;background:var(--panel)}th,td{padding:11px 13px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}th{background:var(--panel2);font-size:12px;letter-spacing:.03em;position:sticky;top:0}td{color:var(--ink)}tbody tr:hover{background:var(--accent-soft)}.good{color:var(--good);font-weight:750}.bad{color:var(--bad);font-weight:750}.warn{color:var(--warn);font-weight:750}.bar{height:9px;background:var(--panel2);border-radius:99px;overflow:hidden;min-width:120px}.bar>i{display:block;height:100%;background:linear-gradient(90deg,var(--accent),var(--violet));border-radius:99px}.callout{border-left:4px solid var(--warn);background:var(--callout);padding:14px 17px;border-radius:0 12px 12px 0}.failure{border-left-color:var(--bad);background:var(--failure-bg)}.success{border-left-color:var(--good);background:var(--success-bg)}pre,code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace}code{color:var(--accent);overflow-wrap:anywhere}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:var(--code-bg);border:1px solid var(--line);padding:18px;border-radius:12px;color:var(--code-ink);max-height:70vh;overflow:auto}.attempt-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:10px}.kv{background:var(--panel);border:1px solid var(--line);border-radius:11px;padding:13px}.kv b{display:block;font-size:12px;color:var(--muted);margin-bottom:4px}.footer{padding:36px 0 64px;color:var(--muted)}.empty{color:var(--muted);font-style:italic}.tiny{font-size:12px;color:var(--muted)}
diff --git a/harness/web_fetch/scoring.py b/harness/web_fetch/scoring.py
index 9cfb9192d..1069fe263 100644
--- a/harness/web_fetch/scoring.py
+++ b/harness/web_fetch/scoring.py
@@ -13,8 +13,6 @@
from .models import EvalCase
from .safety import contained_path
-SCORING_POLICY_VERSION = 2
-
# Bind orchestration as well as the underlying deterministic matchers. Computed
# once at import, shared by live/rescore output; never a reference-freshness claim.
@@ -100,9 +98,8 @@ def score_attempt(
evidence is preserved; the caller must have validated its artifact binding.
"""
case = _case_dict(case)
- policy_version = case.get('acceptance_policy_version', SCORING_POLICY_VERSION)
- if type(policy_version) is not int or policy_version not in {1, 2}:
- raise ValueError('Unsupported scoring policy version')
+ if 'acceptance_policy_version' in case:
+ raise ValueError('Historical policy selectors require their frozen scorer; current scoring has one contract')
contract = case.get('acceptance_contract') or {}
if contract.get('content_scope') == 'full_page_stable_coverage':
if (contract.get('minimum_records', 0) == 0
@@ -116,8 +113,9 @@ def score_attempt(
result = {key: value for key, value in (previous_metrics or {}).items()
if key not in _CONDITIONAL_FIELDS
and not key.startswith(("stable_", "list_", "page_context_"))}
- result['acceptance_policy_version'] = policy_version
- keys = ("required_facts", "forbidden_any", "min_content_chars",
+ for retired in ('acceptance_policy_version', 'forbidden_hits', 'reference_term_exemptions'):
+ result.pop(retired, None)
+ keys = ("required_facts", "forbidden_any", "diagnostic_any", "min_content_chars",
"fact_coverage_threshold", "ground_truth_main_content", "required_patterns",
"acceptance_contract", "body_requirements", "composite_acceptance")
task_kind = case.get("task_kind", "body")
@@ -315,12 +313,7 @@ def score_attempt(
# interpretation of continuous similarity as a pass rate.
if 'ordered_tree_min_similarity' in case:
result['content_pass'] = bool(
- ok and tree_result['score'] >= case['ordered_tree_min_similarity'])
+ ok and tree_result['score'] >= case['ordered_tree_min_similarity']
+ and not result['forbidden_requirement_hits'])
result["scoring_source_sha256"] = SCORING_SOURCE_SHA256
- if policy_version == 1 and (result.get('forbidden_hits') or result.get('blocked_page')
- or result.get('stable_blocked_page')):
- result['content_pass'] = False
- if 'stable_task_pass' in result:
- result['stable_task_pass'] = False
- result['stable_content_pass'] = False
return result
diff --git a/harness/web_fetch/scoring_regressions.py b/harness/web_fetch/scoring_regressions.py
index 84cbb520a..333daec15 100644
--- a/harness/web_fetch/scoring_regressions.py
+++ b/harness/web_fetch/scoring_regressions.py
@@ -32,14 +32,9 @@ def load_scoring_regressions(path: Path | None = None) -> dict[str, dict[str, An
rows: dict[str, dict[str, Any]] = {}
if document.get('control_revisions'):
raise ValueError('Historical controls cannot override inputs or expected results; add new IDs')
- historical = document.get('historical_policy_versions', {})
- if (not isinstance(historical, dict)
- or set(historical) - {row.get('id') for row in document['cases']}
- or any(type(value) is not int or value != 1 for value in historical.values())):
- raise ValueError('Invalid historical scoring policy versions')
+ if document.get('historical_policy_versions'):
+ raise ValueError('Historical policy selectors belong in frozen scorer snapshots')
for index, row in enumerate(document["cases"], 1):
- if row.get('id') in historical:
- row = {**row, 'case': {**row['case'], 'acceptance_policy_version': historical[row['id']]}}
identifier = row.get("id")
if not isinstance(identifier, str) or not identifier.strip() or identifier in rows:
raise ValueError(f"Scoring regression {index} requires a unique nonempty id")
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
index b65eecc1e..e4f2a8f81 100644
--- a/research/native-430-local-report.json
+++ b/research/native-430-local-report.json
@@ -15,8 +15,8 @@
"openclaw-built-in",
"tavily"
],
- "frozen_dataset_sha256": "12952a88e6e0d735830d802436d4030bf9c5d4c31c0f3cf7fa4a8ef77206fd0b",
- "results_sha256": "da69b585066531c4412a0a646e11fd899557964f9b04529a0545cdea7e36e2a4",
+ "frozen_dataset_sha256": "ff578deac41e00056655542c139611d21fd26bbd36a86e9a8628c35d17356490",
+ "results_sha256": "b0bcc2da22ca9fc7e495affb2c25a99c8d56fe89024d12f8c615e97c43b37a1b",
"measured_scoring_source_sha256": "4ce40cdf2ecb1af4088915a2b869ffbdcb65dce5222d1429578e97d175f13bfc",
"evidence_state": "local_measurement",
"certified": false,
@@ -28,7 +28,7 @@
},
{
"path": "result-review.json",
- "sha256": "9f1c5cfd4fee7bf3a6a51490a4c1bfc5a685dcacabbbe35a0fe50581836199e1"
+ "sha256": "0f9190fa85516469740fab16b2786adc092bc26b2a31f13af74a48b1ee4af5bf"
},
{
"path": "archived-reading.json",
@@ -39,14 +39,14 @@
"sha256": "6fdc15530c75676e46e3e44b6213017287d19ddc9c71597493b346072d1ee67c"
}
],
- "evidence_path": "runs/pr51-static-replacement-20261015/assembled-evidence",
+ "evidence_path": "runs/pr51-static-replacement-20261015/review-followup-final-evidence",
"task_replacement": {
"path": "local-evidence/task-replacement/receipt.json",
- "sha256": "fddebdd3e5e5a554fc5e52fbfc806affae7ac77962f5e02b6181f140254af2ee"
+ "sha256": "c68d3b21c3317538089635f8fb0ca5bf4f874f4a03201b27e42c56491780d6e0"
},
"offline_revision": {
- "path": "local-evidence/offline-revisions/6d78a8bc3fd58900/receipt.json",
- "sha256": "6d3a10832c6e5136d4d01a170c5f6fbd05922b836cab8f8acc22065a394c9ada"
+ "path": "local-evidence/offline-revisions/6d78a8bc3fd58900-8d5edbea48da0319/receipt.json",
+ "sha256": "a11c53aae6635ff0915b1e2be1bad7f489ee72c83e2bbb95deddfb38bcd31635"
},
"replacement_case": {
"id": "static-sqlite-quickstart",
@@ -83,17 +83,29 @@
]
}
],
- "forbidden_any": [
- "Page Not Found",
- "Access denied",
- "CAPTCHA"
- ],
"body_coverage": {
"version": 3,
"minimum_coverage": 0.8,
"minimum_floor": 0.6,
"minimum_region_passes": 4,
"regions": 5
+ },
+ "diagnostic_any": [
+ "Page Not Found",
+ "Access denied",
+ "CAPTCHA"
+ ]
+ },
+ "current_scoring_source_sha256": "d4ec10f02e742d6b94d3e9e2aa8a1d605ab3954f59bfd7908a096fc5a4958916",
+ "review_evidence_package": {
+ "url": "https://github.com/lexmount/browser-eval/releases/download/untagged-c84db0e3f874d633407c/PR-51-current-rule-and-sqlite-evidence.zip",
+ "sha256": "3218b22ad388ebb2ae3e084e9378d323ddd9950e75783ae33f75339a4812bdc0",
+ "bytes": 14815846,
+ "access": "GitHub draft release; repository write permission required",
+ "prerequisite": {
+ "url": "https://github.com/lexmount/browser-eval/releases/download/untagged-c84db0e3f874d633407c/PR-51-diagnostic-correction-patch.zip",
+ "sha256": "90d51b8bed3c8c480a45160a3cd8d624753451ee08f04ba816b7e6bb708d1c24",
+ "bytes": 7580469
}
}
}
diff --git a/tests/test_adjudication_evidence.py b/tests/test_adjudication_evidence.py
index 8389026bc..438eb4741 100644
--- a/tests/test_adjudication_evidence.py
+++ b/tests/test_adjudication_evidence.py
@@ -24,3 +24,48 @@ def check():
row['adjudication_evidence']['samples'][0]['text'] = 'invented evidence'
with pytest.raises(ValueError, match='excerpt differs'):
check()
+
+
+@pytest.mark.parametrize('defect', [None, 'opinion', 'verdict', 'excerpt', 'target_rule', 'missing_receipt'])
+def test_diagnostic_field_migration_preserves_historical_review(tmp_path, defect):
+ import copy
+ import json
+ from harness.web_fetch.adjudication_evidence import _require_review_binding_revision
+ raw = b'Required body. CAPTCHA'
+ content_hash = hashlib.sha256(raw).hexdigest()
+ old = {'id': 'a', 'required_facts': [{'id': 'body', 'alternatives': ['Required body']}],
+ 'forbidden_any': ['CAPTCHA']}
+ current = {k: v for k, v in old.items() if k != 'forbidden_any'}
+ current['diagnostic_any'] = ['CAPTCHA']
+ row = {'attempt_id': 'p__a', 'case_id': 'a', 'content_sha256': content_hash,
+ 'agent_verdict': 'fail', 'current_verdict': 'pass', 'final_verdict': 'pass',
+ 'adjudication': 'Required body is present; the word is diagnostic.',
+ 'adjudication_evidence': {'case_sha256': case_hash(old), 'rule_ids': ['required_facts', 'forbidden_any'],
+ 'content_sha256': content_hash, 'samples': [{'text': 'Required body'}]}}
+ def save(name, value):
+ path = tmp_path/name
+ path.write_text(json.dumps(value))
+ return {'path': name, 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
+ old_review = save('old-review.json', {'results_sha256': 'a'*64, 'attempts': [row]})
+ old_dataset = save('old-dataset.json', {'cases': [old]})
+ receipt = save('score-revision.json', {'rule_revision': True, 'provider_calls_added': 0,
+ 'authority': {'preserved_render_input_results_sha256': 'a'*64}, 'input_dataset': old_dataset})
+ migrated = copy.deepcopy(row)
+ proof = migrated['adjudication_evidence']
+ proof['prior_case_sha256'] = proof['case_sha256']
+ proof['case_sha256'] = case_hash(current)
+ proof['rule_ids'] = ['required_facts', 'diagnostic_any']
+ review = {'attempts': [migrated], 'rule_binding_revision': {
+ 'kind': 'unchanged_target_and_verdict_evidence_carryforward', 'new_independent_review': False,
+ 'prior_review': old_review, 'score_revision': receipt, 'attempt_ids': ['p__a']}}
+ if defect == 'opinion': migrated['agent_verdict'] = 'pass'
+ elif defect == 'verdict': migrated['final_verdict'] = 'fail'
+ elif defect == 'excerpt': proof['samples'][0]['text'] = 'Invented body'
+ elif defect == 'target_rule': current['required_facts'] = []
+ elif defect == 'missing_receipt': review.pop('rule_binding_revision')
+ records = [{'attempt_id': 'p__a', 'metrics': {'content_sha256': content_hash}}]
+ if defect:
+ with pytest.raises(ValueError):
+ _require_review_binding_revision(tmp_path, {'cases': [current]}, records, review)
+ else:
+ _require_review_binding_revision(tmp_path, {'cases': [current]}, records, review)
diff --git a/tests/test_anomaly_review.py b/tests/test_anomaly_review.py
index 6edb97832..5dc6c7a27 100644
--- a/tests/test_anomaly_review.py
+++ b/tests/test_anomaly_review.py
@@ -44,7 +44,7 @@ def test_real_stable_structure_forbidden_shape_triggers(self):
'provider':'moli-webfetch','case_id':'dynamic-react-cart','ok':True,
'artifact_path':'answer.txt','metrics':{
'stable_structure_pass':True, 'stable_content_pass':False,
- 'stable_task_pass':False, 'forbidden_hits':['Add to cart'],
+ 'stable_task_pass':False, 'forbidden_requirement_hits':['Add to cart'],
'stable_task_errors':['forbidden_content']}}
self.assertIn('positive-structure-only-forbidden-failed',
detect_anomalies(root,[row])[row['attempt_id']])
@@ -96,7 +96,7 @@ def add(case, text='', *, ok=True, error=None, status=None, route='ordinary_mark
metrics['stable_complete_record_count']=complete
if forbidden:
metrics.update(stable_structure_pass=True,
- forbidden_hits=forbidden,
+ forbidden_requirement_hits=forbidden,
stable_task_errors=['forbidden_content'])
rows.append({'attempt_id':f'moli-webfetch__{case}__r001',
'provider':'moli-webfetch','case_id':case,'ok':ok,
diff --git a/tests/test_dynamic_100_moli_119.py b/tests/test_dynamic_100_moli_119.py
index fc1df4a33..c4a45ab1f 100644
--- a/tests/test_dynamic_100_moli_119.py
+++ b/tests/test_dynamic_100_moli_119.py
@@ -8,7 +8,6 @@
from harness.web_fetch.reference_inputs import input_tree
from harness.web_fetch.report import local_audit_banner, provider_configuration
from harness.web_fetch.storage import data_home
-from harness.web_fetch.scoring_regressions import load_scoring_regressions
from harness.web_fetch.stable_tasks import validate_acceptance_contract
from tests.conftest import read_frozen_dataset_bytes
@@ -53,7 +52,6 @@ def test_dynamic_100_dataset_matches_the_frozen_local_audit(pytestconfig):
assert historical.count(line)==1
historical=historical.replace(line,'',1)
assert hashlib.sha256(historical.encode()).hexdigest()==followup['predecessor_sha256']
- controls = load_scoring_regressions()
for change in revision["dynamic_dataset"]["changes"]:
case = next(case for case in cases if case["id"] == change["case_id"])
assert case[change["field"]] == change["value"]
@@ -92,6 +90,8 @@ def test_dynamic_100_dataset_matches_the_frozen_local_audit(pytestconfig):
(snapshot / "harness/web_fetch" / name).read_bytes() for name in scorer_files)).hexdigest()
assert scorer_revision["scoring_regressions_sha256"] == hashlib.sha256(
(snapshot / "evaluation/scoring-regressions.json").read_bytes()).hexdigest()
+ controls = {row['id']: row for row in json.loads(
+ (snapshot / 'evaluation/scoring-regressions.json').read_text())['cases']}
assert set(scorer_revision["regression_ids"]) <= set(controls)
modes = {}
for identifier in scorer_revision["regression_ids"]:
@@ -136,6 +136,18 @@ def test_dynamic_100_dataset_matches_the_frozen_local_audit(pytestconfig):
assert dataset["comparison"]["local_audit_receipt"] == RECEIPT_PATH.relative_to(ROOT).as_posix()
+def historical_controls_path():
+ revision = json.loads(REVISION_PATH.read_text())
+ scorer = revision['shared_scorer']
+ for followup in revision.get('shared_scorer_followups', []):
+ scorer = {**scorer, **followup}
+ path = data_home() / 'scoring-snapshots' / scorer['revised_sha256'] / 'evaluation/scoring-regressions.json'
+ if not path.is_file():
+ pytest.skip('requires externally installed frozen historical scorer inputs')
+ assert hashlib.sha256(path.read_bytes()).hexdigest() == scorer['scoring_regressions_sha256']
+ return path
+
+
def require_historical_inputs():
try:
input_tree(Path('evaluation/dataset.json'), 'sources')
@@ -158,7 +170,7 @@ def test_historical_scoring_controls_allow_only_new_tail_entries(tmp_path, monke
require_historical_inputs()
dataset = _dataset()
receipt = json.loads(RECEIPT_PATH.read_text(encoding="utf-8"))
- registry = json.loads((ROOT / "evaluation/scoring-regressions.json").read_text())
+ registry = json.loads(historical_controls_path().read_text())
original_file = local_audit._repository_file
replacement = tmp_path / "scoring-regressions.json"
@@ -228,10 +240,14 @@ def test_local_audit_banner_depends_on_dataset_publication_state():
assert "本地审核版" in local_audit_banner(_dataset())
-def test_dynamic_100_repository_evidence_and_scoring_revision_are_exact():
+def test_dynamic_100_repository_evidence_and_scoring_revision_are_exact(monkeypatch):
require_historical_inputs()
dataset = _dataset()
receipt = json.loads(RECEIPT_PATH.read_text(encoding="utf-8"))
+ original = local_audit._repository_file
+ frozen = historical_controls_path()
+ monkeypatch.setattr(local_audit, '_repository_file', lambda relative:
+ frozen if relative == 'evaluation/scoring-regressions.json' else original(relative))
local_audit._require_repository_bindings(dataset, receipt)
diff --git a/tests/test_failure_diagnosis.py b/tests/test_failure_diagnosis.py
index cd5f4cfc0..6f67a036a 100644
--- a/tests/test_failure_diagnosis.py
+++ b/tests/test_failure_diagnosis.py
@@ -110,13 +110,13 @@ def test_prompt_diagnostics_do_not_replace_missing_target_reason(self):
self.assertIsNotNone(result)
self.assertNotIn('blocked_page', result['evidence'])
- def test_qualified_content_with_forbidden_hint_has_no_failure(self):
+ def test_qualified_content_with_diagnostic_hint_has_no_failure(self):
facts = [{'id': 'target', 'alternatives': ['Required target content']}]
metrics = evaluate_content('Required target content\nAccess denied', [], required_facts=facts,
- forbidden_any=['Access denied'])
- self.assertEqual(metrics['forbidden_hits'], ['Access denied'])
+ diagnostic_any=['Access denied'])
+ self.assertEqual(metrics['diagnostic_hits'], ['Access denied'])
self.assertIsNone(diagnose(self.record(metrics), {'task_kind': 'body'}, ''))
- missing = evaluate_content('Access denied', [], required_facts=facts, forbidden_any=['Access denied'])
+ missing = evaluate_content('Access denied', [], required_facts=facts, diagnostic_any=['Access denied'])
self.assertFalse(missing['content_pass'])
self.assertIsNotNone(diagnose(self.record(missing), {'task_kind': 'body'}, ''))
diff --git a/tests/test_fairness.py b/tests/test_fairness.py
index 2f249a78b..457eb3751 100644
--- a/tests/test_fairness.py
+++ b/tests/test_fairness.py
@@ -304,22 +304,23 @@ def test_dynamic_prompt_hits_are_diagnostic_when_target_is_complete(self):
],
}
for identifier, tokens in supplemental.items():
- case = cases[identifier]
+ case = dict(cases[identifier])
+ case['diagnostic_any'] = case.pop('forbidden_any', [])
target_only = case['ground_truth_main_content']
clean = score_attempt(case, target_only.encode(), ok=True)
with self.subTest(case=identifier, variant='target-only'):
self.assertTrue(task_passed(True, clean))
- self.assertEqual(clean['forbidden_hits'], [])
+ self.assertEqual(clean['diagnostic_hits'], [])
for token in tokens:
extended = score_attempt(
case, f'{target_only}\n\n{token}'.encode(), ok=True)
with self.subTest(case=identifier, supplemental=token):
self.assertTrue(extended['stable_structure_pass'])
if identifier == 'dynamic-react-cart' or token == 'Similar Products':
- self.assertEqual(extended['forbidden_hits'], [token])
+ self.assertEqual(extended['diagnostic_hits'], [token])
self.assertTrue(task_passed(True, extended))
else:
- self.assertEqual(extended['forbidden_hits'], [])
+ self.assertEqual(extended['diagnostic_hits'], [])
self.assertTrue(task_passed(True, extended))
def test_end_to_end_processing_is_timed_and_late_answers_rejected(self):
diff --git a/tests/test_local_audit.py b/tests/test_local_audit.py
index cb1171784..8dbc3d40e 100644
--- a/tests/test_local_audit.py
+++ b/tests/test_local_audit.py
@@ -24,6 +24,21 @@ def test_frozen_scoring_controls_allow_additions_but_reject_edits(tmp_path, monk
scoring_path.parent.mkdir(parents=True)
scoring_path.write_bytes((original_root / bindings['scoring_revision_path']).read_bytes())
scoring = json.loads(scoring_path.read_text())
+ # This boundary test owns a frozen synthetic registry. Historical measured
+ # registries are retained with their scorer, not selected by today's runtime.
+ controls = {'cases': [
+ {'id': 'base', 'artifact': 'fixed'},
+ {'id': 'positive', 'artifact': 'complete', 'root_cause': 'fixture', 'failure_mode': 'false-negative'},
+ {'id': 'negative', 'artifact': 'missing', 'root_cause': 'fixture', 'failure_mode': 'false-positive'}]}
+ scoring['preexisting_control_ids'] = ['base']
+ scoring['revision_controls'] = [{k: row[k] for k in ('id', 'root_cause', 'failure_mode')}
+ for row in controls['cases'][1:]]
+ scoring['bindings']['preexisting_controls_sha256'] = local_audit._canonical_sha256(controls['cases'][:1])
+ scoring['bindings']['revision_controls_sha256'] = local_audit._canonical_sha256(controls['cases'][1:])
+ scoring['root_causes'] = [{'id': 'fixture', 'false_positive_ids': ['negative'], 'false_negative_ids': ['positive']}]
+ scoring['scope'] = {'preexisting_controls': 1, 'revision_controls': 2, 'root_causes': 1, 'paired_error_directions': True}
+ _write(scoring_path, scoring)
+ bindings['scoring_revision_sha256'] = _digest(scoring_path)
dataset = {'id': scoring['dataset_id'], 'cases': [{'id': 'fixture'}]}
reference = root / 'evaluation/sources/fixture/reference.txt'
reference.parent.mkdir(parents=True)
@@ -42,7 +57,6 @@ def test_frozen_scoring_controls_allow_additions_but_reject_edits(tmp_path, monk
})
bindings['source_manifest_sha256'] = _digest(source_manifest)
relative = 'evaluation/scoring-regressions.json'
- controls = json.loads((original_root / relative).read_text())
controls['cases'].append({'id': 'new-independent-control'})
path = root / relative
_write(path, controls)
@@ -719,6 +733,13 @@ def test_retrospective_page_review_preserves_call_and_checks_other_requirements(
assert record == before and record['final_url'] is None
assert task_passed(True, scored)
assert scored['page_identity_basis'] == 'retrospective_content_correspondence'
+ # Two substantial matches establish correspondence; additional true short
+ # labels do not invalidate them and cannot replace either long match.
+ with_short = {**annotation, 'matching_samples': annotation['matching_samples'] + ['body']}
+ assert task_passed(True, local_audit.score_reviewed_page_identity(tmp_path, case, record, with_short, captures))
+ with pytest.raises(ValueError, match='substantive text'):
+ local_audit.score_reviewed_page_identity(tmp_path, case, record,
+ {**annotation, 'matching_samples': [annotation['matching_samples'][0], 'body']}, captures)
for samples in ([' ' * 20, ' ' * 21], ['.' * 20, '!' * 21],
['Expected complete body.', '**Expected complete body.**']):
with pytest.raises(ValueError, match='substantive text'):
@@ -751,3 +772,50 @@ def test_retrospective_page_review_requires_bound_receipt(tmp_path):
local_audit._require_page_identity_review(tmp_path, {}, [
{'attempt_id': 'synthetic', 'metrics': {
'page_identity_basis': 'retrospective_content_correspondence'}}])
+
+
+@pytest.mark.parametrize('defect', [None, 'target_changed', 'sample_changed'])
+def test_offline_hint_revision_reapplies_bound_identity_evidence(tmp_path, defect):
+ import copy
+ from harness.web_fetch.scoring import score_attempt
+ body = 'Expected complete body. A second distinctive sentence describing this page.'
+ old = {'id': 'synthetic', 'url': 'https://example.test/start', 'track': 'risk_control',
+ 'task_kind': 'body', 'forbidden_any': ['CAPTCHA'],
+ 'source_benchmark': {'risk_control': {'navigation_contract': {
+ 'requested_url': 'https://example.test/start', 'observed_final_url': 'https://example.test/document'}}},
+ 'acceptance_contract': {'version': 1, 'schema': 'body_text', 'minimum_records': 0,
+ 'stable_text_anchors': [{'id': 'body', 'alternatives': ['Expected complete body']}]}}
+ current = copy.deepcopy(old)
+ current['diagnostic_any'] = current.pop('forbidden_any')
+ (tmp_path/'answer.txt').write_text(body)
+ record = {'attempt_id': 'method__synthetic__r001', 'case_id': 'synthetic',
+ 'artifact_path': 'answer.txt', 'ok': True, 'final_url': None,
+ 'metrics': score_attempt(current, body.encode(), ok=True)}
+ captures = [{'url': 'https://example.test/document', 'text': body}]
+ def save(name, obj):
+ _write(tmp_path/name, obj)
+ return {'path': name, 'sha256': _digest(tmp_path/name)}
+ capture = save('captures.json', captures)
+ annotation = {'attempt_id': record['attempt_id'], 'case_sha256': local_audit._canonical_sha256(old),
+ 'call_sha256': local_audit.local_measurement_call_sha256(record),
+ 'content_sha256': record['metrics']['content_sha256'], 'basis': 'retrospective_content_correspondence',
+ 'reviewed_at': '2026-10-13', 'capture_index': 0,
+ 'capture_sha256': local_audit._canonical_sha256(captures[0]), 'corresponding_url': captures[0]['url'],
+ 'matching_samples': ['Expected complete body.', 'A second distinctive sentence describing this page.']}
+ if defect == 'sample_changed': annotation['matching_samples'] = [' '*20, ' '*21]
+ receipt = save('identity.json', {'kind': 'retrospective_page_identity_review',
+ 'reviewed_at': '2026-10-13', 'basis': 'retained human review', 'provider_calls_added': 0,
+ 'captures_path': capture['path'], 'captures_sha256': capture['sha256'], 'attempts': [annotation]})
+ before = {'cases': [old], 'comparison': {'page_identity_review': receipt}}
+ dataset = {'cases': [current], 'comparison': {}}
+ prior_receipt = (tmp_path/'identity.json').read_bytes()
+ if defect == 'target_changed': current['task_prompt'] = 'A new target requirement'
+ if defect:
+ with pytest.raises(ValueError):
+ local_audit.carry_reviewed_page_identities(tmp_path, before, dataset, [record])
+ else:
+ local_audit.carry_reviewed_page_identities(tmp_path, before, dataset, [record])
+ assert local_audit.task_passed(record['ok'], record['metrics'])
+ assert record['final_url'] is None
+ local_audit._require_page_identity_review(tmp_path, dataset, [record])
+ assert (tmp_path/'identity.json').read_bytes() == prior_receipt
diff --git a/tests/test_local_diagnostic_presentation.py b/tests/test_local_diagnostic_presentation.py
index 7374726c1..cf1cd974b 100644
--- a/tests/test_local_diagnostic_presentation.py
+++ b/tests/test_local_diagnostic_presentation.py
@@ -136,7 +136,8 @@ def test_explicit_sample_ranking_reuses_existing_ranked_table_and_keeps_scope(tm
assert 'z-high' in soup.select_one('tbody tr').get_text()
banner = local_audit_banner(dataset)
assert 'data-ranking-scope="saved-task-pass-rate"' in banner
- assert BeautifulSoup(banner, 'html.parser').get_text() == ''
+ assert '未完成仓库认证流程' in BeautifulSoup(banner, 'html.parser').get_text()
+ assert 'hidden' not in banner
assert 'callout' not in banner
page = tmp_path/'index.html'
page.write_text('Uncertified ',
+ 'Uncertified
',
+ '',
+ '
'])
+def test_sample_ranking_rejects_invisible_scope_disclosure(tmp_path, wrapper):
+ from harness.web_fetch.report_check import check_local_diagnostic_presentation
+ attrs = 'data-evidence-state="local_measurement" data-ranking-scope="saved-task-pass-rate"'
+ (tmp_path/'index.html').write_text('fixture
')
+ reference = save('reference.txt', 'fixture')
+ def capture(stage):
+ observation = save(stage+'.json', {'case_id': 'new', 'url': added['url'],
+ 'independent': True, 'status': 200, 'sha256': source['sha256'],
+ 'reference_sha256': reference['sha256'], 'extracted_sha256': reference['sha256'],
+ 'checked_at': datetime.now(timezone.utc).isoformat()})
+ return {'evidence': observation, 'source': source, 'reference': reference, 'extracted': reference}
+ pre = capture('pre')
+ _, cases = load_dataset(tmp_path/'frozen.json')
+ calls, batches = [], []
+ for method in ['a', 'b']:
+ rows, _ = run_evaluation(output_dir=tmp_path,
+ dataset_meta={'tool': 'web_fetch', 'cases': [added]}, dataset_path=tmp_path/'frozen.json',
+ cases=cases, providers=[build_provider(method, {'type': 'fixture'})],
+ provider_config_path=tmp_path/'providers.json', runs=1, timeout=45, delay=0)
+ batch = json.loads((tmp_path/'manifest.json').read_text())
+ calls.extend(rows)
+ batches.append(batch)
+ post = capture('post')
+ contract = {'schema_version': 1, 'kind': 'uniform_native_local_measurement',
+ 'providers': ['a', 'b'], 'runs': 1, 'timeout_seconds': 45, 'max_failure_retries': 0,
+ 'dataset_storage': 'evidence', 'dataset_sources': [input_binding],
+ 'provider_config_sha256': providers['sha256'],
+ 'runtime_source_sha256': batches[0]['runtime_source_sha256'],
+ 'environment': batches[0]['environment'], 'parameters': batches[0]['parameters'],
+ 'runner_inputs': {'dataset': input_binding, 'providers': providers, 'network': network,
+ 'source_observations': {'new': {'pre': pre, 'post': post}}}}
+ receipt = {'kind': 'local_task_replacement', 'reason': 'private DNS address',
+ 'removed_case_id': 'old', 'added_case_id': 'new', 'provider_calls_added': 2}
receipt['baseline_dataset'] = save('before.json', before)
receipt['baseline_results'] = save('previous.jsonl', previous, True)
receipt['cohort_dataset'] = save('cohort.json', {'cases': [added]})
receipt['cohort_results'] = save('calls.jsonl', calls, True)
+ receipt['execution_contract'] = save('execution-contract.json', contract)
+ receipt['cohort_execution'] = save('execution.json', {'batches': batches})
binding = save('receipt.json', receipt)
records = [r for r in previous if r['case_id'] != 'old']+calls
return binding, dataset, records
@@ -60,12 +97,12 @@ def test_replacement_execution_preserves_history_without_counting_old_task(tmp_p
from harness.web_fetch.report_check import require_replacement_execution
binding, dataset, records = setup_replacement(tmp_path)
physical = Counter({'a__old': 1, 'b__old': 1, 'a__keep': 1, 'b__keep': 1,
- 'a__new': 1, 'b__new': 1})
+ 'a__new__r001': 1, 'b__new__r001': 1})
logical = Counter(r['attempt_id'] for r in records)
if defect == 'missing_history':
del physical['a__old']
elif defect == 'extra_call':
- physical['a__new'] += 1
+ physical['a__new__r001'] += 1
elif defect == 'old_task_counted':
logical['a__old'] = 1
if defect:
@@ -73,3 +110,31 @@ def test_replacement_execution_preserves_history_without_counting_old_task(tmp_p
require_replacement_execution(tmp_path, binding, physical, logical)
else:
require_replacement_execution(tmp_path, binding, physical, logical)
+
+
+@pytest.mark.parametrize('defect', ['missing_contract', 'missing_inputs', 'environment', 'timeout',
+ 'config', 'dns', 'source', 'late_precheck', 'output'])
+def test_replacement_requires_full_physical_execution_evidence(tmp_path, defect):
+ binding, dataset, records = setup_replacement(tmp_path)
+ receipt = json.loads((tmp_path/'receipt.json').read_text())
+ if defect == 'missing_contract':
+ receipt.pop('execution_contract')
+ elif defect in {'missing_inputs', 'environment', 'timeout', 'config'}:
+ contract = json.loads((tmp_path/'execution-contract.json').read_text())
+ if defect == 'missing_inputs': contract.pop('runner_inputs')
+ elif defect == 'environment': contract['environment']['python'] = 'wrong'
+ elif defect == 'timeout': contract['timeout_seconds'] = 44
+ else: contract['provider_config_sha256'] = '0'*64
+ (tmp_path/'execution-contract.json').write_text(json.dumps(contract))
+ receipt['execution_contract']['sha256'] = hashlib.sha256((tmp_path/'execution-contract.json').read_bytes()).hexdigest()
+ elif defect == 'dns': (tmp_path/'network.json').write_text('{}')
+ elif defect == 'source': (tmp_path/'source.html').write_text('wrong page')
+ elif defect == 'late_precheck':
+ observation = json.loads((tmp_path/'pre.json').read_text())
+ observation['checked_at'] = '2099-01-01T00:00:00+00:00'
+ (tmp_path/'pre.json').write_text(json.dumps(observation))
+ else: (tmp_path/records[-1]['artifact_path']).write_text('changed output')
+ (tmp_path/'receipt.json').write_text(json.dumps(receipt))
+ binding['sha256'] = hashlib.sha256((tmp_path/'receipt.json').read_bytes()).hexdigest()
+ with pytest.raises(ValueError):
+ require_local_task_replacement(tmp_path, binding, dataset, records)
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 420a21eec..aa4ed1271 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -133,10 +133,10 @@ def test_ground_truth_requires_every_fact_and_keeps_prompt_diagnostics(self) ->
self.assertEqual(partial["fact_coverage"], 0.5)
blocked = evaluate_content(
"Example Domain documentation examples Access denied",
- ["Example Domain"], required_facts=facts, forbidden_any=["Access denied"],
+ ["Example Domain"], required_facts=facts, diagnostic_any=["Access denied"],
)
self.assertTrue(blocked["content_pass"])
- self.assertEqual(blocked["forbidden_hits"], ["Access denied"])
+ self.assertEqual(blocked["diagnostic_hits"], ["Access denied"])
def test_summary_separates_transport_and_content(self) -> None:
base = {
diff --git a/tests/test_report_article_cases.py b/tests/test_report_article_cases.py
index cdd9c83ea..46ee030ec 100644
--- a/tests/test_report_article_cases.py
+++ b/tests/test_report_article_cases.py
@@ -41,7 +41,7 @@ def test_text_evidence_is_escaped_without_rewriting_its_content(self):
self.assertIn('<script>', left)
self.assertIn('<script>', right)
self.assertNotIn('
diff --git a/harness/web_fetch/providers/subprocess_jsonl.py b/harness/web_fetch/providers/subprocess_jsonl.py
index 8f76d664e..6ab6409cc 100644
--- a/harness/web_fetch/providers/subprocess_jsonl.py
+++ b/harness/web_fetch/providers/subprocess_jsonl.py
@@ -319,6 +319,9 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
error_message=None if ok else str(payload.get("error_message") or "worker failed"),
metadata={
**dict(payload.get("metadata") or {}),
+ **({'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['content'], 'content_format': 'text'}}
+ if isinstance(payload.get('content'), str) else {}),
"_raw_response_b64": base64.b64encode(line.encode("utf-8")).decode("ascii"),
"raw_response_sha256": hashlib.sha256(line.encode("utf-8")).hexdigest(),
"raw_response_bytes": len(line.encode("utf-8")),
diff --git a/harness/web_fetch/response_input.py b/harness/web_fetch/response_input.py
index 7c0e8e8bd..cf1349a56 100644
--- a/harness/web_fetch/response_input.py
+++ b/harness/web_fetch/response_input.py
@@ -53,3 +53,87 @@ def safe_named_links(links, base_url):
if parts.scheme in {'http', 'https'} and parts.hostname and not parts.username and not parts.password:
result.append((url, item['title']))
return result
+
+
+def _projection_value(payload, pointer):
+ if not isinstance(pointer, list) or not pointer:
+ raise ValueError('Native answer projection requires a nonempty field pointer')
+ value = payload
+ try:
+ for key in pointer:
+ if isinstance(value, dict) and isinstance(key, str):
+ value = value[key]
+ elif isinstance(value, list) and type(key) is int and key >= 0:
+ value = value[key]
+ else:
+ raise ValueError('Native answer pointer is not a structural field selection')
+ except (KeyError, IndexError) as error:
+ raise ValueError('Native answer field is missing') from error
+ return value
+
+
+def _attribute_record_urls(groups, base_url):
+ """Project declared HTML URL attributes without classifying their targets."""
+ urls = []
+ for group in groups if isinstance(groups, list) else []:
+ if not isinstance(group, dict):
+ continue
+ elements = group.get('results')
+ for element in elements if isinstance(elements, list) else []:
+ if not isinstance(element, dict):
+ continue
+ attributes = element.get('attributes')
+ for attribute in attributes if isinstance(attributes, list) else []:
+ if not isinstance(attribute, dict):
+ continue
+ name, value = attribute.get('name'), attribute.get('value')
+ if not isinstance(name, str) or not isinstance(value, str):
+ continue
+ name = name.casefold()
+ if name not in {'src', 'srcset', 'href', 'data-src', 'data-original'}:
+ continue
+ values = ([part.strip().split()[0] for part in value.split(',') if part.strip()]
+ if name == 'srcset' else [value])
+ for raw in values:
+ try:
+ urls.append(urljoin(base_url, raw.strip()))
+ except ValueError:
+ # Retain malformed native values; the shared scorer,
+ # not a provider adapter, decides URL eligibility.
+ urls.append(raw)
+ return sorted(set(urls))
+
+
+def project_native_answer(payload, declaration):
+ """Select declared native fields; never inspect references or URL meaning."""
+ if declaration.get('schema_version') != 1:
+ raise ValueError('Unsupported native answer projection')
+ value = _projection_value(payload, declaration.get('content_pointer'))
+ kind = declaration.get('content_format')
+ if kind == 'text' and isinstance(value, str):
+ content = value
+ elif kind == 'json':
+ content = json.dumps(value, ensure_ascii=False, separators=(',', ':'))
+ else:
+ raise ValueError('Native answer field differs from its declared type')
+ fields = {}
+ for name, projection in declaration.get('answer_fields', {}).items():
+ if name not in {'links', 'images', 'videos', 'media', 'extras'}:
+ raise ValueError('Unknown native answer metadata field')
+ selected = _projection_value(payload, projection.get('pointer'))
+ if projection.get('format') == 'html_attribute_urls':
+ fields[name] = _attribute_record_urls(selected, projection['base_url'])
+ elif projection.get('format') == 'identity':
+ fields[name] = selected
+ else:
+ raise ValueError('Unknown native answer field projection')
+ return content, fields
+
+
+def require_native_answer_projection(content, metadata, raw_response):
+ declaration = metadata.get('native_answer_projection')
+ if declaration is None:
+ return
+ expected, fields = project_native_answer(json.loads(raw_response), declaration)
+ if content != expected or any(metadata.get(key) != value for key, value in fields.items()):
+ raise ValueError('Scored answer differs from its declared native field projection')
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index 60c69b696..710c217f4 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -379,6 +379,15 @@ def persist_result_artifacts(
output_dir: Path, artifact_rel: Path, result: ProviderResult
) -> Path:
"""Persist one physical call without changing its scored answer bytes."""
+ discarded_late_answer = (not result.ok and result.error_type == 'timeout'
+ and result.content == '')
+ if result.metadata.get('native_answer_projection') and not discarded_late_answer:
+ from .response_input import require_native_answer_projection
+ raw = result.metadata.get('_raw_response_b64')
+ if not isinstance(raw, str):
+ raise ValueError('Declared native answer lacks its retained raw response')
+ require_native_answer_projection(result.content, result.metadata,
+ base64.b64decode(raw, validate=True))
artifact = contained_path(output_dir, artifact_rel)
artifact.parent.mkdir(parents=True, exist_ok=True)
artifact.write_text(result.content, encoding='utf-8', errors='replace')
diff --git a/tests/test_browser_reference_scope.py b/tests/test_browser_reference_scope.py
index 5a6d9d3e1..0a0a8bb74 100644
--- a/tests/test_browser_reference_scope.py
+++ b/tests/test_browser_reference_scope.py
@@ -253,7 +253,8 @@ def test_resource_observation_matches_every_declared_target_and_cdn_variants():
'id': 'resource', 'url': 'https://example.test/page',
'track': 'image', 'task_kind': 'resource',
'expected_resources': [
- {'id': 'one', 'urls': ['https://images.unsplash.com/photo-one']},
+ {'id': 'one', 'urls': ['https://images.unsplash.com/photo-one'],
+ 'aliases': ['https://images.unsplash.com/photo-one?w=1200']},
{'id': 'two', 'urls': [], 'url_patterns': [r'^https://cdn\.example\.test/two(?:\?|$)']},
],
'resource_output_contract': {'unexpected': 'allow'},
diff --git a/tests/test_fixed_native_policy.py b/tests/test_fixed_native_policy.py
index e8cff000d..860a53f96 100644
--- a/tests/test_fixed_native_policy.py
+++ b/tests/test_fixed_native_policy.py
@@ -126,11 +126,20 @@ def test_all_430_frozen_cases_pass_native_input_guard():
assert count == 430
-def test_native_media_projection_keeps_returned_sources_and_video_links():
- from harness.web_fetch.providers.cloudflare import _resource_urls
- payload = {"result": [
- {"selector": "video", "results": [{"attributes": [{"name": "src", "value": "/a.mp4"}]}]},
- {"selector": "a[href]", "results": [{"attributes": [{"name": "href", "value": "/video/abc"}]}]}]}
- assert _resource_urls(payload, base_url="https://example.com/", track="video",
- include_video_page_links=True, only_video_page_links=False) == [
- "https://example.com/a.mp4", "https://example.com/video/abc"]
+def test_native_media_projection_keeps_all_returned_fields_without_link_guessing():
+ payload = {'success': True, 'result': [
+ {'selector': 'video', 'results': [{'attributes': [{'name': 'src', 'value': '/a.mp4'}]}]},
+ {'selector': 'a[href]', 'results': [{'attributes': [
+ {'name': 'href', 'value': '/opaque'}, {'name': 'href', 'value': '/about'},
+ {'name': 'title', 'value': 'Unchanged native label'}]}]}]}
+ case = EvalCase('control', 'control', 'https://example.com/', 'control', [],
+ track='video', task_kind='resource')
+ provider = build_provider('cf', {'type': 'cloudflare_markdown',
+ 'uniform_native_parameters': True, 'account_id': 'fixture', 'api_token': 'fixture'})
+ with patch('harness.web_fetch.providers.cloudflare.request',
+ return_value=HttpResponse(200, case.url, {}, json.dumps(payload).encode())) as call:
+ result = provider.fetch(case, 45)
+ assert call.call_count == 1
+ assert result.metadata['videos'] == [
+ 'https://example.com/a.mp4', 'https://example.com/about', 'https://example.com/opaque']
+ assert json.loads(result.content) == payload['result']
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index aa4ed1271..48114537a 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -298,7 +298,7 @@ def test_image_scope_ignores_ordinary_page_links(self) -> None:
self.assertEqual(result["resource_candidate_urls"], ["https://cdn.example.test/target.webp"])
self.assertEqual(result["unexpected_resource_urls"], [])
- def test_image_scope_accepts_declared_alias_pattern_and_normalized_thumbnail(self) -> None:
+ def test_image_scope_accepts_declared_alias_pattern_and_thumbnail_alias(self) -> None:
contract = {"candidate_scope": "image_answer_fields", "unexpected": "reject"}
alias = evaluate_resources(
"", {"images": ["/photo-480w.jpg"]},
@@ -318,6 +318,8 @@ def test_image_scope_accepts_declared_alias_pattern_and_normalized_thumbnail(sel
"https://upload.wikimedia.org/wikipedia/commons/thumb/a/a1/Earth.jpg/320px-Earth.jpg"}]}},
[{"id": "earth", "urls": [
"https://upload.wikimedia.org/wikipedia/commons/thumb/a/a1/Earth.jpg/1280px-Earth.jpg"
+ ], "aliases": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a1/Earth.jpg/320px-Earth.jpg"
]}], base_url="https://commons.wikimedia.org/wiki/File:Earth.jpg",
output_contract=contract,
)
@@ -333,14 +335,19 @@ def test_wikimedia_host_equivalence_is_only_for_the_same_thumbnail_asset(self) -
'Moon.jpg/960px-Moon.jpg?utm_campaign=parser')
invalid_original = 'https://thumb.wikimedia.org/wikipedia/commons/4/41/Moon.jpg'
upload_original = 'https://upload.wikimedia.org/wikipedia/commons/4/41/Moon.jpg'
- self.assertEqual(normalize_resource_url(upload_thumb, base),
- normalize_resource_url(migrated_thumb, base))
+ self.assertNotEqual(normalize_resource_url(upload_thumb, base),
+ normalize_resource_url(migrated_thumb, base))
+ declared = [{'id': 'moon', 'urls': [upload_thumb], 'aliases': [migrated_thumb]}]
+ self.assertTrue(evaluate_resources(migrated_thumb, {}, declared,
+ base_url=base)['resource_pass'])
+ self.assertFalse(evaluate_resources(wrong_asset, {}, declared,
+ base_url=base)['resource_pass'])
self.assertNotEqual(normalize_resource_url(upload_thumb, base),
normalize_resource_url(wrong_asset, base))
self.assertNotEqual(normalize_resource_url(invalid_original, base),
normalize_resource_url(upload_original, base))
- def test_image_scope_accepts_bare_declared_extensionless_target(self) -> None:
+ def test_image_scope_rejects_extra_bare_extensionless_url(self) -> None:
target = "https://images.unsplash.com/photo-1493673272479-a20888bcee10"
result = evaluate_resources(
target + "\nhttps://example.test/ordinary-page", {},
@@ -348,8 +355,39 @@ def test_image_scope_accepts_bare_declared_extensionless_target(self) -> None:
base_url="https://unsplash.com/photos/example",
output_contract={"candidate_scope": "image_answer_fields", "unexpected": "reject"},
)
- self.assertTrue(result["resource_pass"])
- self.assertEqual(result["resource_candidate_urls"], [target])
+ self.assertFalse(result["resource_pass"])
+ self.assertEqual(result["resource_candidate_urls"],
+ ["https://example.test/ordinary-page", target])
+ self.assertEqual(result["resource_precision"], 0.5)
+
+ def test_resource_queries_are_not_merged_by_known_host(self):
+ base = 'https://example.test/gallery'
+ for host in ('images.unsplash.com', 'images.pexels.com', 'images-assets.nasa.gov',
+ 'www.nasa.gov', 'cdn.example.test'):
+ with self.subTest(host=host):
+ left, right = f'https://{host}/photo.jpg?crop=left', f'https://{host}/photo.jpg?crop=right'
+ self.assertNotEqual(normalize_resource_url(left, base), normalize_resource_url(right, base))
+ self.assertFalse(evaluate_resources(right, {}, [{'id': 'photo', 'urls': [left]}],
+ base_url=base)['resource_pass'])
+ self.assertTrue(evaluate_resources(right, {}, [{'id': 'photo', 'urls': [left],
+ 'aliases': [right]}], base_url=base)['resource_pass'])
+
+ def test_extensionless_image_candidates_do_not_depend_on_reference(self):
+ right, wrong = 'https://cdn.example/item/RIGHT', 'https://cdn.example/item/WRONG'
+ contract = {'candidate_scope': 'image_answer_fields', 'unexpected': 'reject'}
+ mixed = right + '\n' + wrong
+ candidates = []
+ for target in (right, wrong):
+ result = evaluate_resources(mixed, {}, [{'id': 'image', 'urls': [target]}],
+ base_url='https://example.test/', output_contract=contract)
+ self.assertFalse(result['resource_pass'])
+ self.assertEqual(result['resource_precision'], 0.5)
+ candidates.append(result['resource_candidate_urls'])
+ self.assertEqual(candidates[0], candidates[1])
+ result = evaluate_resources(right, {}, [{'id': 'image', 'urls': [right]}],
+ base_url='https://example.test/', output_contract=contract)
+ self.assertTrue(result['resource_pass'])
+ self.assertEqual(result['resource_precision'], 1.0)
if __name__ == "__main__":
diff --git a/tests/test_providers.py b/tests/test_providers.py
index e2ce37ec9..582d82c7e 100644
--- a/tests/test_providers.py
+++ b/tests/test_providers.py
@@ -8,13 +8,13 @@
from unittest.mock import patch
from harness.web_fetch.metrics import evaluate_resources, task_passed, task_measured
-from harness.web_fetch.models import EvalCase
+from harness.web_fetch.models import EvalCase, ProviderResult
from harness.web_fetch.providers.cloudflare import (
CloudflareContentProvider,
CloudflareMarkdownProvider,
_post_multipart,
)
-from harness.web_fetch.providers.cowagent import CowAgentBuiltinProvider, _sanitize_public_content
+from harness.web_fetch.providers.cowagent import CowAgentBuiltinProvider
from harness.web_fetch.providers.exa import ExaContentsProvider
from harness.web_fetch.providers.firecrawl import FirecrawlProvider
from harness.web_fetch.providers.http import HttpProvider
@@ -1069,7 +1069,7 @@ def test_cloudflare_image_resource_uses_scrape_and_resolves_srcset(
self.assertEqual(selectors, [{"selector": "img"}, {"selector": "picture source"}])
@patch("harness.web_fetch.providers.cloudflare.request")
- def test_cloudflare_video_resource_scrape_filters_ordinary_links(
+ def test_cloudflare_video_resource_preserves_all_declared_links(
self, mock_request
) -> None:
mock_request.return_value = HttpResponse(
@@ -1102,9 +1102,9 @@ def test_cloudflare_video_resource_scrape_filters_ordinary_links(
).fetch(video, 45)
self.assertEqual(result.metadata["videos"], [
- "https://example.com/watch/clip.webm"
+ "https://example.com/about", "https://example.com/watch/clip.webm"
])
- self.assertNotIn("/about", result.content)
+ self.assertEqual(json.loads(result.content), mock_request.return_value.json()['result'])
@patch("harness.web_fetch.providers.cloudflare.request")
def test_cloudflare_video_resource_keeps_packaged_video_downloads(
@@ -1143,11 +1143,11 @@ def test_cloudflare_video_resource_keeps_packaged_video_downloads(
self.assertTrue(result.ok)
self.assertEqual(result.metadata["videos"], [
- "https://example.com/movies/movie.mp4.zip",
+ "https://example.com/", "https://example.com/movies/movie.mp4.zip",
"https://example.com/movies/movie.webm.zip?mirror=1",
+ "https://example.com/movies/notes.zip",
])
- self.assertNotIn("notes.zip", result.content)
- self.assertNotIn("\nhttps://example.com/\n", "\n" + result.content + "\n")
+ self.assertEqual(json.loads(result.content), mock_request.return_value.json()['result'])
@patch("harness.web_fetch.providers.cloudflare.request")
def test_cloudflare_video_link_task_keeps_page_href_without_media_suffix(
@@ -1193,11 +1193,10 @@ def test_cloudflare_video_link_task_keeps_page_href_without_media_suffix(
self.assertTrue(result.ok)
self.assertEqual(result.metadata["videos"], [
+ "https://example.com/about", "https://example.com/short/iframe.html",
"https://example.com/video/BV1234567890",
])
- self.assertIn("https://example.com/video/BV1234567890", result.content)
- self.assertNotIn("https://example.com/about", result.content)
- self.assertNotIn("https://example.com/short/iframe.html", result.content)
+ self.assertEqual(json.loads(result.content), mock_request.return_value.json()['result'])
self.assertEqual(mock_request.call_args.kwargs["json_body"]["elements"],
[{"selector": "a[href]"}])
@@ -1426,15 +1425,18 @@ def test_cowagent_cannot_omit_declared_identity(self) -> None:
self.assertIn("expected_repo_revision", errors)
self.assertIn("required runtime paths", errors)
- def test_cowagent_removes_workspace_path_but_keeps_document_text(self) -> None:
- content = (
- "[Document: guide.pdf | Size: 10KB | Saved to: "
- "/Users/person/.browser-eval/runs/work/tmp/guide.pdf]\n\nBody"
- )
- self.assertEqual(
- _sanitize_public_content(content),
- "[Document: guide.pdf | Size: 10KB]\n\nBody",
- )
+ @patch('harness.web_fetch.providers.subprocess_jsonl.SubprocessJsonlProvider.fetch')
+ def test_cowagent_preserves_native_answer_and_error_before_scoring(self, fetch):
+ content = ('[Document: guide.pdf | Size: 10KB | Saved to: '
+ '/Users/person/work/guide.pdf]\n\nBody')
+ native = ProviderResult(False, content, error_message=content,
+ metadata={'raw_response_sha256': 'native-envelope'})
+ fetch.return_value = native
+ result = CowAgentBuiltinProvider('cowagent', {'repo_path': '.'}).fetch(self.case, 1)
+ self.assertIs(result, native)
+ self.assertEqual(result.content, content)
+ self.assertEqual(result.error_message, content)
+ self.assertEqual(result.metadata, {'raw_response_sha256': 'native-envelope'})
if __name__ == "__main__":
diff --git a/tests/test_report_metrics.py b/tests/test_report_metrics.py
index 3289afa98..a816ce289 100644
--- a/tests/test_report_metrics.py
+++ b/tests/test_report_metrics.py
@@ -24,9 +24,12 @@ def test_cjk_and_markdown(self):
def test_url_echo_is_not_resource_discovery(self):
r=evaluate_resources('https://example.com/a.png',{},[{'urls':['https://example.com/a.png']}],base_url='https://example.com/a.png')
self.assertFalse(r['resource_pass'])
- def test_resized_image_same_asset(self):
- r=evaluate_resources('https://images.pexels.com/photos/123/a.jpeg?w=500',{},[{'urls':['https://images.pexels.com/photos/123/a.jpeg?w=1000']}],base_url='https://example.com')
+ def test_declared_resized_image_alias(self):
+ target='https://images.pexels.com/photos/123/a.jpeg?w=1000'
+ resized='https://images.pexels.com/photos/123/a.jpeg?w=500'
+ r=evaluate_resources(resized,{},[{'urls':[target],'aliases':[resized]}],base_url='https://example.com')
self.assertTrue(r['resource_pass'])
+ self.assertFalse(evaluate_resources(resized,{},[{'urls':[target]}],base_url='https://example.com')['resource_pass'])
def test_generic_query_not_dropped(self):
r=evaluate_resources('https://example.com/file?id=2',{},[{'urls':['https://example.com/file?id=1']}],base_url='https://example.com')
self.assertFalse(r['resource_pass'])
diff --git a/tests/test_subprocess_provider.py b/tests/test_subprocess_provider.py
index af9ec302f..46d458b18 100644
--- a/tests/test_subprocess_provider.py
+++ b/tests/test_subprocess_provider.py
@@ -124,6 +124,68 @@ def test_worker_receives_shared_limits_and_retains_full_response_envelope(self):
persist_result_artifacts(target, Path('artifacts/worker/control.txt'), result)
self.assertEqual(json.loads((target/result.metadata['raw_response_path']).read_bytes()),payload)
+ def test_persistence_rejects_answer_edit_or_candidate_filtering(self):
+ import hashlib
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ from harness.web_fetch.response_input import project_native_answer
+ cases = [
+ ({'content': 'Body | Saved to: /Users/person/work/page.txt'},
+ {'schema_version': 1, 'content_pointer': ['content'], 'content_format': 'text'}),
+ ({'result': [{'selector': 'a[href]', 'results': [{'attributes': [
+ {'name': 'href', 'value': '/opaque-resource'},
+ {'name': 'href', 'value': '/about'},
+ {'name': 'title', 'value': 'Native label'}]}]}]},
+ {'schema_version': 1, 'content_pointer': ['result'], 'content_format': 'json',
+ 'answer_fields': {'videos': {'pointer': ['result'],
+ 'format': 'html_attribute_urls', 'base_url': 'https://example.test/'}}}),
+ ]
+ for payload, declaration in cases:
+ content, fields = project_native_answer(payload, declaration)
+ raw = json.dumps(payload).encode()
+ for defect in (None, 'edited_text', 'filtered_field', 'missing_raw'):
+ if defect == 'filtered_field' and not fields:
+ continue
+ with self.subTest(format=declaration['content_format'], defect=defect):
+ metadata = {**fields, 'native_answer_projection': declaration,
+ '_raw_response_b64': base64.b64encode(raw).decode(),
+ 'raw_response_sha256': hashlib.sha256(raw).hexdigest(),
+ 'raw_response_bytes': len(raw)}
+ answer = content
+ if defect == 'edited_text': answer = content.replace('/about', '') + 'changed'
+ if defect == 'filtered_field': metadata['videos'] = metadata['videos'][:1]
+ if defect == 'missing_raw': metadata.pop('_raw_response_b64')
+ result = ProviderResult(True, answer, metadata=metadata)
+ with tempfile.TemporaryDirectory() as directory:
+ target = Path(directory)/'artifacts/control/result.txt'
+ if defect:
+ with self.assertRaisesRegex(ValueError, 'native'):
+ persist_result_artifacts(Path(directory),
+ Path('artifacts/control/result.txt'), result)
+ self.assertFalse(target.exists())
+ else:
+ persist_result_artifacts(Path(directory),
+ Path('artifacts/control/result.txt'), result)
+ self.assertEqual(target.read_text(), content)
+ if declaration['content_format'] == 'json':
+ self.assertEqual(json.loads(content), payload['result'])
+
+ def test_deadline_discards_answer_without_discarding_native_evidence(self):
+ import hashlib
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ raw = json.dumps({'content': 'late native answer'}).encode()
+ answer = ProviderResult(False, '', error_type='timeout', metadata={
+ 'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['content'], 'content_format': 'text'},
+ '_raw_response_b64': base64.b64encode(raw).decode(),
+ 'raw_response_sha256': hashlib.sha256(raw).hexdigest(),
+ 'raw_response_bytes': len(raw)})
+ with tempfile.TemporaryDirectory() as directory:
+ persist_result_artifacts(Path(directory), Path('artifacts/worker/late.txt'), answer)
+ self.assertEqual((Path(directory)/'artifacts/worker/late.txt').read_bytes(), b'')
+ self.assertEqual((Path(directory)/answer.metadata['raw_response_path']).read_bytes(), raw)
+
def test_native_langchain_document_is_retained_without_changing_answer(self):
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
From ec71b5252eaf164754d0f852d3a62ca4cbb1196e Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 00:08:10 +0800
Subject: [PATCH 26/50] Restore scored answers from bound native envelopes
without new calls
---
evaluation/scoring-regressions.json | 106 ++++++++++++++++++++++++++++
harness/web_fetch/local_revision.py | 84 +++++++++++++++++++++-
harness/web_fetch/metrics.py | 25 ++++---
tests/test_local_revision.py | 84 ++++++++++++++++++++++
4 files changed, 287 insertions(+), 12 deletions(-)
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index 4a05dbb98..c9b81ae89 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -31732,6 +31732,112 @@
],
"resource_pass": true
}
+ },
+ {
+ "id": "native-json-resource-untyped-url-echo-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "native_json_answer_field_scope",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/gallery",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "target",
+ "urls": [
+ "https://cdn.example.test/target.jpg"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "reject"
+ }
+ },
+ "artifact": "{\"echo\": \"https://cdn.example.test/target.jpg\"}",
+ "metadata": {
+ "images": [
+ "https://cdn.example.test/other.jpg"
+ ],
+ "native_answer_projection": {
+ "schema_version": 1,
+ "content_pointer": [
+ "result"
+ ],
+ "content_format": "json",
+ "answer_fields": {
+ "images": {
+ "pointer": [
+ "result",
+ "images"
+ ],
+ "format": "identity"
+ }
+ }
+ }
+ },
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_candidate_urls": [
+ "https://cdn.example.test/other.jpg"
+ ],
+ "resource_precision": 0.0
+ }
+ },
+ {
+ "id": "native-json-resource-declared-image-field-preserved",
+ "category": "acceptance-contract",
+ "root_cause": "native_json_answer_field_scope",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/gallery",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "target",
+ "urls": [
+ "https://cdn.example.test/target.jpg"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "reject"
+ }
+ },
+ "artifact": "{\"echo\": \"https://cdn.example.test/other.jpg\"}",
+ "metadata": {
+ "images": [
+ "https://cdn.example.test/target.jpg"
+ ],
+ "native_answer_projection": {
+ "schema_version": 1,
+ "content_pointer": [
+ "result"
+ ],
+ "content_format": "json",
+ "answer_fields": {
+ "images": {
+ "pointer": [
+ "result",
+ "images"
+ ],
+ "format": "identity"
+ }
+ }
+ }
+ },
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_candidate_urls": [
+ "https://cdn.example.test/target.jpg"
+ ],
+ "resource_precision": 1.0
+ }
}
],
"superseded_controls": {
diff --git a/harness/web_fetch/local_revision.py b/harness/web_fetch/local_revision.py
index 8024a8027..224c82c2d 100644
--- a/harness/web_fetch/local_revision.py
+++ b/harness/web_fetch/local_revision.py
@@ -1,12 +1,14 @@
"""Validate task replacements and offline revisions of a bound local snapshot."""
import hashlib
import json
+from pathlib import Path
from harness.record_io import load_jsonl
from .cohort_exclusions import require_cohort_exclusions
from .safety import contained_path
from .scoring_regressions import require_scoring_regressions
+from .scoring_revision import ALLOWED_FIELDS
def read_bound(source, binding):
@@ -92,6 +94,41 @@ def require_local_revision(source, binding, dataset, records):
return validate_revision_chain(source, binding, dataset, records)[0]
+def _require_native_answer_correction(source, original, corrected, proof):
+ """Verify a derived answer against the same immutable native response."""
+ from .response_input import project_native_answer
+ if proof.get('attempt_id') != original['attempt_id']:
+ raise ValueError('Native answer correction lost its attempt identity')
+ native = read_bound(source, proof['native_response'])
+ metadata = original.get('provider_metadata', {})
+ if (proof['native_response']['sha256'] != metadata.get('raw_response_sha256')
+ or native.stat().st_size != metadata.get('raw_response_bytes')):
+ raise ValueError('Native answer correction changed its original response')
+ before = proof['original_artifact']
+ after = proof['corrected_artifact']
+ if (before['path'] != original['artifact_path']
+ or before['sha256'] != original['metrics']['content_sha256']
+ or after['path'] != corrected['artifact_path']
+ or after['sha256'] != corrected['metrics']['content_sha256']
+ or before['path'] == after['path']):
+ raise ValueError('Native answer correction must preserve its old artifact')
+ read_bound(source, before)
+ text, fields = project_native_answer(json.loads(native.read_bytes()), proof['projection'])
+ if read_bound(source, after).read_bytes() != text.encode('utf-8', errors='replace'):
+ raise ValueError('Native answer correction differs from declared original fields')
+ for field in proof['projection'].get('answer_fields', {}).values():
+ if field.get('format') == 'html_attribute_urls' and field.get('base_url') != original['url']:
+ raise ValueError('Native URL projection must use the original request URL')
+ expected_metadata = {**metadata, **fields, 'native_answer_projection': proof['projection']}
+ if corrected.get('provider_metadata') != expected_metadata:
+ raise ValueError('Native answer correction changed unrelated response metadata')
+ derived = {'metrics', 'scoring', 'diagnosis', 'failure_category',
+ 'artifact_path', 'provider_metadata'}
+ if ({k: v for k, v in original.items() if k not in derived}
+ != {k: v for k, v in corrected.items() if k not in derived}):
+ raise ValueError('Native answer correction changed physical call facts')
+
+
def _require_revision_step(source, receipt, dataset, records, outputs):
if receipt.get('kind') != 'local_offline_score_revision' or receipt.get('provider_calls_added') != 0:
raise ValueError('Local revision must preserve original provider calls')
@@ -122,7 +159,7 @@ def _require_revision_step(source, receipt, dataset, records, outputs):
for identifier, old in old_cases.items():
new = new_cases[identifier]
fields = {key for key in old.keys() | new.keys() if old.get(key) != new.get(key)}
- if fields - {'task_prompt', 'acceptance_contract', 'body_requirements', 'required_patterns', 'required_facts', 'forbidden_any', 'diagnostic_any', 'acceptance_policy_version'}:
+ if fields - (ALLOWED_FIELDS | {'diagnostic_any', 'acceptance_policy_version'}):
raise ValueError('Local score revision changed source or acquisition inputs')
if fields:
changed_cases.add(identifier)
@@ -137,10 +174,17 @@ def _require_revision_step(source, receipt, dataset, records, outputs):
scored = set(receipt.get('rescored_attempt_ids', []))
if not scored <= set(new_rows):
raise ValueError('Local revision rescoring scope differs')
+ native_proofs = receipt.get('native_answer_corrections', [])
+ corrected_answers = {item['attempt_id']: item for item in native_proofs}
+ if (len(corrected_answers) != len(native_proofs)
+ or not set(corrected_answers) <= scored):
+ raise ValueError('Native answer correction scope differs from rescoring scope')
expected_scoring = {}
for identifier, old in old_rows.items():
new = new_rows[identifier]
- if ({k: v for k, v in old.items() if k not in {'metrics', 'scoring', 'diagnosis', 'failure_category'}}
+ if identifier in corrected_answers:
+ _require_native_answer_correction(source, old, new, corrected_answers[identifier])
+ elif ({k: v for k, v in old.items() if k not in {'metrics', 'scoring', 'diagnosis', 'failure_category'}}
!= {k: v for k, v in new.items() if k not in {'metrics', 'scoring', 'diagnosis', 'failure_category'}}
or old['metrics']['content_sha256'] != new['metrics']['content_sha256']):
raise ValueError('Local revision changed measured call facts: ' + identifier)
@@ -224,6 +268,12 @@ def revise_local_snapshot(source, configuration):
history = source / 'local-evidence/offline-revisions' / revision_key
if history.exists():
raise ValueError('Local revision already exists; preserve the prior receipt')
+ native_inputs = config.get('native_answer_corrections', [])
+ native_requests = {item['attempt_id']: item for item in native_inputs}
+ if (len(native_requests) != len(native_inputs)
+ or not set(native_requests) <= {row['attempt_id'] for row in records}):
+ raise ValueError('Native answer correction has an invalid input scope')
+ additional.update(native_requests)
indexed = {case['id']: case for case in dataset['cases']}
input_verdicts = None
if config.get('input_verdicts'):
@@ -234,6 +284,7 @@ def revise_local_snapshot(source, configuration):
or any(type(value) is not bool for value in input_verdicts.values())):
raise ValueError('Frozen input verdicts do not match the complete input results')
selected = []
+ native_proofs = []
before_verdicts = {}
for row in records:
if row['case_id'] not in replacements and row['attempt_id'] not in additional:
@@ -245,9 +296,37 @@ def revise_local_snapshot(source, configuration):
before = (input_verdicts[row['attempt_id']] if input_verdicts is not None
else task_passed(row['ok'], row['metrics']))
before_verdicts[row['attempt_id']] = before
+ if row['attempt_id'] in native_requests:
+ from .response_input import project_native_answer
+ request = native_requests[row['attempt_id']]
+ native_path = read_bound(source, request['native_response'])
+ metadata = row.get('provider_metadata', {})
+ if (request['native_response']['sha256'] != metadata.get('raw_response_sha256')
+ or native_path.stat().st_size != metadata.get('raw_response_bytes')):
+ raise ValueError('Native correction source differs from its original response')
+ answer, fields = project_native_answer(json.loads(native_path.read_bytes()), request['projection'])
+ raw = answer.encode('utf-8', errors='replace')
+ original_artifact = {'path': row['artifact_path'], 'sha256': row['metrics']['content_sha256']}
+ digest = hashlib.sha256(raw).hexdigest()
+ old_path = Path(row['artifact_path'])
+ new_relative = old_path.with_name(old_path.stem + '-native-' + digest[:12] + old_path.suffix)
+ corrected_path = contained_path(source, new_relative, reject_symlinks=True)
+ if corrected_path.exists() and corrected_path.read_bytes() != raw:
+ raise ValueError('Native correction would overwrite a different artifact')
+ corrected_path.parent.mkdir(parents=True, exist_ok=True)
+ corrected_path.write_bytes(raw)
+ row['artifact_path'] = new_relative.as_posix()
+ row['provider_metadata'] = {**metadata, **fields, 'native_answer_projection': request['projection']}
+ native_proofs.append({'attempt_id': row['attempt_id'],
+ 'native_response': request['native_response'], 'projection': request['projection'],
+ 'original_artifact': original_artifact,
+ 'corrected_artifact': {'path': row['artifact_path'], 'sha256': digest}})
row['metrics'] = score_attempt(case, raw, ok=row['ok'], status_code=row.get('status_code'),
title=row.get('title'), final_url=row.get('final_url'), metadata=row.get('provider_metadata'),
previous_metrics=row['metrics'], list_reference=load_list_reference(case, source))
+ # Diagnoses describe the former score and must be regenerated from this answer.
+ row.pop('diagnosis', None)
+ row.pop('failure_category', None)
row['scoring'] = {**row.get('scoring', {}), 'mode': 'offline_rescore',
'scored_at': datetime.now(timezone.utc).isoformat(),
'rule_revision': bool(config.get('rule_revision')),
@@ -276,6 +355,7 @@ def binding(path):
'changed_case_ids': sorted(case['id'] for case in dataset['cases']
if case != next(c for c in original_dataset['cases'] if c['id'] == case['id'])),
'rescored_attempt_ids': selected, 'rule_revision': bool(config.get('rule_revision')),
+ **({'native_answer_corrections': native_proofs} if native_proofs else {}),
'input_verdicts': config.get('input_verdicts'),
**({'predecessor_revision': config['predecessor_revision']} if config.get('predecessor_revision') else {}),
'superseded_controls': config.get('superseded_controls', []),
diff --git a/harness/web_fetch/metrics.py b/harness/web_fetch/metrics.py
index 3c21c5999..e7710b8da 100644
--- a/harness/web_fetch/metrics.py
+++ b/harness/web_fetch/metrics.py
@@ -210,6 +210,9 @@ def evaluate_resources(
raise ValueError(f"Unsupported resource unexpected policy: {contract.get('unexpected')!r}")
candidates: list[tuple[str, str]] = []
+ projection = metadata.get('native_answer_projection') or {}
+ typed_json_answer = (projection.get('content_format') == 'json'
+ and bool(projection.get('answer_fields')))
if eligible and candidate_scope == "image_answer_fields":
if "images" in metadata:
candidates.extend(_metadata_image_candidates(
@@ -219,19 +222,21 @@ def evaluate_resources(
for key in ("media", "extras"):
if key in metadata:
candidates.extend(_metadata_image_candidates(metadata[key], path=f"metadata.{key}"))
- candidates.extend(_image_content_candidates(content))
+ if not typed_json_answer:
+ candidates.extend(_image_content_candidates(content))
elif eligible:
answer_fields = {key: metadata[key] for key in ('links','images','videos','media','extras') if key in metadata}
candidates.extend(_metadata_resource_candidates(answer_fields))
- from .reference_quality import markdown_link_destinations
- candidates.extend((url, "content.bare_url") for url in
- re.findall(r"https?://[^\s<>'\")\]]+", content))
- candidates.extend((match[1], "content.html_url") for match in
- re.findall(r"(?:src|href)\s*=\s*([\"'])(.*?)\1", content, re.I))
- candidates.extend((url, "content.markdown_link")
- for url in markdown_link_destinations(content))
- candidates.extend((url, "content.visible_file")
- for url in _visible_relative_resource_urls(content))
+ if not typed_json_answer:
+ from .reference_quality import markdown_link_destinations
+ candidates.extend((url, "content.bare_url") for url in
+ re.findall(r"https?://[^\s<>'\")\]]+", content))
+ candidates.extend((match[1], "content.html_url") for match in
+ re.findall(r"(?:src|href)\s*=\s*([\"'])(.*?)\1", content, re.I))
+ candidates.extend((url, "content.markdown_link")
+ for url in markdown_link_destinations(content))
+ candidates.extend((url, "content.visible_file")
+ for url in _visible_relative_resource_urls(content))
normalized_candidates, candidate_evidence = _normalized_candidates(candidates, base_url=base_url)
results = []
allowed_urls: set[str] = set()
diff --git a/tests/test_local_revision.py b/tests/test_local_revision.py
index 1a3ed0fac..d30bb7a68 100644
--- a/tests/test_local_revision.py
+++ b/tests/test_local_revision.py
@@ -181,3 +181,87 @@ def test_hash_connected_intermediate_revision_must_pass_full_validation(tmp_path
latest_binding['sha256'] = hashlib.sha256(latest_path.read_bytes()).hexdigest()
with pytest.raises(ValueError, match=message):
validate_revision_chain(tmp_path, latest_binding)[1:]
+
+
+def _native_answer_revision_fixture(root):
+ raw = b'Body fact'
+ (root/'answer.txt').write_bytes(raw)
+ native_content = 'Body fact | Saved to: /Users/person/private/page.txt'
+ native = (json.dumps({'content': native_content})+'\n').encode()
+ native_path = root/'private_artifacts/native.response.bin'
+ native_path.parent.mkdir()
+ native_path.write_bytes(native)
+ case = {'id': 'a', 'url': 'https://example.test/',
+ 'required_facts': [{'id': 'body', 'alternatives': ['Body fact']}]}
+ row = {'attempt_id': 'p__a', 'provider': 'p', 'case_id': 'a', 'ok': True,
+ 'url': case['url'], 'artifact_path': 'answer.txt', 'elapsed_ms': 7,
+ 'diagnosis': {'category': 'obsolete diagnosis', 'evidence': 'old answer'},
+ 'failure_category': 'obsolete failure',
+ 'provider_metadata': {'raw_response_path': 'private_artifacts/native.response.bin',
+ 'raw_response_sha256': hashlib.sha256(native).hexdigest(),
+ 'raw_response_bytes': len(native)},
+ 'metrics': score_attempt(case, raw, ok=True)}
+ (root/'dataset.json').write_text(json.dumps({'cases': [case], 'comparison': {}}))
+ (root/'results.jsonl').write_text(json.dumps(row)+'\n')
+ config = {'cases': [], 'authority': {'description': 'Restore declared native fields from the same call; preserve the former derived answer and measured time.'},
+ 'regression_ids': ['native-json-resource-declared-image-field-preserved',
+ 'native-json-resource-untyped-url-echo-rejected'],
+ 'native_answer_corrections': [{'attempt_id': row['attempt_id'],
+ 'native_response': {'path': row['provider_metadata']['raw_response_path'],
+ 'sha256': row['provider_metadata']['raw_response_sha256']},
+ 'projection': {'schema_version': 1, 'content_pointer': ['content'],
+ 'content_format': 'text'}}]}
+ config_path = root/'correction.json'
+ config_path.write_text(json.dumps(config))
+ receipt = revise_local_snapshot(root, config_path)
+ receipt_path = root/receipt['input_dataset']['path']
+ receipt_path = receipt_path.parent/'receipt.json'
+ binding = {'path': str(receipt_path.relative_to(root)),
+ 'sha256': hashlib.sha256(receipt_path.read_bytes()).hexdigest()}
+ return row, receipt, receipt_path, binding, native_content
+
+
+def test_offline_native_correction_preserves_call_and_both_answers(tmp_path):
+ original, receipt, _, binding, native_content = _native_answer_revision_fixture(tmp_path)
+ current = json.loads((tmp_path/'results.jsonl').read_text())
+ assert receipt['provider_calls_added'] == 0
+ assert receipt['changed_case_ids'] == []
+ assert receipt['task_verdict_changes'] == []
+ assert current['elapsed_ms'] == original['elapsed_ms']
+ assert 'diagnosis' not in current
+ assert 'failure_category' not in current
+ assert (tmp_path/'answer.txt').read_bytes() == b'Body fact'
+ assert current['artifact_path'] != original['artifact_path']
+ assert (tmp_path/current['artifact_path']).read_text() == native_content
+ assert len(receipt['native_answer_corrections']) == 1
+ require_local_revision(tmp_path, binding, json.loads((tmp_path/'dataset.json').read_text()), [current])
+
+
+@pytest.mark.parametrize('defect,message', [
+ ('clock', 'physical call facts'), ('metadata', 'unrelated response metadata'),
+ ('answer', 'declared original fields'), ('source', 'original response'),
+ ('missing_proof', 'measured call facts'),
+])
+def test_native_correction_rejects_self_consistent_but_false_projection(tmp_path, defect, message):
+ _, receipt, path, binding, _ = _native_answer_revision_fixture(tmp_path)
+ current = json.loads((tmp_path/'results.jsonl').read_text())
+ proof = receipt['native_answer_corrections'][0]
+ if defect == 'clock': current['elapsed_ms'] += 1
+ elif defect == 'metadata': current['provider_metadata']['injected'] = 'new answer data'
+ elif defect == 'answer':
+ artifact = tmp_path/current['artifact_path']
+ artifact.write_text('Different answer not supplied by the call')
+ digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
+ current['metrics']['content_sha256'] = digest
+ proof['corrected_artifact']['sha256'] = digest
+ elif defect == 'source':
+ source = tmp_path/proof['native_response']['path']
+ source.write_text(json.dumps({'content': 'Different native response'}))
+ proof['native_response']['sha256'] = hashlib.sha256(source.read_bytes()).hexdigest()
+ else: receipt.pop('native_answer_corrections')
+ (tmp_path/'results.jsonl').write_text(json.dumps(current)+'\n')
+ receipt['output_results_sha256'] = hashlib.sha256((tmp_path/'results.jsonl').read_bytes()).hexdigest()
+ path.write_text(json.dumps(receipt))
+ binding['sha256'] = hashlib.sha256(path.read_bytes()).hexdigest()
+ with pytest.raises(ValueError, match=message):
+ require_local_revision(tmp_path, binding, json.loads((tmp_path/'dataset.json').read_text()), [current])
From 10256084727fd5b499812976c96c9942f95e5686 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 00:18:28 +0800
Subject: [PATCH 27/50] Bind explicit image equivalents without rewriting
source observations
---
evaluation/scoring-regressions.json | 362 +++++++++++++++++++++++
harness/web_fetch/reference_lifecycle.py | 56 +++-
tests/test_reference_lifecycle.py | 53 ++++
3 files changed, 469 insertions(+), 2 deletions(-)
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index c9b81ae89..2fb664fc6 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -31838,6 +31838,368 @@
],
"resource_precision": 1.0
}
+ },
+ {
+ "id": "image-04-declared-display-equivalence-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://en.wikipedia.org/wiki/Apollo_11",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "image-1",
+ "kind": "image-1",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/330px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/(?:thumb/)?4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg(?:/[1-9][0-9]*px-A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg)?(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ },
+ {
+ "id": "image-2",
+ "kind": "image-2",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/250px-Apollo_11_insignia.png?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/(?:thumb/)?2/27/Apollo_11_insignia\\.png(?:/[1-9][0-9]*px-Apollo_11_insignia\\.png)?(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/640px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg\nhttps://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/640px-Apollo_11_insignia.png",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-04-declared-display-equivalence-unlisted-query-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://en.wikipedia.org/wiki/Apollo_11",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "image-1",
+ "kind": "image-1",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/330px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/(?:thumb/)?4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg(?:/[1-9][0-9]*px-A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg)?(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ },
+ {
+ "id": "image-2",
+ "kind": "image-2",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/250px-Apollo_11_insignia.png?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/(?:thumb/)?2/27/Apollo_11_insignia\\.png(?:/[1-9][0-9]*px-Apollo_11_insignia\\.png)?(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/640px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?crop=unapproved\nhttps://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/640px-Apollo_11_insignia.png",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-nasa-article-declared-display-equivalence-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://www.nasa.gov/image-article/historic-engines-take-their-place-on-artemis-iii/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images-assets.nasa.gov/image/KSC-20260825-PH-RCR01_0045/KSC-20260825-PH-RCR01_0045~large.jpg?w=1920&h=1280&fit=clip&crop=faces%2Cfocalpoint"
+ ],
+ "url_patterns": [
+ "^https://images\\-assets\\.nasa\\.gov/image/KSC\\-20260825\\-PH\\-RCR01_0045/KSC\\-20260825\\-PH\\-RCR01_0045\\~large\\.jpg(?:\\?(?:[wh]=[1-9][0-9]*|fit=clip|crop=faces(?:%2[Cc]|,)focalpoint)(?:&(?:[wh]=[1-9][0-9]*|fit=clip|crop=faces(?:%2[Cc]|,)focalpoint))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images-assets.nasa.gov/image/KSC-20260825-PH-RCR01_0045/KSC-20260825-PH-RCR01_0045~large.jpg?w=640&h=480&fit=clip&crop=faces%2Cfocalpoint",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-nasa-article-declared-display-equivalence-unlisted-query-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://www.nasa.gov/image-article/historic-engines-take-their-place-on-artemis-iii/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images-assets.nasa.gov/image/KSC-20260825-PH-RCR01_0045/KSC-20260825-PH-RCR01_0045~large.jpg?w=1920&h=1280&fit=clip&crop=faces%2Cfocalpoint"
+ ],
+ "url_patterns": [
+ "^https://images\\-assets\\.nasa\\.gov/image/KSC\\-20260825\\-PH\\-RCR01_0045/KSC\\-20260825\\-PH\\-RCR01_0045\\~large\\.jpg(?:\\?(?:[wh]=[1-9][0-9]*|fit=clip|crop=faces(?:%2[Cc]|,)focalpoint)(?:&(?:[wh]=[1-9][0-9]*|fit=clip|crop=faces(?:%2[Cc]|,)focalpoint))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images-assets.nasa.gov/image/KSC-20260825-PH-RCR01_0045/KSC-20260825-PH-RCR01_0045~large.jpg?w=640&h=480&fit=clip&crop=faces%2Cfocalpoint&crop=unapproved",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-unsplash-detail-declared-display-equivalence-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://unsplash.com/photos/green-leaf-plants-mSK5nNsAsLY",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.unsplash.com/photo-1493673272479-a20888bcee10",
+ "https://unsplash.com/photos/mSK5nNsAsLY/download"
+ ],
+ "url_patterns": [
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.unsplash.com/photo-1493673272479-a20888bcee10?w=640&auto=format&q=80",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-unsplash-detail-declared-display-equivalence-unlisted-query-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://unsplash.com/photos/green-leaf-plants-mSK5nNsAsLY",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.unsplash.com/photo-1493673272479-a20888bcee10",
+ "https://unsplash.com/photos/mSK5nNsAsLY/download"
+ ],
+ "url_patterns": [
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.unsplash.com/photo-1493673272479-a20888bcee10?w=640&auto=format&q=80&crop=unapproved",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-pexels-detail-declared-display-equivalence-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://www.pexels.com/photo/a-green-trees-in-the-forest-12377231/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg"
+ ],
+ "url_patterns": [
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg?w=640&auto=compress&cs=tinysrgb",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-pexels-detail-declared-display-equivalence-unlisted-query-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://www.pexels.com/photo/a-green-trees-in-the-forest-12377231/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg"
+ ],
+ "url_patterns": [
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg?w=640&auto=compress&cs=tinysrgb&crop=unapproved",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-earth-main-declared-display-equivalence-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://commons.wikimedia.org/wiki/File:The_Earth_seen_from_Apollo_17.jpg",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://thumb.wikimedia.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17.jpg/960px-The_Earth_seen_from_Apollo_17.jpg?utm_source=commons.wikimedia.org&utm_campaign=index&utm_content=thumbnail&_=20200925203546"
+ ],
+ "url_patterns": [
+ "^https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/(?:thumb/)?9/97/The_Earth_seen_from_Apollo_17\\.jpg(?:/[1-9][0-9]*px-The_Earth_seen_from_Apollo_17\\.jpg)?(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17.jpg/640px-The_Earth_seen_from_Apollo_17.jpg",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-earth-main-declared-display-equivalence-unlisted-query-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_image_display_equivalence",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://commons.wikimedia.org/wiki/File:The_Earth_seen_from_Apollo_17.jpg",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://thumb.wikimedia.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17.jpg/960px-The_Earth_seen_from_Apollo_17.jpg?utm_source=commons.wikimedia.org&utm_campaign=index&utm_content=thumbnail&_=20200925203546"
+ ],
+ "url_patterns": [
+ "^https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/(?:thumb/)?9/97/The_Earth_seen_from_Apollo_17\\.jpg(?:/[1-9][0-9]*px-The_Earth_seen_from_Apollo_17\\.jpg)?(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17.jpg/640px-The_Earth_seen_from_Apollo_17.jpg?crop=unapproved",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
}
],
"superseded_controls": {
diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py
index 578e01864..35fd4ef7e 100644
--- a/harness/web_fetch/reference_lifecycle.py
+++ b/harness/web_fetch/reference_lifecycle.py
@@ -215,6 +215,56 @@ def _resource_target_matches(case, candidates: list[dict]) -> list[dict]:
return results
+def _resource_observation_scope(case, root, evidence):
+ """Retain an observation's old scope across an explicit score-only alias revision."""
+ current = resource_source_scope_identity(case)
+ binding = current['resource_output_contract'].pop('equivalence_revision', None)
+ if binding is None:
+ return current
+ identifier = safe_identifier(_get(case, 'id'))
+ if (not isinstance(binding, dict)
+ or binding.get('path') != 'resource-equivalence-revision.json'):
+ raise ReferenceLifecycleError('Invalid resource equivalence revision binding')
+ try:
+ raw = contained_path(root, identifier + '/' + binding['path']).read_bytes()
+ if sha256(raw) != binding.get('sha256'):
+ raise ReferenceLifecycleError('Resource equivalence revision hash changed')
+ receipt = json.loads(raw)
+ before = receipt['before_scope']
+ canonical = json.dumps(current, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
+ if (receipt.get('kind') != 'offline_resource_equivalence_revision'
+ or receipt.get('provider_calls_added') != 0
+ or receipt.get('new_source_observation') is not False
+ or receipt.get('after_scope_sha256') != sha256(canonical)
+ or receipt.get('source_evidence_sha256') != _get(case, 'source_verification', {}).get('evidence_sha256')
+ or receipt.get('browser_observation_sha256') != evidence.get('independent_browser_observation', {}).get('sha256')
+ or not receipt.get('authority', {}).get('description')):
+ raise ReferenceLifecycleError('Resource equivalence revision authority/scope differs')
+ # Only add declared equivalents. Never replace primary source targets,
+ # their identity, candidate policy, task URL, or historical observations.
+ old_targets, new_targets = before['expected_resources'], current['expected_resources']
+ if ({k: v for k, v in before.items() if k != 'expected_resources'}
+ != {k: v for k, v in current.items() if k != 'expected_resources'}
+ or len(old_targets) != len(new_targets)):
+ raise ReferenceLifecycleError('Resource equivalence revision changed source scope')
+ for old, new in zip(old_targets, new_targets):
+ if ({k: v for k, v in old.items() if k not in {'aliases', 'url_patterns'}}
+ != {k: v for k, v in new.items() if k not in {'aliases', 'url_patterns'}}
+ or any(not set(old.get(key, [])) <= set(new.get(key, []))
+ for key in ('aliases', 'url_patterns'))):
+ raise ReferenceLifecycleError('Resource equivalence revision replaced a primary target')
+ from .scoring_regressions import require_scoring_regressions
+ controls = receipt.get('regression_ids')
+ if not isinstance(controls, list) or not controls:
+ raise ReferenceLifecycleError('Resource equivalence revision lacks executable controls')
+ require_scoring_regressions(controls)
+ except ReferenceLifecycleError:
+ raise
+ except (OSError, ValueError, TypeError, KeyError) as error:
+ raise ReferenceLifecycleError('Invalid resource equivalence revision') from error
+ return before
+
+
def validate_resource_source_evidence(case, dataset_path: Path) -> dict | None:
"""Fail closed unless every resource target has independent, hash-bound evidence.
@@ -268,6 +318,7 @@ def validate_resource_source_evidence(case, dataset_path: Path) -> dict | None:
candidates = [*candidates, *observed_candidates]
source_qualified = evidence.get("state") == "captured" and evidence.get("status") == 200
+ observed_scope = _resource_observation_scope(case, root, evidence)
browser_binding = evidence.get("independent_browser_observation")
if browser_binding is not None:
if (not isinstance(browser_binding, dict)
@@ -292,12 +343,13 @@ def validate_resource_source_evidence(case, dataset_path: Path) -> dict | None:
if (observation.get("case_id") != identifier
or observation.get("url") != _get(case, "url")
or observation.get("independent") is not True
- or observation.get("resource_scope_identity") != resource_source_scope_identity(case)
+ or observation.get("resource_scope_identity") != observed_scope
or not isinstance(observed_candidates, list) or len(observed_candidates) > 512
or observation.get("resource_candidates_sha256") != sha256(canonical)):
raise ReferenceLifecycleError(f"{identifier}: browser observation scope/hash mismatch")
timestamp(observation.get("checked_at"))
- browser_matches = _resource_target_matches(case, observed_candidates)
+ browser_matches = _resource_target_matches(
+ {**observed_scope, "id": observed_scope["case_id"]}, observed_candidates)
browser_qualified = (
observation.get("status") == "success"
and observation.get("completion", {}).get("satisfied") is True
diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py
index efd5db3aa..bbf6c13cc 100644
--- a/tests/test_reference_lifecycle.py
+++ b/tests/test_reference_lifecycle.py
@@ -228,6 +228,59 @@ def test_resource_source_browser_observation_must_satisfy_all_targets(self):
with self.assertRaisesRegex(ReferenceLifecycleError, "does not cover every"):
validate_resource_source_evidence(case, self.dataset_path)
+ def test_resource_equivalence_keeps_original_browser_observation_and_primary_target(self):
+ from harness.web_fetch.browser_observations import match_resource_targets
+ original = self.resource_case()
+ scope = resource_source_scope_identity(original)
+ candidates = [{'tag': 'a', 'attribute': 'href', 'url': original.expected_resources[0]['urls'][0]}]
+ browser = {'schema_version': 1, 'case_id': original.id, 'url': original.url,
+ 'status': 'success', 'independent': True, 'checked_at': self.now.isoformat(),
+ 'resource_scope_identity': scope, 'resource_candidates': candidates,
+ 'resource_candidates_sha256': sha256(json.dumps(candidates, ensure_ascii=False,
+ sort_keys=True, separators=(',', ':'))),
+ 'target_matches': match_resource_targets(original, candidates),
+ 'completion': {'satisfied': True}}
+ original = self.resource_case(evidence_updates={'state': 'needs_review', 'status': 403,
+ 'resources': []}, browser=browser)
+ browser_path = self.root/'sources/resource/browser-observation.json'
+ browser_bytes = browser_path.read_bytes()
+ aliases = [{**original.expected_resources[0], 'aliases': ['https://cdn.example.test/a.png?w=800']}]
+ revised = replace(original, expected_resources=aliases)
+ new_scope = resource_source_scope_identity(revised)
+ receipt = {'kind': 'offline_resource_equivalence_revision', 'provider_calls_added': 0,
+ 'new_source_observation': False, 'before_scope': scope,
+ 'after_scope_sha256': sha256(json.dumps(new_scope, ensure_ascii=False,
+ sort_keys=True, separators=(',', ':'))),
+ 'source_evidence_sha256': original.source_verification['evidence_sha256'],
+ 'browser_observation_sha256': sha256(browser_bytes),
+ 'authority': {'description': 'Same observed asset; a score-only display-size equivalent.'},
+ 'regression_ids': ['resource-query-explicit-alias-preserved',
+ 'resource-query-different-resource-rejected']}
+ proof = self.root/'sources/resource/resource-equivalence-revision.json'
+ def bound_case(value):
+ raw = (json.dumps(value)+'\n').encode()
+ proof.write_bytes(raw)
+ return replace(revised, resource_output_contract={
+ **revised.resource_output_contract, 'equivalence_revision': {
+ 'path': proof.name, 'sha256': sha256(raw)}})
+ result = validate_resource_source_evidence(bound_case(receipt), self.dataset_path)
+ self.assertTrue(result['target_matches'][0]['satisfied'])
+ self.assertEqual(browser_path.read_bytes(), browser_bytes)
+ # A new alias cannot silently rewrite or replace the captured scope.
+ for defect in ('primary', 'policy', 'source', 'observation', 'controls', 'fresh'):
+ bad = deepcopy(receipt)
+ if defect == 'primary': bad['before_scope']['expected_resources'][0]['urls'] = ['https://wrong.test/other.png']
+ if defect == 'policy': bad['before_scope']['resource_output_contract'] = {'unexpected': 'reject'}
+ if defect == 'source': bad['source_evidence_sha256'] = '0'*64
+ if defect == 'observation': bad['browser_observation_sha256'] = '0'*64
+ if defect == 'controls': bad['regression_ids'] = []
+ if defect == 'fresh': bad['new_source_observation'] = True
+ with self.subTest(defect=defect), self.assertRaises(ReferenceLifecycleError):
+ validate_resource_source_evidence(bound_case(bad), self.dataset_path)
+ without_receipt = replace(revised, resource_output_contract={})
+ with self.assertRaisesRegex(ReferenceLifecycleError, 'scope/hash mismatch'):
+ validate_resource_source_evidence(without_receipt, self.dataset_path)
+
def test_per_case_rechecks_clock_and_immutable_scope(self):
binding = self.checks()["cases"]["body"]
with self.assertRaisesRegex(ReferenceLifecycleError, "30 minutes"):
From 052dc9c8321c9924725f5252624aaeb8c49506cc Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 00:27:40 +0800
Subject: [PATCH 28/50] Reject invalid thumbnail host paths and retain
immutable rule proofs
---
evaluation/scoring-regressions.json | 156 +++++++++++++++++++++++
harness/web_fetch/reference_lifecycle.py | 3 +-
tests/test_reference_lifecycle.py | 5 +
3 files changed, 163 insertions(+), 1 deletion(-)
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index 2fb664fc6..105d32adf 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -32200,6 +32200,162 @@
"expected_metrics": {
"resource_pass": false
}
+ },
+ {
+ "id": "image-04-declared-display-real-thumbnail-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_thumbnail_host_path_coupling",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://en.wikipedia.org/wiki/Apollo_11",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "image-1",
+ "kind": "image-1",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/330px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg/[1-9][0-9]*px-A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ },
+ {
+ "id": "image-2",
+ "kind": "image-2",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/250px-Apollo_11_insignia.png?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/2/27/Apollo_11_insignia\\.png|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia\\.png/[1-9][0-9]*px-Apollo_11_insignia\\.png)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://thumb.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/640px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg\nhttps://thumb.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/640px-Apollo_11_insignia.png",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-04-declared-display-thumbnail-host-original-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_thumbnail_host_path_coupling",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://en.wikipedia.org/wiki/Apollo_11",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "image-1",
+ "kind": "image-1",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/330px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg/[1-9][0-9]*px-A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ },
+ {
+ "id": "image-2",
+ "kind": "image-2",
+ "urls": [
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/250px-Apollo_11_insignia.png?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail"
+ ],
+ "url_patterns": [
+ "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/2/27/Apollo_11_insignia\\.png|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia\\.png/[1-9][0-9]*px-Apollo_11_insignia\\.png)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://thumb.wikimedia.org/wikipedia/commons/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg\nhttps://thumb.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/640px-Apollo_11_insignia.png",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-earth-main-declared-display-real-thumbnail-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_thumbnail_host_path_coupling",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://commons.wikimedia.org/wiki/File:The_Earth_seen_from_Apollo_17.jpg",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://thumb.wikimedia.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17.jpg/960px-The_Earth_seen_from_Apollo_17.jpg?utm_source=commons.wikimedia.org&utm_campaign=index&utm_content=thumbnail&_=20200925203546"
+ ],
+ "url_patterns": [
+ "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/9/97/The_Earth_seen_from_Apollo_17\\.jpg|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17\\.jpg/[1-9][0-9]*px-The_Earth_seen_from_Apollo_17\\.jpg)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://thumb.wikimedia.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17.jpg/640px-The_Earth_seen_from_Apollo_17.jpg",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-earth-main-declared-display-thumbnail-host-original-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "task_declared_thumbnail_host_path_coupling",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://commons.wikimedia.org/wiki/File:The_Earth_seen_from_Apollo_17.jpg",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://thumb.wikimedia.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17.jpg/960px-The_Earth_seen_from_Apollo_17.jpg?utm_source=commons.wikimedia.org&utm_campaign=index&utm_content=thumbnail&_=20200925203546"
+ ],
+ "url_patterns": [
+ "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/9/97/The_Earth_seen_from_Apollo_17\\.jpg|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/9/97/The_Earth_seen_from_Apollo_17\\.jpg/[1-9][0-9]*px-The_Earth_seen_from_Apollo_17\\.jpg)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://thumb.wikimedia.org/wikipedia/commons/9/97/The_Earth_seen_from_Apollo_17.jpg",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
}
],
"superseded_controls": {
diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py
index 35fd4ef7e..efd9ca9b6 100644
--- a/harness/web_fetch/reference_lifecycle.py
+++ b/harness/web_fetch/reference_lifecycle.py
@@ -223,7 +223,8 @@ def _resource_observation_scope(case, root, evidence):
return current
identifier = safe_identifier(_get(case, 'id'))
if (not isinstance(binding, dict)
- or binding.get('path') != 'resource-equivalence-revision.json'):
+ or re.fullmatch(r'resource-equivalence-(?:revision|[0-9a-f]{64})\.json',
+ str(binding.get('path', ''))) is None):
raise ReferenceLifecycleError('Invalid resource equivalence revision binding')
try:
raw = contained_path(root, identifier + '/' + binding['path']).read_bytes()
diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py
index bbf6c13cc..6ab6c9094 100644
--- a/tests/test_reference_lifecycle.py
+++ b/tests/test_reference_lifecycle.py
@@ -266,6 +266,11 @@ def bound_case(value):
result = validate_resource_source_evidence(bound_case(receipt), self.dataset_path)
self.assertTrue(result['target_matches'][0]['satisfied'])
self.assertEqual(browser_path.read_bytes(), browser_bytes)
+ # Later score-only revisions get immutable proof names, preserving prior receipts.
+ prior_proof = proof
+ proof = proof.with_name('resource-equivalence-'+sha256((json.dumps(receipt)+'\n').encode())+'.json')
+ self.assertIsNotNone(validate_resource_source_evidence(bound_case(receipt), self.dataset_path))
+ self.assertTrue(prior_proof.exists())
# A new alias cannot silently rewrite or replace the captured scope.
for defect in ('primary', 'policy', 'source', 'observation', 'controls', 'fresh'):
bad = deepcopy(receipt)
From e2a012e6bbd1829a4161256933d1fb8fefe62bc6 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 00:38:26 +0800
Subject: [PATCH 29/50] Decode HTML image URL attributes without collapsing
literal queries
---
evaluation/scoring-regressions.json | 90 +++++++++++++++++++++++++++++
harness/web_fetch/metrics.py | 11 ++--
2 files changed, 97 insertions(+), 4 deletions(-)
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index 105d32adf..4818743f8 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -32356,6 +32356,96 @@
"expected_metrics": {
"resource_pass": false
}
+ },
+ {
+ "id": "html-image-attribute-entities-decoded",
+ "category": "format-normalization",
+ "root_cause": "html_resource_attribute_entity_decoding",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/gallery",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "photo",
+ "urls": [
+ "https://cdn.example/photo.png?asset=one&size=640"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "reject"
+ }
+ },
+ "artifact": "]*?\bsrc\s*=\s*([\"'])(.*?)\1", content, re.I | re.S
))
for match in re.findall(r"]*?\bsrcset\s*=\s*([\"'])(.*?)\1", content, re.I | re.S):
- candidates.extend((part.strip().split()[0], "content.html_img_srcset")
+ candidates.extend((html.unescape(part.strip().split()[0]), "content.html_img_srcset")
for part in match[1].split(',') if part.strip())
+ # HTML attributes are decoded above. Do not also admit their raw entity
+ # spelling as a second bare candidate; plain URL answers remain literal.
+ bare_content = re.sub(r"]*>", "", content, flags=re.I | re.S)
# Explicit image markup is typed evidence. A standalone HTTP URL may also
# be the entire answer, including an extensionless CDN address. Every URL
# with that syntax is admitted alike, without consulting target resources.
url_pattern = r"https?://[^\s<>'\")\]]+"
- bare_lines = {line.strip().strip('<>') for line in content.splitlines()
+ bare_lines = {line.strip().strip('<>') for line in bare_content.splitlines()
if re.fullmatch(r"\s*?\s*", line)}
candidates.extend((url, "content.bare_image_url") for url in
- re.findall(url_pattern, content)
+ re.findall(url_pattern, bare_content)
if _looks_like_image_url(url) or url in bare_lines)
return candidates
From d82eee2baa96be891e78ea7d04f1d7bb093dce4b Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 00:48:41 +0800
Subject: [PATCH 30/50] Cover declared photo display variants without ignoring
arbitrary queries
---
evaluation/scoring-regressions.json | 292 ++++++++++++++++++++++++++++
1 file changed, 292 insertions(+)
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index 4818743f8..9a86ff66b 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -32446,6 +32446,298 @@
"expected_metrics": {
"resource_pass": true
}
+ },
+ {
+ "id": "image-unsplash-detail-documented-same-photo-display-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://unsplash.com/photos/green-leaf-plants-mSK5nNsAsLY",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.unsplash.com/photo-1493673272479-a20888bcee10",
+ "https://unsplash.com/photos/mSK5nNsAsLY/download"
+ ],
+ "url_patterns": [
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.unsplash.com/photo-1493673272479-a20888bcee10?fm=jpg&q=80&w=1080&auto=format&fit=crop&ixlib=rb-4.1.0&ixid=example%3D",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-unsplash-detail-documented-undeclared-region-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://unsplash.com/photos/green-leaf-plants-mSK5nNsAsLY",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.unsplash.com/photo-1493673272479-a20888bcee10",
+ "https://unsplash.com/photos/mSK5nNsAsLY/download"
+ ],
+ "url_patterns": [
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.unsplash.com/photo-1493673272479-a20888bcee10?fm=jpg&q=80&w=1080&auto=format&fit=crop&ixlib=rb-4.1.0&ixid=example%3D&h=500&crop=left",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-unsplash-detail-documented-unverified-signature-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://unsplash.com/photos/green-leaf-plants-mSK5nNsAsLY",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.unsplash.com/photo-1493673272479-a20888bcee10",
+ "https://unsplash.com/photos/mSK5nNsAsLY/download"
+ ],
+ "url_patterns": [
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.unsplash.com/photo-1493673272479-a20888bcee10?fm=jpg&q=80&w=1080&auto=format&fit=crop&ixlib=rb-4.1.0&ixid=example%3D&s=unverified",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-unsplash-detail-documented-different-photo-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://unsplash.com/photos/green-leaf-plants-mSK5nNsAsLY",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.unsplash.com/photo-1493673272479-a20888bcee10",
+ "https://unsplash.com/photos/mSK5nNsAsLY/download"
+ ],
+ "url_patterns": [
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=format))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$",
+ "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=format|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|ixlib=[A-Za-z0-9_.-]+|ixid=(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.unsplash.com/photo-0000000000000-different?fm=jpg&q=80&w=1080&auto=format&fit=crop&ixlib=rb-4.1.0&ixid=example%3D",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-pexels-detail-documented-same-photo-display-accepted",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://www.pexels.com/photo/a-green-trees-in-the-forest-12377231/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg"
+ ],
+ "url_patterns": [
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg?cs=srgb&dl=download.jpg&fm=jpg",
+ "ok": true,
+ "expected_task_pass": true,
+ "expected_metrics": {
+ "resource_pass": true
+ }
+ },
+ {
+ "id": "image-pexels-detail-documented-undeclared-region-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://www.pexels.com/photo/a-green-trees-in-the-forest-12377231/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg"
+ ],
+ "url_patterns": [
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg?cs=srgb&dl=download.jpg&fm=jpg&crop=left",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-pexels-detail-documented-unverified-signature-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://www.pexels.com/photo/a-green-trees-in-the-forest-12377231/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg"
+ ],
+ "url_patterns": [
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg?cs=srgb&dl=download.jpg&fm=jpg&s=unverified",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
+ },
+ {
+ "id": "image-pexels-detail-documented-different-photo-rejected",
+ "category": "acceptance-contract",
+ "root_cause": "declared_photo_display_and_download_variants",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://www.pexels.com/photo/a-green-trees-in-the-forest-12377231/",
+ "track": "image",
+ "task_kind": "resource",
+ "content_score_applicable": false,
+ "expected_resources": [
+ {
+ "id": "main-image",
+ "kind": "image",
+ "urls": [
+ "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg"
+ ],
+ "url_patterns": [
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb)(?:&(?:[wh]=[1-9][0-9]*|q=[1-9][0-9]?|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|cs=tinysrgb))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:[wh]=[1-9][0-9]*|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$",
+ "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+)(?:&(?:w=[1-9][0-9]*|fit=(?:crop|max|clip)|q=(?:[1-9][0-9]?|100)|dpr=[1-9](?:\\.[0-9]+)?|auto=compress|fm=(?:jpg|jpeg|png|webp)|cs=(?:srgb|tinysrgb)|dl=[A-Za-z0-9_.%-]+))*)?$"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "allow"
+ }
+ },
+ "artifact": "https://images.pexels.com/photos/99999999/pexels-photo-99999999.jpeg?cs=srgb&dl=download.jpg&fm=jpg",
+ "ok": true,
+ "expected_task_pass": false,
+ "expected_metrics": {
+ "resource_pass": false
+ }
}
],
"superseded_controls": {
From 5c7519940ebbf5a3c7f1757c57fc7a86210adca8 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 01:03:38 +0800
Subject: [PATCH 31/50] Report known-outcome evidence audits separately from
blind review
---
.agents/skills/benchmark-report/SKILL.md | 2 +-
harness/web_fetch/report.py | 13 ++++--
tests/test_review_protocol_presentation.py | 49 ++++++++++++++++++++++
3 files changed, 60 insertions(+), 4 deletions(-)
create mode 100644 tests/test_review_protocol_presentation.py
diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md
index 230200473..7d3b79403 100644
--- a/.agents/skills/benchmark-report/SKILL.md
+++ b/.agents/skills/benchmark-report/SKILL.md
@@ -25,7 +25,7 @@ Public report CLIs default to English where supported and expose `--language en|
6. For heterogeneous same-window latency evidence, preserve the full predeclared task-by-method matrix. Show all-call latency on that common denominator separately from successful-call latency on each method's conditional subset. Treat the all-method shared-pass intersection as a diagnostic, never as a post-outcome selector or ranking sample.
7. At aggregate and publication boundaries, copy only explicit hash-bound evidence and public audit files. Reject symlinks and exclude worker trees, host paths, credentials, and internal run logs. Verify primary, auxiliary, delivered-file, and retained retry artifacts before release.
When a saved-input handoff needs host-metadata redaction, use the shared publication privacy owner. Preserve private originals and distinguish original from public file hashes; rebind dependent metadata references and disclose auxiliary debug-log projections. Never rewrite scored answers, references or response payloads as a privacy repair. Verify unchanged verdicts, timings and review decisions, and rebuild from the public inputs. Keep regenerated package/report wrappers outside the original-input projection receipt so normal rebuilding does not invalidate it or create circular hashes.
-8. Independent result review is opt-in: do not start it unless the user explicitly requests review. When requested without a broader scope, review machine failures only; review machine passes or every attempt only when the user explicitly asks for full review. Keep the initial verdict blind, but do not equate agreement with correctness: run the anomaly investigation in [audit and revision](references/audit-and-revision.md) for triggered results before calling the review complete. For a large independent review that is safely batchable, prefer resumable Codex CLI jobs with an exact model ID, read-only sandbox, strict output schema, non-overlapping packets, and bounded concurrency; persist each completed packet so a retry never repeats substantive review. Record the exact reviewed/total coverage, accepted unreviewed passes, reviewer, disagreements, adjudications, anomalies, and unresolved items. A review summary never replaces per-attempt evidence.
+8. Independent result review is opt-in: do not start it unless the user explicitly requests review. When requested without a broader scope, review machine failures only; review machine passes or every attempt only when the user explicitly asks for full review. Label a review performed with known machine outcomes as an evidence audit, separately from independent blind-review coverage; a fresh review-origin tag alone does not establish blindness. Keep the initial verdict blind, but do not equate agreement with correctness: run the anomaly investigation in [audit and revision](references/audit-and-revision.md) for triggered results before calling the review complete. For a large independent review that is safely batchable, prefer resumable Codex CLI jobs with an exact model ID, read-only sandbox, strict output schema, non-overlapping packets, and bounded concurrency; persist each completed packet so a retry never repeats substantive review. Record the exact reviewed/total coverage, accepted unreviewed passes, reviewer, disagreements, adjudications, anomalies, and unresolved items. A review summary never replaces per-attempt evidence.
9. Rerun only calls whose inputs, provider route/version, runtime contract, or source validity changed. Recompute downstream scores and reports without provider calls when only scoring, annotations, or presentation changed. Never retry merely to obtain a better result. A reference or verdict correction must identify the shared root cause and add executable positive/negative controls through the production scorer; bind those regression IDs in the revision receipt instead of patching stored verdicts.
10. Build the public report from the frozen evidence. Every page should answer a reader question. Show task and run identifiers, the exact expected-versus-actual evidence behind a verdict, denominators and unavailable metrics, method differences, limits, concise reproduction instructions, and a compact review history.
11. Release only after code tests, evidence integrity, report links, responsive/visual checks, secret scans, target-site checks, and deployed-output verification pass. Preserve earlier sites unless replacement was explicitly requested.
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index d4b5d0856..1460f7803 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -1084,9 +1084,10 @@ def review_summary(out, prefix='', *, highlight=True):
if not path.is_file():return ''
audit=json.loads(path.read_text())
if audit.get('retained_reviews'):
+ mode_note = (model_text(' 本次为已知程序结论的证据复核，不是独立盲审。', ' This round checked evidence with known machine outcomes; it was not an independent blind review.') if audit.get('review_protocol',{}).get('blind_initial_verdict') is False else '')
return (''+model_text(
f'当前复核覆盖 {audit.get("reviewed_attempts",0)}/{audit.get("expected_attempts",0)}：本次重新检查 {audit.get("fresh_reviewed_attempts",0)} 条，{audit.get("retained_reviewed_attempts",0)} 条沿用检查依据未变的旧复核。',
- f'Current review coverage: {audit.get("reviewed_attempts",0)}/{audit.get("expected_attempts",0)}: {audit.get("fresh_reviewed_attempts",0)} freshly reviewed; {audit.get("retained_reviewed_attempts",0)} prior reviews retained after verifying unchanged decision inputs.')+' '+link(prefix+'review/index.html','查看核验轮次 →')+'')
+ f'Current review coverage: {audit.get("reviewed_attempts",0)}/{audit.get("expected_attempts",0)}: {audit.get("fresh_reviewed_attempts",0)} freshly reviewed; {audit.get("retained_reviewed_attempts",0)} prior reviews retained after verifying unchanged decision inputs.')+mode_note+' '+link(prefix+'review/index.html','查看核验轮次 →')+'')
if audit.get('adjudication_revision'):
count=len(audit['adjudication_revision']['adjudicated_attempt_ids'])
return (''+model_text(
@@ -2308,8 +2309,12 @@ def current_local_review_findings(out,dataset,records,audit):
if supplements and audit.get('retained_reviews'):
fresh = [row for row in audit.get('attempts', []) if row.get('review_origin') == 'fresh']
conflicts = sum(row.get('agent_verdict') != row.get('final_verdict') for row in fresh)
- rows.append((f'本次新复核先隐藏程序结论；初判与程序有 {conflicts} 条分歧，已对照原始返回和题目要求裁定。原始独立意见保留，未新增网页调用。',
- f'Fresh review withheld machine verdicts. {conflicts} initial disagreements were adjudicated against the original answers and task requirements. Initial independent opinions are preserved; no new webpage calls were added.'))
+ if audit.get('review_protocol',{}).get('blind_initial_verdict') is False:
+ rows.append(('本次复核时已知程序结论，逐条检查了保存的返回、任务要求和原生字段；不称为独立盲审，也未新增被测网页调用。旧复核的原始意见保留。',
+ 'This round inspected saved answers, task requirements and native fields with known machine outcomes. It was not an independent blind review and added no benchmark webpage calls. Original opinions from retained reviews are preserved.'))
+ else:
+ rows.append((f'本次新复核先隐藏程序结论；初判与程序有 {conflicts} 条分歧，已对照原始返回和题目要求裁定。原始独立意见保留，未新增网页调用。',
+ f'Fresh review withheld machine verdicts. {conflicts} initial disagreements were adjudicated against the original answers and task requirements. Initial independent opinions are preserved; no new webpage calls were added.'))
elif supplements:
supplement_rows = [json.loads(read_bound(out, item).read_text()) for item in supplements]
reviewed = sum(item['reviewed_attempts'] for item in supplement_rows)
@@ -2361,6 +2366,8 @@ def coverage(label,item):
'核验时隐藏机器结论和分数，只提供任务契约、标准答案与完整实际输出；分歧回到原始证据裁定，未决非零则禁止发布。任务和输出未变化时沿用原核验；发生变化的调用必须重新盲审。')
if audit.get('adjudication_revision'):
method=''+model_text('原独立判断和补充判断保留在逐条记录中。本次补充裁定查看了已有评分、契约和原始输出，不是新一轮全量盲审。每条裁定列出具体规则和原始证据片段；本地结果不因裁定完成而获得来源认证。','Original independent and supplemental opinions remain in the per-result records. This supplemental adjudication inspected saved scores, contracts and original answers; it is not a new full blind review. Each adjudication records specific rules and original excerpts. Completing adjudication does not certify source validity.')+'
'
+ if audit.get('review_protocol',{}).get('blind_initial_verdict') is False and audit.get('retained_reviews'):
+ method=''+model_text('本次重新检查的记录采用已知程序结论的证据复核，逐条核对完整保存答案、任务要求和原生字段。其余记录经依赖检查沿用旧意见。两类范围分别统计；本次证据复核不算作新的独立盲审，也不补足历史来源认证。','The newly checked records received evidence review with known machine outcomes, checking complete saved answers, task requirements and native fields. Other records retain prior opinions after dependency checks. The two scopes are reported separately; this evidence review is not a new independent blind review and does not complete historical source certification.')+'
'
page+=section('核验口径',method)
dataset_path=out/'dataset.json'
if dataset_path.is_file():
diff --git a/tests/test_review_protocol_presentation.py b/tests/test_review_protocol_presentation.py
new file mode 100644
index 000000000..fb6572630
--- /dev/null
+++ b/tests/test_review_protocol_presentation.py
@@ -0,0 +1,49 @@
+"""A newly checked record must not be presented as a blind review by default."""
+import hashlib
+import json
+
+import pytest
+
+from harness.web_fetch import report
+
+
+@pytest.mark.parametrize('blind', [False, True])
+def test_current_review_text_reports_actual_blindness(tmp_path, blind):
+ replacement = {'provider_calls_added': 0, 'inherited_results': 1}
+ raw = json.dumps(replacement).encode()
+ (tmp_path/'replacement.json').write_bytes(raw)
+ dataset = {'cases': [{'id': 'case', 'track': 'web'}], 'comparison': {
+ 'task_replacement': {'path': 'replacement.json',
+ 'sha256': hashlib.sha256(raw).hexdigest()}}}
+ record = {'attempt_id': 'method__case', 'case_id': 'case', 'ok': True,
+ 'metrics': {'task_kind': 'body', 'content_pass': True}}
+ audit = {'kind': 'local_independent_review', 'review_protocol': {
+ 'scope': 'all', 'blind_initial_verdict': blind},
+ 'reviewer': {}, 'reviewed_attempts': 1, 'expected_attempts': 1,
+ 'fresh_reviewed_attempts': 1, 'retained_reviewed_attempts': 0,
+ 'retained_reviews': {'path': 'prior.json'},
+ 'supplemental_reviews': [{'path': 'supplement.json'}],
+ 'attempts': [{'attempt_id': record['attempt_id'], 'review_origin': 'fresh',
+ 'agent_verdict': 'pass', 'machine_verdict': 'pass',
+ 'final_verdict': 'pass'}],
+ 'review_rounds': [], 'unresolved': 0,
+ 'adjudication_revision': {'adjudicated_attempt_ids': []}}
+ (tmp_path/'result-review.json').write_text(json.dumps(audit))
+ (tmp_path/'dataset.json').write_text(json.dumps(dataset))
+ summary = report.review_summary(tmp_path)
+ findings = report.current_local_review_findings(tmp_path, dataset, [record], audit)
+ text = '\n'.join(row['text']+' '+row['text_en'] for row in findings)
+ if blind:
+ assert '先隐藏程序结论' in text
+ assert 'This round checked evidence with known machine outcomes' not in summary
+ else:
+ assert '已知程序结论' in summary
+ assert 'known machine outcomes' in summary
+ assert '先隐藏程序结论' not in text
+ assert 'Fresh review withheld machine verdicts' not in text
+ assert '不是独立盲审' in summary
+ report.write_review_page(tmp_path, [record])
+ page = (tmp_path/'review/index.html').read_text()
+ if not blind:
+ assert '本次重新检查的记录采用已知程序结论的证据复核' in page
+ assert '发生变化的调用必须重新盲审' not in page
From 1ca7e4505d7822fc28c3e8800ba246b1e2d77ad0 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 01:59:59 +0800
Subject: [PATCH 32/50] Freeze native route and request contracts before cohort
calls
---
.agents/skills/benchmark-report/SKILL.md | 2 +-
harness/web_fetch/README.md | 16 +
harness/web_fetch/artifacts.py | 4 +-
harness/web_fetch/native_routes.py | 385 ++++++++++++++++++
harness/web_fetch/providers/base.py | 29 ++
harness/web_fetch/providers/cloudflare.py | 8 +-
harness/web_fetch/providers/exa.py | 2 +-
harness/web_fetch/providers/firecrawl.py | 2 +-
harness/web_fetch/providers/moli.py | 8 +-
.../web_fetch/providers/subprocess_jsonl.py | 11 +
harness/web_fetch/providers/tavily.py | 2 +-
harness/web_fetch/reference_lifecycle.py | 4 +
harness/web_fetch/report_build.py | 6 +
harness/web_fetch/runner.py | 18 +
harness/web_fetch/workers/crawl4ai_worker.py | 3 +
tests/test_crawl4ai_file_worker.py | 12 +
tests/test_fixed_native_policy.py | 3 +-
tests/test_moli_provider.py | 19 +-
tests/test_native_route_contract.py | 361 ++++++++++++++++
tests/test_scoring.py | 1 +
tests/test_stable_task_lifecycle.py | 1 +
21 files changed, 876 insertions(+), 21 deletions(-)
create mode 100644 harness/web_fetch/native_routes.py
create mode 100644 tests/test_native_route_contract.py
diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md
index 7d3b79403..c34c53439 100644
--- a/.agents/skills/benchmark-report/SKILL.md
+++ b/.agents/skills/benchmark-report/SKILL.md
@@ -19,7 +19,7 @@ Public report CLIs default to English where supported and expose `--language en|
1. Establish the decision being supported, the measured unit, cohort, versions, execution location, freshness policy, and intended deliverable. Inspect the actual dataset, runner, outputs, scorer, prior review evidence, and current report. Do not infer them from screenshots or prose.
2. Freeze a machine-readable evaluation contract before expensive calls. It must identify the exact cases and methods, run count, timeout, concurrency policy, input and output contract, scoring provenance, reference state, and publication target when one exists. For source-dependent tasks, fail closed: verify and freeze references immediately before calls, verify the same declared scope afterward, and certify the cohort only when both checks pass. Partial or uncertified cohorts remain diagnostic evidence and cannot enter a release.
-3. Let each method use its strongest documented route to produce the required output. Equalize the task, inputs, output contract, call budget, and scoring. Do not require identical internals. Record material asymmetries such as local versus remote execution, cold versus warm start, bundled conversion, network transit, account tier, and billable time.
+3. Let each method use its strongest documented route to produce the required output. Equalize the task, inputs, output contract, call budget, and scoring. Do not require identical internals. For fixed-native cohorts, declare the input media type independently of URL patterns and expected answers, freeze the class-level route and transport parameters before calls, and verify the retained request receipts at aggregation and release. Record material asymmetries such as local versus remote execution, cold versus warm start, bundled conversion, network transit, account tier, and billable time.
4. Preserve evidence as an append-only chain: task definition, reference, provider call, raw artifact, normalized artifact, score, review, revision, and release. Keep provider output and call identity immutable. A presentation or scoring change must not masquerade as a new provider result. Declare the native answer fields and validate their structural projection before accepting a scored artifact. Adapters must not remove returned candidates by URL meaning or consult expected answers when selecting candidate fields. Apply privacy redaction only to separately identified public views after scoring, preserving the private originals and distinguishing original, scored and public-view hashes.
5. Separate retrieval success, task acceptance, content quality, latency, cost, and freshness. A missing or partial reference makes the affected metric unavailable or scoped; it does not prove success or failure. Do not derive binary acceptance from an unrelated aggregate similarity score.
6. For heterogeneous same-window latency evidence, preserve the full predeclared task-by-method matrix. Show all-call latency on that common denominator separately from successful-call latency on each method's conditional subset. Treat the all-method shared-pass intersection as a diagnostic, never as a post-outcome selector or ranking sample.
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index ab7b0bd8a..c2a4299a4 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -608,6 +608,22 @@ risk-control family. Case-specific retrieval hints remain prohibited. Scoring
keeps the original case and frozen acceptance contract; family is not a reason
to inject scripts, convert HTML to Markdown, or fetch another URL.
+New fixed-native cohorts declare `source_benchmark.input_media_kind` for every
+case (`html_page`, `direct_pdf`, `direct_docx`, `direct_text`, `direct_image`, or
+`direct_media`). JSONL workers receive the same declaration as
+`input_media_kind`. Native document routes use this frozen type, not URL suffixes
+or expected answers. Historical calls without this declaration are not silently
+upgraded to the new protocol.
+
+Before any case call, the runner writes `native-route-contract.json`, binding
+every method/track/task-kind/input-type cell to its interface, answer fields,
+configuration and normalized transport parameters. Actual arguments are checked
+before transport; changing the task URL, endpoint, parameters or invocation count
+fails the run. Each result binds its request receipt to that pre-call contract.
+Aggregation retains the exact contract bytes; report and release checks verify
+its hash, complete declared scope and per-result receipts. A retrospective audit
+of old logs cannot substitute for this pre-call proof.
+
Resource tasks may select a declared native output: Firecrawl image/link/video
fields, Cloudflare's generic media-element scrape, and Crawl4AI media fields.
Moli uses native HTML for resource discovery and native Markdown for webpage
diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py
index 3f3694d0d..3d35041de 100644
--- a/harness/web_fetch/artifacts.py
+++ b/harness/web_fetch/artifacts.py
@@ -357,7 +357,9 @@ def materialize_bound_artifacts(output: Path, source_root: Path, records: list[d
def materialize_reference_checks(output: Path, source: Path, batches: list[dict]) -> None:
- """Retain only hash-bound observations named by the execution certificates."""
+ """Retain hash-bound source observations and native execution contracts."""
+ from .native_routes import materialize_native_route_contracts
+ materialize_native_route_contracts(output, source, batches)
for batch in batches:
post = batch.get('reference_checks', {}).get('postcheck', {})
if not post:
diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py
new file mode 100644
index 000000000..d6362ff5b
--- /dev/null
+++ b/harness/web_fetch/native_routes.py
@@ -0,0 +1,385 @@
+"""Declared native input and route contracts, independent of URLs and answers."""
+
+INPUT_MEDIA_KINDS = frozenset({
+ 'html_page', 'direct_pdf', 'direct_docx', 'direct_text',
+ 'direct_image', 'direct_media',
+})
+
+
+def declared_input_media_kind(case, *, required=True):
+ """Read a dataset declaration or its worker projection; never inspect a URL.
+
+ Dataset declarations live in ``source_benchmark.input_media_kind``. Workers
+ receive the same value as ``input_media_kind`` in the JSONL request.
+ """
+ get = case.get if isinstance(case, dict) else lambda key, default=None: getattr(case, key, default)
+ source = get('source_benchmark', {}) or {}
+ if not isinstance(source, dict):
+ raise ValueError('Native input media declaration must be an object')
+ nested, direct = source.get('input_media_kind'), get('input_media_kind')
+ if nested is not None and direct is not None and nested != direct:
+ raise ValueError('Conflicting native input media declarations')
+ value = direct if direct is not None else nested
+ if value is None and not required:
+ return None
+ if not isinstance(value, str) or value not in INPUT_MEDIA_KINDS:
+ raise ValueError('Native calls require a declared input_media_kind')
+ return value
+
+
+# Native parameter values come from the immutable provider configuration and its
+# pinned implementation, not a case URL, expected text, or provider outcome.
+def _digest(value):
+ import hashlib
+ import json
+ return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True,
+ separators=(',', ':')).encode()).hexdigest()
+
+
+def native_route_cell(provider, case):
+ media = declared_input_media_kind(case)
+ kind, track = case.task_kind, case.track
+ key = (provider.name, track, kind, media)
+ config, method = provider.config, provider.type
+ endpoint = None
+ if method == 'moli_webfetch':
+ transport = 'cli'
+ interface = ('raw_file' if kind == 'file' else 'native_resource_html'
+ if kind == 'resource' else 'native_document_bytes'
+ if track == 'document' and media in {'direct_pdf', 'direct_docx'}
+ else 'dynamic_native_markdown' if track == 'dynamic' else 'ordinary_markdown')
+ fields = ['original_bytes'] if interface in {'raw_file', 'native_document_bytes'} else ['stdout']
+ elif method in {'firecrawl', 'tavily_extract', 'exa_contents', 'cloudflare_markdown', 'cloudflare_content'}:
+ transport = 'http'
+ if method == 'firecrawl':
+ interface = '/v2/scrape'
+ endpoint = str(config.get('base_url', 'https://api.firecrawl.dev')).rstrip('/') + interface
+ fields = ['data.rawBase64'] if kind == 'file' else ['data.markdown', 'data.links', 'data.images', 'data.video']
+ elif method == 'tavily_extract':
+ interface = '/extract'
+ endpoint = str(config.get('base_url', 'https://api.tavily.com')).rstrip('/') + interface
+ fields = ['results[].raw_content', 'results[].images']
+ elif method == 'exa_contents':
+ interface = '/contents'
+ endpoint = str(config.get('base_url', 'https://api.exa.ai')).rstrip('/') + interface
+ fields = ['results[].text', 'results[].extras']
+ else:
+ interface = ('/scrape' if method == 'cloudflare_markdown' and kind == 'resource'
+ and track in {'image', 'video'} else '/content' if method == 'cloudflare_content'
+ else '/markdown')
+ endpoint = 'https://api.cloudflare.com/client/v4/accounts/' + str(config.get('account_id', '')) + '/browser-rendering' + interface
+ if method=='cloudflare_markdown' and interface=='/markdown' and config.get('browser'):
+ from urllib.parse import urlencode
+ endpoint += '?'+urlencode({'browser': config['browser']})
+ fields = ['result']
+ elif method in {'subprocess_jsonl', 'cowagent_builtin'}:
+ transport, interface = 'jsonl', 'native_worker_command'
+ fields = ['content', 'metadata.links', 'metadata.images', 'metadata.videos', 'files']
+ elif method == 'fixture':
+ transport, interface, fields = 'fixture', 'synthetic_fixture', ['content']
+ else:
+ raise ValueError('Unsupported fixed-native interface: ' + method)
+ options = ('max_bytes', 'max_age', 'max_age_hours', 'only_main_content', 'include_images',
+ 'extract_depth', 'text_verbosity', 'include_resource_links', 'rich_links_limit',
+ 'wait_until', 'block_private_networks', 'dns_mode', 'log_level', 'formats')
+ cell = {'method': key[0], 'provider_type': method, 'track': track, 'task_kind': kind, 'input_media_kind': media,
+ 'transport': transport, 'interface': interface, 'answer_fields': fields,
+ 'configured_parameters': {k: config[k] for k in options if k in config},
+ 'unspecified_parameters': 'native defaults from the bound implementation',
+ **({'command_sha256': _digest(config.get('command'))} if transport == 'jsonl' else {}),
+ 'provider_configuration_sha256': _digest(config),
+ 'endpoint_sha256': _digest(endpoint) if endpoint is not None else None}
+ return key, cell
+
+
+def _request_parameters(provider, case, cell, timeout_seconds):
+ """Explicit pre-call transport contract, separately checked against adapters."""
+ config, method = provider.config, provider.type
+ if cell['transport'] == 'http':
+ if method == 'tavily_extract':
+ body = {'urls': [''], 'extract_depth': config.get('extract_depth', 'advanced'),
+ 'format': 'markdown', 'include_images': bool(config.get('include_images', False))}
+ headers = {'Authorization': 'Bearer '+str(config.get('api_key', ''))}
+ elif method == 'exa_contents':
+ body = {'urls': [''], 'text': {'verbosity': config.get('text_verbosity', 'full')},
+ 'maxAgeHours': int(config.get('max_age_hours', 0))}
+ extras = {'links': 20, 'imageLinks': 20} if config.get('include_resource_links') else {}
+ if config.get('rich_links_limit'):
+ extras['richLinks'] = int(config.get('rich_links_limit', 100))
+ if extras:
+ body['extras'] = extras
+ headers = {'x-api-key': str(config.get('api_key', ''))}
+ elif method == 'firecrawl':
+ formats = ['rawBase64'] if case.task_kind == 'file' else list(config.get('formats', ['markdown']))
+ if case.track == 'video' and case.task_kind == 'resource' and 'video' not in formats:
+ formats.append('video')
+ body = {'url': '', 'formats': formats, 'maxAge': int(config.get('max_age', 0)),
+ 'proxy': str(config.get('proxy', 'auto'))}
+ if 'only_main_content' in config:
+ body['onlyMainContent'] = config['only_main_content']
+ if case.task_kind == 'file':
+ body['parsers'] = []
+ headers = ({'Authorization': 'Bearer '+str(config['api_key'])}
+ if config.get('api_key') else {})
+ else:
+ body = {'url': '', 'gotoOptions': {'waitUntil': config.get('wait_until', 'networkidle2')}}
+ if cell['interface'] == '/scrape':
+ selectors = (['img', 'picture source'] if case.track == 'image' else
+ ['video', 'video source', 'audio', 'audio source', 'iframe', 'a[href]'])
+ body['elements'] = [{'selector': v} for v in selectors]
+ headers = {'Authorization': 'Bearer '+str(config.get('api_token', ''))}
+ return {'method': 'POST', 'body': body, 'headers_sha256': _digest(headers),
+ 'timeout_seconds': timeout_seconds, 'max_bytes': int(config.get('max_bytes', 20_000_000))}
+ if cell['transport'] == 'cli':
+ argv = ['', 'fetch']
+ if cell['interface'] not in {'raw_file', 'native_document_bytes'}:
+ argv += ['--dump', 'html' if case.task_kind == 'resource' else 'markdown',
+ '--wait-until', config.get('wait_until', 'done')]
+ argv += ['--http-max-response-size', str(int(config.get('max_bytes', 20_000_000)))]
+ if config.get('block_private_networks', True):
+ argv.append('--block-private-networks')
+ if config.get('log_level'):
+ argv += ['--log-level', str(config['log_level'])]
+ argv.append('')
+ return {'argv': argv, 'binary_sha256': config.get('expected_binary_sha256')}
+ if cell['transport'] == 'jsonl':
+ request = {'url': '', 'log_level': config.get('log_level'),
+ 'diagnostic_path': '' if config.get('log_level')=='debug' else None,
+ 'timeout_seconds': timeout_seconds, 'uniform_native_parameters': True,
+ 'max_bytes': int(config.get('max_bytes', 20_000_000)),
+ 'dns_mode': config.get('dns_mode', 'system'), 'task_kind': case.task_kind,
+ 'track': case.track, 'input_media_kind': declared_input_media_kind(case),
+ 'retrieval_plan': None,
+ 'retrieval_mode': 'native_default' if case.track == 'dynamic' else 'declared_plan'}
+ command = [provider._resolve_token(part) for part in config.get('command', [])]
+ return {'request': request, 'command_sha256': _digest(command)}
+ return {}
+
+
+class NativeRouteLedger:
+ """Freeze class-level intent before calls; reject observed per-case drift.
+
+ Native defaults are bound by configuration and implementation identities.
+ Interfaces and normalized transport parameters are frozen for every cell
+ before the cohort. Actual arguments are checked before transport. No
+ response, URL pattern, or expected answer selects a route.
+ """
+ def __init__(self, cases, providers, *, timeout_seconds=45):
+ from datetime import datetime, timezone
+ import threading
+ self.cells, self.observed, self.calls, self.errors = {}, {}, {}, []
+ self.lock = threading.Lock()
+ self.providers = {p.name: p for p in providers if p.config.get('uniform_native_parameters')}
+ self.inventory = []
+ for case in cases:
+ media = declared_input_media_kind(case) if self.providers else None
+ if self.providers:
+ self.inventory.append({'case_id': case.id, 'track': case.track,
+ 'task_kind': case.task_kind, 'input_media_kind': media,
+ 'input_url_sha256': _digest(case.url)})
+ for provider in self.providers.values():
+ key, cell = native_route_cell(provider, case)
+ if case.track == 'dynamic':
+ from .retrieval_hints import native_default_case
+ if not native_default_case(case):
+ raise ValueError('Native dynamic inputs require a stable acceptance contract')
+ cell['request_parameters'] = _request_parameters(provider, case, cell, timeout_seconds)
+ if key in self.cells and self.cells[key] != cell:
+ raise ValueError('Native route intent differs within its declared class')
+ self.cells[key] = cell
+ self.document = {'schema_version': 1, 'kind': 'declared_native_route_contract',
+ 'frozen_at': datetime.now(timezone.utc).isoformat(),
+ 'pre_call_frozen': True,
+ 'parameter_policy': 'Interfaces, adapter request parameters and configured options frozen before the cohort; actual transport arguments must match. Unspecified product internals retain pinned native defaults.',
+ 'cases': self.inventory, 'cells': list(self.cells.values())}
+ self.binding = None
+
+ def write(self, source, *, dataset_sha256, runtime_source_sha256):
+ import hashlib
+ import json
+ self.document.update(dataset_sha256=dataset_sha256, runtime_source_sha256=runtime_source_sha256)
+ raw = (json.dumps(self.document, ensure_ascii=False, indent=2)+'\n').encode()
+ path = source/'native-route-contract.json'
+ if path.exists():
+ raise ValueError('Native route contract already exists; do not overwrite a run')
+ path.write_bytes(raw)
+ self.binding = {'path': path.name, 'sha256': hashlib.sha256(raw).hexdigest()}
+ for provider in self.providers.values():
+ provider._native_route_ledger = self
+ return self.binding
+
+ def reject(self, provider, case, reason):
+ with self.lock:
+ self.errors.append({'method': provider.name, 'case_id': case.id, 'reason': reason})
+ raise ValueError(reason)
+
+ def observe(self, provider, case, *, transport, parameters, endpoint=None, interface=None):
+ key, current = native_route_cell(provider, case)
+ cell = self.cells.get(key, {})
+ with self.lock:
+ message = 'Native route protocol was already violated' if self.errors else None
+ if {k:v for k,v in cell.items() if k!='request_parameters'} != current:
+ message = 'Native route configuration changed after the pre-call freeze'
+ elif transport != cell['transport']:
+ message = 'Native transport differs from its frozen route'
+ elif endpoint is not None and _digest(endpoint) != cell['endpoint_sha256']:
+ message = 'Native endpoint differs from its frozen route'
+ elif interface is not None and interface != cell['interface']:
+ message = 'Native command differs from its frozen route'
+ signature = {'transport': transport, 'interface': cell['interface'],
+ 'endpoint_sha256': cell['endpoint_sha256'], 'parameters': parameters}
+ digest = _digest(signature)
+ if not message and parameters != cell.get('request_parameters'):
+ message = 'Native request parameters differ from the pre-call contract'
+ if not message and key in self.observed and self.observed[key] != digest:
+ message = 'Native request signature changed within the same declared class'
+ identity = (provider.name, case.id)
+ if not message and self.calls.get(identity):
+ message = 'Fixed-native calls permit only one adapter invocation per task'
+ if message:
+ self.errors.append({'method': provider.name, 'case_id': case.id, 'reason': message})
+ raise ValueError(message)
+ self.observed[key] = digest
+ self.calls.setdefault(identity, []).append({'signature': signature, 'sha256': digest})
+
+ def receipt(self, provider, case):
+ from copy import deepcopy
+ key, _ = native_route_cell(provider, case)
+ with self.lock:
+ return {'contract_sha256': self.binding['sha256'], 'cell': list(key),
+ 'requests': deepcopy(self.calls.get((provider.name, case.id), [])),
+ 'protocol_errors': [dict(e) for e in self.errors if e['method']==provider.name and e['case_id']==case.id]}
+
+
+def verify_native_route_contract(source, manifest, dataset, records):
+ """Shared execution/release check for newly declared native cohorts only."""
+ import hashlib
+ import json
+ from datetime import datetime
+ from .safety import contained_path
+ if manifest.get('native_route_errors'):
+ raise ValueError('Native route protocol was violated')
+ binding = manifest.get('native_route_contract')
+ if not binding:
+ if manifest.get('native_routes_required'):
+ raise ValueError('Fixed-native cohort lacks its pre-call route contract')
+ return None
+ path = contained_path(source, binding['path'], reject_symlinks=True)
+ raw = path.read_bytes()
+ if hashlib.sha256(raw).hexdigest() != binding.get('sha256'):
+ raise ValueError('Native route contract hash changed')
+ doc = json.loads(raw)
+ if (doc.get('kind') != 'declared_native_route_contract' or doc.get('pre_call_frozen') is not True
+ or doc.get('dataset_sha256') != manifest.get('dataset_sha256')
+ or doc.get('runtime_source_sha256') != manifest.get('runtime_source_sha256')):
+ raise ValueError('Native route contract differs from the frozen execution')
+ cells = {(c['method'], c['track'], c['task_kind'], c['input_media_kind']): c for c in doc['cells']}
+ cases = {c['id']: c for c in dataset['cases']}
+ inventory = {c['case_id']: c for c in doc['cases']}
+ if len(cells) != len(doc['cells']) or len(inventory) != len(doc['cases']):
+ raise ValueError('Native route contract has duplicate declarations')
+ names = {key[0] for key in cells}
+ required = {p['name'] for p in manifest.get('providers', []) if p.get('fixed_native_parameters')}
+ projection = manifest.get('projection', {}).get('kind')
+ allowed_projection = projection in {'complete-provider-case-subset-from-retained-run',
+ 'complete-provider-from-retained-incomplete-matrix', 'completed-case-from-live-preflight'}
+ if (not required or not required <= names or (required != names and not allowed_projection)):
+ raise ValueError('Native route method scope differs from the frozen execution')
+ names = required
+ selected_cases = ({row['case_id'] for row in records if row['provider'] in names}
+ if allowed_projection else set(inventory))
+ if not selected_cases <= set(inventory):
+ raise ValueError('Native route projection changes its frozen input scope')
+ expected = {(name, identifier) for name in names for identifier in selected_cases}
+ actual, signatures = set(), {}
+ for row in records:
+ if row['provider'] not in names:
+ continue
+ identity = row['provider'], row['case_id']
+ if identity in actual or identity not in expected:
+ raise ValueError('Native route result inventory differs')
+ actual.add(identity)
+ case = cases[row['case_id']]
+ media = declared_input_media_kind(case)
+ if (_digest(case['url']) != inventory[row['case_id']]['input_url_sha256']
+ or inventory[row['case_id']] != {'case_id': row['case_id'], 'track': case.get('track', 'web'),
+ 'task_kind': case.get('task_kind', 'body'), 'input_media_kind': media,
+ 'input_url_sha256': _digest(row['url'])}
+ or datetime.fromisoformat(doc['frozen_at']) > datetime.fromisoformat(row['started_at'])):
+ raise ValueError('Native input type or pre-call freeze differs')
+ key = row['provider'], case.get('track', 'web'), case.get('task_kind', 'body'), media
+ receipt = row.get('provider_metadata', {}).get('native_route_receipt', {})
+ if receipt.get('protocol_errors'):
+ raise ValueError('Native request violated its route contract')
+ if receipt.get('contract_sha256') != binding['sha256'] or receipt.get('cell') != list(key) or key not in cells:
+ raise ValueError('Native result lacks its bound route receipt')
+ requests = receipt.get('requests', [])
+ if len(requests) > 1 or (row['ok'] and not requests and cells[key]['transport'] != 'fixture'):
+ raise ValueError('Native result lacks one complete invocation signature')
+ for item in requests:
+ signature = item['signature']
+ if (_digest(signature) != item.get('sha256')
+ or signature['transport'] != cells[key]['transport']
+ or signature['interface'] != cells[key]['interface']
+ or signature['endpoint_sha256'] != cells[key]['endpoint_sha256']
+ or signature.get('parameters') != cells[key]['request_parameters']):
+ raise ValueError('Native invocation differs from its frozen interface')
+ if key in signatures and signatures[key] != item['sha256']:
+ raise ValueError('Native request signature differs within its declared class')
+ signatures[key] = item['sha256']
+ if actual != expected:
+ raise ValueError('Native route contract has an incomplete result scope')
+ return doc
+
+
+def verify_native_execution_manifests(source, manifests, dataset, records):
+ """Check native contracts at every existing aggregate/report boundary."""
+ checked = []
+ for manifest in manifests:
+ if not manifest.get('native_routes_required') and not manifest.get('native_route_contract'):
+ continue
+ if source is None or dataset is None or records is None:
+ raise ValueError('Native route verification requires retained inputs and complete records')
+ identifiers = manifest.get('attempt_ids', [])
+ if len(identifiers) != len(set(identifiers)):
+ raise ValueError('Native execution repeats attempt identities')
+ identifier_set = set(identifiers)
+ selected = [row for row in records if row['attempt_id'] in identifier_set]
+ if {r['attempt_id'] for r in selected} != set(identifiers):
+ raise ValueError('Native execution omits retained attempt evidence')
+ checked.append(verify_native_route_contract(source, manifest, dataset, selected))
+ return checked
+
+
+def materialize_native_route_contracts(output, source, manifests):
+ """Copy hash-bound frozen contracts, deduplicating multiple cohort roots."""
+ import hashlib
+ import re
+ import shutil
+ from .safety import contained_path
+ for manifest in manifests:
+ binding = manifest.get('native_route_contract')
+ if not binding:
+ if manifest.get('native_routes_required'):
+ raise ValueError('Native cohort is missing its frozen route contract')
+ continue
+ digest = binding.get('sha256', '')
+ if not isinstance(digest, str) or re.fullmatch('[0-9a-f]{64}', digest) is None:
+ raise ValueError('Invalid native route contract digest')
+ relative = binding.get('path')
+ if relative not in {'native-route-contract.json', 'native-route-contracts/'+digest+'.json'}:
+ raise ValueError('Invalid native route contract path')
+ path = contained_path(source, relative, reject_symlinks=True)
+ if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != digest:
+ raise ValueError('Native route contract bytes changed')
+ target_relative = 'native-route-contracts/'+digest+'.json'
+ target = contained_path(output, target_relative, reject_symlinks=True)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ if target.exists():
+ if not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest() != digest:
+ raise ValueError('Conflicting native route contract bytes')
+ else:
+ shutil.copy2(path, target)
+ if hashlib.sha256(target.read_bytes()).hexdigest() != digest:
+ raise ValueError('Copied native route contract bytes changed')
+ manifest['native_route_contract'] = {'path': target_relative, 'sha256': digest}
diff --git a/harness/web_fetch/providers/base.py b/harness/web_fetch/providers/base.py
index fe443e3a8..3e0f525a0 100644
--- a/harness/web_fetch/providers/base.py
+++ b/harness/web_fetch/providers/base.py
@@ -21,6 +21,35 @@ def check_native_call(self, case: EvalCase) -> None:
if case.task_kind not in {"body", "structure", "resource", "file"}:
raise ValueError("fixed native calls require a declared native output kind")
+ def capture_native_request(self, case, *, transport, parameters, endpoint=None, interface=None):
+ ledger = getattr(self, '_native_route_ledger', None)
+ if ledger is not None:
+ ledger.observe(self, case, transport=transport, parameters=parameters,
+ endpoint=endpoint, interface=interface)
+
+ def native_http_request(self, case, callback, endpoint, **kwargs):
+ """Observe exact transport arguments before a fixed-native HTTP call."""
+ ledger = getattr(self, '_native_route_ledger', None)
+ if ledger is not None:
+ from copy import deepcopy
+ from ..native_routes import _digest
+ body = deepcopy(kwargs.get('json_body'))
+ if not isinstance(body, dict):
+ ledger.reject(self, case, 'Native HTTP request lacks its declared input field')
+ slots = [key for key in ('url', 'urls') if key in body]
+ if slots == ['url'] and body['url'] == case.url:
+ body['url'] = ''
+ elif slots == ['urls'] and body['urls'] == [case.url]:
+ body['urls'] = ['']
+ else:
+ ledger.reject(self, case, 'Native HTTP input differs from the frozen task URL')
+ self.capture_native_request(case, transport='http', endpoint=endpoint,
+ parameters={'method': kwargs.get('method', 'GET'), 'body': body,
+ 'headers_sha256': _digest(kwargs.get('headers', {})),
+ 'timeout_seconds': kwargs.get('timeout'),
+ 'max_bytes': kwargs.get('max_bytes')})
+ return callback(endpoint, **kwargs)
+
@property
def min_interval_seconds(self) -> float:
"""Minimum cooldown from one response completion to the next request start."""
diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py
index 01b2739b8..7ba21b37a 100644
--- a/harness/web_fetch/providers/cloudflare.py
+++ b/harness/web_fetch/providers/cloudflare.py
@@ -250,7 +250,7 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) ->
account_id=account_id,
http_timeout=http_timeout,
)
- if _uses_direct_document_route(case) and not self.config.get("uniform_native_parameters"):
+ if not self.config.get("uniform_native_parameters") and _uses_direct_document_route(case):
return self._fetch_direct_document(
case,
account_id=account_id,
@@ -346,7 +346,7 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) ->
"client_http_timeout_ms": int(http_timeout * 1000),
"wait_until": wait_until,
}
- response = request(
+ response = self.native_http_request(case, request,
endpoint + ("?" + query if query else ""),
method="POST",
timeout=http_timeout,
@@ -431,7 +431,7 @@ def _fetch_media_resources(
# One native media request for every video-resource task; never
# choose selectors from expected URLs or reference answers.
selectors = ["video", "video source", "audio", "audio source", "iframe", "a[href]"]
- response = request(
+ response = self.native_http_request(case, request,
f"https://api.cloudflare.com/client/v4/accounts/{account_id}/browser-rendering/scrape",
method="POST",
timeout=http_timeout,
@@ -686,7 +686,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
account_id = str(self.config.get("account_id", ""))
http_timeout = inner_timeout_budget(timeout, native_default=bool(self.config.get("uniform_native_parameters")))
service_timeout_ms = min(max(1, int(http_timeout * 1000)), 60_000)
- response = request(
+ response = self.native_http_request(case, request,
f"https://api.cloudflare.com/client/v4/accounts/{account_id}/browser-rendering/content",
method="POST",
timeout=http_timeout,
diff --git a/harness/web_fetch/providers/exa.py b/harness/web_fetch/providers/exa.py
index 393956e51..61a3a9bec 100644
--- a/harness/web_fetch/providers/exa.py
+++ b/harness/web_fetch/providers/exa.py
@@ -72,7 +72,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
# Exa's richLinks route returns the explicit anchor/URL pair needed
# by the shared named-link answer contract.
extras["richLinks"] = int(self.config.get("rich_links_limit", 100))
- response = request(
+ response = self.native_http_request(case, request,
f"{str(self.config.get('base_url', 'https://api.exa.ai')).rstrip('/')}/contents",
method="POST",
timeout=http_timeout,
diff --git a/harness/web_fetch/providers/firecrawl.py b/harness/web_fetch/providers/firecrawl.py
index 543f68735..2949080a2 100644
--- a/harness/web_fetch/providers/firecrawl.py
+++ b/harness/web_fetch/providers/firecrawl.py
@@ -287,7 +287,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
"Firecrawl dynamic_wait_strategy must be actions or wait_for"
),
)
- response = request(
+ response = self.native_http_request(case, request,
f"{base_url}/v2/scrape",
method="POST",
timeout=http_timeout,
diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py
index 3d5425107..d2b799aca 100644
--- a/harness/web_fetch/providers/moli.py
+++ b/harness/web_fetch/providers/moli.py
@@ -357,8 +357,9 @@ def _route_for_case(case: EvalCase, *, native_default: bool = False) -> str:
if native_default:
if case.task_kind == "resource":
return "native_resource_html"
- path = urlsplit(case.url).path.casefold()
- if case.track == "document" and (path.endswith((".pdf", ".docx")) or "/pdf/" in path):
+ from ..native_routes import declared_input_media_kind
+ media = declared_input_media_kind(case, required=case.track == "document")
+ if case.track == "document" and media in {"direct_pdf", "direct_docx"}:
return "native_document_bytes"
return "dynamic_native_markdown" if case.track == "dynamic" else "ordinary_markdown"
if case.task_kind == "resource":
@@ -794,6 +795,9 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
error_message=f"Cannot read Moli binary: {error}",
metadata=metadata,
)
+ self.capture_native_request(case, transport='cli', interface=route,
+ parameters={'argv': ['', *command[1:-1], ''],
+ 'binary_sha256': getattr(self, '_binary_sha256', None)})
# Spool artifacts to files rather than retaining arbitrary CLI output in RAM.
with ExitStack() as dns_stack, ExitStack() as spool_stack:
sink = getattr(self, "_call_diagnostics", None)
diff --git a/harness/web_fetch/providers/subprocess_jsonl.py b/harness/web_fetch/providers/subprocess_jsonl.py
index 6ab6409cc..2888c6bb3 100644
--- a/harness/web_fetch/providers/subprocess_jsonl.py
+++ b/harness/web_fetch/providers/subprocess_jsonl.py
@@ -195,6 +195,9 @@ def _request_stderr(self, cursor: tuple[int, int]) -> dict[str, Any]:
def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
self.check_native_call(case)
+ from ..native_routes import declared_input_media_kind
+ input_media = declared_input_media_kind(case, required=bool(
+ self.config.get("uniform_native_parameters") and case.track == "document"))
answer_begin = time.perf_counter()
try:
retrieval_plan = plan_for_case(case)
@@ -225,9 +228,17 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
"dns_mode": self.config.get("dns_mode", "system"),
"task_kind": case.task_kind,
"track": case.track,
+ **({"input_media_kind": input_media} if input_media is not None else {}),
"retrieval_plan": retrieval_plan.worker_dict() if retrieval_plan else None,
"retrieval_mode": "native_default" if native_default_case(case) else "declared_plan",
}
+ if getattr(self, '_native_route_ledger', None) is not None:
+ from ..native_routes import _digest
+ projected = {**request, 'url': '',
+ 'diagnostic_path': '' if request['diagnostic_path'] else None}
+ self.capture_native_request(case, transport='jsonl', interface='native_worker_command',
+ parameters={'request': projected,
+ 'command_sha256': _digest([self._resolve_token(part) for part in self.config['command']])})
if self._active_diagnostics: self._active_diagnostics.emit("caller", "worker_request_sent", request=request)
try:
process.stdin.write(json.dumps(request, ensure_ascii=False) + "\n")
diff --git a/harness/web_fetch/providers/tavily.py b/harness/web_fetch/providers/tavily.py
index b826d15b3..7b8c13893 100644
--- a/harness/web_fetch/providers/tavily.py
+++ b/harness/web_fetch/providers/tavily.py
@@ -48,7 +48,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
# deadline remains authoritative and we omit an invalid API value.
if extract_timeout >= 1.0 and not self.config.get("uniform_native_parameters"):
request_body["timeout"] = round(extract_timeout, 3)
- response = request(
+ response = self.native_http_request(case, request,
f"{str(self.config.get('base_url', 'https://api.tavily.com')).rstrip('/')}/extract",
method="POST",
timeout=http_timeout,
diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py
index efd9ca9b6..556f3369d 100644
--- a/harness/web_fetch/reference_lifecycle.py
+++ b/harness/web_fetch/reference_lifecycle.py
@@ -1356,6 +1356,10 @@ def current_protocol(batch):
def valid_hash(value):
return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None
+ from .native_routes import verify_native_execution_manifests
+ verify_native_execution_manifests(source, manifests,
+ {'cases': [dict(c) if isinstance(c, dict) else vars(c) for c in case_map.values()]}
+ if case_map is not None else None, records)
allow_scoring_rule_revision = (allow_scoring_rule_revision or valid_scoring_revision(source))
revised_references = reference_revision_cases(source)
original_latency = latency_correction_original_elapsed(source, records=records)
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 258a78ba5..d6c30db3a 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -411,6 +411,8 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
raise ValueError('A report requires tasks and measured records')
execution = source/'execution.json'
manifests = json.loads(execution.read_text())['batches'] if execution.exists() else [json.loads((source/'manifest.json').read_text())]
+ from .native_routes import verify_native_execution_manifests
+ verify_native_execution_manifests(source, manifests, dataset, records)
if not local_audit and not diagnostic_preview:
require_verified_reference_checks(manifests, cases=dataset, records=records, source=source,
equivalent_capture_bindings=dataset.get('comparison',{}).get('reference_binding_mode') ==
@@ -1304,6 +1306,10 @@ def validate_local_presentation(output: Path, binding_path: Path):
require_host_metadata_projection(output, binding['publication_projection'])
dataset = json.loads((output / 'dataset.json').read_text())
records = load_jsonl(contained_path(output, 'results.jsonl'))
+ from .native_routes import verify_native_execution_manifests
+ execution = output/'execution.json'
+ if execution.is_file():
+ verify_native_execution_manifests(output, json.loads(execution.read_text())['batches'], dataset, records)
scoring_sources = {row['attempt_id']: binding.get('measured_scoring_source_sha256') for row in records}
if binding.get('offline_revision'):
from .local_revision import validate_revision_chain, read_bound
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index 710c217f4..65be6fe70 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -495,8 +495,10 @@ def run_evaluation(
if provider.config.get("uniform_native_parameters"):
if runs != 1 or max_failure_retries != 0:
raise ValueError("fixed native calls require one run and zero caller retries")
+ from .native_routes import declared_input_media_kind
for case in cases:
provider.check_native_call(case)
+ declared_input_media_kind(case)
safe_identifier(provider.name)
for case in cases:
validate_case(case)
@@ -504,6 +506,9 @@ def run_evaluation(
raise ValueError('Provider names must be unique within an execution')
if len({case.id for case in cases}) != len(cases):
raise ValueError('Task IDs must be unique within an execution')
+ from .native_routes import NativeRouteLedger
+ native_routes = (NativeRouteLedger(cases, providers, timeout_seconds=timeout)
+ if any(p.config.get('uniform_native_parameters') for p in providers) else None)
for filename in ['artifacts', PUBLIC_CALL_LOG_ROOT, 'manifest.json', 'dataset.json',
'results.jsonl', 'summary.json']:
contained_path(output_dir, filename)
@@ -562,6 +567,7 @@ def run_evaluation(
"name": provider.name,
"type": provider.type,
"min_interval_seconds": provider.min_interval_seconds,
+ "fixed_native_parameters": bool(provider.config.get("uniform_native_parameters")),
**({"implementation_identity": provider._implementation_identity}
if isinstance(getattr(provider, '_implementation_identity', None), dict) else {}),
}
@@ -596,6 +602,11 @@ def run_evaluation(
manifest['evidence_state'] = 'diagnostic'
manifest['unqualified_reference_cases'] = sorted(
set(comparison.get('unqualified_reference_cases', [])) & {case.id for case in cases})
+ if native_routes is not None:
+ manifest['native_routes_required'] = True
+ manifest['native_route_contract'] = native_routes.write(output_dir,
+ dataset_sha256=manifest['dataset_sha256'],
+ runtime_source_sha256=manifest['runtime_source_sha256'])
(output_dir / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
@@ -694,6 +705,8 @@ def run_provider(provider: Provider) -> None:
suffix = (f'run-{run_number:03d}.txt' if max_failure_retries == 0 else
f'run-{run_number:03d}-physical-{physical_attempt:03d}.txt')
artifact_rel = Path('artifacts') / provider.name / case.id / suffix
+ if native_routes is not None and provider.name in native_routes.providers:
+ result.metadata['native_route_receipt'] = native_routes.receipt(provider, case)
artifact = persist_result_artifacts(output_dir, artifact_rel, result)
artifact_bytes = result.content.encode('utf-8', errors='replace')
measured = score_attempt(
@@ -858,12 +871,17 @@ def run_provider(provider: Provider) -> None:
manifest["completed_at"] = utc_now()
manifest["attempt_count"] = len(records)
manifest["attempt_ids"] = [r['attempt_id'] for r in records]
+ if native_routes is not None:
+ manifest['native_route_errors'] = list(native_routes.errors)
(output_dir / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
(output_dir / "summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
+ if native_routes is not None:
+ from .native_routes import verify_native_route_contract
+ verify_native_route_contract(output_dir, manifest, dataset_meta, records)
expected_attempts = len(cases) * len(providers) * runs
if len(records) != expected_attempts or manifest["unexecuted"]:
raise IncompleteEvaluationError(
diff --git a/harness/web_fetch/workers/crawl4ai_worker.py b/harness/web_fetch/workers/crawl4ai_worker.py
index 5887b7485..043b46e3e 100755
--- a/harness/web_fetch/workers/crawl4ai_worker.py
+++ b/harness/web_fetch/workers/crawl4ai_worker.py
@@ -145,6 +145,9 @@ def is_direct_pdf_document(request):
return False
if request.get("task_kind", "body") != "body":
return False
+ if request.get("uniform_native_parameters"):
+ from harness.web_fetch.native_routes import declared_input_media_kind
+ return declared_input_media_kind(request) == "direct_pdf"
path = urlsplit(request["url"]).path.lower().rstrip("/")
return path.endswith(".pdf") or "/pdf/" in path
diff --git a/tests/test_crawl4ai_file_worker.py b/tests/test_crawl4ai_file_worker.py
index 69ecc25e0..86f2da606 100644
--- a/tests/test_crawl4ai_file_worker.py
+++ b/tests/test_crawl4ai_file_worker.py
@@ -162,6 +162,18 @@ def test_direct_pdf_selection_is_limited_to_document_body_tasks(self):
):
self.assertFalse(worker.is_direct_pdf_document(request))
+ def test_fixed_native_pdf_route_uses_declared_input_not_url_shape(self):
+ worker = self.load_worker(object)
+ request = {'url': 'https://example.test/no-extension', 'track': 'document',
+ 'task_kind': 'body', 'uniform_native_parameters': True,
+ 'input_media_kind': 'direct_pdf'}
+ self.assertTrue(worker.is_direct_pdf_document(request))
+ request.update(url='https://example.test/looks-like.pdf', input_media_kind='html_page')
+ self.assertFalse(worker.is_direct_pdf_document(request))
+ request.pop('input_media_kind')
+ with self.assertRaisesRegex(ValueError, 'declared input_media_kind'):
+ worker.is_direct_pdf_document(request)
+
def test_direct_pdf_uses_native_pdf_strategies_and_returns_markdown(self):
calls = []
pdf_crawler_instance = object()
diff --git a/tests/test_fixed_native_policy.py b/tests/test_fixed_native_policy.py
index 860a53f96..973d72e5c 100644
--- a/tests/test_fixed_native_policy.py
+++ b/tests/test_fixed_native_policy.py
@@ -85,6 +85,7 @@ def test_native_cloudflare_document_never_downloads_in_caller():
provider = build_provider("cf", {"type": "cloudflare_markdown", "uniform_native_parameters": True})
case = EvalCase("pdf", "pdf", "https://example.com/a.pdf", "control", [], track="document")
with patch("harness.web_fetch.providers.cloudflare._get_document", side_effect=AssertionError("extra download")), \
+ patch("harness.web_fetch.providers.cloudflare._uses_direct_document_route", side_effect=AssertionError("native URL guessing")), \
patch("harness.web_fetch.providers.cloudflare.request") as call:
call.return_value = HttpResponse(200, case.url, {}, b'{"success":true,"result":"native text"}')
assert provider.fetch(case, 45).content == "native text"
@@ -109,7 +110,7 @@ def test_native_cloudflare_media_has_no_answer_dependent_selectors_or_timers():
assert "timeout" not in bodies[0]["gotoOptions"]
-def test_all_430_frozen_cases_pass_native_input_guard():
+def test_historical_430_cases_pass_native_hint_and_output_guard():
from harness.web_fetch.config import load_dataset
provider = build_provider("fixture", {"type": "fixture", "uniform_native_parameters": True})
count = 0
diff --git a/tests/test_moli_provider.py b/tests/test_moli_provider.py
index d1f16ee85..a6c7dae46 100644
--- a/tests/test_moli_provider.py
+++ b/tests/test_moli_provider.py
@@ -211,15 +211,16 @@ def test_native_families_never_use_caller_scripts_or_conversion(self):
" assert sys.argv[sys.argv.index('--dump') + 1] in ('markdown', 'html')\n"
"print('native output')\n", uniform_native_parameters=True)
with patch("harness.web_fetch.providers.moli.extract_document_text", side_effect=AssertionError("caller conversion")):
- for track, kind, url in [("challenge", "structure", "https://example.com/"),
- ("risk_control", "body", "https://example.com/"),
- ("document", "body", "https://example.com/a.docx"),
- ("document", "body", "https://example.com/a.pdf"),
- ("document", "body", "https://example.com/viewer.html"),
- ("image", "resource", "https://example.com/"),
- ("video", "resource", "https://example.com/"),
- ("video", "body", "https://example.com/")]:
- result = provider.fetch(self.case_with(track=track, task_kind=kind, url=url), 5)
+ for track, kind, url, media in [("challenge", "structure", "https://example.com/", "html_page"),
+ ("risk_control", "body", "https://example.com/", "html_page"),
+ ("document", "body", "https://example.com/a.docx", "direct_docx"),
+ ("document", "body", "https://example.com/a.pdf", "direct_pdf"),
+ ("document", "body", "https://example.com/viewer.html", "html_page"),
+ ("image", "resource", "https://example.com/", "html_page"),
+ ("video", "resource", "https://example.com/", "html_page"),
+ ("video", "body", "https://example.com/", "html_page")]:
+ result = provider.fetch(self.case_with(track=track, task_kind=kind, url=url,
+ source_benchmark={"input_media_kind": media}), 5)
self.assertTrue(result.ok, result.error_message)
if url.endswith((".pdf", ".docx")):
self.assertEqual(result.content, "")
diff --git a/tests/test_native_route_contract.py b/tests/test_native_route_contract.py
new file mode 100644
index 000000000..a19e5fe34
--- /dev/null
+++ b/tests/test_native_route_contract.py
@@ -0,0 +1,361 @@
+"""Declared input types select native routes without URL or answer inspection."""
+from types import SimpleNamespace
+
+import pytest
+
+from harness.web_fetch.native_routes import INPUT_MEDIA_KINDS, declared_input_media_kind
+from harness.web_fetch.providers.moli import _route_for_case
+
+
+@pytest.mark.parametrize('media', sorted(INPUT_MEDIA_KINDS))
+def test_dataset_and_worker_use_the_same_explicit_input_type(media):
+ assert declared_input_media_kind({'source_benchmark': {'input_media_kind': media}}) == media
+ assert declared_input_media_kind({'input_media_kind': media}) == media
+
+
+@pytest.mark.parametrize('value', [None, '', 'guess_from_url', [], {}])
+def test_invalid_or_missing_input_type_is_not_guessed(value):
+ with pytest.raises(ValueError, match='declared input_media_kind'):
+ declared_input_media_kind({'url': 'https://example.test/actual.pdf',
+ 'input_media_kind': value})
+
+
+def test_conflicting_input_declarations_are_rejected():
+ with pytest.raises(ValueError, match='Conflicting'):
+ declared_input_media_kind({'input_media_kind': 'html_page',
+ 'source_benchmark': {'input_media_kind': 'direct_pdf'}})
+
+
+@pytest.mark.parametrize('media,expected', [
+ ('direct_pdf', 'native_document_bytes'), ('direct_docx', 'native_document_bytes'),
+ ('html_page', 'ordinary_markdown'), ('direct_text', 'ordinary_markdown'),
+])
+def test_moli_document_selection_cannot_read_url_or_reference(media, expected):
+ class DeclaredDocument:
+ track = 'document'
+ task_kind = 'body'
+ source_benchmark = {'input_media_kind': media}
+ @property
+ def url(self):
+ raise AssertionError('route selection inspected the URL')
+ @property
+ def expected_resources(self):
+ raise AssertionError('route selection inspected expected answers')
+ assert _route_for_case(DeclaredDocument(), native_default=True) == expected
+
+
+def test_fixed_native_cohort_rejects_undeclared_media_before_preflight(tmp_path):
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.runner import run_evaluation
+ provider = build_provider('fixture', {'type': 'fixture', 'uniform_native_parameters': True})
+ case = EvalCase('case', 'Case', 'https://example.test/a.pdf', 'control', [])
+ with pytest.raises(ValueError, match='declared input_media_kind'):
+ run_evaluation(output_dir=tmp_path/'run', dataset_meta={}, dataset_path=tmp_path/'absent.json',
+ cases=[case], providers=[provider], provider_config_path=tmp_path/'absent-providers.json',
+ runs=1, timeout=45, delay=0)
+ assert not (tmp_path/'run').exists()
+
+
+def test_plain_provider_use_does_not_infer_optional_media():
+ assert declared_input_media_kind(SimpleNamespace(source_benchmark={}), required=False) is None
+
+
+def _route_http_fixture(tmp_path):
+ from dataclasses import asdict
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.native_routes import NativeRouteLedger
+ from harness.web_fetch.providers.base import build_provider
+ provider = build_provider('tavily', {'type': 'tavily_extract', 'api_key': 'fixture',
+ 'uniform_native_parameters': True})
+ cases = [EvalCase(name, name, 'https://example.test/'+name, 'control', [],
+ source_benchmark={'input_media_kind': 'html_page'})
+ for name in ('one', 'two')]
+ ledger = NativeRouteLedger(cases, [provider])
+ binding = ledger.write(tmp_path, dataset_sha256='dataset', runtime_source_sha256='runtime')
+ manifest = {'native_routes_required': True, 'native_route_contract': binding,
+ 'dataset_sha256': 'dataset', 'runtime_source_sha256': 'runtime',
+ 'providers': [{'name': provider.name, 'fixed_native_parameters': True}]}
+ return provider, cases, ledger, manifest, {'cases': [asdict(c) for c in cases]}
+
+
+def test_actual_native_http_requests_are_locked_before_transport(tmp_path):
+ import json
+ from datetime import datetime, timezone
+ from unittest.mock import patch
+ from harness.web_fetch.native_routes import verify_native_route_contract
+ from harness.web_fetch.providers.http_client import HttpResponse
+ provider, cases, ledger, manifest, dataset = _route_http_fixture(tmp_path)
+ records = []
+ seen = []
+ def transport(endpoint, **kwargs):
+ assert (tmp_path/'native-route-contract.json').exists()
+ seen.append(kwargs['json_body'])
+ return HttpResponse(200, endpoint, {}, json.dumps({'results': [
+ {'url': kwargs['json_body']['urls'][0], 'raw_content': 'answer'}]}).encode())
+ with patch('harness.web_fetch.providers.tavily.request', side_effect=transport):
+ for case in cases:
+ started = datetime.now(timezone.utc).isoformat()
+ result = provider.fetch(case, 45)
+ records.append({'provider': provider.name, 'case_id': case.id,
+ 'url': case.url, 'started_at': started, 'ok': result.ok,
+ 'provider_metadata': {'native_route_receipt': ledger.receipt(provider, case)}})
+ assert len(seen) == 2 and seen[0]['urls'] != seen[1]['urls']
+ assert len(ledger.observed) == 1
+ assert all('fixture' not in str(row['provider_metadata']) for row in records)
+ assert verify_native_route_contract(tmp_path, manifest, dataset, records)
+
+
+@pytest.mark.parametrize('defect', ['endpoint', 'parameters', 'input_url', 'configuration'])
+def test_native_request_drift_is_rejected_before_second_transport(tmp_path, defect):
+ from unittest.mock import Mock
+ provider, cases, ledger, _, _ = _route_http_fixture(tmp_path)
+ callback = Mock(return_value=object())
+ endpoint = 'https://api.tavily.com/extract'
+ kwargs = {'method': 'POST', 'json_body': {'urls': [cases[0].url], 'format': 'markdown', 'extract_depth': 'advanced', 'include_images': False},
+ 'headers': {'Authorization': 'Bearer fixture'}, 'timeout': 45, 'max_bytes': 20000000}
+ provider.native_http_request(cases[0], callback, endpoint, **kwargs)
+ kwargs['json_body'] = {'urls': [cases[1].url], 'format': 'markdown', 'extract_depth': 'advanced', 'include_images': False}
+ if defect == 'endpoint': endpoint = 'https://api.tavily.com/other'
+ if defect == 'parameters': kwargs['json_body']['format'] = 'html'
+ if defect == 'input_url': kwargs['json_body']['urls'] = ['https://wrong.test/']
+ if defect == 'configuration': provider.config['include_images'] = True
+ with pytest.raises(ValueError, match='Native'):
+ provider.native_http_request(cases[1], callback, endpoint, **kwargs)
+ assert callback.call_count == 1
+ assert ledger.errors
+
+
+@pytest.mark.parametrize('defect', ['missing_receipt', 'signature', 'media', 'freeze', 'scope'])
+def test_release_check_rejects_changed_native_route_evidence(tmp_path, defect):
+ from copy import deepcopy
+ from datetime import datetime, timezone
+ from unittest.mock import Mock
+ from harness.web_fetch.native_routes import verify_native_route_contract
+ provider, cases, ledger, manifest, dataset = _route_http_fixture(tmp_path)
+ records = []
+ for case in cases:
+ provider.native_http_request(case, Mock(), 'https://api.tavily.com/extract',
+ method='POST', json_body={'urls': [case.url], 'format': 'markdown',
+ 'extract_depth': 'advanced', 'include_images': False},
+ headers={'Authorization': 'Bearer fixture'}, timeout=45, max_bytes=20000000)
+ records.append({'provider': provider.name, 'case_id': case.id, 'url': case.url,
+ 'started_at': datetime.now(timezone.utc).isoformat(), 'ok': True,
+ 'provider_metadata': {'native_route_receipt': ledger.receipt(provider, case)}})
+ records = deepcopy(records)
+ if defect == 'missing_receipt': records[0]['provider_metadata'] = {}
+ if defect == 'signature': records[0]['provider_metadata']['native_route_receipt']['requests'][0]['sha256'] = '0'*64
+ if defect == 'media': dataset['cases'][0]['source_benchmark']['input_media_kind'] = 'direct_pdf'
+ if defect == 'freeze': records[0]['started_at'] = '2000-01-01T00:00:00+00:00'
+ if defect == 'scope': records.pop()
+ with pytest.raises(ValueError, match='Native'):
+ verify_native_route_contract(tmp_path, manifest, dataset, records)
+
+
+def test_runner_freezes_route_matrix_before_mocked_provider_calls(tmp_path):
+ import json
+ from dataclasses import asdict
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.providers.http_client import HttpResponse
+ from harness.web_fetch.runner import run_evaluation
+ cases = [EvalCase(label, label, 'https://example.test/'+label, 'control', [],
+ source_benchmark={'input_media_kind': 'html_page'}) for label in ('one', 'two')]
+ dataset = {'tool': 'web_fetch', 'id': 'fixture', 'cases': [asdict(c) for c in cases]}
+ source = tmp_path/'dataset.json'
+ source.write_text(json.dumps(dataset))
+ provider = build_provider('tavily', {'type': 'tavily_extract', 'api_key': 'fixture',
+ 'uniform_native_parameters': True})
+ output = tmp_path/'run'
+ def call(endpoint, **kwargs):
+ frozen = json.loads((output/'native-route-contract.json').read_text())
+ assert frozen['pre_call_frozen'] and len(frozen['cases']) == 2
+ target = kwargs['json_body']['urls'][0]
+ return HttpResponse(200, endpoint, {}, json.dumps({'results': [
+ {'url': target, 'raw_content': 'complete answer'}]}).encode())
+ with patch('harness.web_fetch.providers.tavily.request', side_effect=call) as transport:
+ records, _ = run_evaluation(output_dir=output, dataset_meta=dataset, dataset_path=source,
+ cases=cases, providers=[provider], provider_config_path=tmp_path/'providers.json',
+ runs=1, timeout=45, delay=0, reference_policy='frozen_existing')
+ assert transport.call_count == 2
+ assert len(records) == 2
+ manifest = json.loads((output/'manifest.json').read_text())
+ assert manifest['native_routes_required'] and not manifest['native_route_errors']
+ assert all(r['provider_metadata']['native_route_receipt']['requests'] for r in records)
+
+
+def test_jsonl_worker_receives_declared_media_and_records_actual_packet(tmp_path):
+ import sys
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.native_routes import NativeRouteLedger
+ from harness.web_fetch.providers.base import build_provider
+ script = ('import sys,json\nfor line in sys.stdin:\n'
+ ' r=json.loads(line);print(json.dumps({"ok":True,"content":r["input_media_kind"]}),flush=True)\n')
+ provider = build_provider('worker', {'type': 'subprocess_jsonl', 'command': [sys.executable, '-u', '-c', script],
+ 'repo_path': str(tmp_path), 'uniform_native_parameters': True})
+ case = EvalCase('case', 'Case', 'https://example.test/no-extension', 'control', [], track='document',
+ source_benchmark={'input_media_kind': 'direct_pdf'})
+ ledger = NativeRouteLedger([case], [provider], timeout_seconds=5)
+ ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture')
+ try:
+ result = provider.fetch(case, 5)
+ assert result.ok and result.content == 'direct_pdf'
+ request = ledger.receipt(provider, case)['requests'][0]['signature']['parameters']['request']
+ assert request['input_media_kind'] == 'direct_pdf'
+ assert request['url'] == ''
+ assert str(tmp_path) not in str(request)
+ finally:
+ provider.cancel()
+
+
+@pytest.mark.parametrize('method,module', [
+ ('tavily_extract', 'tavily'), ('exa_contents', 'exa'),
+ ('firecrawl', 'firecrawl'), ('cloudflare_markdown', 'cloudflare'),
+ ('cloudflare_content', 'cloudflare'),
+])
+@pytest.mark.parametrize('track,kind,media', [
+ ('web', 'body', 'html_page'), ('document', 'body', 'direct_pdf'),
+ ('image', 'resource', 'html_page'), ('video', 'resource', 'html_page'),
+ ('file', 'file', 'direct_text'),
+])
+def test_each_http_adapter_matches_its_pre_call_contract(tmp_path, method, module, track, kind, media):
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.native_routes import NativeRouteLedger
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.providers.http_client import HttpResponse
+ config = {'type': method, 'uniform_native_parameters': True,
+ 'api_key': 'fixture', 'api_token': 'fixture', 'account_id': 'fixture',
+ 'include_resource_links': True, 'rich_links_limit': 12, 'include_images': True,
+ 'formats': ['markdown', 'links', 'images'], 'only_main_content': False,
+ 'browser': 'chrome'}
+ provider = build_provider('method', config)
+ case = EvalCase('case', 'Case', 'https://example.test/no-extension', 'control', [],
+ track=track, task_kind=kind, source_benchmark={'input_media_kind': media})
+ ledger = NativeRouteLedger([case], [provider], timeout_seconds=17)
+ ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture')
+ # Deliberately empty native output: this test proves transport selection,
+ # not retrieval success, and cannot accidentally make a live request.
+ with patch('harness.web_fetch.providers.'+module+'.request',
+ return_value=HttpResponse(200, case.url, {}, b'{}')) as transport:
+ provider.fetch(case, 17)
+ assert transport.call_count == 1
+ receipt = ledger.receipt(provider, case)
+ assert not ledger.errors and len(receipt['requests']) == 1
+ cell = next(iter(ledger.cells.values()))
+ assert receipt['requests'][0]['signature']['parameters'] == cell['request_parameters']
+
+
+def test_custom_firecrawl_without_auth_still_matches_actual_request(tmp_path):
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.native_routes import NativeRouteLedger
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.providers.http_client import HttpResponse
+ provider = build_provider('firecrawl', {'type': 'firecrawl', 'base_url': 'https://fixture.test',
+ 'uniform_native_parameters': True})
+ case = EvalCase('case', 'Case', 'https://example.test/', 'control', [],
+ source_benchmark={'input_media_kind': 'html_page'})
+ ledger = NativeRouteLedger([case], [provider])
+ ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture')
+ with patch('harness.web_fetch.providers.firecrawl.request',
+ return_value=HttpResponse(200, case.url, {}, b'{}')) as transport:
+ provider.fetch(case, 45)
+ assert transport.call_count == 1 and not ledger.errors
+
+
+def test_contract_survives_aggregation_and_complete_declared_subset(tmp_path):
+ import json
+ from copy import deepcopy
+ from datetime import datetime, timezone
+ from unittest.mock import Mock
+ from harness.web_fetch.native_routes import (
+ materialize_native_route_contracts, verify_native_execution_manifests,
+ )
+ source, output = tmp_path/'source', tmp_path/'aggregate'
+ source.mkdir()
+ output.mkdir()
+ provider, cases, ledger, manifest, dataset = _route_http_fixture(source)
+ records = []
+ for case in cases:
+ provider.native_http_request(case, Mock(), 'https://api.tavily.com/extract',
+ method='POST', json_body={'urls': [case.url], 'format': 'markdown',
+ 'extract_depth': 'advanced', 'include_images': False},
+ headers={'Authorization': 'Bearer fixture'}, timeout=45, max_bytes=20000000)
+ records.append({'attempt_id': case.id, 'provider': provider.name, 'case_id': case.id,
+ 'url': case.url, 'started_at': datetime.now(timezone.utc).isoformat(), 'ok': True,
+ 'provider_metadata': {'native_route_receipt': ledger.receipt(provider, case)}})
+ manifest['attempt_ids'] = [r['attempt_id'] for r in records]
+ original = (source/'native-route-contract.json').read_bytes()
+ materialize_native_route_contracts(output, source, [manifest])
+ assert (output/manifest['native_route_contract']['path']).read_bytes() == original
+ assert verify_native_execution_manifests(output, [manifest], dataset, records)
+ subset = deepcopy(manifest)
+ subset['attempt_ids'] = [records[0]['attempt_id']]
+ with pytest.raises(ValueError, match='incomplete result scope'):
+ verify_native_execution_manifests(output, [subset], dataset, records)
+ subset['projection'] = {'kind': 'complete-provider-case-subset-from-retained-run'}
+ assert verify_native_execution_manifests(output, [subset], dataset, records)
+ # Even a self-consistent forged request hash cannot change frozen parameters.
+ from harness.web_fetch.native_routes import _digest
+ bad = deepcopy(records)
+ item = bad[0]['provider_metadata']['native_route_receipt']['requests'][0]
+ item['signature']['parameters']['body']['format'] = 'html'
+ item['sha256'] = _digest(item['signature'])
+ with pytest.raises(ValueError, match='frozen interface'):
+ verify_native_execution_manifests(output, [manifest], dataset, bad)
+ contract = output/manifest['native_route_contract']['path']
+ doc = json.loads(contract.read_text())
+ doc['pre_call_frozen'] = False
+ contract.write_text(json.dumps(doc))
+ with pytest.raises(ValueError, match='hash changed'):
+ verify_native_execution_manifests(output, [manifest], dataset, records)
+ with pytest.raises(ValueError, match='bytes changed'):
+ materialize_native_route_contracts(tmp_path/'another', output, [deepcopy(manifest)])
+
+
+@pytest.mark.parametrize('media,kind', [('html_page', 'body'), ('direct_pdf', 'body'),
+ ('html_page', 'resource'), ('direct_text', 'file')])
+def test_moli_actual_cli_obeys_declared_route_without_network(tmp_path, media, kind):
+ import hashlib
+ import sys
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.native_routes import NativeRouteLedger
+ from harness.web_fetch.providers.base import build_provider
+ binary = tmp_path/'fake-moli'
+ binary.write_text('#!'+sys.executable+'\nimport sys\n'
+ 'print("moli 1.1.4" if sys.argv[1:]==["--version"] else "answer")\n')
+ binary.chmod(0o700)
+ provider = build_provider('moli', {'type': 'moli_webfetch', 'binary': str(binary),
+ 'expected_reported_version': 'moli 1.1.4',
+ 'expected_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(),
+ 'uniform_native_parameters': True, 'block_private_networks': False})
+ assert provider.validate() == []
+ case = EvalCase('case', 'Case', 'https://example.test/no-extension', 'control', [],
+ track='document' if kind=='body' else 'image' if kind=='resource' else 'file',
+ task_kind=kind, source_benchmark={'input_media_kind': media})
+ ledger = NativeRouteLedger([case], [provider], timeout_seconds=5)
+ ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture')
+ provider.fetch(case, 5)
+ assert not ledger.errors and len(ledger.receipt(provider, case)['requests']) == 1
+
+
+def test_route_verifier_preserves_optional_dataset_defaults(tmp_path):
+ from datetime import datetime, timezone
+ from unittest.mock import Mock
+ from harness.web_fetch.native_routes import verify_native_route_contract
+ provider, cases, ledger, manifest, dataset = _route_http_fixture(tmp_path)
+ records = []
+ for case in cases:
+ provider.native_http_request(case, Mock(), 'https://api.tavily.com/extract',
+ method='POST', json_body={'urls': [case.url], 'format': 'markdown',
+ 'extract_depth': 'advanced', 'include_images': False},
+ headers={'Authorization': 'Bearer fixture'}, timeout=45, max_bytes=20000000)
+ records.append({'provider': provider.name, 'case_id': case.id, 'url': case.url,
+ 'started_at': datetime.now(timezone.utc).isoformat(), 'ok': True,
+ 'provider_metadata': {'native_route_receipt': ledger.receipt(provider, case)}})
+ for case in dataset['cases']:
+ del case['track']
+ del case['task_kind']
+ assert verify_native_route_contract(tmp_path, manifest, dataset, records)
diff --git a/tests/test_scoring.py b/tests/test_scoring.py
index 5dcb728f4..8a74aec1f 100644
--- a/tests/test_scoring.py
+++ b/tests/test_scoring.py
@@ -699,6 +699,7 @@ def setUp(self):
"legacy": LINKS, "unverified": LINKS.replace("Alpha", "Different"),
"redirected": "[Redirected story](item/1)"}
self.provider = Mock()
+ self.provider.config = {}
self.provider.name, self.provider.type, self.provider.min_interval_seconds = "p", "fixture", 0
self.provider.validate.return_value = []
self.provider.fetch.side_effect = lambda c, timeout: ProviderResult(
diff --git a/tests/test_stable_task_lifecycle.py b/tests/test_stable_task_lifecycle.py
index 389a224cd..37b46fefc 100644
--- a/tests/test_stable_task_lifecycle.py
+++ b/tests/test_stable_task_lifecycle.py
@@ -43,6 +43,7 @@ def test_runner_scores_new_values_and_binds_rule_not_old_answer(self):
path = root/'dataset.json'
path.write_text(json.dumps(dataset))
provider = Mock()
+ provider.config = {}
provider.name, provider.type, provider.min_interval_seconds = 'p', 'synthetic', 0
provider.validate.return_value = []
provider.fetch.return_value = ProviderResult(ok=True, content='# Catalog\n\nNew item\n$99')
From 4e140a1df84a93ce916f45a1cc935e91cd3045a2 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 02:05:03 +0800
Subject: [PATCH 33/50] Create separately bound post-score public evidence
views
---
.agents/skills/benchmark-report/SKILL.md | 2 +-
harness/web_fetch/README.md | 11 ++
harness/web_fetch/publication_privacy.py | 122 +++++++++++++++++++++++
tests/test_public_evidence_views.py | 106 ++++++++++++++++++++
4 files changed, 240 insertions(+), 1 deletion(-)
create mode 100644 tests/test_public_evidence_views.py
diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md
index c34c53439..a7badf6da 100644
--- a/.agents/skills/benchmark-report/SKILL.md
+++ b/.agents/skills/benchmark-report/SKILL.md
@@ -24,7 +24,7 @@ Public report CLIs default to English where supported and expose `--language en|
5. Separate retrieval success, task acceptance, content quality, latency, cost, and freshness. A missing or partial reference makes the affected metric unavailable or scoped; it does not prove success or failure. Do not derive binary acceptance from an unrelated aggregate similarity score.
6. For heterogeneous same-window latency evidence, preserve the full predeclared task-by-method matrix. Show all-call latency on that common denominator separately from successful-call latency on each method's conditional subset. Treat the all-method shared-pass intersection as a diagnostic, never as a post-outcome selector or ranking sample.
7. At aggregate and publication boundaries, copy only explicit hash-bound evidence and public audit files. Reject symlinks and exclude worker trees, host paths, credentials, and internal run logs. Verify primary, auxiliary, delivered-file, and retained retry artifacts before release.
- When a saved-input handoff needs host-metadata redaction, use the shared publication privacy owner. Preserve private originals and distinguish original from public file hashes; rebind dependent metadata references and disclose auxiliary debug-log projections. Never rewrite scored answers, references or response payloads as a privacy repair. Verify unchanged verdicts, timings and review decisions, and rebuild from the public inputs. Keep regenerated package/report wrappers outside the original-input projection receipt so normal rebuilding does not invalidate it or create circular hashes.
+ When a saved-input handoff needs host-metadata redaction, use the shared publication privacy owner. Preserve private originals and distinguish original from public file hashes; rebind dependent metadata references and disclose auxiliary debug-log projections. Never rewrite original scored answers, references or response payloads as a privacy repair. When a scored answer or native envelope itself contains private values, create a separately identified post-score public view through the shared privacy owner, verify its native-field projection and unchanged task verdict, and disclose that omitted original bytes cannot be reconstructed from the public copy. Verify unchanged verdicts, timings and review decisions, and rebuild from the public inputs. Keep regenerated package/report wrappers outside the original-input projection receipt so normal rebuilding does not invalidate it or create circular hashes.
8. Independent result review is opt-in: do not start it unless the user explicitly requests review. When requested without a broader scope, review machine failures only; review machine passes or every attempt only when the user explicitly asks for full review. Label a review performed with known machine outcomes as an evidence audit, separately from independent blind-review coverage; a fresh review-origin tag alone does not establish blindness. Keep the initial verdict blind, but do not equate agreement with correctness: run the anomaly investigation in [audit and revision](references/audit-and-revision.md) for triggered results before calling the review complete. For a large independent review that is safely batchable, prefer resumable Codex CLI jobs with an exact model ID, read-only sandbox, strict output schema, non-overlapping packets, and bounded concurrency; persist each completed packet so a retry never repeats substantive review. Record the exact reviewed/total coverage, accepted unreviewed passes, reviewer, disagreements, adjudications, anomalies, and unresolved items. A review summary never replaces per-attempt evidence.
9. Rerun only calls whose inputs, provider route/version, runtime contract, or source validity changed. Recompute downstream scores and reports without provider calls when only scoring, annotations, or presentation changed. Never retry merely to obtain a better result. A reference or verdict correction must identify the shared root cause and add executable positive/negative controls through the production scorer; bind those regression IDs in the revision receipt instead of patching stored verdicts.
10. Build the public report from the frozen evidence. Every page should answer a reader question. Show task and run identifiers, the exact expected-versus-actual evidence behind a verdict, denominators and unavailable metrics, method differences, limits, concise reproduction instructions, and a compact review history.
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index c2a4299a4..c0c8a853d 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -653,6 +653,17 @@ product retrieval failure. Full native tool/worker envelopes are retained privat
with hashes, separately from scored answer text. Historical outputs without those
envelopes remain insufficient for an exact raw-response audit.
+When those retained bytes contain private host paths or credentials, the shared
+`publication_privacy.project_evidence_view` creates a separate post-score copy
+under `public-evidence/`. Keep the originals private. The bound receipt records
+original and public hashes and byte sizes; `require_public_evidence_view` checks
+the delivered copy without claiming it can reconstruct omitted values. Native
+JSON envelopes are projected structurally to preserve field types. Before using
+such views in a handoff, verify that the public native-field projection still
+matches the public answer and that rescoring it preserves the task verdict.
+Do not replace an original scored binding with a public-view hash or describe a
+redacted reproduction as byte-for-byte recovery of the private call.
+
For native caller comparisons, `dns_mode` is explicitly `system` (default) or
`public_doh`. All six local methods (Moli, Crawl4AI, OpenClaw, DeepSeek, CowAgent and
diff --git a/harness/web_fetch/publication_privacy.py b/harness/web_fetch/publication_privacy.py
index 855292198..f8fb2eb6d 100644
--- a/harness/web_fetch/publication_privacy.py
+++ b/harness/web_fetch/publication_privacy.py
@@ -354,3 +354,125 @@ def require_host_metadata_projection(source, binding):
if public.stat().st_size != item['published_bytes']:
raise ValueError('Published metadata size differs from projection receipt')
return receipt
+
+
+def project_evidence_view(name, data, *, role, host_prefixes=(), host_names=()):
+ """Create a separately identified public view after scoring.
+
+ The caller retains original bytes privately. This function never changes a
+ result, score, original binding or source file. Its receipt identifies both
+ versions; verifying the public hash does not verify unavailable originals.
+ JSON envelopes are traversed structurally so escaped paths are redacted
+ without corrupting strings or rewriting numeric measurements.
+ """
+ path = PurePosixPath(name)
+ if (path.is_absolute() or not path.parts or path.parts[0] != 'public-evidence'
+ or '..' in path.parts or any(part.startswith('.') for part in path.parts)):
+ raise ValueError('Public evidence views require a separate safe public-evidence path')
+ if role not in {'scored_answer', 'native_response'}:
+ raise ValueError('Unknown public evidence role')
+ if any(not p.startswith('/') or p == '/' for p in host_prefixes):
+ raise ValueError('Public evidence host prefixes must be explicit non-root paths')
+ if any(not host for host in host_names):
+ raise ValueError('Public evidence host names must be explicit nonempty identities')
+ counts = {}
+ patterns = [re.compile(re.escape(p.rstrip('/')) + r'''(?=/|$)[^\s"'<>\\\x00]*''')
+ for p in sorted(set(host_prefixes), key=len, reverse=True)]
+ # User-home paths in retained diagnostics need a complete replacement, not
+ # just a username replacement that would disclose the remaining directories.
+ local = re.compile(LOCAL_USER_PATH.pattern + rb'''[^\s"'<>\\\x00]*''')
+
+ def text_view(value):
+ before = value
+ for pattern in patterns:
+ value, count = pattern.subn('[HOST_PATH_OMITTED]', value)
+ counts['local_path'] = counts.get('local_path', 0) + count
+ for host in host_names:
+ count = value.count(host)
+ value = value.replace(host, '[HOSTNAME_OMITTED]')
+ counts['hostname'] = counts.get('hostname', 0) + count
+ raw = value.encode('utf-8')
+ raw, count = local.subn(b'[HOST_PATH_OMITTED]', raw)
+ counts['local_path'] = counts.get('local_path', 0) + count
+ raw, count = OTHER_SECRET.subn(b'[REDACTED_CREDENTIAL]', raw)
+ counts['credential_shape'] = counts.get('credential_shape', 0) + count
+ projected, receipt = project_file(name, raw)
+ if receipt:
+ counts['sensitive_values'] = counts.get('sensitive_values', 0) + receipt['replacements']
+ result = projected.decode('utf-8')
+ return result if result != before else before
+
+ def node_view(value):
+ if isinstance(value, dict):
+ result = {}
+ for key, item in value.items():
+ public_key = text_view(key)
+ if public_key in result:
+ raise ValueError('Public view would merge distinct native fields')
+ result[public_key] = node_view(item)
+ return result
+ if isinstance(value, list):
+ return [node_view(item) for item in value]
+ return text_view(value) if isinstance(value, str) else value
+
+ text = data.decode('utf-8')
+ try:
+ native = json.loads(text)
+ except ValueError:
+ projected = text_view(text).encode('utf-8')
+ encoding = 'utf-8-text'
+ else:
+ public = node_view(native)
+ projected = (json.dumps(public, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
+ if public != native else data)
+ encoding = 'json'
+ risk = publication_risk(projected)
+ if risk:
+ raise ValueError('Public evidence still has a publication risk: '+risk)
+ receipt = {'kind': 'post_score_public_evidence_view', 'role': role, 'path': name,
+ 'encoding': encoding, 'provider_calls_added': 0,
+ 'original_sha256': hashlib.sha256(data).hexdigest(),
+ 'original_bytes': len(data),
+ 'published_sha256': hashlib.sha256(projected).hexdigest(),
+ 'published_bytes': len(projected),
+ 'redactions': {key: count for key, count in counts.items() if count},
+ 'original_retained_privately': True,
+ 'meaning': 'Public-view bytes are separately identified; original scoring bytes and measurements are unchanged. The public view cannot reconstruct omitted private values.'}
+ return projected, receipt
+
+
+def require_public_evidence_view(source, binding):
+ """Verify a declared public view, without claiming original bytes are public."""
+ from .local_revision import read_bound
+ receipt = json.loads(read_bound(source, binding).read_text())
+ name = receipt.get('path', '')
+ path = PurePosixPath(name)
+ redactions = receipt.get('redactions')
+ if (receipt.get('kind') != 'post_score_public_evidence_view'
+ or receipt.get('role') not in {'scored_answer', 'native_response'}
+ or receipt.get('encoding') not in {'json', 'utf-8-text'}
+ or receipt.get('provider_calls_added') != 0
+ or receipt.get('original_retained_privately') is not True
+ or path.is_absolute() or not path.parts or path.parts[0] != 'public-evidence'
+ or '..' in path.parts or any(p.startswith('.') for p in path.parts)
+ or not isinstance(redactions, dict)
+ or any(key not in {'local_path', 'hostname', 'credential_shape', 'sensitive_values'}
+ or type(count) is not int or count < 1 for key, count in redactions.items())):
+ raise ValueError('Invalid separately identified public evidence view')
+ for label in ('original', 'published'):
+ if (not re.fullmatch('[0-9a-f]{64}', str(receipt.get(label+'_sha256', '')))
+ or type(receipt.get(label+'_bytes')) is not int or receipt[label+'_bytes'] < 0):
+ raise ValueError('Invalid original/public evidence identity')
+ same = receipt['original_sha256'] == receipt['published_sha256']
+ if (same and (redactions or receipt['original_bytes'] != receipt['published_bytes'])
+ or not same and not redactions):
+ raise ValueError('Public view identity does not match its declared redactions')
+ public = read_bound(source, {'path': name, 'sha256': receipt['published_sha256']})
+ raw = public.read_bytes()
+ if len(raw) != receipt['published_bytes'] or publication_risk(raw):
+ raise ValueError('Public evidence view bytes or privacy state differ')
+ if receipt['encoding'] == 'json':
+ json.loads(raw)
+ else:
+ raw.decode('utf-8')
+ return receipt
diff --git a/tests/test_public_evidence_views.py b/tests/test_public_evidence_views.py
new file mode 100644
index 000000000..474098f8e
--- /dev/null
+++ b/tests/test_public_evidence_views.py
@@ -0,0 +1,106 @@
+import hashlib
+import json
+
+import pytest
+
+from harness.web_fetch.publication_privacy import project_evidence_view
+from harness.web_fetch.response_input import project_native_answer
+
+
+def test_native_envelope_view_redacts_decoded_paths_without_changing_other_fields():
+ original = json.dumps({'content': 'Article\nSaved to: /Users/example/private/run/body.txt\nDone',
+ 'elapsed_ms': 1234.5, 'count': 7, 'success': True}, ensure_ascii=True).encode()
+ declaration = {'schema_version': 1, 'content_pointer': ['content'], 'content_format': 'text'}
+ scored, _ = project_native_answer(json.loads(original), declaration)
+ public, receipt = project_evidence_view('public-evidence/native/response.json', original,
+ role='native_response')
+ payload = json.loads(public)
+ assert payload['elapsed_ms'] == 1234.5 and payload['count'] == 7 and payload['success'] is True
+ assert payload['content'] == 'Article\nSaved to: [HOST_PATH_OMITTED]\nDone'
+ assert 'private/run' not in public.decode()
+ assert '/Users/example' in scored
+ assert receipt['original_sha256'] == hashlib.sha256(original).hexdigest()
+ assert receipt['published_sha256'] == hashlib.sha256(public).hexdigest()
+ assert receipt['original_sha256'] != receipt['published_sha256']
+ assert receipt['provider_calls_added'] == 0
+
+
+def test_answer_and_envelope_public_views_preserve_native_field_projection():
+ raw = json.dumps({'body': 'Answer\nSaved to: /Users/example/private/file.md\nEnd'}).encode()
+ declaration = {'schema_version': 1, 'content_pointer': ['body'], 'content_format': 'text'}
+ answer, _ = project_native_answer(json.loads(raw), declaration)
+ public_response, _ = project_evidence_view('public-evidence/response.json', raw, role='native_response')
+ public_answer, receipt = project_evidence_view('public-evidence/answer.txt', answer.encode(), role='scored_answer')
+ assert project_native_answer(json.loads(public_response), declaration)[0].encode() == public_answer
+ assert receipt['role'] == 'scored_answer'
+ assert receipt['original_sha256'] == hashlib.sha256(answer.encode()).hexdigest()
+
+
+def test_view_handles_json_escaped_paths_and_retains_clean_original_bytes():
+ raw = br'{"content":"failed at \u002fUsers\u002fexample\u002fprivate\u002frun.txt","count":4}'
+ public, _ = project_evidence_view('public-evidence/response.json', raw, role='native_response')
+ assert json.loads(public)['content'] == 'failed at [HOST_PATH_OMITTED]'
+ clean = b'{ "count": 4, "content": "Normal body" }\n'
+ assert project_evidence_view('public-evidence/response.json', clean, role='native_response')[0] == clean
+
+
+def test_view_hides_explicit_temp_paths_and_credential_shapes():
+ raw = b'Failed at /private/tmp/example/session.log\nkey: ghp_' + b'X'*32
+ public, receipt = project_evidence_view('public-evidence/answer.txt', raw, role='scored_answer',
+ host_prefixes=['/private/tmp'])
+ assert b'/private/tmp' not in public and b'example/session' not in public
+ assert b'ghp_' not in public
+ assert receipt['redactions']['credential_shape'] == 1
+
+
+@pytest.mark.parametrize('name', ['private_artifacts/response.bin', '../response.txt',
+ '/public-evidence/response.txt', 'public-evidence/.env'])
+def test_view_rejects_paths_that_could_confuse_original_and_public_evidence(name):
+ with pytest.raises(ValueError, match='separate safe'):
+ project_evidence_view(name, b'answer', role='scored_answer')
+
+
+def test_view_does_not_accept_unknown_roles_or_binary_guessing():
+ with pytest.raises(ValueError, match='role'):
+ project_evidence_view('public-evidence/a.txt', b'a', role='reference')
+ with pytest.raises(UnicodeDecodeError):
+ project_evidence_view('public-evidence/a.txt', b'\xff\xfe', role='native_response')
+
+
+def test_redaction_cannot_merge_distinct_native_metadata_fields():
+ raw = json.dumps({'/Users/one/a': 1, '/Users/two/b': 2}).encode()
+ with pytest.raises(ValueError, match='merge distinct'):
+ project_evidence_view('public-evidence/response.json', raw, role='native_response')
+
+
+def test_saved_public_view_verifier_rejects_changed_bytes_or_false_identity(tmp_path):
+ from harness.web_fetch.publication_privacy import require_public_evidence_view
+ name = 'public-evidence/answer.txt'
+ public, receipt = project_evidence_view(name, b'Answer /Users/example/private/a.txt', role='scored_answer')
+ path = tmp_path/name
+ path.parent.mkdir()
+ path.write_bytes(public)
+ proof = tmp_path/'view-receipt.json'
+ def binding():
+ proof.write_text(json.dumps(receipt))
+ return {'path': proof.name, 'sha256': hashlib.sha256(proof.read_bytes()).hexdigest()}
+ assert require_public_evidence_view(tmp_path, binding()) == receipt
+ path.write_bytes(b'different')
+ with pytest.raises(ValueError, match='binding changed'):
+ require_public_evidence_view(tmp_path, binding())
+ path.write_bytes(public)
+ receipt['original_sha256'] = receipt['published_sha256']
+ with pytest.raises(ValueError, match='identity does not match'):
+ require_public_evidence_view(tmp_path, binding())
+
+
+def test_public_view_verifier_rejects_symlink_even_with_matching_bytes(tmp_path):
+ from harness.web_fetch.publication_privacy import require_public_evidence_view
+ public, receipt = project_evidence_view('public-evidence/a.txt', b'answer', role='scored_answer')
+ (tmp_path/'public-evidence').mkdir()
+ (tmp_path/'original.txt').write_bytes(public)
+ (tmp_path/'public-evidence/a.txt').symlink_to(tmp_path/'original.txt')
+ raw = json.dumps(receipt).encode()
+ (tmp_path/'receipt.json').write_bytes(raw)
+ with pytest.raises(ValueError):
+ require_public_evidence_view(tmp_path, {'path': 'receipt.json', 'sha256': hashlib.sha256(raw).hexdigest()})
From f43a06ea543fe4bdf3bab7e4685a9dbb77fb44a6 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 03:54:43 +0800
Subject: [PATCH 34/50] Bind redacted public evidence through frozen report
rebuilds
---
.agents/skills/benchmark-report/SKILL.md | 2 +-
harness/web_fetch/README.md | 10 +-
harness/web_fetch/adjudication_evidence.py | 27 +-
harness/web_fetch/artifacts.py | 30 +-
harness/web_fetch/local_revision.py | 9 +-
harness/web_fetch/publication_privacy.py | 427 ++++++++++++++++++---
harness/web_fetch/report.py | 32 +-
harness/web_fetch/report_build.py | 21 +-
harness/web_fetch/report_check.py | 60 ++-
harness/web_fetch/safety.py | 23 ++
tests/test_frozen_evidence_copy.py | 38 ++
tests/test_public_evidence_presentation.py | 35 ++
tests/test_public_evidence_views.py | 320 +++++++++++++++
13 files changed, 925 insertions(+), 109 deletions(-)
create mode 100644 tests/test_frozen_evidence_copy.py
create mode 100644 tests/test_public_evidence_presentation.py
diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md
index a7badf6da..d4fa7bae1 100644
--- a/.agents/skills/benchmark-report/SKILL.md
+++ b/.agents/skills/benchmark-report/SKILL.md
@@ -24,7 +24,7 @@ Public report CLIs default to English where supported and expose `--language en|
5. Separate retrieval success, task acceptance, content quality, latency, cost, and freshness. A missing or partial reference makes the affected metric unavailable or scoped; it does not prove success or failure. Do not derive binary acceptance from an unrelated aggregate similarity score.
6. For heterogeneous same-window latency evidence, preserve the full predeclared task-by-method matrix. Show all-call latency on that common denominator separately from successful-call latency on each method's conditional subset. Treat the all-method shared-pass intersection as a diagnostic, never as a post-outcome selector or ranking sample.
7. At aggregate and publication boundaries, copy only explicit hash-bound evidence and public audit files. Reject symlinks and exclude worker trees, host paths, credentials, and internal run logs. Verify primary, auxiliary, delivered-file, and retained retry artifacts before release.
- When a saved-input handoff needs host-metadata redaction, use the shared publication privacy owner. Preserve private originals and distinguish original from public file hashes; rebind dependent metadata references and disclose auxiliary debug-log projections. Never rewrite original scored answers, references or response payloads as a privacy repair. When a scored answer or native envelope itself contains private values, create a separately identified post-score public view through the shared privacy owner, verify its native-field projection and unchanged task verdict, and disclose that omitted original bytes cannot be reconstructed from the public copy. Verify unchanged verdicts, timings and review decisions, and rebuild from the public inputs. Keep regenerated package/report wrappers outside the original-input projection receipt so normal rebuilding does not invalidate it or create circular hashes.
+ When a saved-input handoff needs host-metadata redaction, use the shared publication privacy owner. Preserve private originals and distinguish original from public file hashes; rebind dependent metadata references and disclose auxiliary debug-log projections. Never rewrite original scored answers, references or response payloads as a privacy repair. When a scored answer or native envelope itself contains private values, create a separately identified post-score public view through the shared privacy owner, verify its native-field projection and unchanged task verdict, and disclose that omitted original bytes cannot be reconstructed from the public copy. A public frozen presentation must explicitly bind its complete view inventory, verify displayed bytes against those views, keep acquisition gates strict, and distinguish preserved original diagnostics from what redacted text can reproduce. Verify unchanged verdicts, timings and review decisions, and rebuild from the public inputs. Keep regenerated package/report wrappers outside the original-input projection receipt so normal rebuilding does not invalidate it or create circular hashes.
8. Independent result review is opt-in: do not start it unless the user explicitly requests review. When requested without a broader scope, review machine failures only; review machine passes or every attempt only when the user explicitly asks for full review. Label a review performed with known machine outcomes as an evidence audit, separately from independent blind-review coverage; a fresh review-origin tag alone does not establish blindness. Keep the initial verdict blind, but do not equate agreement with correctness: run the anomaly investigation in [audit and revision](references/audit-and-revision.md) for triggered results before calling the review complete. For a large independent review that is safely batchable, prefer resumable Codex CLI jobs with an exact model ID, read-only sandbox, strict output schema, non-overlapping packets, and bounded concurrency; persist each completed packet so a retry never repeats substantive review. Record the exact reviewed/total coverage, accepted unreviewed passes, reviewer, disagreements, adjudications, anomalies, and unresolved items. A review summary never replaces per-attempt evidence.
9. Rerun only calls whose inputs, provider route/version, runtime contract, or source validity changed. Recompute downstream scores and reports without provider calls when only scoring, annotations, or presentation changed. Never retry merely to obtain a better result. A reference or verdict correction must identify the shared root cause and add executable positive/negative controls through the production scorer; bind those regression IDs in the revision receipt instead of patching stored verdicts.
10. Build the public report from the frozen evidence. Every page should answer a reader question. Show task and run identifiers, the exact expected-versus-actual evidence behind a verdict, denominators and unavailable metrics, method differences, limits, concise reproduction instructions, and a compact review history.
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index c0c8a853d..8fdb93fbe 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -662,7 +662,15 @@ JSON envelopes are projected structurally to preserve field types. Before using
such views in a handoff, verify that the public native-field projection still
matches the public answer and that rescoring it preserves the task verdict.
Do not replace an original scored binding with a public-view hash or describe a
-redacted reproduction as byte-for-byte recovery of the private call.
+redacted reproduction as byte-for-byte recovery of the private call. Public
+frozen presentations declare `publication_evidence_views` in their presentation
+binding. The shared gate requires complete current answer/native coverage,
+checks the actual displayed aliases, rejects private artifact trees, and keeps
+ordinary acquisition/aggregation checks strict. Redacted excerpts can verify
+visible text, not original private coordinates. Report rebuilding uses saved
+original scores; changed public text does not reproduce every original text
+quality diagnostic. Disk-backed metadata projection and independent file clones
+keep large histories practical without hard-linking writable outputs to inputs.
For native caller comparisons, `dns_mode` is explicitly `system` (default) or
diff --git a/harness/web_fetch/adjudication_evidence.py b/harness/web_fetch/adjudication_evidence.py
index c58519e55..3ae306ac9 100644
--- a/harness/web_fetch/adjudication_evidence.py
+++ b/harness/web_fetch/adjudication_evidence.py
@@ -2,7 +2,7 @@
import hashlib
import json
-from .safety import contained_path
+from .publication_privacy import read_evidence_binding, evidence_excerpt_matches
TEMPLATE = '核对已有原始返回、冻结要求和相应修正证据；当前结论见评分明细。'
@@ -131,9 +131,8 @@ def require_adjudication_evidence(source, dataset, records, review):
or not proof.get('rule_ids')):
raise ValueError('Overridden review lacks specific rule/evidence: ' + row['attempt_id'])
require_rule_assertions(cases[record['case_id']], record, row, proof)
- raw = contained_path(source, record['artifact_path'], reject_symlinks=True).read_bytes()
- if hashlib.sha256(raw).hexdigest() != proof['content_sha256']:
- raise ValueError('Adjudication answer changed')
+ answer_binding = {'path': record['artifact_path'], 'sha256': proof['content_sha256']}
+ raw = read_evidence_binding(source, answer_binding).read_bytes()
text = raw.decode('utf-8', errors='replace')
samples = proof.get('samples', [])
if not text:
@@ -142,10 +141,7 @@ def require_adjudication_evidence(source, dataset, records, review):
elif not samples:
raise ValueError('Adjudication lacks an answer excerpt')
for sample in samples:
- start, end = sample.get('start'), sample.get('end')
- if (type(start) is not int or type(end) is not int
- or not 0 <= start < end <= len(text)
- or text[start:end] != sample.get('text')):
+ if not evidence_excerpt_matches(source, answer_binding, sample, text):
raise ValueError('Adjudication excerpt differs from original answer')
checked += 1
return checked
@@ -234,17 +230,13 @@ def _require_supplemental_reviews(source, dataset, records, review):
or not row.get('agent_evidence', '').strip()
or row.get('agent_verdict') not in {'pass', 'fail', 'insufficient'}):
raise ValueError('Supplemental review lacks bound task and answer evidence')
- raw = contained_path(source, record['artifact_path'], reject_symlinks=True).read_bytes()
- if hashlib.sha256(raw).hexdigest() != row['content_sha256']:
- raise ValueError('Supplemental review answer changed')
+ answer_binding = {'path': record['artifact_path'], 'sha256': row['content_sha256']}
+ raw = read_evidence_binding(source, answer_binding).read_bytes()
text = raw.decode('utf-8', errors='replace')
if text and not proof.get('samples'):
raise ValueError('Supplemental review lacks an answer excerpt')
for sample in proof.get('samples', []):
- start, end = sample.get('start'), sample.get('end')
- if (type(start) is not int or type(end) is not int
- or not 0 <= start < end <= len(text)
- or text[start:end] != sample.get('text')):
+ if not evidence_excerpt_matches(source, answer_binding, sample, text):
raise ValueError('Supplemental review excerpt differs')
seen.update(identifiers)
if fresh and seen != fresh:
@@ -317,9 +309,8 @@ def require_retained_reviews(source, dataset, records, review):
or opinion.get('final_verdict') not in {'pass', 'fail'}
or row.get('final_verdict') != opinion['final_verdict']):
raise ValueError('Retained review answer, opinion or verdict changed')
- raw = contained_path(source, record['artifact_path'], reject_symlinks=True).read_bytes()
- if hashlib.sha256(raw).hexdigest() != row.get('content_sha256'):
- raise ValueError('Retained review answer changed')
+ read_evidence_binding(source, {'path': record['artifact_path'],
+ 'sha256': row.get('content_sha256')})
fresh = sum(r.get('review_origin') == 'fresh' for r in review['attempts'])
if (review.get('fresh_reviewed_attempts') != fresh
or fresh + len(ids) != len(review['attempts'])):
diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py
index 3d35041de..97ca6c3a4 100644
--- a/harness/web_fetch/artifacts.py
+++ b/harness/web_fetch/artifacts.py
@@ -59,7 +59,7 @@ def record_public_call_log_bindings(record: dict[str, Any]) -> list[dict[str, An
for path, (digest, size) in seen.items()]
-def verify_public_call_log(root: Path, binding: dict[str, Any]) -> Path:
+def verify_public_call_log(root: Path, binding: dict[str, Any], *, allow_public_views=False) -> Path:
"""Verify one public call log's containment, bytes, digest and closed schema."""
normalized = public_call_log_binding(binding)
if normalized is None: # pragma: no cover - caller supplied an object
@@ -130,21 +130,29 @@ def verify_public_call_log(root: Path, binding: dict[str, Any]) -> Path:
output_path = Path(str(payload["output"]["path"]))
if not output_path.parts or output_path.parts[0] != "artifacts":
raise ValueError("Public call log output must stay under artifacts")
- output = contained_path(root, output_path)
- if not output.is_file() or output.is_symlink():
- raise ValueError("Public call log output is missing or unsafe")
- output_bytes = output.read_bytes()
claimed = payload["output"]
- if (claimed["bytes"] != len(output_bytes)
- or claimed["sha256"] != hashlib.sha256(output_bytes).hexdigest()
- or claimed["empty"] is not (len(output_bytes) == 0)):
- raise ValueError("Public call log output binding changed")
+ if allow_public_views:
+ from .publication_privacy import read_evidence_binding, original_evidence_size
+ answer = {'path': output_path.as_posix(), 'sha256': claimed['sha256']}
+ read_evidence_binding(root, answer)
+ size = original_evidence_size(root, answer)
+ if claimed['bytes'] != size or claimed['empty'] is not (size == 0):
+ raise ValueError("Public call log original output binding changed")
+ else:
+ output = contained_path(root, output_path)
+ if not output.is_file() or output.is_symlink():
+ raise ValueError("Public call log output is missing or unsafe")
+ output_bytes = output.read_bytes()
+ if (claimed["bytes"] != len(output_bytes)
+ or claimed["sha256"] != hashlib.sha256(output_bytes).hexdigest()
+ or claimed["empty"] is not (len(output_bytes) == 0)):
+ raise ValueError("Public call log output binding changed")
return path
-def verify_record_public_call_logs(root: Path, record: dict[str, Any]) -> None:
+def verify_record_public_call_logs(root: Path, record: dict[str, Any], *, allow_public_views=False) -> None:
for binding in record_public_call_log_bindings(record):
- verify_public_call_log(root, binding)
+ verify_public_call_log(root, binding, allow_public_views=allow_public_views)
def raw_response_binding(metadata: dict[str, Any]) -> dict[str, Any] | None:
diff --git a/harness/web_fetch/local_revision.py b/harness/web_fetch/local_revision.py
index 224c82c2d..5a4e2431c 100644
--- a/harness/web_fetch/local_revision.py
+++ b/harness/web_fetch/local_revision.py
@@ -12,10 +12,8 @@
def read_bound(source, binding):
- path = contained_path(source, binding['path'], reject_symlinks=True)
- if hashlib.sha256(path.read_bytes()).hexdigest() != binding.get('sha256'):
- raise ValueError('Local revision evidence binding changed: ' + binding['path'])
- return path
+ from .publication_privacy import read_evidence_binding
+ return read_evidence_binding(source, binding)
def require_local_task_replacement(source, binding, dataset, records):
@@ -97,12 +95,13 @@ def require_local_revision(source, binding, dataset, records):
def _require_native_answer_correction(source, original, corrected, proof):
"""Verify a derived answer against the same immutable native response."""
from .response_input import project_native_answer
+ from .publication_privacy import original_evidence_size
if proof.get('attempt_id') != original['attempt_id']:
raise ValueError('Native answer correction lost its attempt identity')
native = read_bound(source, proof['native_response'])
metadata = original.get('provider_metadata', {})
if (proof['native_response']['sha256'] != metadata.get('raw_response_sha256')
- or native.stat().st_size != metadata.get('raw_response_bytes')):
+ or original_evidence_size(source, proof['native_response']) != metadata.get('raw_response_bytes')):
raise ValueError('Native answer correction changed its original response')
before = proof['original_artifact']
after = proof['corrected_artifact']
diff --git a/harness/web_fetch/publication_privacy.py b/harness/web_fetch/publication_privacy.py
index f8fb2eb6d..30771c20c 100644
--- a/harness/web_fetch/publication_privacy.py
+++ b/harness/web_fetch/publication_privacy.py
@@ -4,6 +4,7 @@
import json
import re
from copy import deepcopy
+from functools import lru_cache
from pathlib import PurePosixPath, PureWindowsPath
from urllib.parse import unquote_plus
@@ -76,7 +77,7 @@ def project_execution_manifest(manifest: dict) -> dict:
)
JSON_SENSITIVE_VALUE = re.compile(
rb'(?i)(["\']' + SENSITIVE_KEY + rb'["\']\s*:\s*["\'])(?!<|\[)'
- rb'((?:\\.|[^"\'\r\n])*)(["\'])'
+ rb'((?:\\.|[^"\'\r\n])+)(["\'])'
)
LINE_SENSITIVE_VALUE = re.compile(
rb'(?im)(^|[ \t])(' + SENSITIVE_KEY + rb'[ \t]*[:=])'
@@ -98,33 +99,88 @@ def project_execution_manifest(manifest: dict) -> dict:
)
-def _redact_plain_url_queries(text):
- """Redact sensitive parameters in URLs embedded in diagnostic prose."""
- replacements = 0
+_URL_ENTITY = re.compile(r'&(?:\#[xX][0-9A-Fa-f]+;?|\#[0-9]+;?|[A-Za-z][A-Za-z0-9]+;)')
+
- def redact_url(match):
- nonlocal replacements
- url = match.group()
- query = url.partition('?')[2].partition('#')[0]
- if not query:
- return url
- parts = []
- for item in query.split('&'):
- key, separator, value = item.partition('=')
- # HTML attributes can spell a separator as &.
- logical_key = key.removeprefix('amp;')
- decoded_key = unquote_plus(logical_key).casefold()
- decoded_value = unquote_plus(value)
- sensitive = bool(QUERY_SENSITIVE_KEY.fullmatch(decoded_key))
- placeholder = (decoded_value.startswith('<') and decoded_value.endswith('>')
- or decoded_value == '[REDACTED]')
- if separator and sensitive and value and not placeholder:
- item = key + '=[REDACTED]'
- replacements += 1
- parts.append(item)
- return url.replace('?' + query, '?' + '&'.join(parts), 1)
-
- return EMBEDDED_URL.sub(redact_url, text), replacements
+def _url_entity_surface(raw):
+ """Decode URL entities while retaining each character's original span."""
+ units = [(char, index, index+1) for index, char in enumerate(raw)]
+ for _ in range(8):
+ text = ''.join(unit[0] for unit in units)
+ matches = [m for m in _URL_ENTITY.finditer(text) if html.unescape(m.group()) != m.group()]
+ if not matches:
+ return text, units
+ result, cursor = [], 0
+ for match in matches:
+ result.extend(units[cursor:match.start()])
+ start, end = units[match.start()][1], units[match.end()-1][2]
+ result.extend((char, start, end) for char in html.unescape(match.group()))
+ cursor = match.end()
+ result.extend(units[cursor:])
+ units = result
+ raise ValueError('Excessively nested URL entities require explicit privacy review')
+
+
+def _redact_plain_url_queries(text):
+ """Redact actual query values, preserving URL/Markdown/entity syntax."""
+ replacements, cursor, output = 0, 0, []
+ while match := EMBEDDED_URL.search(text, cursor):
+ start, end = match.span()
+ markdown = bool(re.search(r'\]\([ \t]*$', text[max(0, start-100):start]))
+ if markdown:
+ depth = 0
+ for index in range(start, end):
+ if text[index] == '(':
+ depth += 1
+ elif text[index] == ')':
+ if depth == 0:
+ end = index
+ break
+ depth -= 1
+ raw = text[start:end]
+ suffix = ''
+ if not markdown:
+ for opening, closing in (('(', ')'), ('[', ']'), ('{', '}')):
+ while raw.endswith(closing) and raw.count(closing) > raw.count(opening):
+ raw, suffix = raw[:-1], closing + suffix
+ if _URL_ENTITY.search(raw):
+ url, units = _url_entity_surface(raw)
+ stop = next((i for i, ch in enumerate(url) if ch in '\"\'<>\\\x00' or ch.isspace()), len(url))
+ if stop < len(url):
+ # An encoded HTML attribute delimiter is not query content.
+ used = units[stop][1]
+ raw, end, suffix = raw[:used], start+used, ''
+ url, units = url[:stop], units[:stop]
+ else:
+ url, units = raw, None
+ query_start = url.find('?')
+ changes = []
+ if query_start >= 0:
+ query_end = url.find('#', query_start)
+ if query_end < 0:
+ query_end = len(url)
+ position = query_start+1
+ for item in url[position:query_end].split('&'):
+ key, separator, value = item.partition('=')
+ logical_key = key
+ while logical_key.startswith('amp;'):
+ logical_key = logical_key[4:]
+ decoded_key, decoded_value = unquote_plus(logical_key).casefold(), unquote_plus(value)
+ sensitive = bool(QUERY_SENSITIVE_KEY.fullmatch(decoded_key))
+ placeholder = (decoded_value.startswith('<') and decoded_value.endswith('>')
+ or decoded_value == '[REDACTED]')
+ if separator and sensitive and value and not placeholder:
+ left, right = position+len(key)+1, position+len(item)
+ changes.append((units[left][1] if units else left,
+ units[right-1][2] if units else right))
+ replacements += 1
+ position += len(item)+1
+ for left, right in reversed(changes):
+ raw = raw[:left]+'[REDACTED]'+raw[right:]
+ output.extend((text[cursor:start], raw+suffix))
+ cursor = end
+ output.append(text[cursor:])
+ return ''.join(output), replacements
_JSON_STRING = re.compile(r'"(?:\\.|[^"\\\x00-\x1f])*"')
@@ -173,20 +229,75 @@ def auth_matches(data):
return distinct
-def publication_risk(data):
- """Return a public-bundle risk class without exposing the matched value."""
+def _plain_publication_risk(data):
+ """Inspect actual text, excluding serialization delimiters."""
decoded=html.unescape(data.decode('utf-8',errors='surrogateescape')).encode('utf-8',errors='surrogateescape')
if GOOGLE_KEY.search(decoded) or OTHER_SECRET.search(decoded):
return 'credential-shaped value'
if auth_matches(decoded):
return 'sensitive field value'
- if redact_url_queries(decoded.decode('utf-8', errors='surrogateescape'))[1]:
+ # Bare query parameters such as ×tamp must not be interpreted as the
+ # semicolon-less HTML entity ×. Decode explicit named entities and
+ # numeric references for URL inspection while preserving raw separators.
+ text = data.decode('utf-8', errors='surrogateescape')
+ url_surface = re.sub(r'&(?:\#[xX][0-9A-Fa-f]+;?|\#[0-9]+;?|[A-Za-z][A-Za-z0-9]+;)',
+ lambda match: html.unescape(match.group()), text)
+ if redact_url_queries(url_surface)[1]:
return 'sensitive URL query value'
if LOCAL_USER_PATH.search(decoded):
return 'machine-local user path'
return None
+def _scan_publication_risk(data):
+ """Inspect JSON values structurally, otherwise inspect the actual text.
+
+ A serialized quote after a string such as ``Authorization:`` is not an
+ authorization value. Nonempty sensitive fields and real header values still
+ fail closed, including values hidden by JSON string escapes.
+ """
+ try:
+ payload = json.loads(data)
+ except (ValueError, UnicodeDecodeError):
+ return _plain_publication_risk(data)
+
+ def inspect(value, *, secret=False):
+ if isinstance(value, dict):
+ for key, item in value.items():
+ risk = _plain_publication_risk(key.encode('utf-8'))
+ if risk:
+ return risk
+ sensitive = secret or bool(re.fullmatch(SENSITIVE_KEY.decode('ascii'), key, re.I))
+ risk = inspect(item, secret=sensitive)
+ if risk:
+ return risk
+ elif isinstance(value, list):
+ for item in value:
+ risk = inspect(item, secret=secret)
+ if risk:
+ return risk
+ elif isinstance(value, str):
+ if (secret and value and not ((value.startswith('<') and value.endswith('>'))
+ or (value.startswith('[') and value.endswith(']')))):
+ return 'sensitive field value'
+ return _plain_publication_risk(value.encode('utf-8'))
+ return None
+
+ return inspect(payload)
+
+
+@lru_cache(maxsize=8)
+def _cached_publication_risk(data):
+ return _scan_publication_risk(data)
+
+
+def publication_risk(data):
+ """Reuse checks only for identical immutable bytes; bound retained memory."""
+ data = bytes(data)
+ return (_cached_publication_risk(data) if len(data) <= 2_000_000
+ else _scan_publication_risk(data))
+
+
def project_file(name, data):
"""Redact only auxiliary HTML. Any match elsewhere requires human review."""
path = PurePosixPath(name)
@@ -254,7 +365,7 @@ def _host_metadata_projection_path(name):
return metadata or auxiliary_log
-def project_host_metadata_files(files, host_prefixes, *, host_names=()):
+def project_host_metadata_files(files, host_prefixes, *, host_names=(), public_excerpt_views=False, output_store=None):
"""Project host metadata/auxiliary logs and explicitly rebind dependencies.
Inputs and measured payloads are immutable. Returned receipts distinguish
@@ -263,24 +374,33 @@ def project_host_metadata_files(files, host_prefixes, *, host_names=()):
"""
if not host_prefixes or any(not p.startswith('/') or p == '/' for p in host_prefixes):
raise ValueError('Explicit non-root host prefixes are required')
- patterns = [re.compile(re.escape(p.rstrip('/')) + r"(?=/|$)[^\s\"'<>\x00]*")
+ patterns = [re.compile(re.escape(p.rstrip('/')) + r"(?=/|$)[^\s\"'<>\\\x00]*")
for p in sorted(set(host_prefixes), key=len, reverse=True)]
json_names = {name for name in files if name.endswith(('.json', '.jsonl'))}
- parsed = {}
+ original_hashes = {name: hashlib.sha256(files[name]).hexdigest() for name in files}
+ by_hash = {digest: name for name, digest in original_hashes.items()}
+ # Prewarm hash dependencies before parsing a large parent. Disk-backed
+ # mappings can therefore project multi-GB histories one document at a time.
+ dependencies = {}
for name in json_names:
- text = files[name].decode('utf-8')
- parsed[name] = ([json.loads(line) for line in text.splitlines() if line.strip()]
- if name.endswith('.jsonl') else json.loads(text))
- by_hash = {hashlib.sha256(raw).hexdigest(): name for name, raw in files.items()}
- projected, published_hashes, changed_fields, visiting = {}, {}, {}, set()
+ hashes = {match.group(1).decode('ascii') for match in
+ re.finditer(rb'"([0-9a-f]{64})"(?!\s*:)', files[name])}
+ dependencies[name] = {by_hash[digest] for digest in hashes
+ if digest in by_hash and by_hash[digest] != name}
+ projected = output_store if output_store is not None else {}
+ published_hashes, changed_fields, visiting = {}, {}, set()
+ original_sizes, published_sizes, changed_names = {}, {}, set()
def project(name):
- if name in projected:
- return projected[name]
+ if name in published_hashes:
+ return
if name in visiting:
raise ValueError('Cyclic publication metadata dependencies')
visiting.add(name)
- raw = files[name]
+ for dependency in sorted(dependencies.get(name, set())):
+ project(dependency)
+ original = raw = files[name]
+ original_sizes[name] = len(original)
fields = []
def value(node, path=''):
if isinstance(node, dict):
@@ -304,35 +424,46 @@ def value(node, path=''):
if not host:
raise ValueError('Host names must be explicit nonempty identities')
result = result.replace(host, '[HOSTNAME_OMITTED]')
+ if public_excerpt_views and (path.endswith('.samples[].text')
+ or path.endswith('.agent_evidence_quotes[]') or path.endswith('.evidence_quotes[]')):
+ public, _ = project_evidence_view('public-evidence/excerpt.txt', result.encode('utf-8'),
+ role='scored_answer', host_prefixes=host_prefixes, host_names=host_names,
+ content_format='text')
+ result = public.decode('utf-8')
if result != node:
fields.append(path)
return result
- if name in parsed:
- result = value(parsed[name])
- if result != parsed[name]:
+ if name in json_names:
+ text = raw.decode('utf-8')
+ parsed = ([json.loads(line) for line in text.splitlines() if line.strip()]
+ if name.endswith('.jsonl') else json.loads(text))
+ result = value(parsed)
+ if result != parsed:
raw = ((''.join(json.dumps(row, ensure_ascii=False, separators=(',', ':')) + '\n'
for row in result)) if name.endswith('.jsonl')
else json.dumps(result, ensure_ascii=False, indent=2) + '\n').encode()
elif host_names:
raw = value(raw.decode('utf-8'), 'auxiliary_log').encode('utf-8')
- if raw != files[name] and not _host_metadata_projection_path(name):
+ if raw != original and not _host_metadata_projection_path(name):
raise ValueError('Host projection must not alter scored or response payloads')
+ if raw != original:
+ changed_names.add(name)
projected[name] = raw
+ published_sizes[name] = len(raw)
published_hashes[name] = hashlib.sha256(raw).hexdigest()
changed_fields[name] = sorted(set(fields))
visiting.remove(name)
- return raw
for name in files:
project(name)
receipt = {'kind': 'host_metadata_publication_projection',
'provider_calls_added': 0,
'meaning': 'Original identities are retained below; published hashes identify redacted copies only.',
- 'files': [{'path': name, 'original_sha256': hashlib.sha256(files[name]).hexdigest(),
- 'published_sha256': hashlib.sha256(projected[name]).hexdigest(),
- 'original_bytes': len(files[name]), 'published_bytes': len(projected[name]),
+ 'files': [{'path': name, 'original_sha256': original_hashes[name],
+ 'published_sha256': published_hashes[name],
+ 'original_bytes': original_sizes[name], 'published_bytes': published_sizes[name],
'redacted_fields': changed_fields[name]}
- for name in sorted(files) if files[name] != projected[name]]}
+ for name in sorted(changed_names)]}
return projected, receipt
@@ -356,7 +487,7 @@ def require_host_metadata_projection(source, binding):
return receipt
-def project_evidence_view(name, data, *, role, host_prefixes=(), host_names=()):
+def project_evidence_view(name, data, *, role, host_prefixes=(), host_names=(), content_format="auto"):
"""Create a separately identified public view after scoring.
The caller retains original bytes privately. This function never changes a
@@ -371,6 +502,8 @@ def project_evidence_view(name, data, *, role, host_prefixes=(), host_names=()):
raise ValueError('Public evidence views require a separate safe public-evidence path')
if role not in {'scored_answer', 'native_response'}:
raise ValueError('Unknown public evidence role')
+ if content_format not in {'auto', 'text', 'json'}:
+ raise ValueError('Unknown public evidence content format')
if any(not p.startswith('/') or p == '/' for p in host_prefixes):
raise ValueError('Public evidence host prefixes must be explicit non-root paths')
if any(not host for host in host_names):
@@ -409,23 +542,36 @@ def node_view(value):
public_key = text_view(key)
if public_key in result:
raise ValueError('Public view would merge distinct native fields')
- result[public_key] = node_view(item)
+ if (re.fullmatch(SENSITIVE_KEY.decode('ascii'), key, re.I)
+ and isinstance(item, str) and item
+ and not ((item.startswith('<') and item.endswith('>'))
+ or (item.startswith('[') and item.endswith(']')))):
+ result[public_key] = '[REDACTED]'
+ counts['sensitive_values'] = counts.get('sensitive_values', 0) + 1
+ else:
+ result[public_key] = node_view(item)
return result
if isinstance(value, list):
return [node_view(item) for item in value]
return text_view(value) if isinstance(value, str) else value
text = data.decode('utf-8')
- try:
- native = json.loads(text)
- except ValueError:
+ if content_format == 'text':
projected = text_view(text).encode('utf-8')
encoding = 'utf-8-text'
else:
- public = node_view(native)
- projected = (json.dumps(public, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
- if public != native else data)
- encoding = 'json'
+ try:
+ native = json.loads(text)
+ except ValueError:
+ if content_format == 'json':
+ raise ValueError('Declared JSON public evidence is not JSON')
+ projected = text_view(text).encode('utf-8')
+ encoding = 'utf-8-text'
+ else:
+ public = node_view(native)
+ projected = (json.dumps(public, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
+ if public != native else data)
+ encoding = 'json'
risk = publication_risk(projected)
if risk:
raise ValueError('Public evidence still has a publication risk: '+risk)
@@ -476,3 +622,166 @@ def require_public_evidence_view(source, binding):
else:
raw.decode('utf-8')
return receipt
+
+
+@lru_cache(maxsize=4)
+def _public_view_inventory_index(raw):
+ # Cache parsing by exact file bytes, never by a path or timestamp. Every
+ # caller still reads and checks the inventory's declared SHA-256.
+ index = json.loads(raw)
+ if (index.get('kind') != 'post_score_public_evidence_view_inventory'
+ or index.get('provider_calls_added') != 0):
+ raise ValueError('Invalid public evidence view inventory')
+ entries = {}
+ for entry in index.get('files', []):
+ key = entry.get('original_path'), entry.get('original_sha256')
+ if key in entries:
+ raise ValueError('Ambiguous public evidence view identity')
+ entries[key] = entry
+ return entries
+
+
+def bound_public_view(source, binding):
+ """Resolve only a view explicitly bound by this snapshot's public profile.
+
+ A returned view verifies public bytes plus the recorded original identity;
+ it does not claim that omitted original bytes have been reconstructed.
+ """
+ from pathlib import Path
+ from .safety import contained_path
+ source = Path(source)
+ profile_path = contained_path(source, 'presentation-binding.json', reject_symlinks=True)
+ if not profile_path.is_file():
+ return None
+ profile = json.loads(profile_path.read_text())
+ declared = profile.get('publication_evidence_views')
+ if not declared:
+ return None
+ index_path = contained_path(source, declared['path'], reject_symlinks=True)
+ raw = index_path.read_bytes()
+ if hashlib.sha256(raw).hexdigest() != declared.get('sha256'):
+ raise ValueError('Public evidence view inventory hash changed')
+ entries = _public_view_inventory_index(raw)
+ entry = entries.get((binding.get('path'), binding.get('sha256')))
+ if entry is None:
+ return None
+ receipt = require_public_evidence_view(source, entry['view_receipt'])
+ original_root = PurePosixPath(binding['path']).parts[0]
+ expected_root = {'scored_answer': 'artifacts', 'native_response': 'private_artifacts'}[receipt['role']]
+ if original_root != expected_root:
+ raise ValueError('Public payload views cannot replace metadata or source authority')
+ if receipt['original_sha256'] != binding['sha256']:
+ raise ValueError('Public view does not identify the requested original')
+ return contained_path(source, receipt['path'], reject_symlinks=True), receipt
+
+
+def read_evidence_binding(source, binding):
+ """Verify original bytes, or return the explicitly disclosed public view."""
+ from .safety import contained_path
+ path = contained_path(source, binding['path'], reject_symlinks=True)
+ if path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == binding.get('sha256'):
+ return path
+ public = bound_public_view(source, binding)
+ if public is None:
+ raise ValueError('Local revision evidence binding changed: ' + binding['path'])
+ return public[0]
+
+
+def original_evidence_size(source, binding):
+ """Return verified original size, or its explicitly recorded private size."""
+ path = read_evidence_binding(source, binding)
+ if hashlib.sha256(path.read_bytes()).hexdigest() == binding['sha256']:
+ return path.stat().st_size
+ public = bound_public_view(source, binding)
+ if public is None:
+ raise ValueError('Public evidence lacks its original-size disclosure')
+ return public[1]['original_bytes']
+
+
+def evidence_excerpt_matches(source, binding, sample, text):
+ """Validate exact original offsets, or a disclosed redacted-view excerpt.
+
+ Original coordinates remain recorded. A redacted public view can verify the
+ visible excerpt's presence, not its former private-byte/character position.
+ """
+ start, end = sample.get('start'), sample.get('end')
+ if type(start) is not int or type(end) is not int or not 0 <= start < end:
+ return False
+ excerpt = sample.get('text')
+ if not isinstance(excerpt, str) or not excerpt:
+ return False
+ if end <= len(text) and text[start:end] == excerpt:
+ return True
+ public = bound_public_view(source, binding)
+ return bool(public and public[1]['redactions'] and excerpt in text)
+
+
+def require_public_evidence_view_inventory(source, binding, records):
+ """Require complete current-answer/native-response view coverage at release."""
+ from .local_revision import read_bound
+ from .safety import contained_path
+ if (source/'private_artifacts').exists():
+ raise ValueError('Public evidence snapshot must not contain private artifact trees')
+ index = json.loads(read_bound(source, binding).read_text())
+ if (index.get('kind') != 'post_score_public_evidence_view_inventory'
+ or index.get('provider_calls_added') != 0):
+ raise ValueError('Invalid public evidence view inventory')
+ entries = index.get('files', [])
+ wanted = {(row['artifact_path'], row['metrics']['content_sha256']) for row in records}
+ for row in records:
+ metadata = row.get('provider_metadata', {})
+ if metadata.get('raw_response_path'):
+ wanted.add((metadata['raw_response_path'], metadata['raw_response_sha256']))
+ identities = set()
+ for entry in entries:
+ identity = entry.get('original_path'), entry.get('original_sha256')
+ if identity in identities:
+ raise ValueError('Public evidence view inventory repeats an original')
+ identities.add(identity)
+ receipt = require_public_evidence_view(source, entry['view_receipt'])
+ root = PurePosixPath(identity[0]).parts
+ expected_root = {'scored_answer': 'artifacts', 'native_response': 'private_artifacts'}[receipt['role']]
+ if (not root or root[0] != expected_root or '..' in root
+ or receipt['original_sha256'] != identity[1]):
+ raise ValueError('Public evidence view inventory changes an original authority')
+ if identity in wanted and receipt['role'] == 'scored_answer':
+ alias = contained_path(source, identity[0], reject_symlinks=True)
+ if (not alias.is_file()
+ or hashlib.sha256(alias.read_bytes()).hexdigest() != receipt['published_sha256']):
+ raise ValueError('Displayed answer differs from its declared public view')
+ if not wanted <= identities:
+ raise ValueError('Public evidence view inventory omits current evidence')
+ return index
+
+
+def public_render_views(source, records):
+ """Read bound display annotations; never rewrite saved result records."""
+ from .local_revision import read_bound
+ profile_path = source/'presentation-binding.json'
+ if not profile_path.is_file():
+ return {}
+ profile = json.loads(profile_path.read_text())
+ declared = profile.get('publication_evidence_views')
+ if not declared:
+ return {}
+ index = json.loads(read_bound(source, declared).read_text())
+ entries = {(e['original_path'], e['original_sha256']): e for e in index['files']}
+ result = {}
+ for row in records:
+ metadata = row.get('provider_metadata', {})
+ wanted = {'scored_answer': (row['artifact_path'], row['metrics']['content_sha256'])}
+ if metadata.get('raw_response_path'):
+ wanted['native_response'] = (metadata['raw_response_path'], metadata['raw_response_sha256'])
+ views = {}
+ for role, identity in wanted.items():
+ if identity not in entries:
+ raise ValueError('Public rendering omits a declared evidence view')
+ binding = entries[identity]['view_receipt']
+ receipt = json.loads(read_bound(source, binding).read_text())
+ if receipt['role'] != role or receipt['original_sha256'] != identity[1]:
+ raise ValueError('Public rendering changed its original evidence identity')
+ views[role] = {key: receipt[key] for key in ('path', 'original_sha256', 'published_sha256',
+ 'original_bytes', 'published_bytes', 'redactions')}
+ views[role]['receipt'] = binding['path']
+ result[row['attempt_id']] = views
+ return result
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 1460f7803..bab784f94 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -590,6 +590,8 @@ def run_log_section(out,record,output):
payload=logs[-1]['payload'] if logs else {}
process=payload.get('process') if isinstance(payload.get('process'),dict) else {}
timing=payload.get('timing') if isinstance(payload.get('timing'),dict) else {}
+ native_public=metadata.get('publication_views',{}).get('native_response')
+ native_public_link=('下载{esc(log["label"])}记录 ' for log in logs)
if artifact_relative:
if raw_record:raw_record+=' · '
- raw_record+=f'下载本次保存的原始输出 '
+ _answer_view=metadata.get('publication_views',{}).get('scored_answer')
+ if _answer_view and _answer_view['original_sha256']!=_answer_view['published_sha256']:
+ raw_record+=f''+model_text('下载本次输出公开副本','Download public output view')+' '
+ else:
+ raw_record+=f'下载本次保存的原始输出 '
+ raw_record+=native_public_link
public_response=metadata.get('public_response_path')
if public_response:
response_view=diagnostic_view_page(out,public_response,'完整原始响应','产品实际返回；公开副本已隐藏凭据与本机路径。')
@@ -683,16 +690,17 @@ def run_log_section(out,record,output):
raw_record+=(f'下载完整原始 debug 日志 '
f'（{diagnostic_size:,} 字节 · SHA-256：{esc(diagnostic_sha)} · 截断：{diagnostic_truncated}）')
if not raw_record:raw_record='这次历史调用没有可下载的原始记录'
+ evidence_identity_note=(model_text('评分原件私下保留；公开下载隐藏目录或敏感值，校验记录分别标明原件与公开副本。','Scoring originals remain private; public downloads hide paths or sensitive values and bind original/public identities separately.') if any(view.get('redactions') for view in metadata.get('publication_views',{}).values()) else '原始输出保持调用后保存的字节。')
return (f'运行日志与证据 '
f'
{fields} '
f'
查看完整运行字段 {full_fields} '
f'
结论说明： {result_note}
'
f'
查看本次真实日志 {diagnostic}
'
f'
原始运行证据 '
- f'
{raw_record}。运行记录保留请求状态、耗时和证据路径；原始输出保持调用后保存的字节。
{raw_record}。运行记录保留请求状态、耗时和证据路径；{evidence_identity_note}
')
def pct(n):return '—' if n is None else f'{n*100:.1f}%'
def ms(n):return '—' if n is None else f'{n:,.0f} ms'
-def link(url,label):return f'{esc(label)} '
+def link(url,label,english=None):return f'{model_text(label,english)} '
def cell(value,sort=None):return '{value}'
def model_text(value,english=None):
return esc(value) if english is None else ''+esc(value)+' '
@@ -1875,6 +1883,11 @@ def local_audit_banner(dataset):
return '本地审核版 本页合并两个不同日期的 Moli 调用批次，保留每例的真实调用时间。本次仅更换报告展示代码，没有新增 Moli 调用，也不代表正式发布。
{esc(m.get("content_sha256"))}'))
+ _answer_view=r.get('provider_metadata',{}).get('publication_views',{}).get('scored_answer')
+ if _answer_view and _answer_view['original_sha256']!=_answer_view['published_sha256']:
+ rows[7]=('',cell(model_text('公开副本字符','Public-view characters'))+cell(f'{len(output):,}'))
+ rows[8]=('',cell(model_text('评分时输出校验值','Original scored output hash'))+cell(f'{esc(m.get("content_sha256"))}'))
+ rows.append(('',cell(model_text('公开副本校验值','Public-view hash'))+cell(f'{esc(_answer_view["published_sha256"])}')))
if 'content_check_total' in m:
rows.append(('',cell('文字核对覆盖')+cell(pct(m.get('content_check_coverage'))+f' · {m["content_check_hits"]}/{m["content_check_total"]} 组')))
if c.get('task_kind')=='resource':
@@ -2184,7 +2202,11 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
corruption=(f'实际输出存在解码损坏： '
f'含 {m.get("replacement_character_count",0):,} 个 �，占 {pct(m.get("replacement_character_ratio"))}。'
'这些字符在保存前已丢失，不能从当前 Markdown 还原。
')
+ answer_view=r.get('provider_metadata',{}).get('publication_views',{}).get('scored_answer')
output_note=(f'{len(output):,} 字符 · '+link('../'+r['artifact_path'],'下载原始输出'))
+ if answer_view and answer_view['original_sha256']!=answer_view['published_sha256']:
+ output_note=(f'{len(output):,} 字符 · '+link('../'+answer_view['path'],'下载公开副本','Download public view')+' · '+link('../'+answer_view['receipt'],'查看脱敏记录','View redaction record')+
+ '{esc(readable_output(output))} ' if output else
f'{esc(empty_output_reason(r,c,output))}
')
@@ -2542,7 +2564,9 @@ def option_text(value):
if local_matrix and frozen_binding:
public_profile=json.loads((out/'presentation-binding.json').read_text())
if public_profile.get('publication_projection'):
- reproduce+=''+model_text('公开副本已去掉本机目录信息。清单分别记录原始文件与公开副本的校验值；公开校验值不冒充原始调用标识。网页返回、成绩、耗时和复核结论保持不变。','This public copy omits host paths. The manifest distinguishes original and public file hashes; public hashes do not claim to be original call identities. Answers, scores, timings and review decisions are unchanged.')+' '+link('../'+public_profile['publication_projection']['path'],'PUBLICATION-PROJECTION.json')+'
'
+ reproduce+=''+model_text('公开副本已去掉本机目录信息。清单分别记录原始文件与公开副本的校验值；公开校验值不冒充原始调用标识。成绩、耗时和复核结论保持不变。','This public copy omits host paths. The manifest distinguishes original and public file hashes; public hashes do not claim to be original call identities. Scores, timings and review decisions are unchanged.')+' '+link('../'+public_profile['publication_projection']['path'],'PUBLICATION-PROJECTION.json')+'
'
+ if public_profile.get('publication_evidence_views'):
+ reproduce+=''+model_text('评分时的原件私下保留；公开回答和完整原生响应隐藏了目录或敏感值。清单保留评分原件的标识，并单独记录公开副本。公开文件可重建本报告，不能还原隐藏内容；正文质量等指标保留原件的计算值，不保证用脱敏文字逐字重算一致。复核片段的坐标记录原件位置；公开副本仅能核对仍可见的片段。','Scoring originals remain private. Public answers and complete native responses hide paths or sensitive values. The inventory preserves original scoring identities and separately binds public views. Public files rebuild this report but cannot reconstruct omitted content. Text-quality diagnostics retain the original calculations and may not reproduce exactly from redacted text. Review coordinates refer to originals; public views verify only the visible excerpts.')+' '+link('../'+public_profile['publication_evidence_views']['path'],'PUBLIC-EVIDENCE-VIEWS.json')+'
'
support+=section('核心复现指令',reproduce)
write(out,'reproduce/index.html','环境与复现',support)
if not authored_interpretation and not local_matrix:
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index d6c30db3a..bf661fea5 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -206,7 +206,7 @@ def _build_frozen_presentation(source, output, presentation_binding, *,
show_local_status_banners):
"""Render a bound historical report using saved scores and public evidence."""
from harness.report_languages import select_report_language
- from .safety import evidence_tree_identity
+ from .safety import evidence_tree_identity, copy_evidence_file
if source in output.parents:
raise ValueError("Frozen presentation output must be outside its source tree")
before = evidence_tree_identity(source)
@@ -224,7 +224,7 @@ def _build_frozen_presentation(source, output, presentation_binding, *,
output.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".frozen-presentation-", dir=output.parent) as work:
staging = Path(work) / "report"
- shutil.copytree(source, staging)
+ shutil.copytree(source, staging, copy_function=copy_evidence_file)
shutil.rmtree(staging / "en", ignore_errors=True)
shutil.copy2(presentation_binding, staging / "presentation-binding.json")
batches = json.loads((source / "execution.json").read_text())["batches"]
@@ -1306,6 +1306,9 @@ def validate_local_presentation(output: Path, binding_path: Path):
require_host_metadata_projection(output, binding['publication_projection'])
dataset = json.loads((output / 'dataset.json').read_text())
records = load_jsonl(contained_path(output, 'results.jsonl'))
+ if binding.get('publication_evidence_views'):
+ from .publication_privacy import require_public_evidence_view_inventory
+ require_public_evidence_view_inventory(output, binding['publication_evidence_views'], records)
from .native_routes import verify_native_execution_manifests
execution = output/'execution.json'
if execution.is_file():
@@ -1339,10 +1342,16 @@ def validate_local_presentation(output: Path, binding_path: Path):
or any(item['case_id'] not in excluded for item in reliability['unjudged']))):
raise ValueError('Presentation refresh requires the complete declared matrix')
for row in records:
- artifact = contained_path(output, row['artifact_path'])
- if hashlib.sha256(artifact.read_bytes()).hexdigest() != row['metrics']['content_sha256']:
- raise ValueError('Saved output changed: ' + row['attempt_id'])
- verify_record_public_call_logs(output, row)
+ from .publication_privacy import read_evidence_binding
+ try:
+ read_evidence_binding(output, {'path': row['artifact_path'],
+ 'sha256': row['metrics']['content_sha256']})
+ except ValueError as error:
+ if binding.get('publication_evidence_views'):
+ raise
+ raise ValueError('Saved output changed: ' + row['attempt_id']) from error
+ verify_record_public_call_logs(output, row,
+ allow_public_views=bool(binding.get('publication_evidence_views')))
return binding, dataset, records, manifest
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index b31966cfe..b1b8102f7 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -18,6 +18,49 @@
from harness.web_fetch.storage import require_canonical_inputs, require_current_report_profile
+def require_public_evidence_page(content, relative, views):
+ """Require inspectable public downloads and honest original/public hashes."""
+ from bs4 import BeautifulSoup
+ from urllib.parse import unquote, urljoin
+ content = re.sub(r'<(pre|script|style)\b[^>]*>.*?', '', content, flags=re.I | re.S)
+ soup = BeautifulSoup(content, 'html.parser')
+ visible = soup.get_text(' ', strip=True)
+ if ' None:
f"Rollback failed; original data retained at {backup}"
) from recovery_error
raise
+
+
+def copy_evidence_file(source, destination):
+ """Copy independent evidence bytes, using APFS copy-on-write when available.
+
+ Never hard-link frozen inputs: writing a generated copy must not alter its
+ source. Unsupported filesystems/platforms retain ordinary copy semantics.
+ """
+ import shutil
+ source, destination = unlinked_path(Path(source)), unlinked_path(Path(destination))
+ if sys.platform == 'darwin':
+ import ctypes
+ try:
+ clone = ctypes.CDLL(None, use_errno=True).clonefile
+ except (AttributeError, OSError):
+ clone = None
+ if clone is not None:
+ clone.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int]
+ clone.restype = ctypes.c_int
+ if clone(os.fsencode(source), os.fsencode(destination), 0) == 0:
+ shutil.copystat(source, destination)
+ return str(destination)
+ return shutil.copy2(source, destination)
diff --git a/tests/test_frozen_evidence_copy.py b/tests/test_frozen_evidence_copy.py
new file mode 100644
index 000000000..627f3883b
--- /dev/null
+++ b/tests/test_frozen_evidence_copy.py
@@ -0,0 +1,38 @@
+from unittest.mock import patch
+
+import pytest
+
+from harness.web_fetch.safety import copy_evidence_file
+
+
+def test_frozen_copy_has_independent_identity_and_writes(tmp_path):
+ source, target = tmp_path/'source.txt', tmp_path/'target.txt'
+ source.write_text('saved measurement')
+ copy_evidence_file(source, target)
+ assert target.read_bytes() == source.read_bytes()
+ assert target.stat().st_ino != source.stat().st_ino
+ target.write_text('generated presentation')
+ assert source.read_text() == 'saved measurement'
+
+
+def test_unsupported_clone_falls_back_to_an_independent_copy(tmp_path):
+ source, target = tmp_path/'source.txt', tmp_path/'target.txt'
+ source.write_bytes(b'saved bytes')
+ with patch('harness.web_fetch.safety.sys.platform', 'darwin'), \
+ patch('ctypes.CDLL', side_effect=OSError('clone unavailable')):
+ copy_evidence_file(source, target)
+ assert target.read_bytes() == b'saved bytes'
+ assert target.stat().st_ino != source.stat().st_ino
+
+
+@pytest.mark.parametrize('side', ['source', 'destination'])
+def test_frozen_copy_rejects_symlinked_inputs_or_outputs(tmp_path, side):
+ real = tmp_path/'real.txt'
+ real.write_bytes(b'private original')
+ link = tmp_path/'link.txt'
+ link.symlink_to(real)
+ source = link if side == 'source' else real
+ target = link if side == 'destination' else tmp_path/'copy.txt'
+ with pytest.raises(ValueError, match='symlink'):
+ copy_evidence_file(source, target)
+ assert real.read_bytes() == b'private original'
diff --git a/tests/test_public_evidence_presentation.py b/tests/test_public_evidence_presentation.py
new file mode 100644
index 000000000..128af5910
--- /dev/null
+++ b/tests/test_public_evidence_presentation.py
@@ -0,0 +1,35 @@
+import pytest
+from harness.web_fetch.report_check import require_public_evidence_page
+
+
+@pytest.fixture
+def specimen():
+ answer = {'original_sha256': 'a'*64, 'published_sha256': 'b'*64,
+ 'path': 'public-evidence/answer.txt', 'receipt': 'public-evidence/answer-receipt.json'}
+ native = {'path': 'public-evidence/native.json', 'receipt': 'public-evidence/native-receipt.json'}
+ return {'scored_answer': answer, 'native_response': native}
+
+
+def page(views, prefix='../'):
+ answer = views['scored_answer']
+ links = ''.join('Public download '
+ for item in views.values() for key,v in item.items() if key in {'path','receipt'})
+ return ''+answer['original_sha256']+' '+answer['published_sha256']+'
'+links+''
+
+
+@pytest.mark.parametrize('relative,prefix', [('attempts/p.html','../'),('en/attempts/p.html','../../')])
+def test_public_evidence_links_work_in_both_editions(specimen,relative,prefix):
+ require_public_evidence_page(page(specimen,prefix),relative,specimen)
+
+
+@pytest.mark.parametrize('defect', ['original_hash','public_hash','native_link','receipt_link','escaped_markup'])
+def test_release_rejects_missing_identity_or_download_evidence(specimen,defect):
+ html = page(specimen)
+ if defect == 'original_hash': html = html.replace('a'*64, 'wrong original')
+ if defect == 'public_hash': html = html.replace('b'*64, 'wrong public')
+ if defect == 'native_link': html = html.replace('../public-evidence/native.json','../unrelated.json')
+ if defect == 'receipt_link': html = html.replace('../public-evidence/answer-receipt.json','../unrelated.json')
+ if defect == 'escaped_markup': html = html.replace('Public download','<span data-report-en="Download">Download</span>',1)
+ # Payload text cannot stand in for a working reader-facing evidence link.
+ html += ''+('a'*64)+' '+('b'*64)+' public-evidence/native.json '
+ with pytest.raises(ValueError):require_public_evidence_page(html,'attempts/p.html',specimen)
diff --git a/tests/test_public_evidence_views.py b/tests/test_public_evidence_views.py
index 474098f8e..00aa4349e 100644
--- a/tests/test_public_evidence_views.py
+++ b/tests/test_public_evidence_views.py
@@ -104,3 +104,323 @@ def test_public_view_verifier_rejects_symlink_even_with_matching_bytes(tmp_path)
(tmp_path/'receipt.json').write_bytes(raw)
with pytest.raises(ValueError):
require_public_evidence_view(tmp_path, {'path': 'receipt.json', 'sha256': hashlib.sha256(raw).hexdigest()})
+
+
+def _declared_view_snapshot(tmp_path):
+ original = b'Beginning\nSaved to: /Users/example/private/result.txt\nStable ending'
+ public_name = 'public-evidence/answer.txt'
+ public, receipt = project_evidence_view(public_name, original, role='scored_answer')
+ (tmp_path/'public-evidence').mkdir()
+ (tmp_path/public_name).write_bytes(public)
+ receipt_raw = json.dumps(receipt).encode()
+ (tmp_path/'receipt.json').write_bytes(receipt_raw)
+ original_binding = {'path': 'artifacts/method/case/run.txt',
+ 'sha256': hashlib.sha256(original).hexdigest()}
+ alias = tmp_path/original_binding['path']
+ alias.parent.mkdir(parents=True, exist_ok=True)
+ alias.write_bytes(public)
+ index = {'kind': 'post_score_public_evidence_view_inventory', 'provider_calls_added': 0,
+ 'files': [{'original_path': original_binding['path'],
+ 'original_sha256': original_binding['sha256'],
+ 'view_receipt': {'path': 'receipt.json', 'sha256': hashlib.sha256(receipt_raw).hexdigest()}}]}
+ index_raw = json.dumps(index).encode()
+ (tmp_path/'public-views.json').write_bytes(index_raw)
+ profile = {'publication_evidence_views': {'path': 'public-views.json',
+ 'sha256': hashlib.sha256(index_raw).hexdigest()}}
+ (tmp_path/'presentation-binding.json').write_text(json.dumps(profile))
+ return original_binding, original, public, index, profile
+
+
+def test_bound_view_requires_explicit_profile_and_preserves_original_identity(tmp_path):
+ from harness.web_fetch.publication_privacy import read_evidence_binding, original_evidence_size
+ binding, original, public, _, _ = _declared_view_snapshot(tmp_path)
+ assert read_evidence_binding(tmp_path, binding).read_bytes() == public
+ assert original_evidence_size(tmp_path, binding) == len(original)
+ assert len(public) != len(original)
+ (tmp_path/'presentation-binding.json').unlink()
+ with pytest.raises(ValueError, match='binding changed'):
+ read_evidence_binding(tmp_path, binding)
+
+
+@pytest.mark.parametrize('defect', ['inventory_hash', 'wrong_original', 'metadata_alias', 'duplicate'])
+def test_bound_view_rejects_changed_inventory_and_unrelated_authority(tmp_path, defect):
+ from harness.web_fetch.publication_privacy import read_evidence_binding
+ binding, _, _, index, profile = _declared_view_snapshot(tmp_path)
+ if defect == 'inventory_hash':
+ (tmp_path/'public-views.json').write_text('{}')
+ else:
+ if defect == 'wrong_original': binding['sha256'] = '0'*64
+ if defect == 'metadata_alias':
+ binding['path'] = 'dataset.json'
+ index['files'][0]['original_path'] = binding['path']
+ if defect == 'duplicate': index['files'].append(index['files'][0].copy())
+ raw = json.dumps(index).encode()
+ (tmp_path/'public-views.json').write_bytes(raw)
+ profile['publication_evidence_views']['sha256'] = hashlib.sha256(raw).hexdigest()
+ (tmp_path/'presentation-binding.json').write_text(json.dumps(profile))
+ with pytest.raises(ValueError):
+ read_evidence_binding(tmp_path, binding)
+
+
+def test_public_excerpt_keeps_original_coordinates_without_claiming_reconstruction(tmp_path):
+ from harness.web_fetch.publication_privacy import evidence_excerpt_matches
+ binding, original, public, _, _ = _declared_view_snapshot(tmp_path)
+ excerpt = 'Stable ending'
+ start = original.decode().index(excerpt)
+ sample = {'start': start, 'end': start+len(excerpt), 'text': excerpt}
+ assert evidence_excerpt_matches(tmp_path, binding, sample, public.decode())
+ assert public.decode()[start:start+len(excerpt)] != excerpt
+ assert not evidence_excerpt_matches(tmp_path, binding, {**sample, 'text': 'invented evidence'}, public.decode())
+ (tmp_path/'presentation-binding.json').unlink()
+ assert not evidence_excerpt_matches(tmp_path, binding, sample, public.decode())
+
+
+def test_native_custom_secret_fields_are_redacted_structurally():
+ raw = json.dumps({'content': 'Stable answer', 'headers': {'authorization': 'Custom opaque-value'},
+ 'meta': {'access_token': 'opaque-session', 'api_key': ''},
+ 'token_count': 28}).encode()
+ public, receipt = project_evidence_view('public-evidence/response.json', raw, role='native_response')
+ payload = json.loads(public)
+ assert payload['headers']['authorization'] == '[REDACTED]'
+ assert payload['meta']['access_token'] == '[REDACTED]'
+ assert payload['meta']['api_key'] == ''
+ assert payload['content'] == 'Stable answer' and payload['token_count'] == 28
+ assert receipt['redactions']['sensitive_values'] == 2
+
+
+def test_text_projection_keeps_native_json_looking_body_format():
+ body = '{\n "path": "/Users/example/private/result.txt",\n "value": 3\n}'
+ raw = json.dumps({'content': body}).encode()
+ declaration = {'schema_version': 1, 'content_pointer': ['content'], 'content_format': 'text'}
+ public_native, _ = project_evidence_view('public-evidence/response.json', raw, role='native_response')
+ public_answer, receipt = project_evidence_view('public-evidence/answer.txt', body.encode(),
+ role='scored_answer', content_format='text')
+ assert project_native_answer(json.loads(public_native), declaration)[0].encode() == public_answer
+ assert receipt['encoding'] == 'utf-8-text'
+ assert b'\n "value": 3\n' in public_answer
+
+
+def test_release_inventory_requires_complete_current_evidence(tmp_path):
+ from harness.web_fetch.publication_privacy import require_public_evidence_view_inventory
+ original, _, _, index, profile = _declared_view_snapshot(tmp_path)
+ records = [{'artifact_path': original['path'], 'metrics': {'content_sha256': original['sha256']}}]
+ assert require_public_evidence_view_inventory(tmp_path, profile['publication_evidence_views'], records)
+ records[0]['provider_metadata'] = {'raw_response_path': 'private_artifacts/response.bin',
+ 'raw_response_sha256': '1'*64}
+ with pytest.raises(ValueError, match='omits current evidence'):
+ require_public_evidence_view_inventory(tmp_path, profile['publication_evidence_views'], records)
+
+
+def test_empty_native_token_is_not_a_secret_but_nonempty_value_is_masked():
+ from harness.web_fetch.publication_privacy import publication_risk
+ empty = b'{"metadata":{"csrf-token":""},"content":"Stable body"}'
+ assert publication_risk(empty) is None
+ assert project_evidence_view('public-evidence/response.json', empty, role='native_response')[0] == empty
+ nonempty = b'{"metadata":{"csrf-token":"opaque-private-value"},"content":"Stable body"}'
+ assert publication_risk(nonempty) == 'sensitive field value'
+ public, _ = project_evidence_view('public-evidence/response.json', nonempty, role='native_response')
+ assert json.loads(public)['metadata']['csrf-token'] == '[REDACTED]'
+ assert publication_risk(public) is None
+
+
+def test_serialized_header_name_has_no_value_but_real_header_is_sensitive():
+ from harness.web_fetch.publication_privacy import publication_risk
+ raw = json.dumps({'header_names': ['Authorization:', 'Proxy-Authorization:'], 'count': 3}).encode()
+ assert publication_risk(raw) is None
+ assert project_evidence_view('public-evidence/response.json', raw, role='native_response')[0] == raw
+ private = json.dumps({'header': 'Authorization: Custom opaque-private-value'}).encode()
+ assert publication_risk(private) == 'sensitive field value'
+ public, _ = project_evidence_view('public-evidence/response.json', private, role='native_response')
+ assert b'opaque-private-value' not in public and publication_risk(public) is None
+
+
+def test_json_escaped_sensitive_values_are_still_detected():
+ from harness.web_fetch.publication_privacy import publication_risk
+ assert publication_risk(br'{"content":"failed at \u002fUsers\u002fexample\u002fprivate\u002fa"}') == 'machine-local user path'
+ assert publication_risk(br'{"\u0061pi_key":"private-value"}') == 'sensitive field value'
+
+
+def test_public_call_log_view_support_does_not_weaken_acquisition_checks(tmp_path):
+ from tests.test_public_call_logs import run, ResultProvider
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.artifacts import verify_record_public_call_logs
+ binding, original, public, index, profile = _declared_view_snapshot(tmp_path)
+ row = run(tmp_path/'run', ResultProvider(ProviderResult(True, original.decode())))
+ root = tmp_path/'run'
+ # Install only the explicit public view proof into the simulated public root.
+ import shutil
+ shutil.copytree(tmp_path/'public-evidence', root/'public-evidence')
+ shutil.copy2(tmp_path/'receipt.json', root/'receipt.json')
+ index['files'][0]['original_path'] = row['artifact_path']
+ raw = json.dumps(index).encode()
+ (root/'public-views.json').write_bytes(raw)
+ profile['publication_evidence_views']['sha256'] = hashlib.sha256(raw).hexdigest()
+ (root/'presentation-binding.json').write_text(json.dumps(profile))
+ (root/row['artifact_path']).write_bytes(public)
+ with pytest.raises(ValueError, match='output binding changed'):
+ verify_record_public_call_logs(root, row)
+ verify_record_public_call_logs(root, row, allow_public_views=True)
+ (root/'presentation-binding.json').unlink()
+ with pytest.raises(ValueError):
+ verify_record_public_call_logs(root, row, allow_public_views=True)
+
+
+def test_report_display_annotations_do_not_replace_original_score_hash(tmp_path):
+ from harness.web_fetch.publication_privacy import public_render_views
+ binding, _, _, _, _ = _declared_view_snapshot(tmp_path)
+ row = {'attempt_id': 'p__case', 'artifact_path': binding['path'],
+ 'metrics': {'content_sha256': binding['sha256']}}
+ before = json.dumps(row, sort_keys=True)
+ fields = public_render_views(tmp_path, [row])['p__case']['scored_answer']
+ assert fields['original_sha256'] == row['metrics']['content_sha256']
+ assert fields['published_sha256'] != fields['original_sha256']
+ assert json.dumps(row, sort_keys=True) == before
+
+
+@pytest.mark.parametrize('url', ['https://example.test/item?auth_key=opaque-private',
+ 'https://example.test/wiki/Function_(math)?token=opaque-private'])
+def test_query_redaction_preserves_markdown_link_boundary(url):
+ from harness.web_fetch.publication_privacy import redact_url_queries
+ original = '[Stable title]('+url+')'
+ public, count = redact_url_queries(original)
+ assert count == 1 and 'opaque-private' not in public
+ assert public.startswith('[Stable title](') and public.endswith(')')
+ assert public.count('(') == original.count('(')
+ assert public.count(')') == original.count(')')
+ assert public == original.replace('opaque-private', '[REDACTED]')
+ assert redact_url_queries(public) == (public, 0)
+
+
+def test_redacting_one_url_cannot_remove_adjacent_markdown_records():
+ from harness.web_fetch.publication_privacy import redact_url_queries
+ from harness.web_fetch.reference_quality import markdown_links
+ original = ('[Photo](https://cdn.test/image?auth_key=private-value)'
+ '[Item](https://example.test/pins/123)'
+ '[Other](https://example.test/pins/456)')
+ public, count = redact_url_queries(original)
+ assert count == 1
+ assert public == original.replace('private-value', '[REDACTED]')
+ assert len(markdown_links(public)) == len(markdown_links(original))
+ assert 'https://example.test/pins/123' in public and 'https://example.test/pins/456' in public
+
+
+def test_unwrapped_url_token_with_parenthesis_is_fully_redacted():
+ from harness.web_fetch.publication_privacy import redact_url_queries
+ public, count = redact_url_queries('https://example.test/api?token=private)continued')
+ assert count == 1 and 'continued' not in public and 'private' not in public
+
+
+def test_public_review_excerpts_use_view_policy_without_changing_task_rules():
+ from harness.web_fetch.publication_privacy import project_host_metadata_files
+ excerpt = 'Example /home/example/private/file.txt and stable body'
+ files = {'review.json': json.dumps({'attempts': [{'adjudication_evidence': {
+ 'samples': [{'start': 0, 'end': len(excerpt), 'text': excerpt}]}}]}).encode(),
+ 'dataset.json': json.dumps({'cases': [{'required_facts': [{'alternatives': [excerpt]}]}]}).encode()}
+ public, receipt = project_host_metadata_files(files, ['/Users/actual-user'], public_excerpt_views=True)
+ sample = json.loads(public['review.json'])['attempts'][0]['adjudication_evidence']['samples'][0]
+ assert sample['text'] == 'Example [HOST_PATH_OMITTED] and stable body'
+ assert sample['start'] == 0 and sample['end'] == len(excerpt)
+ assert public['dataset.json'] == files['dataset.json']
+ assert receipt['files'][0]['path'] == 'review.json'
+
+
+def test_query_timestamp_is_not_a_semicolonless_html_times_entity():
+ from harness.web_fetch.publication_privacy import publication_risk
+ original = '[Image](https://example.test/photo?token=opaque-private×tamp=123456)'
+ assert publication_risk(original.encode()) == 'sensitive URL query value'
+ public, _ = project_evidence_view('public-evidence/answer.txt', original.encode(),
+ role='scored_answer', content_format='text')
+ assert b'×tamp=123456' in public and b'opaque-private' not in public
+ assert publication_risk(public) is None
+ assert publication_risk(b'https://example.test/?api_key=opaque-private') == 'sensitive URL query value'
+
+
+@pytest.mark.parametrize('defect', ['different_display', 'original_leak', 'private_tree'])
+def test_release_rejects_wrong_display_or_private_originals(tmp_path, defect):
+ from harness.web_fetch.publication_privacy import require_public_evidence_view_inventory
+ binding, original, _, _, profile = _declared_view_snapshot(tmp_path)
+ rows = [{'artifact_path': binding['path'], 'metrics': {'content_sha256': binding['sha256']}}]
+ if defect == 'different_display': (tmp_path/binding['path']).write_bytes(b'wrong answer')
+ if defect == 'original_leak': (tmp_path/binding['path']).write_bytes(original)
+ if defect == 'private_tree': (tmp_path/'private_artifacts').mkdir()
+ with pytest.raises(ValueError):
+ require_public_evidence_view_inventory(tmp_path, profile['publication_evidence_views'], rows)
+
+
+@pytest.mark.parametrize('source,expected', [
+ ('https://example.test/api?apiKey=opaque-private',
+ 'https://example.test/api?apiKey=[REDACTED]'),
+ ('https://example.test/api?x=1&apiKey=opaque-private&view=full',
+ 'https://example.test/api?x=1&apiKey=[REDACTED]&view=full'),
+ ('https://example.test/api?apiKey=opaque+private">safe',
+ 'https://example.test/api?apiKey=[REDACTED]">safe'),
+])
+def test_encoded_query_delimiters_are_redacted_without_rewriting_html(source, expected):
+ from harness.web_fetch.publication_privacy import redact_url_queries, publication_risk
+ public, count = redact_url_queries(source)
+ assert public == expected and count == 1
+ assert publication_risk(public.encode()) is None
+ assert redact_url_queries(public) == (public, 0)
+
+
+def test_metadata_projection_supports_disk_backed_inputs_and_outputs(tmp_path):
+ from collections.abc import MutableMapping
+ from harness.web_fetch.publication_privacy import project_host_metadata_files
+ class Files(MutableMapping):
+ def __init__(self, root): self.root, self.names = root, set()
+ def __getitem__(self, key): return (self.root/key).read_bytes()
+ def __setitem__(self, key, value):
+ path = self.root/key
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(value)
+ self.names.add(key)
+ def __delitem__(self, key): (self.root/key).unlink(); self.names.remove(key)
+ def __iter__(self): return iter(sorted(self.names))
+ def __len__(self): return len(self.names)
+ source, destination = Files(tmp_path/'original'), Files(tmp_path/'public')
+ source['child.json'] = b'{"file":"/Users/example/private/input.json"}'
+ source['parent.json'] = json.dumps({'child_sha256': hashlib.sha256(source['child.json']).hexdigest()}).encode()
+ public, receipt = project_host_metadata_files(source, ['/Users/example'], output_store=destination)
+ assert public is destination
+ assert json.loads(destination['child.json'])['file'] == '[HOST_PATH_OMITTED]'
+ assert json.loads(destination['parent.json'])['child_sha256'] == hashlib.sha256(destination['child.json']).hexdigest()
+ assert '/Users/example' in source['child.json'].decode()
+ assert len(receipt['files']) == 2
+
+
+def test_metadata_path_redaction_preserves_following_json_escaped_body():
+ from harness.web_fetch.publication_privacy import project_host_metadata_files
+ packet = json.dumps({'content': 'Saved to /Users/example/private/a.txt\nStill visible'})
+ original = json.dumps({'stdout': packet}).encode()
+ public, _ = project_host_metadata_files({'metadata.json': original}, ['/Users/example'])
+ inner = json.loads(json.loads(public['metadata.json'])['stdout'])
+ assert inner['content'] == 'Saved to [HOST_PATH_OMITTED]\nStill visible'
+
+
+def test_bilingual_download_link_displays_text_instead_of_escaped_markup():
+ from bs4 import BeautifulSoup
+ from pathlib import PurePosixPath
+ from harness.web_fetch.report import link
+ from harness.web_fetch.report_i18n import localize_html
+ raw = ''+link('../public-evidence/a.json',
+ '下载完整原生响应（公开副本）','Download complete native response (public view)')+''
+ chinese = BeautifulSoup(raw, 'html.parser').select_one('a')
+ assert chinese.text == '下载完整原生响应（公开副本）'
+ english = BeautifulSoup(localize_html(raw, PurePosixPath('attempts/a.html')), 'html.parser').select_one('a')
+ assert english.text == 'Download complete native response (public view)'
+ assert english['href'] == '../../public-evidence/a.json'
+
+
+def test_privacy_check_reuses_only_identical_bytes():
+ from unittest.mock import patch
+ from harness.web_fetch import publication_privacy as owner
+ owner._cached_publication_risk.cache_clear()
+ safe = b'Unique unchanged evidence cache fixture'
+ unsafe = safe+b'\nAuthorization: Custom private-credential'
+ with patch.object(owner, '_scan_publication_risk', wraps=owner._scan_publication_risk) as scan:
+ assert owner.publication_risk(safe) is None
+ assert owner.publication_risk(safe) is None
+ assert scan.call_count == 1
+ assert owner.publication_risk(unsafe) == 'sensitive field value'
+ assert scan.call_count == 2
+ owner._cached_publication_risk.cache_clear()
From c759ff3bcea41323742ef2dadf9b931b3de89f78 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 04:14:57 +0800
Subject: [PATCH 35/50] Describe redacted answers as public views in evidence
panes
---
harness/web_fetch/report.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index bab784f94..100fbbff3 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -2226,7 +2226,10 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
output_note='实际交付 0 个文件'
reference_scope=('' if not (c.get('track')=='document' and 0 < len(ref) < 100) else
f'参考范围： 这是最小文档解析样本。冻结源文件的全部可提取正文共 {len(ref):,} 个字符，标准答案没有省略。它只验证短文档文字提取，不用于证明多页、表格或附录完整性。
')
- evidence=('左边是判定时使用的标准答案或明确验收规则，右边是该次调用原样保存的完整输出。两栏都是真实内容，不是摘要；长内容可在栏内滚动。
'+reference_scope+
+ evidence_intro='左边是判定时使用的标准答案或明确验收规则，右边是该次调用原样保存的完整输出。两栏都是真实内容，不是摘要；长内容可在栏内滚动。'
+ if answer_view and answer_view['original_sha256']!=answer_view['published_sha256']:
+ evidence_intro=model_text('左边是判定时的标准答案或验收规则，右边是本次输出的公开副本，目录或敏感值已隐藏。成绩保留评分原件的结论；副本与原件分别标明校验值。长内容可在栏内滚动。','The left pane contains the scoring reference or acceptance rules. The right pane contains the public output view with paths or sensitive values hidden. Scores retain the original verdict; original and public hashes are identified separately. Long content scrolls within each pane.')
+ evidence=(''+evidence_intro+'
'+reference_scope+
f'{esc(reference_heading)} {source_meta}
{source_body}
'
f'
{actual_body}
按同一任务逐项比较，不用各方法不同的成功样本拼接结论。成功与失败均沿用主榜已核验判定；点击方法可查看标准答案、实际返回和原因。
'+method_comparisons(cases,bycase,names),'method-comparison')
- write(out,'analysis/index.html','结果分析',hero('结果分析','先比较方法在同一任务上的成败组合，再筛选全部执行记录。')+comparison+section('全部执行记录',controls+table(['方法']+headers,[(case_attrs(r,cases[r['case_id']]),cell(link('../providers/'+r['provider']+'.html',provider_label(r['provider'])))+attempt_row(r,cases[r['case_id']])[1]) for r in records],'filtered-table')))
+ write(out,'analysis/index.html','结果分析',hero('结果分析','先比较方法在同一任务上的成败组合，再筛选全部执行记录。')+comparison+section('全部执行记录',controls+table(['方法']+headers,[(case_attrs(r,cases[r['case_id']]),cell(link('../providers/'+r['provider']+'.html',provider_label(r['provider'])))+attempt_row(r,cases[r['case_id']])[1]) for r in records],'filtered-table',cls='all-method-results')))
write_support(out,records,manifests,authored_interpretation=authored_interpretation)
from .report_changes import render_changes
if records and all(row.get('provider_type') == 'fixture' for row in records):
From 4ef472ff151274e9c526be58bf0e056a1591f420 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 05:03:48 +0800
Subject: [PATCH 37/50] Scan malformed escaped strings without quadratic suffix
retries
---
harness/web_fetch/publication_privacy.py | 43 +++++++++++++++++++++---
tests/test_publication_privacy.py | 30 +++++++++++++++++
2 files changed, 69 insertions(+), 4 deletions(-)
diff --git a/harness/web_fetch/publication_privacy.py b/harness/web_fetch/publication_privacy.py
index 30771c20c..ec738b1e5 100644
--- a/harness/web_fetch/publication_privacy.py
+++ b/harness/web_fetch/publication_privacy.py
@@ -186,16 +186,51 @@ def _redact_plain_url_queries(text):
_JSON_STRING = re.compile(r'"(?:\\.|[^"\\\x00-\x1f])*"')
+def _json_string_spans(text):
+ """Match the JSON-string grammar without retrying long invalid suffixes.
+
+ An unterminated escaped string can make an unanchored regex rescan the same
+ suffix at every quote. Backward states remember the next valid closing
+ quote; forward selection preserves ordinary finditer's nonoverlap rule.
+ """
+ if '"' not in text:
+ return
+ if len(text) <= 4096:
+ yield from (match.span() for match in _JSON_STRING.finditer(text))
+ return
+ candidates = []
+ next_end = after_next_end = None
+ for index in range(len(text)-1, -1, -1):
+ char = text[index]
+ if char == '"':
+ if next_end is not None:
+ candidates.append((index, next_end+1))
+ end = index
+ elif char == '\\':
+ end = (after_next_end if index+1 < len(text) and text[index+1] != '\n'
+ else None)
+ elif ord(char) < 32:
+ end = None
+ else:
+ end = next_end
+ after_next_end, next_end = next_end, end
+ cursor = 0
+ for start, end in reversed(candidates):
+ if start >= cursor:
+ yield start, end
+ cursor = end
+
+
def redact_url_queries(text):
"""Decode JSON string escapes before checking URLs; preserve clean bytes."""
parts = []
cursor = 0
replacements = 0
- for match in _JSON_STRING.finditer(text):
- prefix, count = _redact_plain_url_queries(text[cursor:match.start()])
+ for start, end in _json_string_spans(text):
+ prefix, count = _redact_plain_url_queries(text[cursor:start])
parts.append(prefix)
replacements += count
- raw = match.group()
+ raw = text[start:end]
try:
decoded = json.loads(raw)
except json.JSONDecodeError:
@@ -205,7 +240,7 @@ def redact_url_queries(text):
projected = json.dumps(projected, ensure_ascii=True) if count else raw
parts.append(projected)
replacements += count
- cursor = match.end()
+ cursor = end
suffix, count = _redact_plain_url_queries(text[cursor:])
parts.append(suffix)
return ''.join(parts), replacements + count
diff --git a/tests/test_publication_privacy.py b/tests/test_publication_privacy.py
index 9b95c4d5d..0c0f8a6ea 100644
--- a/tests/test_publication_privacy.py
+++ b/tests/test_publication_privacy.py
@@ -547,3 +547,33 @@ def test_host_projection_refuses_to_rewrite_score_bearing_or_response_payloads()
raw=b'{"host":"fixture.local"}'
with pytest.raises(ValueError,match='must not alter scored'):
project_host_metadata_files({name:raw},['/Users/fixture'],host_names=['fixture.local'])
+def test_long_unterminated_escaped_string_preserves_content_without_rescanning_suffixes():
+ from harness.web_fetch.publication_privacy import redact_url_queries
+ text = '"' + ('\\"fragment' * 20000)
+ assert redact_url_queries(text) == (text, 0)
+
+
+def test_long_escaped_url_still_redacts_sensitive_query_value():
+ from harness.web_fetch.publication_privacy import redact_url_queries
+ text = 'ordinary log ' * 500 + r'"https:\/\/example.com\/image?apiKey=private-value&width=300"'
+ public, count = redact_url_queries(text)
+ assert count == 1
+ assert 'private-value' not in public
+ assert 'width=300' in public
+
+
+def test_large_and_small_string_tokenization_have_identical_public_redaction_semantics():
+ import random
+ from harness.web_fetch.publication_privacy import redact_url_queries
+ rng = random.Random(51)
+ prefix = ' ' * 4097
+ fragments = [
+ '"abc\\"def" tail "https://example.com/?token=private-value"',
+ '"unterminated\\"suffix', '"line\nfeed"', '"line\\\nfeed"',
+ '"line\\\rfeed"', '"\\u0068ttps://example.com/?token=private-value"',
+ '"nul\x00boundary"', '"backslash\\\\"',
+ ]
+ fragments += [''.join(rng.choice('ab"\\\r\n\t\x00') for _ in range(60)) for _ in range(64)]
+ for text in fragments:
+ expected, count = redact_url_queries(text)
+ assert redact_url_queries(prefix+text) == (prefix+expected, count)
From 7a8731ffddcfed76ff3c4ab9968e0b58a6ab1407 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 05:20:49 +0800
Subject: [PATCH 38/50] Bind complete public native evidence handoff and final
review scope
---
research/native-430-local-report.json | 71 ++++++++++++++++++---------
1 file changed, 47 insertions(+), 24 deletions(-)
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
index 4851d3b8d..861ac4607 100644
--- a/research/native-430-local-report.json
+++ b/research/native-430-local-report.json
@@ -15,8 +15,8 @@
"openclaw-built-in",
"tavily"
],
- "frozen_dataset_sha256": "5fd8f69886bb1919e592bec5a88becc3f1eb555a10c912e2724652e3219d4fb3",
- "results_sha256": "21a64bec19d9b5614643a013592cb9bfb35b341301ac45814ce763b58cde6690",
+ "frozen_dataset_sha256": "28e3b3e9ec89ce545aae44355bd28eb0dc383ad6f64aed4dc91f327b14a528ab",
+ "results_sha256": "ece94c0911a769c84ce9d896b00d97706fd4435d6d194f03cff77391710934a4",
"measured_scoring_source_sha256": "4ce40cdf2ecb1af4088915a2b869ffbdcb65dce5222d1429578e97d175f13bfc",
"evidence_state": "local_measurement",
"certified": false,
@@ -28,7 +28,7 @@
},
{
"path": "result-review.json",
- "sha256": "3c0be7dedda1a31ea749a989fa622b7aa8907a6580bed6c00512ec91871fa035"
+ "sha256": "8bfc48ba7b19ec244e6118e4620cd460d2fe425cd74b878fe1e9003f0f23f1fe"
},
{
"path": "archived-reading.json",
@@ -40,7 +40,7 @@
},
{
"path": "local-evidence/result-ledger.json",
- "sha256": "28b6b78515fe6521a4ea564b7de16935982e86b4855fd5af0bf2d342961db6fb"
+ "sha256": "738944823525d424a879572aa894e81a384802692d78ed52ca95400f27f94d6b"
},
{
"path": "frozen-method-options.json",
@@ -87,14 +87,14 @@
"sha256": "bb1bd4f9b3bf0b5704b730d3a84d01dc6f4698947de2fd2d3da2667e2a8b08b9"
}
],
- "evidence_path": "runs/pr51-block-chain-20261015/evidence",
+ "evidence_path": "runs/pr51-native-fairness-20261015/evidence-reviewed",
"task_replacement": {
"path": "local-evidence/task-replacement/receipt.json",
"sha256": "c68d3b21c3317538089635f8fb0ca5bf4f874f4a03201b27e42c56491780d6e0"
},
"offline_revision": {
- "path": "local-evidence/offline-revisions/f1884b4ae2f0ba4d-3aa4e30c449b3731/receipt.json",
- "sha256": "a230fff72209ab675f3e93587947dc041ffb964fafea37e82da643fd9de2cace"
+ "path": "local-evidence/offline-revisions/246d255fbf52a069-063e7324042d60b4/receipt.json",
+ "sha256": "f633998e82f266156902c81dc9d7f33892c588ce2ca2bcab24f8c8b8f30eaa22"
},
"replacement_case": {
"id": "static-sqlite-quickstart",
@@ -149,36 +149,59 @@
"authority": "Frozen pre-call official source; preserve code tokens while allowing whitespace and HTML entities."
}
},
- "current_scoring_source_sha256": "0c182d0c4d7f83b680dfddea0115e614721fc9590735411bf84bc7b013311eb1",
+ "current_scoring_source_sha256": "d042394ea0e8c3aa7aed5851663eaf73772fc4357781999e20a87f48e857e00a",
"supplemental_review": {
- "path": "local-evidence/current-contract-review/supplement.json",
- "sha256": "0b356816e4f1b9c9e013754ea79f42bd83a69e9be82a73ae9062b81e573da18a"
+ "path": "local-evidence/native-answer-review/supplement.json",
+ "sha256": "5b5a9a22755fe35e8852195d5dfc1b6d1edd4286b50c4e8c5ed8774bc54f7c1f"
},
"retained_review": {
- "path": "local-evidence/review-retention/receipt.json",
- "sha256": "eadbd3e71ccaec5e9817acd4890f7b7b08f980caf1f7a5e09d20e8fecace7ae6"
+ "path": "local-evidence/native-answer-review/retention.json",
+ "sha256": "b3275c808947af2312fa48861db1a49b46c62cc8998c6c3afa7965c8f1f4e12f"
},
"review_scope": {
- "fresh": 66,
- "retained": 4234,
- "total": 4300
+ "fresh": 72,
+ "retained": 4228,
+ "total": 4300,
+ "fresh_independent_blind_review": false
},
+ "implementation_commit": "4ef472ff151274e9c526be58bf0e056a1591f420",
"review_evidence_package": {
- "archive_name": "PR-51-public-saved-inputs-20261015.tar.xz",
- "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261015/PR-51-public-saved-inputs-20261015.tar.xz",
- "sha256": "ed17643b6a232221a8f0b892f817cbe2d8b92b61d5dd2524cc448e24359bc745",
- "bytes": 255231492,
+ "archive_name": "PR-51-public-saved-inputs-20261016.tar.xz",
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-public-saved-inputs-20261016.tar.xz",
+ "sha256": "6ceb6a8bb35bc0e1d4ba40ff183dbb66e975d37426d7277ec6fca27ef58ad1ea",
+ "bytes": 295677260,
"self_contained_restore_verified": true,
"full_report_rebuild_verified": true,
- "logical_report_routes": 25796,
- "access": "Read-only public host-metadata projection. Rebuild with the included public presentation binding. Original host metadata remains private; original and public file hashes are distinguished. No earlier draft assets are required.",
+ "logical_report_routes": 25760,
+ "anonymous_download_verified": true,
+ "access": "Public read-only saved inputs; no repository write access or earlier draft assets required.",
"presentation_binding": {
"path": "presentation-binding.json",
- "sha256": "a604a44bc612ba0357a31155bdc8e21957049669e81661b7364fbf8da80674bc"
+ "sha256": "89b0622a86b2713bc670903600931b2cc5e3851b757afa4012c10b3f0cc763a4"
},
"projection_receipt": {
"path": "PUBLICATION-PROJECTION.json",
- "sha256": "08001ffd26611ae7bc0d8e676afeaa3a97a9e383dbfa03cb1c8eddd407ee210b"
- }
+ "sha256": "b836c8da535c5eb106b223099b15bed11023098545ee38209e39b3b96c223b66"
+ },
+ "public_evidence_views": {
+ "path": "PUBLIC-EVIDENCE-VIEWS.json",
+ "sha256": "3b4ad900100bfc94cc004ee2f67c0fae54a2199ea78265c60c623d91db6fd79b"
+ },
+ "public_dataset_sha256": "e2a5c68fa7c2640414c56c0949a8f344f8091e602e6479f8fd2bb07fec5b9681",
+ "public_results_sha256": "2d2a35f6b51cd1023ee940f9a310c05610806f36d9c6363410200ffb26556dbe",
+ "verification": {
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-verification-20261016.json",
+ "sha256": "2fe714e6e8b410890f8be0329da3cb01bb02796c051b5477de0c2f5e3f4a6b40"
+ },
+ "report_renderer_commit": "ffe4a03367dc2b2d2fd168600042c7a3ead10d73",
+ "validation_commit": "4ef472ff151274e9c526be58bf0e056a1591f420"
+ },
+ "historical_native_route_audit": {
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-route-audit-20261016.json",
+ "sha256": "54a8b3a1370192a31d6d606aca21ef84e8f4e57d4ce6052c9d299e2ab5c0b69e",
+ "cells": 140,
+ "observed_signature_drift": 0,
+ "timeouts_without_route_confirmation": 19,
+ "pre_call_frozen": false
}
}
From c2490db3225d2dcff5968a5d84de50795927a461 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 12:45:44 +0800
Subject: [PATCH 39/50] Validate native answer fields and retain transport
failure stages
---
evaluation/scoring-regressions.json | 140 +++++++++++++++
harness/web_fetch/artifacts.py | 2 +
harness/web_fetch/call_boundary.py | 38 ++--
harness/web_fetch/native_routes.py | 111 +++++-------
harness/web_fetch/providers/base.py | 32 ++--
harness/web_fetch/providers/cloudflare.py | 31 ++++
harness/web_fetch/providers/exa.py | 19 ++
harness/web_fetch/providers/firecrawl.py | 26 +++
harness/web_fetch/providers/moli.py | 33 +++-
.../web_fetch/providers/subprocess_jsonl.py | 26 ++-
harness/web_fetch/providers/tavily.py | 12 ++
harness/web_fetch/report.py | 4 +
harness/web_fetch/response_input.py | 169 ++++++++++++++++--
harness/web_fetch/runner.py | 27 ++-
harness/web_fetch/scoring.py | 16 +-
research/native-430-local-report.json | 14 ++
tests/test_native_route_contract.py | 107 +++++++++++
tests/test_provider_evidence_boundary.py | 7 +-
tests/test_scoring.py | 35 +++-
tests/test_scoring_regressions.py | 16 +-
tests/test_subprocess_provider.py | 86 +++++++++
21 files changed, 813 insertions(+), 138 deletions(-)
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index 9a86ff66b..4e67c14d3 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -32738,6 +32738,146 @@
"expected_metrics": {
"resource_pass": false
}
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_answer_fields",
+ "case": {
+ "url": "https://example.test/",
+ "track": "image",
+ "task_kind": "resource",
+ "expected_resources": [
+ {
+ "id": "image",
+ "kind": "image",
+ "urls": [
+ "https://example.test/answer.png"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "reject"
+ }
+ },
+ "artifact": "",
+ "ok": true,
+ "native_response": "{\"content\": \"\", \"metadata\": {\"images\": [\"https://example.test/answer.png\"]}}",
+ "id": "native-declared-image-projection-passes",
+ "failure_mode": "false-negative",
+ "metadata": {
+ "images": [
+ "https://example.test/answer.png"
+ ],
+ "native_answer_projection": {
+ "schema_version": 1,
+ "content_pointer": [
+ "content"
+ ],
+ "content_format": "text",
+ "answer_fields": {
+ "images": {
+ "pointer": [
+ "metadata",
+ "images"
+ ],
+ "format": "identity"
+ }
+ }
+ }
+ },
+ "expected_task_pass": true
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_answer_fields",
+ "case": {
+ "url": "https://example.test/",
+ "track": "image",
+ "task_kind": "resource",
+ "expected_resources": [
+ {
+ "id": "image",
+ "kind": "image",
+ "urls": [
+ "https://example.test/answer.png"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "reject"
+ }
+ },
+ "artifact": "",
+ "ok": true,
+ "native_response": "{\"content\": \"\", \"metadata\": {\"images\": [\"https://example.test/native.png\"]}}",
+ "id": "native-undeclared-image-cannot-supply-answer",
+ "failure_mode": "false-positive",
+ "metadata": {
+ "images": [
+ "https://example.test/answer.png"
+ ],
+ "native_answer_projection": {
+ "schema_version": 1,
+ "content_pointer": [
+ "content"
+ ],
+ "content_format": "text"
+ }
+ },
+ "expected_task_pass": false,
+ "expected_validation_error": "undeclared"
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_answer_fields",
+ "case": {
+ "url": "https://example.test/",
+ "track": "image",
+ "task_kind": "resource",
+ "expected_resources": [
+ {
+ "id": "image",
+ "kind": "image",
+ "urls": [
+ "https://example.test/answer.png"
+ ]
+ }
+ ],
+ "resource_output_contract": {
+ "candidate_scope": "image_answer_fields",
+ "unexpected": "reject"
+ }
+ },
+ "artifact": "",
+ "ok": true,
+ "native_response": "{\"content\": \"\", \"metadata\": {\"images\": [\"https://example.test/native.png\"]}}",
+ "id": "native-replaced-image-cannot-supply-answer",
+ "failure_mode": "false-positive",
+ "metadata": {
+ "images": [
+ "https://example.test/answer.png"
+ ],
+ "native_answer_projection": {
+ "schema_version": 1,
+ "content_pointer": [
+ "content"
+ ],
+ "content_format": "text",
+ "answer_fields": {
+ "images": {
+ "pointer": [
+ "metadata",
+ "images"
+ ],
+ "format": "identity"
+ }
+ }
+ }
+ },
+ "expected_task_pass": false,
+ "expected_validation_error": "projection"
}
],
"superseded_controls": {
diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py
index 97ca6c3a4..2093c618c 100644
--- a/harness/web_fetch/artifacts.py
+++ b/harness/web_fetch/artifacts.py
@@ -338,6 +338,8 @@ def verify_record_artifacts(root: Path, record: dict[str, Any]) -> None:
if not source.is_file() or hashlib.sha256(source.read_bytes()).hexdigest() != digest:
raise ValueError('Artifact differs from its result binding: ' + relative)
verify_record_public_call_logs(root, record)
+ from .response_input import verify_saved_native_answer
+ verify_saved_native_answer(root, record)
def materialize_bound_artifacts(output: Path, source_root: Path, records: list[dict]) -> None:
diff --git a/harness/web_fetch/call_boundary.py b/harness/web_fetch/call_boundary.py
index 6b2eb2846..ff6c54da6 100644
--- a/harness/web_fetch/call_boundary.py
+++ b/harness/web_fetch/call_boundary.py
@@ -9,6 +9,25 @@
from .response_input import answer_input
+def evaluate_returned_files(returned_files, source_verification):
+ files = []
+ for item in returned_files:
+ raw = item.get('data')
+ if not isinstance(raw, bytes):
+ raise ValueError('Provider file data must be bytes')
+ mime = ('image/png' if raw.startswith(b'\x89PNG\r\n\x1a\n') else
+ 'video/webm' if raw.startswith(b'\x1aE\xdf\xa3') else
+ 'video/mp4' if len(raw) >= 12 and raw[4:8] == b'ftyp' else None)
+ expected = source_verification.get('mime', '').split(';')[0]
+ digest = hashlib.sha256(raw).hexdigest()
+ expected_hash = source_verification.get('file_sha256')
+ files.append({'bytes': len(raw), 'sha256': digest,
+ 'mime': mime, 'pass': bool(len(raw) >= 32 and mime and (not expected or mime == expected)
+ and (not expected_hash or digest == expected_hash))})
+ return {'files': files, 'pass': any(f['pass'] for f in files),
+ 'scope': 'returned_bytes_signature_length_sha256; not semantic media validation'}
+
+
def prepare_answer(result, case):
# This required representation conversion is inside the measured call.
result.prepared_answer = answer_input(result.content.encode('utf-8', errors='replace'), result.metadata,
@@ -17,22 +36,7 @@ def prepare_answer(result, case):
# downloads produced while a body task is being fetched. Only ``file``
# tasks use this evidence for acceptance; body tasks still require text.
if case.task_kind == 'file' or result.files:
- files = []
- for item in result.files:
- raw = item.get('data')
- if not isinstance(raw, bytes):
- raise ValueError('Provider file data must be bytes')
- mime = ('image/png' if raw.startswith(b'\x89PNG\r\n\x1a\n') else
- 'video/webm' if raw.startswith(b'\x1aE\xdf\xa3') else
- 'video/mp4' if len(raw) >= 12 and raw[4:8] == b'ftyp' else None)
- expected = case.source_verification.get('mime', '').split(';')[0]
- digest = hashlib.sha256(raw).hexdigest()
- expected_hash = case.source_verification.get('file_sha256')
- files.append({'bytes': len(raw), 'sha256': digest,
- 'mime': mime, 'pass': bool(len(raw) >= 32 and mime and (not expected or mime == expected)
- and (not expected_hash or digest == expected_hash))})
- result.metadata['file_evaluation'] = {'files': files, 'pass': any(f['pass'] for f in files),
- 'scope': 'returned_bytes_signature_length_sha256; not semantic media validation'}
+ result.metadata['file_evaluation'] = evaluate_returned_files(result.files, case.source_verification)
return result
@@ -84,6 +88,8 @@ def work():
late=None
if isinstance(late,ProviderResult):
metadata.update(late.metadata)
+ for key in ('links', 'images', 'videos', 'media', 'extras', 'file_evaluation'):
+ metadata.pop(key, None)
metadata['_answer_elapsed_ms']=answer_elapsed_ms
return ProviderResult(
False,
diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py
index d6362ff5b..734809fc5 100644
--- a/harness/web_fetch/native_routes.py
+++ b/harness/web_fetch/native_routes.py
@@ -54,19 +54,17 @@ def native_route_cell(provider, case):
if method == 'firecrawl':
interface = '/v2/scrape'
endpoint = str(config.get('base_url', 'https://api.firecrawl.dev')).rstrip('/') + interface
- fields = ['data.rawBase64'] if kind == 'file' else ['data.markdown', 'data.links', 'data.images', 'data.video']
+ fields = ['data.rawBase64'] if kind == 'file' else ['data.markdown', 'data.links', 'data.images', 'data.videos', 'data.video']
elif method == 'tavily_extract':
interface = '/extract'
endpoint = str(config.get('base_url', 'https://api.tavily.com')).rstrip('/') + interface
- fields = ['results[].raw_content', 'results[].images']
+ fields = ['results[].raw_content', 'results[].images', 'images']
elif method == 'exa_contents':
interface = '/contents'
endpoint = str(config.get('base_url', 'https://api.exa.ai')).rstrip('/') + interface
fields = ['results[].text', 'results[].extras']
else:
- interface = ('/scrape' if method == 'cloudflare_markdown' and kind == 'resource'
- and track in {'image', 'video'} else '/content' if method == 'cloudflare_content'
- else '/markdown')
+ interface = provider.native_http_interface(case)
endpoint = 'https://api.cloudflare.com/client/v4/accounts/' + str(config.get('account_id', '')) + '/browser-rendering' + interface
if method=='cloudflare_markdown' and interface=='/markdown' and config.get('browser'):
from urllib.parse import urlencode
@@ -74,7 +72,7 @@ def native_route_cell(provider, case):
fields = ['result']
elif method in {'subprocess_jsonl', 'cowagent_builtin'}:
transport, interface = 'jsonl', 'native_worker_command'
- fields = ['content', 'metadata.links', 'metadata.images', 'metadata.videos', 'files']
+ fields = ['content', 'metadata.links', 'metadata.images', 'metadata.videos', 'metadata.media', 'metadata.extras', 'files']
elif method == 'fixture':
transport, interface, fields = 'fixture', 'synthetic_fixture', ['content']
else:
@@ -94,68 +92,42 @@ def native_route_cell(provider, case):
def _request_parameters(provider, case, cell, timeout_seconds):
"""Explicit pre-call transport contract, separately checked against adapters."""
- config, method = provider.config, provider.type
if cell['transport'] == 'http':
- if method == 'tavily_extract':
- body = {'urls': [''], 'extract_depth': config.get('extract_depth', 'advanced'),
- 'format': 'markdown', 'include_images': bool(config.get('include_images', False))}
- headers = {'Authorization': 'Bearer '+str(config.get('api_key', ''))}
- elif method == 'exa_contents':
- body = {'urls': [''], 'text': {'verbosity': config.get('text_verbosity', 'full')},
- 'maxAgeHours': int(config.get('max_age_hours', 0))}
- extras = {'links': 20, 'imageLinks': 20} if config.get('include_resource_links') else {}
- if config.get('rich_links_limit'):
- extras['richLinks'] = int(config.get('rich_links_limit', 100))
- if extras:
- body['extras'] = extras
- headers = {'x-api-key': str(config.get('api_key', ''))}
- elif method == 'firecrawl':
- formats = ['rawBase64'] if case.task_kind == 'file' else list(config.get('formats', ['markdown']))
- if case.track == 'video' and case.task_kind == 'resource' and 'video' not in formats:
- formats.append('video')
- body = {'url': '', 'formats': formats, 'maxAge': int(config.get('max_age', 0)),
- 'proxy': str(config.get('proxy', 'auto'))}
- if 'only_main_content' in config:
- body['onlyMainContent'] = config['only_main_content']
- if case.task_kind == 'file':
- body['parsers'] = []
- headers = ({'Authorization': 'Bearer '+str(config['api_key'])}
- if config.get('api_key') else {})
- else:
- body = {'url': '', 'gotoOptions': {'waitUntil': config.get('wait_until', 'networkidle2')}}
- if cell['interface'] == '/scrape':
- selectors = (['img', 'picture source'] if case.track == 'image' else
- ['video', 'video source', 'audio', 'audio source', 'iframe', 'a[href]'])
- body['elements'] = [{'selector': v} for v in selectors]
- headers = {'Authorization': 'Bearer '+str(config.get('api_token', ''))}
- return {'method': 'POST', 'body': body, 'headers_sha256': _digest(headers),
- 'timeout_seconds': timeout_seconds, 'max_bytes': int(config.get('max_bytes', 20_000_000))}
+ endpoint, arguments = provider.native_http_arguments(case, timeout_seconds)
+ if _digest(endpoint) != cell['endpoint_sha256']:
+ raise ValueError('Native request builder differs from its declared endpoint')
+ return normalized_http_parameters(case, arguments)
if cell['transport'] == 'cli':
- argv = ['', 'fetch']
- if cell['interface'] not in {'raw_file', 'native_document_bytes'}:
- argv += ['--dump', 'html' if case.task_kind == 'resource' else 'markdown',
- '--wait-until', config.get('wait_until', 'done')]
- argv += ['--http-max-response-size', str(int(config.get('max_bytes', 20_000_000)))]
- if config.get('block_private_networks', True):
- argv.append('--block-private-networks')
- if config.get('log_level'):
- argv += ['--log-level', str(config['log_level'])]
- argv.append('')
- return {'argv': argv, 'binary_sha256': config.get('expected_binary_sha256')}
+ command = provider.native_cli_command(case)
+ return {'argv': ['', *command[1:-1], ''],
+ 'binary_sha256': provider.config.get('expected_binary_sha256')}
if cell['transport'] == 'jsonl':
- request = {'url': '', 'log_level': config.get('log_level'),
- 'diagnostic_path': '' if config.get('log_level')=='debug' else None,
- 'timeout_seconds': timeout_seconds, 'uniform_native_parameters': True,
- 'max_bytes': int(config.get('max_bytes', 20_000_000)),
- 'dns_mode': config.get('dns_mode', 'system'), 'task_kind': case.task_kind,
- 'track': case.track, 'input_media_kind': declared_input_media_kind(case),
- 'retrieval_plan': None,
- 'retrieval_mode': 'native_default' if case.track == 'dynamic' else 'declared_plan'}
- command = [provider._resolve_token(part) for part in config.get('command', [])]
+ request = provider.native_worker_request(case, timeout_seconds,
+ '' if provider.config.get('log_level') == 'debug' else None)
+ request['url'] = ''
+ command = [provider._resolve_token(part) for part in provider.config.get('command', [])]
return {'request': request, 'command_sha256': _digest(command)}
return {}
+def normalized_http_parameters(case, arguments):
+ from copy import deepcopy
+ body = deepcopy(arguments.get('json_body'))
+ if not isinstance(body, dict):
+ raise ValueError('Native HTTP request lacks its declared input field')
+ slots = [key for key in ('url', 'urls') if key in body]
+ if slots == ['url'] and body['url'] == case.url:
+ body['url'] = ''
+ elif slots == ['urls'] and body['urls'] == [case.url]:
+ body['urls'] = ['']
+ else:
+ raise ValueError('Native HTTP input differs from the frozen task URL')
+ return {'method': arguments.get('method', 'GET'), 'body': body,
+ 'headers_sha256': _digest(arguments.get('headers', {})),
+ 'timeout_seconds': arguments.get('timeout'), 'max_bytes': arguments.get('max_bytes')}
+
+
+
class NativeRouteLedger:
"""Freeze class-level intent before calls; reject observed per-case drift.
@@ -242,12 +214,17 @@ def observe(self, provider, case, *, transport, parameters, endpoint=None, inter
self.observed[key] = digest
self.calls.setdefault(identity, []).append({'signature': signature, 'sha256': digest})
- def receipt(self, provider, case):
+ def receipt(self, provider, case, *, ok=None):
+ """Transport started means entry to the outbound call, not server receipt."""
from copy import deepcopy
key, _ = native_route_cell(provider, case)
with self.lock:
return {'contract_sha256': self.binding['sha256'], 'cell': list(key),
+ 'answer_fields': list(self.cells[key]['answer_fields']),
'requests': deepcopy(self.calls.get((provider.name, case.id), [])),
+ 'transport_state': 'started' if self.calls.get((provider.name, case.id)) else 'not_started',
+ **({'failure_stage': 'after_transport' if self.calls.get((provider.name, case.id))
+ else 'before_transport'} if ok is False else {}),
'protocol_errors': [dict(e) for e in self.errors if e['method']==provider.name and e['case_id']==case.id]}
@@ -313,7 +290,17 @@ def verify_native_route_contract(source, manifest, dataset, records):
raise ValueError('Native request violated its route contract')
if receipt.get('contract_sha256') != binding['sha256'] or receipt.get('cell') != list(key) or key not in cells:
raise ValueError('Native result lacks its bound route receipt')
+ if receipt.get('answer_fields') != cells[key]['answer_fields']:
+ raise ValueError('Native answer fields differ from the frozen route')
requests = receipt.get('requests', [])
+ state = receipt.get('transport_state')
+ if state not in {'started', 'not_started'}:
+ raise ValueError('Native result lacks its transport stage')
+ if bool(requests) != (state == 'started'):
+ raise ValueError('Native transport stage lacks its invocation signature')
+ if not row['ok'] and receipt.get('failure_stage') != (
+ 'after_transport' if requests else 'before_transport'):
+ raise ValueError('Native failed result lacks an explicit failure stage')
if len(requests) > 1 or (row['ok'] and not requests and cells[key]['transport'] != 'fixture'):
raise ValueError('Native result lacks one complete invocation signature')
for item in requests:
diff --git a/harness/web_fetch/providers/base.py b/harness/web_fetch/providers/base.py
index 3e0f525a0..aea0d8953 100644
--- a/harness/web_fetch/providers/base.py
+++ b/harness/web_fetch/providers/base.py
@@ -30,26 +30,26 @@ def capture_native_request(self, case, *, transport, parameters, endpoint=None,
def native_http_request(self, case, callback, endpoint, **kwargs):
"""Observe exact transport arguments before a fixed-native HTTP call."""
ledger = getattr(self, '_native_route_ledger', None)
+ if self.config.get('uniform_native_parameters'):
+ native_endpoint, native_arguments = self.native_http_arguments(case, kwargs.get('timeout'))
+ if endpoint != native_endpoint or kwargs != native_arguments:
+ if ledger is not None:
+ ledger.reject(self, case, 'Native adapter arguments differ from its canonical request builder')
+ raise ValueError('Native adapter arguments differ from its canonical request builder')
+ endpoint, kwargs = native_endpoint, native_arguments
if ledger is not None:
- from copy import deepcopy
- from ..native_routes import _digest
- body = deepcopy(kwargs.get('json_body'))
- if not isinstance(body, dict):
- ledger.reject(self, case, 'Native HTTP request lacks its declared input field')
- slots = [key for key in ('url', 'urls') if key in body]
- if slots == ['url'] and body['url'] == case.url:
- body['url'] = ''
- elif slots == ['urls'] and body['urls'] == [case.url]:
- body['urls'] = ['']
- else:
- ledger.reject(self, case, 'Native HTTP input differs from the frozen task URL')
+ from ..native_routes import normalized_http_parameters
self.capture_native_request(case, transport='http', endpoint=endpoint,
- parameters={'method': kwargs.get('method', 'GET'), 'body': body,
- 'headers_sha256': _digest(kwargs.get('headers', {})),
- 'timeout_seconds': kwargs.get('timeout'),
- 'max_bytes': kwargs.get('max_bytes')})
+ parameters=normalized_http_parameters(case, kwargs))
return callback(endpoint, **kwargs)
+ def native_http_arguments(self, case, timeout):
+ raise ValueError('Provider has no fixed-native HTTP request builder')
+
+ def native_http_options(self, body, headers, timeout):
+ return {'method': 'POST', 'json_body': body, 'headers': headers,
+ 'timeout': timeout, 'max_bytes': int(self.config.get('max_bytes', 20_000_000))}
+
@property
def min_interval_seconds(self) -> float:
"""Minimum cooldown from one response completion to the next request start."""
diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py
index 7ba21b37a..c33c76920 100644
--- a/harness/web_fetch/providers/cloudflare.py
+++ b/harness/web_fetch/providers/cloudflare.py
@@ -215,6 +215,22 @@ def _interface_error(*, response_status: int,
class CloudflareMarkdownProvider(Provider):
+ def native_http_interface(self, case):
+ return ('/content' if self.type == 'cloudflare_content' else '/scrape'
+ if case.task_kind == 'resource' and case.track in {'image', 'video'} else '/markdown')
+
+ def native_http_arguments(self, case, timeout):
+ interface = self.native_http_interface(case)
+ body = {'url': case.url, 'gotoOptions': {'waitUntil': self.config.get('wait_until', 'networkidle2')}}
+ if interface == '/scrape':
+ selectors = (['img', 'picture source'] if case.track == 'image' else
+ ['video', 'video source', 'audio', 'audio source', 'iframe', 'a[href]'])
+ body['elements'] = [{'selector': value} for value in selectors]
+ endpoint = 'https://api.cloudflare.com/client/v4/accounts/'+str(self.config.get('account_id', ''))+'/browser-rendering'+interface
+ if interface == '/markdown' and self.config.get('browser'):
+ endpoint += '?' + urlencode({'browser': self.config['browser']})
+ return endpoint, self.native_http_options(body, {'Authorization': 'Bearer '+str(self.config.get('api_token', ''))}, timeout)
+
def validate(self) -> list[str]:
return [name for name in ("api_token", "account_id") if not self.config.get(name)]
@@ -392,6 +408,9 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) ->
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
+ **({'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
+ if envelope_success and isinstance(payload.get('result', ''), str) else {}),
"route": route,
"wait": wait_metadata,
"browser": self.config.get("browser"),
@@ -483,6 +502,9 @@ def _fetch_media_resources(
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
+ **({'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
+ if envelope_success and isinstance(payload.get('result', ''), str) else {}),
"route": "browser_rendering_scrape_resources",
**({'native_answer_projection': projection} if envelope_success else {}),
"request_method": "POST",
@@ -657,6 +679,9 @@ def _fetch_direct_document(
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
+ **({'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
+ if envelope_success and isinstance(payload.get('result', ''), str) else {}),
"route": "workers_ai_tomarkdown",
"source_status": source.status,
"source_sha256": hashlib.sha256(source.body).hexdigest(),
@@ -676,6 +701,9 @@ def _fetch_direct_document(
class CloudflareContentProvider(Provider):
+ native_http_interface = CloudflareMarkdownProvider.native_http_interface
+ native_http_arguments = CloudflareMarkdownProvider.native_http_arguments
+
"""Rendered HTML diagnostic. It is deliberately excluded from the readable-text leaderboard."""
def validate(self) -> list[str]:
@@ -731,6 +759,9 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
+ **({'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
+ if envelope_success and isinstance(payload.get('result', ''), str) else {}),
"output_contract": "rendered_html",
"upstream_status": upstream_status,
"empty_content": envelope_success and not content,
diff --git a/harness/web_fetch/providers/exa.py b/harness/web_fetch/providers/exa.py
index 61a3a9bec..b141a4af6 100644
--- a/harness/web_fetch/providers/exa.py
+++ b/harness/web_fetch/providers/exa.py
@@ -55,6 +55,17 @@ def _public_url_error(value: object) -> dict[str, object]:
class ExaContentsProvider(Provider):
+ def native_http_arguments(self, case, timeout):
+ body = {'urls': [case.url], 'text': {'verbosity': self.config.get('text_verbosity', 'full')},
+ 'maxAgeHours': int(self.config.get('max_age_hours', 0))}
+ extras = {'links': 20, 'imageLinks': 20} if self.config.get('include_resource_links') else {}
+ if self.config.get('rich_links_limit'):
+ extras['richLinks'] = int(self.config.get('rich_links_limit', 100))
+ if extras:
+ body['extras'] = extras
+ return (str(self.config.get('base_url', 'https://api.exa.ai')).rstrip('/') + '/contents',
+ self.native_http_options(body, {'x-api-key': str(self.config.get('api_key', ''))}, timeout))
+
def validate(self) -> list[str]:
return [] if self.config.get("api_key") else ["api_key"]
@@ -156,5 +167,13 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
),
"extras": result_extras,
"links": _rich_answer_links(result_extras),
+ **({'native_answer_projection': {
+ 'schema_version': 1, 'content_pointer': ['results', 0, 'text'],
+ 'content_format': 'text', 'optional_content': True,
+ 'answer_fields': ({
+ 'extras': {'pointer': ['results', 0, 'extras'], 'format': 'identity', 'default': {}},
+ 'links': {'pointer': ['results', 0, 'extras', 'richLinks'],
+ 'format': 'named_links', 'default': []}} if ok else {})}}
+ if ok or content else {}),
},
)
diff --git a/harness/web_fetch/providers/firecrawl.py b/harness/web_fetch/providers/firecrawl.py
index 2949080a2..2efbc1f01 100644
--- a/harness/web_fetch/providers/firecrawl.py
+++ b/harness/web_fetch/providers/firecrawl.py
@@ -16,6 +16,20 @@ class FirecrawlProvider(Provider):
_PROXY_MAX_CREDITS = {"basic": 1, "enhanced": 1, "auto": 1}
_VIDEO_FORMAT_CREDITS = 5
+ def native_http_arguments(self, case, timeout):
+ formats = ['rawBase64'] if case.task_kind == 'file' else list(self.config.get('formats', ['markdown']))
+ if case.track == 'video' and case.task_kind == 'resource' and 'video' not in formats:
+ formats.append('video')
+ body = {'url': case.url, 'formats': formats, 'maxAge': int(self.config.get('max_age', 0)),
+ 'proxy': str(self.config.get('proxy', 'auto'))}
+ if 'only_main_content' in self.config:
+ body['onlyMainContent'] = self.config['only_main_content']
+ if case.task_kind == 'file':
+ body['parsers'] = []
+ headers = {'Authorization': 'Bearer '+str(self.config['api_key'])} if self.config.get('api_key') else {}
+ return (str(self.config.get('base_url', 'https://api.firecrawl.dev')).rstrip('/') + '/v2/scrape',
+ self.native_http_options(body, headers, timeout))
+
def validate(self) -> list[str]:
if "only_main_content" in self.config and not isinstance(self.config["only_main_content"], bool):
return ["only_main_content must be boolean"]
@@ -437,6 +451,18 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
"links": data.get("links") if ok and isinstance(data.get("links"), list) else [],
"images": data.get("images") if ok and isinstance(data.get("images"), list) else [],
"videos": videos if ok else [],
+ **({'native_answer_projection': {
+ 'schema_version': 1,
+ 'content_pointer': ['data', 'markdown'],
+ 'content_format': 'empty' if file_route else 'text', 'optional_content': True,
+ 'answer_fields': ({} if file_route else {
+ 'links': {'pointer': ['data', 'links'], 'format': 'identity', 'default': []},
+ 'images': {'pointer': ['data', 'images'], 'format': 'identity', 'default': []},
+ **({'videos': {'pointer': ['data', 'videos'], 'format': 'video_values', 'default': [],
+ 'legacy': {'pointer': ['data', 'video'], 'default': ''}}}
+ if native_video_route else {})}),
+ **({'files': {'pointer': ['data', 'rawBase64'], 'format': 'base64_file'}}
+ if file_bytes else {})}} if ok else {}),
**(
{
"raw_file": {
diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py
index d2b799aca..40b83cdda 100644
--- a/harness/web_fetch/providers/moli.py
+++ b/harness/web_fetch/providers/moli.py
@@ -522,6 +522,19 @@ def _validate_resolved_binary(self, binary: str | None) -> list[str]:
self._implementation_identity["native_dns"] = binding
return errors
+ def native_cli_command(self, case):
+ route = _route_for_case(case, native_default=True)
+ argv = [str(self.config.get('binary', 'moli')), 'fetch']
+ if route not in {'raw_file', 'native_document_bytes'}:
+ argv += ['--dump', 'html' if case.task_kind == 'resource' else 'markdown',
+ '--wait-until', self.config.get('wait_until', 'done')]
+ argv += ['--http-max-response-size', str(int(self.config.get('max_bytes', 20_000_000)))]
+ if self.config.get('block_private_networks', True):
+ argv.append('--block-private-networks')
+ if self.config.get('log_level'):
+ argv += ['--log-level', str(self.config['log_level'])]
+ return [*argv, case.url]
+
def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
self.check_native_call(case)
validate_http_url(case.url)
@@ -739,6 +752,11 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
if log_level:
command.extend(["--log-level", str(log_level)])
command.append(case.url)
+ if self.config.get('uniform_native_parameters'):
+ canonical = self.native_cli_command(case)
+ if command[1:] != canonical[1:]:
+ raise ValueError('Native CLI arguments differ from its canonical request builder')
+ command = [command[0], *canonical[1:]]
limit = int(self.config.get("max_bytes", 20_000_000))
retain_diagnostic = bool(self.config.get("retain_complete_diagnostic_log", False))
diagnostic_limit = int(self.config.get("max_diagnostic_bytes", 100_000_000))
@@ -795,9 +813,6 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
error_message=f"Cannot read Moli binary: {error}",
metadata=metadata,
)
- self.capture_native_request(case, transport='cli', interface=route,
- parameters={'argv': ['', *command[1:-1], ''],
- 'binary_sha256': getattr(self, '_binary_sha256', None)})
# Spool artifacts to files rather than retaining arbitrary CLI output in RAM.
with ExitStack() as dns_stack, ExitStack() as spool_stack:
sink = getattr(self, "_call_diagnostics", None)
@@ -820,6 +835,9 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
process_environment = dns_broker.environment(library)
metadata["native_dns_binding"] = binding
try:
+ self.capture_native_request(case, transport='cli', interface=route,
+ parameters={'argv': ['', *command[1:-1], ''],
+ 'binary_sha256': getattr(self, '_binary_sha256', None)})
process = subprocess.Popen(
command,
stdin=subprocess.DEVNULL,
@@ -927,6 +945,11 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
metadata=metadata,
)
if case.task_kind == "file" or native_document_download:
+ if self.config.get('uniform_native_parameters'):
+ metadata.update(_raw_stdout_b64=base64.b64encode(raw).decode('ascii'),
+ raw_stdout_sha256=hashlib.sha256(raw).hexdigest(),
+ native_answer_projection={'schema_version': 1, 'source_format': 'bytes',
+ 'content_format': 'empty', 'files': {'format': 'raw_bytes'}})
if native_document_download:
metadata["native_output_contract"] = "original_document_bytes_without_extracted_text"
ok = process.returncode == 0
@@ -1274,6 +1297,10 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
error_message="Moli markdown output is not UTF-8",
metadata=metadata,
)
+ if self.config.get('uniform_native_parameters'):
+ metadata.update(_raw_stdout_b64=base64.b64encode(raw).decode('ascii'),
+ raw_stdout_sha256=hashlib.sha256(raw).hexdigest(),
+ native_answer_projection={'schema_version': 1, 'source_format': 'bytes', 'content_format': 'utf8'})
parsed_error_status = _markdown_http_error_status(content)
ok = process.returncode == 0 and parsed_error_status is None
final_url = _diagnostic_main_document_url(diagnostic_raw, case.url)
diff --git a/harness/web_fetch/providers/subprocess_jsonl.py b/harness/web_fetch/providers/subprocess_jsonl.py
index 2888c6bb3..588736bb4 100644
--- a/harness/web_fetch/providers/subprocess_jsonl.py
+++ b/harness/web_fetch/providers/subprocess_jsonl.py
@@ -193,6 +193,18 @@ def _request_stderr(self, cursor: tuple[int, int]) -> dict[str, Any]:
"stderr_truncated": retained_bytes < actual_bytes,
}
+ def native_worker_request(self, case, timeout, diagnostic_path=None):
+ from ..native_routes import declared_input_media_kind
+ media = declared_input_media_kind(case, required=False)
+ return {'url': case.url, 'log_level': self.config.get('log_level'),
+ 'diagnostic_path': diagnostic_path, 'timeout_seconds': timeout,
+ 'uniform_native_parameters': True,
+ 'max_bytes': int(self.config.get('max_bytes', 20_000_000)),
+ 'dns_mode': self.config.get('dns_mode', 'system'),
+ 'task_kind': case.task_kind, 'track': case.track,
+ **({'input_media_kind': media} if media is not None else {}), 'retrieval_plan': None,
+ 'retrieval_mode': 'native_default' if case.track == 'dynamic' else 'declared_plan'}
+
def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
self.check_native_call(case)
from ..native_routes import declared_input_media_kind
@@ -232,6 +244,8 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
"retrieval_plan": retrieval_plan.worker_dict() if retrieval_plan else None,
"retrieval_mode": "native_default" if native_default_case(case) else "declared_plan",
}
+ if self.config.get('uniform_native_parameters'):
+ request = self.native_worker_request(case, timeout, request['diagnostic_path'])
if getattr(self, '_native_route_ledger', None) is not None:
from ..native_routes import _digest
projected = {**request, 'url': '',
@@ -312,6 +326,8 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
worker_logs_truncated = True
continue
payload = candidate
+ if payload.get("content") is not None and not isinstance(payload["content"], str):
+ raise ValueError("worker content must be text or absent")
ok = bool(payload.get("ok"))
files = []
for item in payload.get("files") or []:
@@ -331,8 +347,14 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
metadata={
**dict(payload.get("metadata") or {}),
**({'native_answer_projection': {'schema_version': 1,
- 'content_pointer': ['content'], 'content_format': 'text'}}
- if isinstance(payload.get('content'), str) else {}),
+ 'content_pointer': ['content'],
+ 'content_format': 'text' if isinstance(payload.get('content'), str) else 'empty',
+ 'answer_fields': {key: {'pointer': ['metadata', key], 'format': 'identity'}
+ for key in ('links', 'images', 'videos', 'media', 'extras')
+ if key in (payload.get('metadata') or {})},
+ **({'files': {'pointer': ['files'], 'format': 'worker_files'}}
+ if payload.get('files') else {})}}
+ ),
"_raw_response_b64": base64.b64encode(line.encode("utf-8")).decode("ascii"),
"raw_response_sha256": hashlib.sha256(line.encode("utf-8")).hexdigest(),
"raw_response_bytes": len(line.encode("utf-8")),
diff --git a/harness/web_fetch/providers/tavily.py b/harness/web_fetch/providers/tavily.py
index 7b8c13893..2742cb676 100644
--- a/harness/web_fetch/providers/tavily.py
+++ b/harness/web_fetch/providers/tavily.py
@@ -27,6 +27,12 @@ def _public_failure_errors(values: list[object]) -> list[str]:
class TavilyExtractProvider(Provider):
+ def native_http_arguments(self, case, timeout):
+ body = {'urls': [case.url], 'extract_depth': self.config.get('extract_depth', 'advanced'),
+ 'format': 'markdown', 'include_images': bool(self.config.get('include_images', False))}
+ return (str(self.config.get('base_url', 'https://api.tavily.com')).rstrip('/') + '/extract',
+ self.native_http_options(body, {'Authorization': 'Bearer '+str(self.config.get('api_key', ''))}, timeout))
+
def validate(self) -> list[str]:
return [] if self.config.get("api_key") else ["api_key"]
@@ -109,5 +115,11 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
request_body.get("timeout")
),
"images": (result.get("images") or payload.get("images") or []) if ok else [],
+ **({'native_answer_projection': {
+ 'schema_version': 1, 'content_pointer': ['results', 0, 'raw_content'],
+ 'content_format': 'text', 'optional_content': True,
+ 'answer_fields': {'images': {
+ 'pointer': ['results', 0, 'images'] if result.get('images') else ['images'],
+ 'format': 'identity', 'default': []}}}} if ok or content else {}),
},
)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 7426bcbc0..fafc13f8c 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -702,6 +702,10 @@ def pct(n):return '—' if n is None else f'{n*100:.1f}%'
def ms(n):return '—' if n is None else f'{n:,.0f} ms'
def link(url,label,english=None):return f'{model_text(label,english)} '
def cell(value,sort=None):return '{value}'
+EXTRA+='''
+table.ranking:not([data-row-index="false"]) tbody td:nth-child(4),
+table.ranking[data-row-index="false"] tbody td:nth-child(3){color:var(--good);font-weight:800}
+'''
def model_text(value,english=None):
return esc(value) if english is None else ''+esc(value)+' '
def rate(a,b):return pct(a/b) if b else '—'
diff --git a/harness/web_fetch/response_input.py b/harness/web_fetch/response_input.py
index cf1349a56..da29c6cf1 100644
--- a/harness/web_fetch/response_input.py
+++ b/harness/web_fetch/response_input.py
@@ -104,36 +104,171 @@ def _attribute_record_urls(groups, base_url):
return sorted(set(urls))
+SCORE_ANSWER_FIELDS = frozenset({'links', 'images', 'videos', 'media', 'extras'})
+
+
+def _selected(payload, spec):
+ try:
+ return _projection_value(payload, spec['pointer'])
+ except ValueError:
+ if 'default' not in spec:
+ raise
+ return spec['default']
+
+
+def _project_field(payload, spec):
+ value = _selected(payload, spec)
+ kind = spec.get('format', 'identity')
+ if kind == 'identity':
+ return value
+ if kind == 'html_attribute_urls':
+ return _attribute_record_urls(value, spec['base_url'])
+ if kind == 'named_links':
+ return [{'text': item['anchor'], 'href': item['url']}
+ for item in value if isinstance(item, dict)
+ and isinstance(item.get('anchor'), str) and item['anchor'].strip()
+ and isinstance(item.get('url'), str) and item['url'].strip()]
+ if kind == 'video_values':
+ values = [item for item in value if isinstance(item, (str, dict))] if isinstance(value, list) else []
+ legacy = _selected(payload, spec['legacy'])
+ return values + ([legacy] if isinstance(legacy, str) and legacy else [])
+ raise ValueError('Unknown native answer field projection')
+
+
def project_native_answer(payload, declaration):
"""Select declared native fields; never inspect references or URL meaning."""
if declaration.get('schema_version') != 1:
raise ValueError('Unsupported native answer projection')
- value = _projection_value(payload, declaration.get('content_pointer'))
kind = declaration.get('content_format')
- if kind == 'text' and isinstance(value, str):
- content = value
- elif kind == 'json':
- content = json.dumps(value, ensure_ascii=False, separators=(',', ':'))
+ if declaration.get('source_format') == 'bytes':
+ if declaration.get('answer_fields'):
+ raise ValueError('Native byte projection cannot declare structured answer fields')
+ if not isinstance(payload, bytes):
+ raise ValueError('Native byte projection requires bytes')
+ if kind == 'utf8':
+ return payload.decode('utf-8'), {}
+ if kind == 'empty':
+ return '', {}
+ raise ValueError('Unknown native byte projection')
+ if kind == 'empty':
+ content = ''
else:
- raise ValueError('Native answer field differs from its declared type')
+ value = _selected(payload, {'pointer': declaration.get('content_pointer'),
+ **({'default': ''} if declaration.get('optional_content') else {})})
+ if kind == 'text' and isinstance(value, str):
+ content = value
+ elif kind == 'json':
+ content = json.dumps(value, ensure_ascii=False, separators=(',', ':'))
+ else:
+ raise ValueError('Native answer field differs from its declared type')
fields = {}
- for name, projection in declaration.get('answer_fields', {}).items():
- if name not in {'links', 'images', 'videos', 'media', 'extras'}:
+ for name, spec in declaration.get('answer_fields', {}).items():
+ if name not in SCORE_ANSWER_FIELDS:
raise ValueError('Unknown native answer metadata field')
- selected = _projection_value(payload, projection.get('pointer'))
- if projection.get('format') == 'html_attribute_urls':
- fields[name] = _attribute_record_urls(selected, projection['base_url'])
- elif projection.get('format') == 'identity':
- fields[name] = selected
- else:
- raise ValueError('Unknown native answer field projection')
+ fields[name] = _project_field(payload, spec)
return content, fields
-def require_native_answer_projection(content, metadata, raw_response):
+def project_native_files(payload, declaration):
+ """Decode native delivered bytes; filenames and logs cannot supply answers."""
+ import base64
+ spec = declaration.get('files')
+ if spec is None:
+ return []
+ value = _selected(payload, spec) if spec.get('format') != 'raw_bytes' else payload
+ if spec['format'] == 'raw_bytes' and isinstance(payload, bytes):
+ return [payload] if payload else []
+ if spec['format'] == 'base64_file':
+ return [base64.b64decode(value, validate=True)] if value else []
+ if spec['format'] == 'worker_files':
+ return [base64.b64decode(item['base64'], validate=True) for item in value]
+ raise ValueError('Unknown native file projection')
+
+
+def require_native_answer_projection(content, metadata, raw_response, *, files=()):
+ """Reject every undeclared score-bearing input, then check exact provenance."""
declaration = metadata.get('native_answer_projection')
+ present = {key for key in SCORE_ANSWER_FIELDS if metadata.get(key)}
if declaration is None:
+ if present or files or (raw_response is not None and content):
+ raise ValueError('Scored native answer has undeclared answer fields')
return
- expected, fields = project_native_answer(json.loads(raw_response), declaration)
+ if raw_response is None:
+ raise ValueError('Declared native answer lacks its retained raw response')
+ if present - set(declaration.get('answer_fields', {})):
+ raise ValueError('Scored native answer has undeclared answer fields')
+ allowed = metadata.get('native_route_receipt', {}).get('answer_fields')
+ if allowed is not None:
+ def check_source(spec):
+ pointer = spec.get('pointer')
+ if pointer:
+ name = '.'.join(str(key) for key in pointer if not isinstance(key, int))
+ allowed_names = [field.replace('[]', '') for field in allowed]
+ if not any(name == field or name.startswith(field+'.') for field in allowed_names):
+ raise ValueError('Native projection reads outside its frozen answer fields')
+ if 'legacy' in spec:
+ check_source(spec['legacy'])
+ if declaration.get('source_format') == 'bytes':
+ required = 'original_bytes' if declaration.get('files') else 'stdout'
+ if required not in allowed:
+ raise ValueError('Native byte projection differs from its frozen answer fields')
+ else:
+ if declaration.get('content_format') != 'empty':
+ check_source({'pointer': declaration.get('content_pointer')})
+ for spec in declaration.get('answer_fields', {}).values():
+ check_source(spec)
+ if declaration.get('files'):
+ check_source(declaration['files'])
+ payload = raw_response if declaration.get('source_format') == 'bytes' else json.loads(raw_response)
+ expected, fields = project_native_answer(payload, declaration)
if content != expected or any(metadata.get(key) != value for key, value in fields.items()):
raise ValueError('Scored answer differs from its declared native field projection')
+ if [item['data'] for item in files] != project_native_files(payload, declaration):
+ raise ValueError('Scored native files differ from their declared byte projection')
+
+
+def verify_saved_native_answer(root, record):
+ """Validate newly declared saved answers at aggregation/release boundaries.
+
+ Older calls with no declaration need a separately recorded historical audit;
+ their missing acquisition declarations must never be invented in place.
+ """
+ from .publication_privacy import read_evidence_binding
+ from .safety import contained_path
+ metadata = record.get('provider_metadata', {})
+ declaration = metadata.get('native_answer_projection')
+ if not declaration:
+ if metadata.get('native_answer_required'):
+ raise ValueError('Scored native answer has no projection declaration')
+ return
+ answer = read_evidence_binding(root, {'path': record['artifact_path'],
+ 'sha256': record['metrics']['content_sha256']}).read_bytes()
+ # Deadlines discard answers while preserving late raw evidence.
+ if not record['ok'] and record.get('error_type') == 'timeout' and not answer:
+ if any(metadata.get(key) for key in SCORE_ANSWER_FIELDS) or metadata.get('file_evaluation', {}).get('files'):
+ raise ValueError('Discarded late native answer retains score-bearing fields')
+ return
+ raw_path = metadata.get('raw_stdout_path') if declaration.get('source_format') == 'bytes' else metadata.get('raw_response_path')
+ raw_hash = metadata.get('raw_stdout_sha256') if declaration.get('source_format') == 'bytes' else metadata.get('raw_response_sha256')
+ if not raw_path or not raw_hash:
+ raise ValueError('Declared native answer lacks its retained raw response')
+ raw = read_evidence_binding(root, {'path': raw_path, 'sha256': raw_hash}).read_bytes()
+ files = [{'data': contained_path(root, item['artifact_path'], reject_symlinks=True).read_bytes()}
+ for item in metadata.get('file_evaluation', {}).get('files', [])]
+ require_native_answer_projection(answer.decode('utf-8'), metadata, raw, files=files)
+ if files or 'file_evaluation' in metadata:
+ from .call_boundary import evaluate_returned_files
+ source_verification = {}
+ dataset_path = root/'dataset.json'
+ if dataset_path.is_file():
+ cases = json.loads(dataset_path.read_bytes()).get('cases', [])
+ case = next((case for case in cases if case.get('id') == record.get('case_id')), None)
+ if case is None:
+ raise ValueError('Native file result lacks its frozen case')
+ source_verification = case.get('source_verification') or {}
+ expected = evaluate_returned_files(files, source_verification)
+ saved = metadata.get('file_evaluation', {})
+ compared = {**saved, 'files': [{k: v for k, v in item.items() if k != 'artifact_path'}
+ for item in saved.get('files', [])]}
+ if compared != expected:
+ raise ValueError('Native file evaluation differs from its retained bytes')
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index 65be6fe70..21a02bd2e 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -381,13 +381,24 @@ def persist_result_artifacts(
"""Persist one physical call without changing its scored answer bytes."""
discarded_late_answer = (not result.ok and result.error_type == 'timeout'
and result.content == '')
- if result.metadata.get('native_answer_projection') and not discarded_late_answer:
- from .response_input import require_native_answer_projection
- raw = result.metadata.get('_raw_response_b64')
- if not isinstance(raw, str):
- raise ValueError('Declared native answer lacks its retained raw response')
+ from .response_input import require_native_answer_projection
+ raw = result.metadata.get('_raw_response_b64')
+ if result.metadata.get('native_answer_projection', {}).get('source_format') == 'bytes':
+ raw = result.metadata.get('_raw_stdout_b64')
+ needs_projection = (result.metadata.get('native_answer_required')
+ or result.metadata.get('native_answer_projection')
+ or any(result.metadata.get(key) for key in ('links', 'images', 'videos', 'media', 'extras'))
+ or (raw is not None and result.files))
+ if not discarded_late_answer and needs_projection:
+ if result.metadata.get('native_answer_required') and not result.metadata.get('native_answer_projection'):
+ raise ValueError('Scored native answer has no projection declaration')
require_native_answer_projection(result.content, result.metadata,
- base64.b64decode(raw, validate=True))
+ base64.b64decode(raw, validate=True) if isinstance(raw, str) else None,
+ files=result.files if raw is not None or result.metadata.get('native_answer_projection') else ())
+ elif discarded_late_answer and (result.files or any(result.metadata.get(key) for key in ('images', 'links', 'videos', 'media', 'extras'))):
+ raise ValueError('Discarded late native answer retains score-bearing fields')
+ if result.metadata.get('native_answer_projection'):
+ result.metadata['native_answer_required'] = True
artifact = contained_path(output_dir, artifact_rel)
artifact.parent.mkdir(parents=True, exist_ok=True)
artifact.write_text(result.content, encoding='utf-8', errors='replace')
@@ -706,7 +717,9 @@ def run_provider(provider: Provider) -> None:
f'run-{run_number:03d}-physical-{physical_attempt:03d}.txt')
artifact_rel = Path('artifacts') / provider.name / case.id / suffix
if native_routes is not None and provider.name in native_routes.providers:
- result.metadata['native_route_receipt'] = native_routes.receipt(provider, case)
+ result.metadata['native_route_receipt'] = native_routes.receipt(provider, case, ok=result.ok)
+ if provider.config.get('uniform_native_parameters') and (result.content or result.files):
+ result.metadata['native_answer_required'] = True
artifact = persist_result_artifacts(output_dir, artifact_rel, result)
artifact_bytes = result.content.encode('utf-8', errors='replace')
measured = score_attempt(
diff --git a/harness/web_fetch/scoring.py b/harness/web_fetch/scoring.py
index 1069fe263..82c1b4803 100644
--- a/harness/web_fetch/scoring.py
+++ b/harness/web_fetch/scoring.py
@@ -21,7 +21,7 @@
"scoring.py", "metrics.py", "stable_tasks.py", "dynamic_lists.py",
"list_verification.py", "reference_quality.py", "dynamic_entry_matching.py",
"response_input.py", "body_coverage.py", "structured_tables.py",
- "ordered_tree.py", "composite_tasks.py",
+ "ordered_tree.py", "composite_tasks.py", "call_boundary.py",
)
)).hexdigest()
@@ -85,6 +85,8 @@ def score_attempt(
list_reference: dict | None = None,
status_code: int | None = None,
prepared_answer: tuple | None = None,
+ native_response: bytes | None = None,
+ native_files: list | None = None,
title: str | None = None,
final_url: str | None = None,
) -> dict[str, Any]:
@@ -97,7 +99,17 @@ def score_attempt(
Previous non-text
evidence is preserved; the caller must have validated its artifact binding.
"""
+ if native_files is not None and native_response is None:
+ raise ValueError('Native file scoring requires its retained response bytes')
+ if native_response is not None:
+ from .response_input import require_native_answer_projection
+ require_native_answer_projection(artifact.decode('utf-8'), metadata or {},
+ native_response, files=native_files or ())
case = _case_dict(case)
+ if native_files is not None:
+ from .call_boundary import evaluate_returned_files
+ metadata = dict(metadata or {})
+ metadata['file_evaluation'] = evaluate_returned_files(native_files, case.get('source_verification') or {})
if 'acceptance_policy_version' in case:
raise ValueError('Historical policy selectors require their frozen scorer; current scoring has one contract')
contract = case.get('acceptance_contract') or {}
@@ -249,7 +261,7 @@ def score_attempt(
if "declared_truncation" not in result["stable_task_errors"]:
result["stable_task_errors"].append("declared_truncation")
file_evaluation = (metadata or {}).get('file_evaluation')
- if result['task_kind'] == 'file' and isinstance(file_evaluation, dict) and 'raw_file_pass' not in result:
+ if result['task_kind'] == 'file' and isinstance(file_evaluation, dict) and ('raw_file_pass' not in result or native_files is not None):
result['raw_file_pass'] = bool(file_evaluation.get('pass'))
result["content_score_applicable"] = case.get("content_score_applicable", True)
contract = scoring_values.get("acceptance_contract")
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
index 861ac4607..0eccfbdc1 100644
--- a/research/native-430-local-report.json
+++ b/research/native-430-local-report.json
@@ -203,5 +203,19 @@
"observed_signature_drift": 0,
"timeouts_without_route_confirmation": 19,
"pre_call_frozen": false
+ },
+ "native_answer_field_audit": {
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-native-answer-field-audit-20261016.json",
+ "sha256": "4bd896daebca2b479c8e1b84fc41a58465385f063962edba55b3dc0629c156f8",
+ "bytes": 1468237,
+ "results_sha256": "ece94c0911a769c84ce9d896b00d97706fd4435d6d194f03cff77391710934a4",
+ "provider_calls_added": 0
+ },
+ "historical_timeout_stage_audit": {
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-timeout-stage-audit-20261016.json",
+ "sha256": "369441bf256f753791ead3f08ead503cf91bc20577ab8cb4e00e88c7bfa202eb",
+ "bytes": 27272,
+ "results_sha256": "ece94c0911a769c84ce9d896b00d97706fd4435d6d194f03cff77391710934a4",
+ "provider_calls_added": 0
}
}
diff --git a/tests/test_native_route_contract.py b/tests/test_native_route_contract.py
index a19e5fe34..74ecb984d 100644
--- a/tests/test_native_route_contract.py
+++ b/tests/test_native_route_contract.py
@@ -359,3 +359,110 @@ def test_route_verifier_preserves_optional_dataset_defaults(tmp_path):
del case['track']
del case['task_kind']
assert verify_native_route_contract(tmp_path, manifest, dataset, records)
+
+
+@pytest.mark.parametrize('stage', ['before_transport', 'after_transport'])
+def test_failed_native_calls_require_an_explicit_transport_stage(tmp_path, stage):
+ from datetime import datetime, timezone
+ from unittest.mock import Mock
+ from copy import deepcopy
+ from harness.web_fetch.native_routes import verify_native_route_contract
+ provider, cases, ledger, manifest, dataset = _route_http_fixture(tmp_path)
+ records = []
+ for case in cases:
+ if stage == 'after_transport':
+ provider.native_http_request(case, Mock(), *provider.native_http_arguments(case, 45)[:1],
+ **provider.native_http_arguments(case, 45)[1])
+ records.append({'provider': provider.name, 'case_id': case.id, 'url': case.url,
+ 'started_at': datetime.now(timezone.utc).isoformat(), 'ok': False,
+ 'provider_metadata': {'native_route_receipt': ledger.receipt(provider, case, ok=False)}})
+ assert verify_native_route_contract(tmp_path, manifest, dataset, records)
+ for defect in ('unknown_stage', 'missing_failure_stage', 'missing_signature', 'wrong_stage'):
+ bad = deepcopy(records)
+ receipt = bad[0]['provider_metadata']['native_route_receipt']
+ if defect == 'unknown_stage': receipt.pop('transport_state')
+ if defect == 'missing_failure_stage': receipt.pop('failure_stage')
+ if defect == 'missing_signature':
+ receipt['requests'] = []
+ receipt['transport_state'] = 'started'
+ if defect == 'wrong_stage': receipt['failure_stage'] = 'unknown'
+ with pytest.raises(ValueError, match='Native'):
+ verify_native_route_contract(tmp_path, manifest, dataset, bad)
+
+
+def test_runner_persists_pre_transport_failure_without_inventing_request(tmp_path):
+ import json
+ from dataclasses import asdict
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase, ProviderResult
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.runner import run_evaluation
+ case = EvalCase('case', 'Case', 'https://example.test/', 'control', [],
+ source_benchmark={'input_media_kind': 'html_page'})
+ dataset = {'tool': 'web_fetch', 'id': 'fixture', 'cases': [asdict(case)]}
+ source = tmp_path/'dataset.json'
+ source.write_text(json.dumps(dataset))
+ provider = build_provider('tavily', {'type': 'tavily_extract', 'api_key': 'fixture',
+ 'uniform_native_parameters': True})
+ with patch.object(provider, 'fetch', return_value=ProviderResult(False, '', error_type='configuration')):
+ records, _ = run_evaluation(output_dir=tmp_path/'run', dataset_meta=dataset, dataset_path=source,
+ cases=[case], providers=[provider], provider_config_path=tmp_path/'providers.json',
+ runs=1, timeout=45, delay=0, reference_policy='frozen_existing')
+ receipt = records[0]['provider_metadata']['native_route_receipt']
+ assert receipt['failure_stage'] == 'before_transport'
+ assert receipt['transport_state'] == 'not_started' and receipt['requests'] == []
+
+
+@pytest.mark.parametrize('method,module,payload,track,kind', [
+ ('tavily_extract', 'tavily', {'results': [{'raw_content': 'answer', 'images': ['https://example.test/image.png']}]}, 'image', 'resource'),
+ ('tavily_extract', 'tavily', {'results': [{'raw_content': 'answer'}], 'images': ['https://example.test/image.png']}, 'image', 'resource'),
+ ('exa_contents', 'exa', {'results': [{'text': 'answer', 'extras': {'richLinks': [
+ {'anchor': 'Article', 'url': 'https://example.test/article'}], 'imageLinks': ['https://example.test/image.png']}}]}, 'image', 'resource'),
+ ('firecrawl', 'firecrawl', {'success': True, 'data': {'markdown': 'answer', 'images': ['https://example.test/image.png'],
+ 'links': ['https://example.test/article'], 'videos': ['https://example.test/video.mp4'], 'video': 'https://example.test/legacy.mp4'}}, 'video', 'resource'),
+ ('firecrawl', 'firecrawl', {'success': True, 'data': {'rawBase64': 'b3JpZ2luYWwgZmlsZSBieXRlcw=='}}, 'file', 'file'),
+ ('cloudflare_markdown', 'cloudflare', {'success': True, 'result': 'answer'}, 'web', 'body'),
+ ('cloudflare_content', 'cloudflare', {'success': True, 'result': 'answer '}, 'web', 'body'),
+ ('cloudflare_markdown', 'cloudflare', {'success': True, 'result': [{'selector': 'img', 'results': [
+ {'attributes': [{'name': 'src', 'value': '/image.png'}, {'name': 'src', 'value': '/wrong.png'}]}]}]}, 'image', 'resource'),
+])
+def test_http_native_answers_survive_persistence_and_reject_tampering(tmp_path, method, module, payload, track, kind):
+ from pathlib import Path
+ import json
+ from copy import deepcopy
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.providers.http_client import HttpResponse
+ from harness.web_fetch.native_routes import NativeRouteLedger
+ from harness.web_fetch.runner import persist_result_artifacts
+ from harness.web_fetch.call_boundary import prepare_answer
+ provider = build_provider('method', {'type': method, 'api_key': 'fixture', 'api_token': 'fixture',
+ 'account_id': 'fixture', 'uniform_native_parameters': True, 'include_images': True,
+ 'include_resource_links': True, 'rich_links_limit': 12, 'formats': ['markdown', 'images', 'links']})
+ case = EvalCase('case', 'Case', 'https://example.test/', 'control', [], track=track, task_kind=kind,
+ source_benchmark={'input_media_kind': 'html_page'})
+ ledger = NativeRouteLedger([case], [provider])
+ ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture')
+ with patch('harness.web_fetch.providers.'+module+'.request',
+ return_value=HttpResponse(200, case.url, {}, json.dumps(payload).encode())):
+ result = prepare_answer(provider.fetch(case, 45), case)
+ assert result.ok
+ result.metadata['native_route_receipt'] = ledger.receipt(provider, case, ok=result.ok)
+ altered = deepcopy(result)
+ altered.metadata['images'] = ['https://example.test/forged.png']
+ with pytest.raises(ValueError, match='native|projection'):
+ persist_result_artifacts(tmp_path, Path('artifacts/forged.txt'), altered)
+ persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result)
+
+
+def test_projection_cannot_read_request_echo_outside_declared_answer_fields():
+ import json
+ from harness.web_fetch.response_input import require_native_answer_projection
+ raw = json.dumps({'result': 'answer', 'request': {'images': ['https://example.test/forged.png']}}).encode()
+ metadata = {'images': ['https://example.test/forged.png'],
+ 'native_route_receipt': {'answer_fields': ['result']},
+ 'native_answer_projection': {'schema_version': 1, 'content_pointer': ['result'], 'content_format': 'text',
+ 'answer_fields': {'images': {'pointer': ['request', 'images'], 'format': 'identity'}}}}
+ with pytest.raises(ValueError, match='frozen answer fields'):
+ require_native_answer_projection('answer', metadata, raw)
diff --git a/tests/test_provider_evidence_boundary.py b/tests/test_provider_evidence_boundary.py
index 4406bf68f..250f6e1a9 100644
--- a/tests/test_provider_evidence_boundary.py
+++ b/tests/test_provider_evidence_boundary.py
@@ -164,9 +164,12 @@ class Fixture(Provider):
def fetch(self, case, timeout):
self.calls[case.id] = self.calls.get(case.id, 0) + 1
- private = b'{"synthetic_private_response":true}'
+ answer = 'Expected fixture' if self.calls[case.id] == 2 else 'Wrong'
+ private = json.dumps({'synthetic_private_response': True, 'content': answer}).encode()
return ProviderResult(True, 'Expected fixture' if self.calls[case.id] == 2 else 'Wrong',
- metadata={'_rendered_html': 'fixture ',
+ metadata={'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['content'], 'content_format': 'text'},
+ '_rendered_html': 'fixture ',
'rendered_html_sha256': hashlib.sha256(b'fixture ').hexdigest(),
'_raw_stdout_b64': base64.b64encode(b'fixture stdout').decode(),
'raw_stdout_sha256': hashlib.sha256(b'fixture stdout').hexdigest(),
diff --git a/tests/test_scoring.py b/tests/test_scoring.py
index 8a74aec1f..3b9d3c9eb 100644
--- a/tests/test_scoring.py
+++ b/tests/test_scoring.py
@@ -1,4 +1,5 @@
"""Shared scoring uses saved bytes only; all calls and references are synthetic."""
+import base64
import hashlib
import io
import json
@@ -657,7 +658,7 @@ def test_source_hash_includes_shared_helper(self):
"scoring.py", "metrics.py", "stable_tasks.py", "dynamic_lists.py",
"list_verification.py", "reference_quality.py", "dynamic_entry_matching.py",
"response_input.py", "body_coverage.py", "structured_tables.py",
- "ordered_tree.py", "composite_tasks.py"))).hexdigest()
+ "ordered_tree.py", "composite_tasks.py", "call_boundary.py"))).hexdigest()
self.assertEqual(SCORING_SOURCE_SHA256, expected)
self.assertEqual(score_attempt(case(), b"Text", ok=True)["scoring_source_sha256"], expected)
@@ -702,11 +703,19 @@ def setUp(self):
self.provider.config = {}
self.provider.name, self.provider.type, self.provider.min_interval_seconds = "p", "fixture", 0
self.provider.validate.return_value = []
- self.provider.fetch.side_effect = lambda c, timeout: ProviderResult(
- ok=c.id != "failed", content=texts[c.id], status_code=200 if c.id != "failed" else 502,
- final_url=("https://new.example.test/news/" if c.id == "redirected" else c.url),
- error_type="synthetic_failure" if c.id == "failed" else None,
- metadata={"images": [{"src": "/image.png"}], "request_id": "original-" + c.id})
+ def response(c, timeout):
+ fields = {'images': [{'src': '/image.png'}], 'request_id': 'original-'+c.id}
+ raw = json.dumps({'content': texts[c.id], 'metadata': fields}).encode()
+ return ProviderResult(ok=c.id != 'failed', content=texts[c.id],
+ status_code=200 if c.id != 'failed' else 502,
+ final_url='https://new.example.test/news/' if c.id == 'redirected' else c.url,
+ error_type='synthetic_failure' if c.id == 'failed' else None,
+ metadata={**fields, '_raw_response_b64': base64.b64encode(raw).decode(),
+ 'raw_response_sha256': hashlib.sha256(raw).hexdigest(), 'raw_response_bytes': len(raw),
+ 'native_answer_projection': {'schema_version': 1, 'content_pointer': ['content'],
+ 'content_format': 'text', 'answer_fields': {'images': {
+ 'pointer': ['metadata', 'images'], 'format': 'identity'}}}})
+ self.provider.fetch.side_effect = response
def live(self):
with redirect_stdout(io.StringIO()):
@@ -909,6 +918,20 @@ def test_both_rescores_verify_every_declared_evidence_before_writes(self):
snapshots = ("results.jsonl", "summary.json", "dataset.json", "manifest.json")
for kind, declaration in declarations.items():
record["provider_metadata"] = {**baseline, **declaration}
+ if kind == 'downloaded_file':
+ from harness.web_fetch.call_boundary import evaluate_returned_files
+ native = json.loads((self.run/baseline['raw_response_path']).read_bytes())
+ native['files'] = [{'base64': base64.b64encode(original).decode()}]
+ raw = json.dumps(native).encode()
+ raw_path = self.run/'private_artifacts/auxiliary-native.response.bin'
+ raw_path.write_bytes(raw)
+ fields = record['provider_metadata']
+ fields.update(raw_response_path=raw_path.relative_to(self.run).as_posix(),
+ raw_response_sha256=hashlib.sha256(raw).hexdigest(), raw_response_bytes=len(raw))
+ fields['native_answer_projection'] = {**baseline['native_answer_projection'],
+ 'files': {'pointer': ['files'], 'format': 'worker_files'}}
+ fields['file_evaluation'] = evaluate_returned_files([{'data': original}], {})
+ fields['file_evaluation']['files'][0]['artifact_path'] = relative
artifact.unlink(missing_ok=True)
artifact.write_bytes(original)
(self.run / "results.jsonl").write_text(
diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py
index 346114d9e..596ebdaab 100644
--- a/tests/test_scoring_regressions.py
+++ b/tests/test_scoring_regressions.py
@@ -62,11 +62,17 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self)
root_modes = {}
for identifier, row in rows.items():
with self.subTest(regression=identifier):
- metrics = score_attempt(
- row["case"], row["artifact"].encode(), ok=row["ok"],
- metadata=row.get("metadata"), title=row.get("title"),
- final_url=row.get("final_url"),
- )
+ arguments = dict(ok=row['ok'], metadata=row.get('metadata'), title=row.get('title'),
+ final_url=row.get('final_url'))
+ if 'native_response' in row:
+ arguments['native_response'] = row['native_response'].encode()
+ if row.get('expected_validation_error'):
+ with self.assertRaisesRegex(ValueError, row['expected_validation_error']):
+ score_attempt(row['case'], row['artifact'].encode(), **arguments)
+ self.assertFalse(row['expected_task_pass'])
+ root_modes.setdefault(row['root_cause'], set()).add(row['failure_mode'])
+ continue
+ metrics = score_attempt(row['case'], row['artifact'].encode(), **arguments)
verdict = task_passed(
row["ok"], metrics, row["case"].get("task_kind", "body")
)
diff --git a/tests/test_subprocess_provider.py b/tests/test_subprocess_provider.py
index 46d458b18..375dc8f4d 100644
--- a/tests/test_subprocess_provider.py
+++ b/tests/test_subprocess_provider.py
@@ -322,3 +322,89 @@ def test_stderr_keeps_more_than_legacy_two_hundred_lines(self) -> None:
if __name__ == "__main__":
unittest.main()
+
+
+# These controls operate only on synthetic saved responses; no provider calls.
+import pytest
+
+
+@pytest.mark.parametrize('field', ['images', 'links', 'videos', 'media', 'extras'])
+@pytest.mark.parametrize('declare_content', [False, True])
+def test_undeclared_score_fields_cannot_enter_persistence(tmp_path, field, declare_content):
+ import hashlib
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ raw = b'{"content":"answer","metadata":{"images":["https://example.test/native.png"]}}'
+ metadata = {field: ['https://example.test/forged.png'],
+ '_raw_response_b64': base64.b64encode(raw).decode(),
+ 'raw_response_sha256': hashlib.sha256(raw).hexdigest(), 'raw_response_bytes': len(raw)}
+ if declare_content:
+ metadata['native_answer_projection'] = {'schema_version': 1,
+ 'content_pointer': ['content'], 'content_format': 'text'}
+ with pytest.raises(ValueError, match='undeclared'):
+ persist_result_artifacts(tmp_path, Path('artifacts/test.txt'), ProviderResult(True, 'answer', metadata=metadata))
+ assert not (tmp_path/'artifacts/test.txt').exists()
+
+
+@pytest.mark.parametrize('defect', [None, 'replace_image', 'drop_wrong_image', 'forge_file'])
+def test_native_worker_fields_and_files_are_exactly_bound(tmp_path, defect):
+ import hashlib
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ from harness.web_fetch.call_boundary import prepare_answer
+ payload = {'content': 'answer', 'metadata': {'images': ['https://example.test/good.png',
+ 'https://example.test/wrong.png']},
+ 'files': [{'base64': base64.b64encode(b'original file bytes').decode()}]}
+ raw = json.dumps(payload).encode()
+ metadata = {**payload['metadata'], 'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['content'], 'content_format': 'text',
+ 'answer_fields': {'images': {'pointer': ['metadata', 'images'], 'format': 'identity'}},
+ 'files': {'pointer': ['files'], 'format': 'worker_files'}},
+ '_raw_response_b64': base64.b64encode(raw).decode(),
+ 'raw_response_sha256': hashlib.sha256(raw).hexdigest(), 'raw_response_bytes': len(raw)}
+ files = [{'data': b'original file bytes'}]
+ if defect == 'replace_image': metadata['images'] = ['https://example.test/forged.png']
+ if defect == 'drop_wrong_image': metadata['images'] = metadata['images'][:1]
+ if defect == 'forge_file': files = [{'data': b'forged file bytes'}]
+ result = prepare_answer(ProviderResult(True, 'answer', metadata=metadata, files=files),
+ EvalCase('case', 'Case', 'https://example.test/', 'control', []))
+ if defect:
+ with pytest.raises(ValueError, match='projection'):
+ persist_result_artifacts(tmp_path, Path('artifacts/test.txt'), result)
+ else:
+ persist_result_artifacts(tmp_path, Path('artifacts/test.txt'), result)
+ from harness.web_fetch.artifacts import verify_record_artifacts
+ record = {'attempt_id': 'test', 'ok': True, 'artifact_path': 'artifacts/test.txt',
+ 'metrics': {'content_sha256': hashlib.sha256(b'answer').hexdigest()},
+ 'provider_metadata': result.metadata}
+ verify_record_artifacts(tmp_path, record)
+ result.metadata['file_evaluation']['pass'] = True
+ with pytest.raises(ValueError, match='file evaluation'):
+ verify_record_artifacts(tmp_path, record)
+ result.metadata['file_evaluation']['pass'] = False
+ result.metadata['images'] = ['https://example.test/forged.png']
+ with pytest.raises(ValueError, match='projection'):
+ verify_record_artifacts(tmp_path, record)
+
+
+def test_file_scoring_recomputes_acceptance_from_native_bytes():
+ from harness.web_fetch.scoring import score_attempt
+ from harness.web_fetch.metrics import task_passed
+ raw = json.dumps({'content': '', 'files': [{'base64': base64.b64encode(b'not a PNG').decode()}]}).encode()
+ metadata = {'file_evaluation': {'pass': True}, 'native_answer_projection': {'schema_version': 1,
+ 'content_pointer': ['content'], 'content_format': 'text',
+ 'files': {'pointer': ['files'], 'format': 'worker_files'}}}
+ metrics = score_attempt({'url': 'https://example.test/', 'task_kind': 'file'}, b'', ok=True,
+ metadata=metadata, previous_metrics={'raw_file_pass': True}, native_response=raw,
+ native_files=[{'data': b'not a PNG'}])
+ assert metrics['raw_file_pass'] is False
+ assert not task_passed(True, metrics, 'file')
+
+
+def test_byte_projection_cannot_claim_unverified_structured_metadata():
+ from harness.web_fetch.response_input import require_native_answer_projection
+ metadata = {'images': ['https://example.test/forged.png'], 'native_answer_projection': {
+ 'schema_version': 1, 'source_format': 'bytes', 'content_format': 'utf8',
+ 'answer_fields': {'images': {'pointer': ['images'], 'format': 'identity'}}}}
+ with pytest.raises(ValueError, match='cannot declare'):
+ require_native_answer_projection('native stdout', metadata, b'native stdout')
From 9eb9d968192c86790d7b1ade7017b259c5fa5e42 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 12:50:35 +0800
Subject: [PATCH 40/50] Keep timed-out native file tasks in the measured
failure count
---
harness/web_fetch/call_boundary.py | 2 ++
tests/test_runner_concurrency.py | 28 ++++++++++++++++++++++++++++
2 files changed, 30 insertions(+)
diff --git a/harness/web_fetch/call_boundary.py b/harness/web_fetch/call_boundary.py
index ff6c54da6..d3a553f5d 100644
--- a/harness/web_fetch/call_boundary.py
+++ b/harness/web_fetch/call_boundary.py
@@ -90,6 +90,8 @@ def work():
metadata.update(late.metadata)
for key in ('links', 'images', 'videos', 'media', 'extras', 'file_evaluation'):
metadata.pop(key, None)
+ if case.task_kind == 'file':
+ metadata['file_evaluation'] = evaluate_returned_files([], case.source_verification)
metadata['_answer_elapsed_ms']=answer_elapsed_ms
return ProviderResult(
False,
diff --git a/tests/test_runner_concurrency.py b/tests/test_runner_concurrency.py
index 7d18d76aa..6f26a25df 100644
--- a/tests/test_runner_concurrency.py
+++ b/tests/test_runner_concurrency.py
@@ -47,6 +47,34 @@ def cancel(self):self.release.set()
self.assertIn('connection pending',result.metadata['stderr'])
self.assertLess(result.metadata['_answer_elapsed_ms'],100)
+ def test_discarded_late_file_remains_a_measured_failure(self):
+ from harness.web_fetch.call_boundary import fetch_with_deadline
+ from harness.web_fetch.scoring import score_attempt
+ from harness.web_fetch.metrics import task_measured, task_passed
+ class LateFile(Provider):
+ def __init__(self):
+ super().__init__('late', {})
+ self.release = threading.Event()
+ @property
+ def cancellation_cleanup_seconds(self):
+ return .2
+ def cancel(self):
+ self.release.set()
+ def fetch(self, case, timeout):
+ self.release.wait(1)
+ return ProviderResult(True, '', files=[{'data': b'\x89PNG\r\n\x1a\n'+b'x'*40}],
+ metadata={'images': ['https://example.test/late.png']})
+ case = EvalCase('case', 'Case', 'https://example.test/file', 'static', [], task_kind='file')
+ result, worker = fetch_with_deadline(LateFile(), case, .02)
+ self.assertFalse(worker.is_alive())
+ self.assertEqual(result.error_type, 'timeout')
+ self.assertEqual(result.files, [])
+ self.assertNotIn('images', result.metadata)
+ metrics = score_attempt(case, b'', ok=result.ok, metadata=result.metadata)
+ self.assertFalse(metrics['raw_file_pass'])
+ self.assertTrue(task_measured(metrics))
+ self.assertFalse(task_passed(result.ok, metrics))
+
def test_capacity_hook_is_optional_and_runs_once_per_logical_provider(self):
calls = []
From c6c004f7840f3ed10ed370d2abc2ce621eec429c Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 14:13:26 +0800
Subject: [PATCH 41/50] Validate native title and final URL provenance
---
evaluation/scoring-regressions.json | 192 ++++++++++++++++++
harness/web_fetch/README.md | 18 ++
harness/web_fetch/local_revision.py | 22 +-
harness/web_fetch/native_routes.py | 12 +-
harness/web_fetch/providers/cloudflare.py | 43 ++--
harness/web_fetch/providers/exa.py | 8 +-
harness/web_fetch/providers/firecrawl.py | 11 +-
harness/web_fetch/providers/moli.py | 82 +-------
.../web_fetch/providers/subprocess_jsonl.py | 4 +-
harness/web_fetch/providers/tavily.py | 9 +-
harness/web_fetch/response_input.py | 143 ++++++++++++-
harness/web_fetch/runner.py | 14 +-
harness/web_fetch/scoring.py | 7 +-
tests/test_cowagent_budget.py | 3 +-
tests/test_local_revision.py | 38 ++++
tests/test_native_route_contract.py | 128 +++++++++++-
tests/test_public_call_logs.py | 4 +-
tests/test_scoring.py | 6 +-
tests/test_subprocess_provider.py | 2 +-
19 files changed, 612 insertions(+), 134 deletions(-)
diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json
index 4e67c14d3..2c5d5e66e 100644
--- a/evaluation/scoring-regressions.json
+++ b/evaluation/scoring-regressions.json
@@ -32878,6 +32878,198 @@
},
"expected_task_pass": false,
"expected_validation_error": "projection"
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_title_or_final_url",
+ "ok": true,
+ "id": "native-title-must-match-original",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://wrong.example.test/page",
+ "task_kind": "body",
+ "required_facts": [
+ {
+ "alternatives": [
+ "SECRET"
+ ]
+ }
+ ],
+ "expected_any": [],
+ "min_content_chars": 0
+ },
+ "artifact": "",
+ "native_response": "{\"content\": \"\"}",
+ "metadata": {
+ "native_answer_projection": {
+ "schema_version": 2,
+ "content_format": "text",
+ "content_pointer": [
+ "content"
+ ],
+ "result_fields": {
+ "title": {
+ "pointer": [
+ "title"
+ ]
+ },
+ "final_url": {
+ "pointer": [
+ "final_url"
+ ],
+ "fallback": "request_url"
+ }
+ }
+ }
+ },
+ "title": "SECRET",
+ "final_url": "https://wrong.example.test/page",
+ "expected_task_pass": false,
+ "expected_validation_error": "Scored title or final URL"
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_title_or_final_url",
+ "ok": true,
+ "id": "native-returned-title-remains-answer",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://wrong.example.test/page",
+ "task_kind": "body",
+ "required_facts": [
+ {
+ "alternatives": [
+ "SECRET"
+ ]
+ }
+ ],
+ "expected_any": [],
+ "min_content_chars": 0
+ },
+ "artifact": "",
+ "native_response": "{\"content\": \"\", \"title\": \"SECRET\"}",
+ "metadata": {
+ "native_answer_projection": {
+ "schema_version": 2,
+ "content_format": "text",
+ "content_pointer": [
+ "content"
+ ],
+ "result_fields": {
+ "title": {
+ "pointer": [
+ "title"
+ ]
+ },
+ "final_url": {
+ "pointer": [
+ "final_url"
+ ],
+ "fallback": "request_url"
+ }
+ }
+ }
+ },
+ "title": "SECRET",
+ "final_url": "https://wrong.example.test/page",
+ "expected_task_pass": true
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_title_or_final_url",
+ "ok": true,
+ "id": "native-final-url-must-match-original",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://wrong.example.test/page",
+ "task_kind": "resource",
+ "track": "image",
+ "expected_any": [],
+ "expected_resources": [
+ {
+ "id": "asset",
+ "urls": [
+ "https://good.example.test/asset.png"
+ ]
+ }
+ ]
+ },
+ "artifact": "",
+ "native_response": "{\"content\": \"\"}",
+ "metadata": {
+ "native_answer_projection": {
+ "schema_version": 2,
+ "content_format": "text",
+ "content_pointer": [
+ "content"
+ ],
+ "result_fields": {
+ "title": {
+ "pointer": [
+ "title"
+ ]
+ },
+ "final_url": {
+ "pointer": [
+ "final_url"
+ ],
+ "fallback": "request_url"
+ }
+ }
+ }
+ },
+ "title": null,
+ "final_url": "https://good.example.test/page",
+ "expected_task_pass": false,
+ "expected_validation_error": "Scored title or final URL"
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_title_or_final_url",
+ "ok": true,
+ "id": "native-returned-redirect-resolves-relative-image",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://wrong.example.test/page",
+ "task_kind": "resource",
+ "track": "image",
+ "expected_any": [],
+ "expected_resources": [
+ {
+ "id": "asset",
+ "urls": [
+ "https://good.example.test/asset.png"
+ ]
+ }
+ ]
+ },
+ "artifact": "",
+ "native_response": "{\"content\": \"\", \"final_url\": \"https://good.example.test/page\"}",
+ "metadata": {
+ "native_answer_projection": {
+ "schema_version": 2,
+ "content_format": "text",
+ "content_pointer": [
+ "content"
+ ],
+ "result_fields": {
+ "title": {
+ "pointer": [
+ "title"
+ ]
+ },
+ "final_url": {
+ "pointer": [
+ "final_url"
+ ],
+ "fallback": "request_url"
+ }
+ }
+ }
+ },
+ "title": null,
+ "final_url": "https://good.example.test/page",
+ "expected_task_pass": true
}
],
"superseded_controls": {
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index 8fdb93fbe..17b4a9586 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -939,3 +939,21 @@ Local uncertified reports retain a disclosure on every page and omit ordinal
rankings. The CLI cannot hide the disclosure. Historical article HTML is limited
to inert allowlisted markup and safe links/raster images; its translations apply
only inside the article. A diagnostic snapshot cannot certify a formal release.
+
+
+### Native scored result context
+
+Native projection schema 2 includes `result_fields.title` and `result_fields.final_url`
+alongside text, resource metadata and delivered files. Validate these fields before
+persisting a call, before native-response scoring, and when aggregating saved calls.
+JSON fields use declared structural pointers. A final address may also be replayed
+from a retained same-call main-document log or a source HTTP response receipt;
+request-URL fallback applies only when the selected native address is absent.
+Resolve relative resource attributes against that verified address. A title absent
+from the native response cannot supply required facts.
+
+Immutable schema-1 acquisitions retain their original text/resource/file scope;
+they do not retroactively prove title/address provenance. Bind a separate offline
+context audit to historical results, replay the complete affected route when a
+context difference is found, and retain the original records and exact score effects.
+These checks add no provider calls.
diff --git a/harness/web_fetch/local_revision.py b/harness/web_fetch/local_revision.py
index 5a4e2431c..cd8e1e07d 100644
--- a/harness/web_fetch/local_revision.py
+++ b/harness/web_fetch/local_revision.py
@@ -94,7 +94,7 @@ def require_local_revision(source, binding, dataset, records):
def _require_native_answer_correction(source, original, corrected, proof):
"""Verify a derived answer against the same immutable native response."""
- from .response_input import project_native_answer
+ from .response_input import project_native_answer, project_native_result_fields
from .publication_privacy import original_evidence_size
if proof.get('attempt_id') != original['attempt_id']:
raise ValueError('Native answer correction lost its attempt identity')
@@ -112,17 +112,22 @@ def _require_native_answer_correction(source, original, corrected, proof):
or before['path'] == after['path']):
raise ValueError('Native answer correction must preserve its old artifact')
read_bound(source, before)
- text, fields = project_native_answer(json.loads(native.read_bytes()), proof['projection'])
+ payload = json.loads(native.read_bytes())
+ text, fields = project_native_answer(payload, proof['projection'])
+ result_fields = (project_native_result_fields(payload, proof['projection'], request_url=original.get('url'))
+ if 'result_fields' in proof['projection'] else {})
+ if any(corrected.get(key) != value for key, value in result_fields.items()):
+ raise ValueError('Native result context correction differs from original fields')
if read_bound(source, after).read_bytes() != text.encode('utf-8', errors='replace'):
raise ValueError('Native answer correction differs from declared original fields')
for field in proof['projection'].get('answer_fields', {}).values():
- if field.get('format') == 'html_attribute_urls' and field.get('base_url') != original['url']:
- raise ValueError('Native URL projection must use the original request URL')
+ if field.get('format') == 'html_attribute_urls' and field.get('base_url') != (result_fields.get('final_url') or original['url']):
+ raise ValueError('Native URL projection must use the verified result or request URL')
expected_metadata = {**metadata, **fields, 'native_answer_projection': proof['projection']}
if corrected.get('provider_metadata') != expected_metadata:
raise ValueError('Native answer correction changed unrelated response metadata')
derived = {'metrics', 'scoring', 'diagnosis', 'failure_category',
- 'artifact_path', 'provider_metadata'}
+ 'artifact_path', 'provider_metadata'} | set(result_fields)
if ({k: v for k, v in original.items() if k not in derived}
!= {k: v for k, v in corrected.items() if k not in derived}):
raise ValueError('Native answer correction changed physical call facts')
@@ -296,14 +301,17 @@ def revise_local_snapshot(source, configuration):
else task_passed(row['ok'], row['metrics']))
before_verdicts[row['attempt_id']] = before
if row['attempt_id'] in native_requests:
- from .response_input import project_native_answer
+ from .response_input import project_native_answer, project_native_result_fields
request = native_requests[row['attempt_id']]
native_path = read_bound(source, request['native_response'])
metadata = row.get('provider_metadata', {})
if (request['native_response']['sha256'] != metadata.get('raw_response_sha256')
or native_path.stat().st_size != metadata.get('raw_response_bytes')):
raise ValueError('Native correction source differs from its original response')
- answer, fields = project_native_answer(json.loads(native_path.read_bytes()), request['projection'])
+ payload = json.loads(native_path.read_bytes())
+ answer, fields = project_native_answer(payload, request['projection'])
+ if 'result_fields' in request['projection']:
+ row.update(project_native_result_fields(payload, request['projection'], request_url=row.get('url')))
raw = answer.encode('utf-8', errors='replace')
original_artifact = {'path': row['artifact_path'], 'sha256': row['metrics']['content_sha256']}
digest = hashlib.sha256(raw).hexdigest()
diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py
index 734809fc5..858c63872 100644
--- a/harness/web_fetch/native_routes.py
+++ b/harness/web_fetch/native_routes.py
@@ -48,31 +48,33 @@ def native_route_cell(provider, case):
if kind == 'resource' else 'native_document_bytes'
if track == 'document' and media in {'direct_pdf', 'direct_docx'}
else 'dynamic_native_markdown' if track == 'dynamic' else 'ordinary_markdown')
- fields = ['original_bytes'] if interface in {'raw_file', 'native_document_bytes'} else ['stdout']
+ fields = (['original_bytes'] if interface in {'raw_file', 'native_document_bytes'}
+ else ['stdout', 'diagnostic.main_document_url'])
elif method in {'firecrawl', 'tavily_extract', 'exa_contents', 'cloudflare_markdown', 'cloudflare_content'}:
transport = 'http'
if method == 'firecrawl':
interface = '/v2/scrape'
endpoint = str(config.get('base_url', 'https://api.firecrawl.dev')).rstrip('/') + interface
fields = ['data.rawBase64'] if kind == 'file' else ['data.markdown', 'data.links', 'data.images', 'data.videos', 'data.video']
+ fields += ['data.metadata.title', 'data.metadata.url']
elif method == 'tavily_extract':
interface = '/extract'
endpoint = str(config.get('base_url', 'https://api.tavily.com')).rstrip('/') + interface
- fields = ['results[].raw_content', 'results[].images', 'images']
+ fields = ['results[].raw_content', 'results[].images', 'images', 'results[].url']
elif method == 'exa_contents':
interface = '/contents'
endpoint = str(config.get('base_url', 'https://api.exa.ai')).rstrip('/') + interface
- fields = ['results[].text', 'results[].extras']
+ fields = ['results[].text', 'results[].extras', 'results[].title', 'results[].url']
else:
interface = provider.native_http_interface(case)
endpoint = 'https://api.cloudflare.com/client/v4/accounts/' + str(config.get('account_id', '')) + '/browser-rendering' + interface
if method=='cloudflare_markdown' and interface=='/markdown' and config.get('browser'):
from urllib.parse import urlencode
endpoint += '?'+urlencode({'browser': config['browser']})
- fields = ['result']
+ fields = ['result', 'meta.title', 'meta.finalUrl']
elif method in {'subprocess_jsonl', 'cowagent_builtin'}:
transport, interface = 'jsonl', 'native_worker_command'
- fields = ['content', 'metadata.links', 'metadata.images', 'metadata.videos', 'metadata.media', 'metadata.extras', 'files']
+ fields = ['content', 'metadata.links', 'metadata.images', 'metadata.videos', 'metadata.media', 'metadata.extras', 'files', 'title', 'final_url']
elif method == 'fixture':
transport, interface, fields = 'fixture', 'synthetic_fixture', ['content']
else:
diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py
index c33c76920..2f36f2e4f 100644
--- a/harness/web_fetch/providers/cloudflare.py
+++ b/harness/web_fetch/providers/cloudflare.py
@@ -408,9 +408,11 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) ->
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
- **({'native_answer_projection': {'schema_version': 1,
- 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
- if envelope_success and isinstance(payload.get('result', ''), str) else {}),
+ **({'native_answer_projection': {'schema_version': 2,
+ 'content_pointer': ['result'], 'content_format': 'text' if envelope_success and isinstance(payload.get('result', ''), str) else 'empty', 'optional_content': True,
+ 'result_fields': {'title': {'pointer': ['meta', 'title']},
+ 'final_url': {'pointer': ['meta', 'finalUrl'], 'fallback': 'request_url'}}}}
+ if (envelope_success and isinstance(payload.get('result', ''), str)) or meta else {}),
"route": route,
"wait": wait_metadata,
"browser": self.config.get("browser"),
@@ -475,10 +477,14 @@ def _fetch_media_resources(
envelope_success = payload.get("success") is True
from ..response_input import project_native_answer
answer_key = 'images' if case.track == 'image' else 'videos'
- projection = {'schema_version': 1, 'content_pointer': ['result'],
- 'content_format': 'json', 'answer_fields': {answer_key: {
+ meta = payload.get('meta') if isinstance(payload.get('meta'), dict) else {}
+ effective_url = meta.get('finalUrl') or case.url
+ projection = {'schema_version': 2, 'content_pointer': ['result'],
+ 'content_format': 'json', 'result_fields': {'final_url': {
+ 'pointer': ['meta', 'finalUrl'], 'fallback': 'request_url'}},
+ 'answer_fields': {answer_key: {
'pointer': ['result'], 'format': 'html_attribute_urls',
- 'base_url': case.url}}}
+ 'base_url': effective_url}}}
answer, fields = ('', {})
if envelope_success:
answer, fields = project_native_answer(payload, projection)
@@ -495,18 +501,17 @@ def _fetch_media_resources(
ok=error_type is None,
content=answer,
status_code=response.status,
- final_url=case.url,
+ final_url=effective_url,
error_type=error_type,
error_message=error_message,
metadata={
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
- **({'native_answer_projection': {'schema_version': 1,
- 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
- if envelope_success and isinstance(payload.get('result', ''), str) else {}),
"route": "browser_rendering_scrape_resources",
- **({'native_answer_projection': projection} if envelope_success else {}),
+ **({'native_answer_projection': (projection if envelope_success else {
+ **projection, 'content_format': 'empty', 'answer_fields': {}})}
+ if envelope_success or meta else {}),
"request_method": "POST",
"request_summary": {
"selectors": selectors,
@@ -679,8 +684,9 @@ def _fetch_direct_document(
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
- **({'native_answer_projection': {'schema_version': 1,
- 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
+ **({'native_answer_projection': {'schema_version': 2,
+ 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True,
+ 'result_fields': {'final_url': {'format': 'transport_response_url'}}}}
if envelope_success and isinstance(payload.get('result', ''), str) else {}),
"route": "workers_ai_tomarkdown",
"source_status": source.status,
@@ -688,6 +694,8 @@ def _fetch_direct_document(
"source_bytes": len(source.body),
"source_content_type": content_type,
"source_final_url": source.url,
+ "source_transport_receipt": {"response_url": source.url,
+ "body_sha256": hashlib.sha256(source.body).hexdigest()},
"source_elapsed_ms": source_elapsed_ms,
"conversion_elapsed_ms": conversion_elapsed_ms,
"conversion_attempted": True,
@@ -752,16 +760,17 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
ok=ok,
content=content if envelope_success else "",
status_code=response.status,
- final_url=case.url,
+ final_url=meta.get("finalUrl") or case.url,
error_type=error_type,
error_message=error_message,
metadata={
"_raw_response_b64": base64.b64encode(response.body).decode("ascii"),
"raw_response_sha256": hashlib.sha256(response.body).hexdigest(),
"raw_response_bytes": len(response.body),
- **({'native_answer_projection': {'schema_version': 1,
- 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True}}
- if envelope_success and isinstance(payload.get('result', ''), str) else {}),
+ **({'native_answer_projection': {'schema_version': 2,
+ 'content_pointer': ['result'], 'content_format': 'text' if envelope_success and isinstance(payload.get('result', ''), str) else 'empty', 'optional_content': True,
+ 'result_fields': {'final_url': {'pointer': ['meta', 'finalUrl'], 'fallback': 'request_url'}}}}
+ if (envelope_success and isinstance(payload.get('result', ''), str)) or meta else {}),
"output_contract": "rendered_html",
"upstream_status": upstream_status,
"empty_content": envelope_success and not content,
diff --git a/harness/web_fetch/providers/exa.py b/harness/web_fetch/providers/exa.py
index b141a4af6..2cc79b95c 100644
--- a/harness/web_fetch/providers/exa.py
+++ b/harness/web_fetch/providers/exa.py
@@ -143,7 +143,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
ok=ok,
content=content,
status_code=response.status,
- final_url=(result.get("url") or case.url) if ok else case.url,
+ final_url=result.get("url") or case.url,
title=result.get("title") if ok else None,
error_type=error_type,
error_message=error_message,
@@ -168,12 +168,14 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
"extras": result_extras,
"links": _rich_answer_links(result_extras),
**({'native_answer_projection': {
- 'schema_version': 1, 'content_pointer': ['results', 0, 'text'],
+ 'schema_version': 2, 'content_pointer': ['results', 0, 'text'],
'content_format': 'text', 'optional_content': True,
+ 'result_fields': {'final_url': {'pointer': ['results', 0, 'url'], 'fallback': 'request_url'},
+ **({'title': {'pointer': ['results', 0, 'title']}} if ok else {})},
'answer_fields': ({
'extras': {'pointer': ['results', 0, 'extras'], 'format': 'identity', 'default': {}},
'links': {'pointer': ['results', 0, 'extras', 'richLinks'],
'format': 'named_links', 'default': []}} if ok else {})}}
- if ok or content else {}),
+ if ok or content or result.get('url') else {}),
},
)
diff --git a/harness/web_fetch/providers/firecrawl.py b/harness/web_fetch/providers/firecrawl.py
index 2efbc1f01..a2d721cc7 100644
--- a/harness/web_fetch/providers/firecrawl.py
+++ b/harness/web_fetch/providers/firecrawl.py
@@ -452,17 +452,20 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
"images": data.get("images") if ok and isinstance(data.get("images"), list) else [],
"videos": videos if ok else [],
**({'native_answer_projection': {
- 'schema_version': 1,
+ 'schema_version': 2,
+ 'result_fields': {
+ 'final_url': {'pointer': ['data', 'metadata', 'url'], 'format': 'text_or_none', 'nonempty': True},
+ **({'title': {'pointer': ['data', 'metadata', 'title'], 'format': 'text_or_none'}} if ok else {})},
'content_pointer': ['data', 'markdown'],
- 'content_format': 'empty' if file_route else 'text', 'optional_content': True,
- 'answer_fields': ({} if file_route else {
+ 'content_format': 'empty' if file_route or not ok else 'text', 'optional_content': True,
+ 'answer_fields': ({} if file_route or not ok else {
'links': {'pointer': ['data', 'links'], 'format': 'identity', 'default': []},
'images': {'pointer': ['data', 'images'], 'format': 'identity', 'default': []},
**({'videos': {'pointer': ['data', 'videos'], 'format': 'video_values', 'default': [],
'legacy': {'pointer': ['data', 'video'], 'default': ''}}}
if native_video_route else {})}),
**({'files': {'pointer': ['data', 'rawBase64'], 'format': 'base64_file'}}
- if file_bytes else {})}} if ok else {}),
+ if file_bytes else {})}} if ok or final_url is not None else {}),
**(
{
"raw_file": {
diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py
index 40b83cdda..beaf628fd 100644
--- a/harness/web_fetch/providers/moli.py
+++ b/harness/web_fetch/providers/moli.py
@@ -16,7 +16,9 @@
import threading
import time
from pathlib import Path
-from urllib.parse import parse_qsl, quote, urlsplit
+from urllib.parse import urlsplit
+
+from ..response_input import main_document_url_from_fetch_log as _diagnostic_main_document_url
from ..document_text import extract_document_text
from ..markdown import CONVERTER, converter_command, require_converter
@@ -52,76 +54,6 @@ def _markdown_http_error_status(content: str) -> int | None:
return int(match.group("status")) if match else None
-def _diagnostic_main_document_url(stderr: str | bytes, requested_url: str) -> str | None:
- """Read Moli's effective main-document URL from the same call's debug log."""
- pattern = re.compile(
- rb"fetch request complete method=(?:GET|POST) url=(?P\S+) "
- rb"final_url=(?P\S+) status=(?P\d{3})\b"
- )
-
- diagnostic = stderr.encode("utf-8") if isinstance(stderr, str) else stderr
- navigation_pattern = re.compile(
- rb"following pending location navigation asynchronously[^\n]* url=(?P\S+)"
- )
-
- def field(row: re.Match[bytes], name: str) -> str:
- return row.group(name).decode("utf-8", "replace")
-
- def identity(value: str) -> tuple | None:
- try:
- validate_http_url(value)
- parts = urlsplit(value)
- return (
- parts.scheme.casefold(),
- (parts.hostname or "").casefold(),
- parts.port if parts.port is not None else (443 if parts.scheme.casefold() == "https" else 80),
- re.sub(r"%[0-9a-fA-F]{2}", lambda match: match[0].upper(),
- quote(parts.path, safe="/%:@!$&'()*+,;=")).rstrip("/") or "/",
- sorted(parse_qsl(parts.query, keep_blank_values=True)),
- )
- except ValueError:
- return None
-
- requested_identity = identity(requested_url)
- if requested_identity is None:
- return None
- rows = list(pattern.finditer(diagnostic))
-
- # Legacy completion records do not identify redirect hops or request owners.
- # Never infer the document from GET ordering or challenge query parameters.
- row = next((candidate for candidate in rows
- if identity(field(candidate, "url")) == requested_identity), None)
- if row is None or 300 <= int(field(row, "status")) < 400:
- return None
- navigations = list(navigation_pattern.finditer(diagnostic))
- for index, navigation in enumerate(navigations):
- boundary = navigations[index + 1].start() if index + 1 < len(navigations) else len(diagnostic)
- committed = diagnostic.find(
- b"committed main navigation before replacement realm bootstrap",
- navigation.end(), boundary,
- )
- if committed < 0:
- continue
- navigation_identity = identity(field(navigation, "url"))
- if navigation_identity is None:
- return None
- matches = [candidate for candidate in rows
- if navigation.end() <= candidate.start() < boundary
- and identity(field(candidate, "url")) == navigation_identity]
- if len(matches) != 1 or 300 <= int(field(matches[0], "status")) < 400:
- return None
- row = matches[0]
- candidate = field(row, "final_url")
- if identity(candidate) is None:
- return None
- try:
- validate_http_url(candidate)
- except ValueError:
- return None
- return candidate
-
-
-
_RESOURCE_PAGE_EXPRESSION = r"""
(async () => {
const urls = new Set();
@@ -758,7 +690,8 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
raise ValueError('Native CLI arguments differ from its canonical request builder')
command = [command[0], *canonical[1:]]
limit = int(self.config.get("max_bytes", 20_000_000))
- retain_diagnostic = bool(self.config.get("retain_complete_diagnostic_log", False))
+ retain_diagnostic = bool(self.config.get("retain_complete_diagnostic_log", False)
+ or self.config.get("uniform_native_parameters"))
diagnostic_limit = int(self.config.get("max_diagnostic_bytes", 100_000_000))
diagnostic_read_limit = diagnostic_limit if retain_diagnostic else limit
metadata = {
@@ -948,7 +881,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
if self.config.get('uniform_native_parameters'):
metadata.update(_raw_stdout_b64=base64.b64encode(raw).decode('ascii'),
raw_stdout_sha256=hashlib.sha256(raw).hexdigest(),
- native_answer_projection={'schema_version': 1, 'source_format': 'bytes',
+ native_answer_projection={'schema_version': 2, 'source_format': 'bytes',
'content_format': 'empty', 'files': {'format': 'raw_bytes'}})
if native_document_download:
metadata["native_output_contract"] = "original_document_bytes_without_extracted_text"
@@ -1300,7 +1233,8 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
if self.config.get('uniform_native_parameters'):
metadata.update(_raw_stdout_b64=base64.b64encode(raw).decode('ascii'),
raw_stdout_sha256=hashlib.sha256(raw).hexdigest(),
- native_answer_projection={'schema_version': 1, 'source_format': 'bytes', 'content_format': 'utf8'})
+ native_answer_projection={'schema_version': 2, 'source_format': 'bytes', 'content_format': 'utf8',
+ 'result_fields': {'final_url': {'format': 'fetch_log_final_url'}}})
parsed_error_status = _markdown_http_error_status(content)
ok = process.returncode == 0 and parsed_error_status is None
final_url = _diagnostic_main_document_url(diagnostic_raw, case.url)
diff --git a/harness/web_fetch/providers/subprocess_jsonl.py b/harness/web_fetch/providers/subprocess_jsonl.py
index 588736bb4..960dabf1c 100644
--- a/harness/web_fetch/providers/subprocess_jsonl.py
+++ b/harness/web_fetch/providers/subprocess_jsonl.py
@@ -346,8 +346,10 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
error_message=None if ok else str(payload.get("error_message") or "worker failed"),
metadata={
**dict(payload.get("metadata") or {}),
- **({'native_answer_projection': {'schema_version': 1,
+ **({'native_answer_projection': {'schema_version': 2,
'content_pointer': ['content'],
+ 'result_fields': {'title': {'pointer': ['title']},
+ 'final_url': {'pointer': ['final_url'], 'fallback': 'request_url'}},
'content_format': 'text' if isinstance(payload.get('content'), str) else 'empty',
'answer_fields': {key: {'pointer': ['metadata', key], 'format': 'identity'}
for key in ('links', 'images', 'videos', 'media', 'extras')
diff --git a/harness/web_fetch/providers/tavily.py b/harness/web_fetch/providers/tavily.py
index 2742cb676..47d7a58d8 100644
--- a/harness/web_fetch/providers/tavily.py
+++ b/harness/web_fetch/providers/tavily.py
@@ -89,7 +89,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
ok=ok,
content=content,
status_code=response.status,
- final_url=(result.get("url") or case.url) if ok else case.url,
+ final_url=result.get("url") or case.url,
error_type=error_type,
error_message=error_message,
metadata={
@@ -116,10 +116,11 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult:
),
"images": (result.get("images") or payload.get("images") or []) if ok else [],
**({'native_answer_projection': {
- 'schema_version': 1, 'content_pointer': ['results', 0, 'raw_content'],
+ 'schema_version': 2, 'content_pointer': ['results', 0, 'raw_content'],
'content_format': 'text', 'optional_content': True,
- 'answer_fields': {'images': {
+ 'result_fields': {'final_url': {'pointer': ['results', 0, 'url'], 'fallback': 'request_url'}},
+ 'answer_fields': ({'images': {
'pointer': ['results', 0, 'images'] if result.get('images') else ['images'],
- 'format': 'identity', 'default': []}}}} if ok or content else {}),
+ 'format': 'identity', 'default': []}} if ok else {})}} if ok or content or result.get('url') else {}),
},
)
diff --git a/harness/web_fetch/response_input.py b/harness/web_fetch/response_input.py
index da29c6cf1..b4e886f75 100644
--- a/harness/web_fetch/response_input.py
+++ b/harness/web_fetch/response_input.py
@@ -1,6 +1,9 @@
"""Provider-neutral answer envelope. Never infer answers from logs or requests."""
import json
-from urllib.parse import urljoin, urlsplit
+import re
+from urllib.parse import parse_qsl, quote, urljoin, urlsplit
+
+from .safety import validate_http_url
def answer_input(artifact, metadata=None, *, ok=True, status_code=None):
@@ -137,7 +140,7 @@ def _project_field(payload, spec):
def project_native_answer(payload, declaration):
"""Select declared native fields; never inspect references or URL meaning."""
- if declaration.get('schema_version') != 1:
+ if declaration.get('schema_version') not in {1, 2}:
raise ValueError('Unsupported native answer projection')
kind = declaration.get('content_format')
if declaration.get('source_format') == 'bytes':
@@ -185,12 +188,121 @@ def project_native_files(payload, declaration):
raise ValueError('Unknown native file projection')
-def require_native_answer_projection(content, metadata, raw_response, *, files=()):
+def main_document_url_from_fetch_log(stderr: str | bytes, requested_url: str) -> str | None:
+ """Replay the effective main-document URL from the same call's debug log."""
+ pattern = re.compile(
+ rb"fetch request complete method=(?:GET|POST) url=(?P\S+) "
+ rb"final_url=(?P\S+) status=(?P\d{3})\b"
+ )
+
+ diagnostic = stderr.encode("utf-8") if isinstance(stderr, str) else stderr
+ navigation_pattern = re.compile(
+ rb"following pending location navigation asynchronously[^\n]* url=(?P\S+)"
+ )
+
+ def field(row: re.Match[bytes], name: str) -> str:
+ return row.group(name).decode("utf-8", "replace")
+
+ def identity(value: str) -> tuple | None:
+ try:
+ validate_http_url(value)
+ parts = urlsplit(value)
+ return (
+ parts.scheme.casefold(),
+ (parts.hostname or "").casefold(),
+ parts.port if parts.port is not None else (443 if parts.scheme.casefold() == "https" else 80),
+ re.sub(r"%[0-9a-fA-F]{2}", lambda match: match[0].upper(),
+ quote(parts.path, safe="/%:@!$&'()*+,;=")).rstrip("/") or "/",
+ sorted(parse_qsl(parts.query, keep_blank_values=True)),
+ )
+ except ValueError:
+ return None
+
+ requested_identity = identity(requested_url)
+ if requested_identity is None:
+ return None
+ rows = list(pattern.finditer(diagnostic))
+
+ # Legacy completion records do not identify redirect hops or request owners.
+ # Never infer the document from GET ordering or challenge query parameters.
+ row = next((candidate for candidate in rows
+ if identity(field(candidate, "url")) == requested_identity), None)
+ if row is None or 300 <= int(field(row, "status")) < 400:
+ return None
+ navigations = list(navigation_pattern.finditer(diagnostic))
+ for index, navigation in enumerate(navigations):
+ boundary = navigations[index + 1].start() if index + 1 < len(navigations) else len(diagnostic)
+ committed = diagnostic.find(
+ b"committed main navigation before replacement realm bootstrap",
+ navigation.end(), boundary,
+ )
+ if committed < 0:
+ continue
+ navigation_identity = identity(field(navigation, "url"))
+ if navigation_identity is None:
+ return None
+ matches = [candidate for candidate in rows
+ if navigation.end() <= candidate.start() < boundary
+ and identity(field(candidate, "url")) == navigation_identity]
+ if len(matches) != 1 or 300 <= int(field(matches[0], "status")) < 400:
+ return None
+ row = matches[0]
+ candidate = field(row, "final_url")
+ if identity(candidate) is None:
+ return None
+ try:
+ validate_http_url(candidate)
+ except ValueError:
+ return None
+ return candidate
+
+
+
+def project_native_result_fields(payload, declaration, *, request_url=None,
+ diagnostic=None, transport_receipt=None):
+ """Replay scored title/address from native fields or same-call evidence."""
+ fields = {'title': None, 'final_url': None}
+ for name, spec in declaration.get('result_fields', {}).items():
+ if name not in fields:
+ raise ValueError('Unknown native result field')
+ kind = spec.get('format', 'identity')
+ if kind == 'request_url':
+ if name != 'final_url' or request_url is None:
+ raise ValueError('Native URL fallback requires its declared request URL')
+ value = request_url
+ elif kind == 'fetch_log_final_url':
+ if name != 'final_url' or diagnostic is None or request_url is None:
+ raise ValueError('Native final URL lacks its same-call diagnostic evidence')
+ value = main_document_url_from_fetch_log(diagnostic, request_url)
+ elif kind == 'transport_response_url':
+ if name != 'final_url' or not transport_receipt:
+ raise ValueError('Native final URL lacks its source transport receipt')
+ value = transport_receipt['response_url']
+ else:
+ value = _selected(payload, {**spec, 'default': None})
+ if kind == 'text_or_none':
+ value = value if isinstance(value, str) else None
+ elif kind != 'identity':
+ raise ValueError('Unknown native result field projection')
+ if spec.get('nonempty') and not value:
+ value = None
+ if not value and spec.get('fallback') == 'request_url':
+ if name != 'final_url' or request_url is None:
+ raise ValueError('Native URL fallback requires its declared request URL')
+ value = request_url
+ fields[name] = value
+ return fields
+
+
+def require_native_answer_projection(content, metadata, raw_response, *, files=(), title=None, final_url=None,
+ request_url=None, diagnostic=None, transport_receipt=None):
"""Reject every undeclared score-bearing input, then check exact provenance."""
declaration = metadata.get('native_answer_projection')
present = {key for key in SCORE_ANSWER_FIELDS if metadata.get(key)}
if declaration is None:
- if present or files or (raw_response is not None and content):
+ if (present or files or title is not None
+ or (final_url is not None and final_url != request_url)
+ or (raw_response is not None and content)):
raise ValueError('Scored native answer has undeclared answer fields')
return
if raw_response is None:
@@ -219,7 +331,19 @@ def check_source(spec):
check_source(spec)
if declaration.get('files'):
check_source(declaration['files'])
+ for spec in declaration.get('result_fields', {}).values():
+ check_source(spec)
+ if spec.get('format') == 'fetch_log_final_url' and 'diagnostic.main_document_url' not in allowed:
+ raise ValueError('Native final URL conversion is outside its frozen sources')
payload = raw_response if declaration.get('source_format') == 'bytes' else json.loads(raw_response)
+ projected_result = project_native_result_fields(payload, declaration, request_url=request_url,
+ diagnostic=diagnostic, transport_receipt=transport_receipt)
+ if projected_result != {'title': title, 'final_url': final_url}:
+ raise ValueError('Scored title or final URL differs from its declared native projection')
+ if declaration.get('schema_version') == 2:
+ for spec in declaration.get('answer_fields', {}).values():
+ if spec.get('format') == 'html_attribute_urls' and spec.get('base_url') != (final_url or request_url):
+ raise ValueError('Native resource base URL differs from its verified result URL')
expected, fields = project_native_answer(payload, declaration)
if content != expected or any(metadata.get(key) != value for key, value in fields.items()):
raise ValueError('Scored answer differs from its declared native field projection')
@@ -255,7 +379,16 @@ def verify_saved_native_answer(root, record):
raw = read_evidence_binding(root, {'path': raw_path, 'sha256': raw_hash}).read_bytes()
files = [{'data': contained_path(root, item['artifact_path'], reject_symlinks=True).read_bytes()}
for item in metadata.get('file_evaluation', {}).get('files', [])]
- require_native_answer_projection(answer.decode('utf-8'), metadata, raw, files=files)
+ diagnostic = None
+ if any(spec.get('format') == 'fetch_log_final_url'
+ for spec in declaration.get('result_fields', {}).values()):
+ diagnostic = read_evidence_binding(root, {'path': metadata['diagnostic_log_path'],
+ 'sha256': metadata['diagnostic_log_sha256']}).read_bytes()
+ require_native_answer_projection(answer.decode('utf-8'), metadata, raw, files=files,
+ title=record.get('title') if declaration.get('schema_version') == 2 or 'result_fields' in declaration else None,
+ final_url=record.get('final_url') if declaration.get('schema_version') == 2 or 'result_fields' in declaration else None,
+ request_url=record.get('url'),
+ diagnostic=diagnostic, transport_receipt=metadata.get('source_transport_receipt'))
if files or 'file_evaluation' in metadata:
from .call_boundary import evaluate_returned_files
source_verification = {}
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index 21a02bd2e..ba10264e4 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -376,7 +376,7 @@ def persist_public_call_log(
def persist_result_artifacts(
- output_dir: Path, artifact_rel: Path, result: ProviderResult
+ output_dir: Path, artifact_rel: Path, result: ProviderResult, *, request_url: str | None = None
) -> Path:
"""Persist one physical call without changing its scored answer bytes."""
discarded_late_answer = (not result.ok and result.error_type == 'timeout'
@@ -388,13 +388,17 @@ def persist_result_artifacts(
needs_projection = (result.metadata.get('native_answer_required')
or result.metadata.get('native_answer_projection')
or any(result.metadata.get(key) for key in ('links', 'images', 'videos', 'media', 'extras'))
- or (raw is not None and result.files))
+ or (raw is not None and (result.files or result.title is not None or result.final_url is not None)))
if not discarded_late_answer and needs_projection:
if result.metadata.get('native_answer_required') and not result.metadata.get('native_answer_projection'):
raise ValueError('Scored native answer has no projection declaration')
require_native_answer_projection(result.content, result.metadata,
base64.b64decode(raw, validate=True) if isinstance(raw, str) else None,
- files=result.files if raw is not None or result.metadata.get('native_answer_projection') else ())
+ files=result.files if raw is not None or result.metadata.get('native_answer_projection') else (),
+ title=result.title, final_url=result.final_url, request_url=request_url,
+ diagnostic=base64.b64decode(result.metadata['_diagnostic_log_b64'], validate=True)
+ if isinstance(result.metadata.get('_diagnostic_log_b64'), str) else None,
+ transport_receipt=result.metadata.get('source_transport_receipt'))
elif discarded_late_answer and (result.files or any(result.metadata.get(key) for key in ('images', 'links', 'videos', 'media', 'extras'))):
raise ValueError('Discarded late native answer retains score-bearing fields')
if result.metadata.get('native_answer_projection'):
@@ -718,9 +722,9 @@ def run_provider(provider: Provider) -> None:
artifact_rel = Path('artifacts') / provider.name / case.id / suffix
if native_routes is not None and provider.name in native_routes.providers:
result.metadata['native_route_receipt'] = native_routes.receipt(provider, case, ok=result.ok)
- if provider.config.get('uniform_native_parameters') and (result.content or result.files):
+ if provider.config.get('uniform_native_parameters') and (result.content or result.files or result.title is not None):
result.metadata['native_answer_required'] = True
- artifact = persist_result_artifacts(output_dir, artifact_rel, result)
+ artifact = persist_result_artifacts(output_dir, artifact_rel, result, request_url=case.url)
artifact_bytes = result.content.encode('utf-8', errors='replace')
measured = score_attempt(
case, artifact_bytes, ok=result.ok, metadata=result.metadata,
diff --git a/harness/web_fetch/scoring.py b/harness/web_fetch/scoring.py
index 82c1b4803..3824beb86 100644
--- a/harness/web_fetch/scoring.py
+++ b/harness/web_fetch/scoring.py
@@ -87,6 +87,7 @@ def score_attempt(
prepared_answer: tuple | None = None,
native_response: bytes | None = None,
native_files: list | None = None,
+ native_diagnostic: bytes | None = None,
title: str | None = None,
final_url: str | None = None,
) -> dict[str, Any]:
@@ -99,13 +100,15 @@ def score_attempt(
Previous non-text
evidence is preserved; the caller must have validated its artifact binding.
"""
+ case = _case_dict(case)
if native_files is not None and native_response is None:
raise ValueError('Native file scoring requires its retained response bytes')
if native_response is not None:
from .response_input import require_native_answer_projection
require_native_answer_projection(artifact.decode('utf-8'), metadata or {},
- native_response, files=native_files or ())
- case = _case_dict(case)
+ native_response, files=native_files or (), title=title, final_url=final_url,
+ request_url=case.get('url'), diagnostic=native_diagnostic,
+ transport_receipt=(metadata or {}).get('source_transport_receipt'))
if native_files is not None:
from .call_boundary import evaluate_returned_files
metadata = dict(metadata or {})
diff --git a/tests/test_cowagent_budget.py b/tests/test_cowagent_budget.py
index eb037313f..9af000ba1 100644
--- a/tests/test_cowagent_budget.py
+++ b/tests/test_cowagent_budget.py
@@ -37,7 +37,8 @@ def execute(self, arguments):
continue
assert answer.ok and answer.content == 'native answer'
assert answer.metadata['effective_timeout_seconds'] == 30
- persist_result_artifacts(tmp_path, Path('artifacts/cow/' + suffix + '.txt'), answer)
+ persist_result_artifacts(tmp_path, Path('artifacts/cow/' + suffix + '.txt'), answer,
+ request_url='https://example.com/' + suffix)
envelope = json.loads((tmp_path / answer.metadata['raw_response_path']).read_bytes())
assert envelope['raw_tool_response'] == {
'status': 'success', 'result': 'native answer',
diff --git a/tests/test_local_revision.py b/tests/test_local_revision.py
index d30bb7a68..efa3425a0 100644
--- a/tests/test_local_revision.py
+++ b/tests/test_local_revision.py
@@ -265,3 +265,41 @@ def test_native_correction_rejects_self_consistent_but_false_projection(tmp_path
binding['sha256'] = hashlib.sha256(path.read_bytes()).hexdigest()
with pytest.raises(ValueError, match=message):
require_local_revision(tmp_path, binding, json.loads((tmp_path/'dataset.json').read_text()), [current])
+
+
+@pytest.mark.parametrize('field,changed', [('title', 'forged title'), ('final_url', 'https://forged.example.test/')])
+def test_native_context_revision_replays_native_fields_and_rejects_tampering(tmp_path, field, changed):
+ original, _, _, _, _ = _native_answer_revision_fixture(tmp_path)
+ current = json.loads((tmp_path/'results.jsonl').read_text())
+ native_path = tmp_path/current['provider_metadata']['raw_response_path']
+ # Prepare a new immutable native acquisition fixture for this transition.
+ payload = json.loads(native_path.read_bytes())
+ payload.update(title='native title', final_url='https://returned.example.test/')
+ native = json.dumps(payload).encode()
+ second_native = tmp_path/'private_artifacts/context.response.bin'
+ second_native.write_bytes(native)
+ current['provider_metadata'].update(raw_response_path='private_artifacts/context.response.bin',
+ raw_response_sha256=hashlib.sha256(native).hexdigest(), raw_response_bytes=len(native))
+ current.update(title=None, final_url=current['url'])
+ (tmp_path/'results.jsonl').write_text(json.dumps(current)+'\n')
+ config = {'cases': [], 'authority': {'description': 'Recover native title and redirect without new calls.'},
+ 'regression_ids': ['native-title-must-match-original', 'native-returned-title-remains-answer',
+ 'native-final-url-must-match-original', 'native-returned-redirect-resolves-relative-image'],
+ 'native_answer_corrections': [{'attempt_id': current['attempt_id'],
+ 'native_response': {'path': 'private_artifacts/context.response.bin', 'sha256': hashlib.sha256(native).hexdigest()},
+ 'projection': {'schema_version': 2, 'content_format': 'text', 'content_pointer': ['content'],
+ 'result_fields': {key: {'pointer': [key]} for key in ('title', 'final_url')}}}]}
+ config_path = tmp_path/'context-correction.json'
+ config_path.write_text(json.dumps(config))
+ receipt = revise_local_snapshot(tmp_path, config_path)
+ fixed = json.loads((tmp_path/'results.jsonl').read_text())
+ assert fixed['title'] == 'native title'
+ assert fixed['final_url'] == 'https://returned.example.test/'
+ assert fixed['elapsed_ms'] == original['elapsed_ms']
+ assert receipt['provider_calls_added'] == 0
+ from harness.web_fetch.local_revision import _require_native_answer_correction
+ proof = receipt['native_answer_corrections'][0]
+ _require_native_answer_correction(tmp_path, current, fixed, proof)
+ fixed[field] = changed
+ with pytest.raises(ValueError, match='context correction'):
+ _require_native_answer_correction(tmp_path, current, fixed, proof)
diff --git a/tests/test_native_route_contract.py b/tests/test_native_route_contract.py
index 74ecb984d..a8e697e64 100644
--- a/tests/test_native_route_contract.py
+++ b/tests/test_native_route_contract.py
@@ -452,8 +452,8 @@ def test_http_native_answers_survive_persistence_and_reject_tampering(tmp_path,
altered = deepcopy(result)
altered.metadata['images'] = ['https://example.test/forged.png']
with pytest.raises(ValueError, match='native|projection'):
- persist_result_artifacts(tmp_path, Path('artifacts/forged.txt'), altered)
- persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result)
+ persist_result_artifacts(tmp_path, Path('artifacts/forged.txt'), altered, request_url=case.url)
+ persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result, request_url=case.url)
def test_projection_cannot_read_request_echo_outside_declared_answer_fields():
@@ -466,3 +466,127 @@ def test_projection_cannot_read_request_echo_outside_declared_answer_fields():
'answer_fields': {'images': {'pointer': ['request', 'images'], 'format': 'identity'}}}}
with pytest.raises(ValueError, match='frozen answer fields'):
require_native_answer_projection('answer', metadata, raw)
+
+
+@pytest.mark.parametrize('field,forged', [('title', 'forged title'), ('final_url', 'https://forged.example.test/')])
+def test_native_title_and_address_are_checked_at_live_and_saved_boundaries(tmp_path, field, forged):
+ import base64
+ import hashlib
+ import json
+ from copy import deepcopy
+ from pathlib import Path
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ from harness.web_fetch.response_input import verify_saved_native_answer
+ raw = json.dumps({'content': 'answer', 'title': 'native title',
+ 'final_url': 'https://returned.example.test/'}).encode()
+ result = ProviderResult(True, 'answer', title='native title', final_url='https://returned.example.test/',
+ metadata={'_raw_response_b64': base64.b64encode(raw).decode(),
+ 'raw_response_sha256': hashlib.sha256(raw).hexdigest(), 'raw_response_bytes': len(raw),
+ 'native_answer_projection': {'schema_version': 2, 'content_pointer': ['content'],
+ 'content_format': 'text', 'result_fields': {name: {'pointer': [name]} for name in ('title', 'final_url')}}})
+ altered = deepcopy(result)
+ setattr(altered, field, forged)
+ with pytest.raises(ValueError, match='title or final URL'):
+ persist_result_artifacts(tmp_path, Path('artifacts/forged.txt'), altered, request_url='https://requested.example.test/')
+ persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result, request_url='https://requested.example.test/')
+ record = {'ok': True, 'artifact_path': 'artifacts/answer.txt', 'title': result.title,
+ 'url': 'https://requested.example.test/', 'final_url': result.final_url,
+ 'provider_metadata': result.metadata, 'metrics': {'content_sha256': hashlib.sha256(b'answer').hexdigest()}}
+ verify_saved_native_answer(tmp_path, record)
+ record[field] = forged
+ with pytest.raises(ValueError, match='title or final URL'):
+ verify_saved_native_answer(tmp_path, record)
+
+
+def test_native_url_fallback_cannot_override_a_returned_redirect():
+ import json
+ from harness.web_fetch.response_input import require_native_answer_projection
+ request = 'https://requested.example.test/'
+ returned = 'https://returned.example.test/'
+ metadata = {'native_answer_projection': {'schema_version': 2, 'content_format': 'empty',
+ 'result_fields': {'final_url': {'pointer': ['url'], 'fallback': 'request_url'}}}}
+ require_native_answer_projection('', metadata, b'{}', request_url=request, final_url=request)
+ require_native_answer_projection('', metadata, json.dumps({'url': returned}).encode(), request_url=request, final_url=returned)
+ with pytest.raises(ValueError, match='title or final URL'):
+ require_native_answer_projection('', metadata, json.dumps({'url': returned}).encode(), request_url=request, final_url=request)
+
+
+def test_cli_final_url_replays_only_same_call_main_document_log():
+ from harness.web_fetch.response_input import require_native_answer_projection
+ request = 'https://requested.example.test/'
+ returned = 'https://returned.example.test/'
+ log = f'fetch request complete method=GET url={request} final_url={returned} status=200'.encode()
+ metadata = {'native_answer_projection': {'schema_version': 2, 'source_format': 'bytes',
+ 'content_format': 'utf8', 'result_fields': {'final_url': {'format': 'fetch_log_final_url'}}}}
+ require_native_answer_projection('answer', metadata, b'answer', request_url=request, final_url=returned, diagnostic=log)
+ with pytest.raises(ValueError, match='title or final URL'):
+ require_native_answer_projection('answer', metadata, b'answer', request_url=request, final_url=request, diagnostic=log)
+ with pytest.raises(ValueError, match='diagnostic evidence'):
+ require_native_answer_projection('answer', metadata, b'answer', request_url=request, final_url=returned)
+
+
+@pytest.mark.parametrize('method,module,payload', [
+ ('cloudflare_markdown', 'cloudflare', {'success': False, 'meta': {'title': 'Origin unavailable', 'finalUrl': 'https://returned.example.test/'}}),
+ ('cloudflare_content', 'cloudflare', {'success': False, 'meta': {'finalUrl': 'https://returned.example.test/'}}),
+ ('firecrawl', 'firecrawl', {'success': False, 'data': {'metadata': {'url': 'https://returned.example.test/'}}}),
+])
+def test_failed_native_envelope_retains_provable_result_context(tmp_path, method, module, payload):
+ import json
+ from pathlib import Path
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.providers.http_client import HttpResponse
+ from harness.web_fetch.runner import persist_result_artifacts
+ provider = build_provider('method', {'type': method, 'api_key': 'fixture', 'api_token': 'fixture',
+ 'account_id': 'fixture', 'uniform_native_parameters': True})
+ case = EvalCase('case', 'Case', 'https://requested.example.test/', 'control', [])
+ with patch('harness.web_fetch.providers.'+module+'.request',
+ return_value=HttpResponse(502, 'https://api.example.test/', {}, json.dumps(payload).encode())):
+ result = provider.fetch(case, 45)
+ assert not result.ok
+ assert result.final_url == 'https://returned.example.test/'
+ persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result, request_url=case.url)
+
+
+def test_resource_adapter_uses_returned_address_before_resolving_relative_attributes(tmp_path):
+ import json
+ from pathlib import Path
+ from unittest.mock import patch
+ from harness.web_fetch.models import EvalCase
+ from harness.web_fetch.providers.base import build_provider
+ from harness.web_fetch.providers.http_client import HttpResponse
+ from harness.web_fetch.runner import persist_result_artifacts
+ provider = build_provider('method', {'type': 'cloudflare_markdown', 'api_token': 'fixture',
+ 'account_id': 'fixture', 'uniform_native_parameters': True})
+ case = EvalCase('case', 'Case', 'https://requested.example.test/', 'control', [], track='image', task_kind='resource')
+ payload = {'success': True, 'meta': {'finalUrl': 'https://returned.example.test/page'},
+ 'result': [{'results': [{'attributes': [{'name': 'src', 'value': '/asset.png'}]}]}]}
+ with patch('harness.web_fetch.providers.cloudflare.request',
+ return_value=HttpResponse(200, 'https://api.example.test/', {}, json.dumps(payload).encode())):
+ result = provider.fetch(case, 45)
+ assert result.final_url == 'https://returned.example.test/page'
+ assert result.metadata['images'] == ['https://returned.example.test/asset.png']
+ persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result, request_url=case.url)
+
+
+
+def test_empty_same_call_diagnostic_is_retained_without_inventing_final_url(tmp_path):
+ import base64
+ import hashlib
+ from pathlib import Path
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ from harness.web_fetch.response_input import verify_saved_native_answer
+ result = ProviderResult(True, 'answer', metadata={
+ '_raw_stdout_b64': base64.b64encode(b'answer').decode(), 'raw_stdout_sha256': hashlib.sha256(b'answer').hexdigest(),
+ '_diagnostic_log_b64': '', 'diagnostic_log_sha256': hashlib.sha256(b'').hexdigest(), 'diagnostic_log_bytes': 0,
+ 'native_answer_projection': {'schema_version': 2, 'source_format': 'bytes', 'content_format': 'utf8',
+ 'result_fields': {'final_url': {'format': 'fetch_log_final_url'}}}})
+ persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result, request_url='https://example.test/')
+ record = {'ok': True, 'url': 'https://example.test/', 'title': None, 'final_url': None,
+ 'artifact_path': 'artifacts/answer.txt', 'provider_metadata': result.metadata,
+ 'metrics': {'content_sha256': hashlib.sha256(b'answer').hexdigest()}}
+ verify_saved_native_answer(tmp_path, record)
+ assert (tmp_path/result.metadata['diagnostic_log_path']).read_bytes() == b''
diff --git a/tests/test_public_call_logs.py b/tests/test_public_call_logs.py
index eb041c0be..abf8f1af7 100644
--- a/tests/test_public_call_logs.py
+++ b/tests/test_public_call_logs.py
@@ -64,10 +64,12 @@ def test_common_sensitive_fields_are_redacted_from_public_log(self):
self.assertIn('view=full', serialized)
def test_success_has_closed_complete_hash_bound_log(self):
- raw = b'{"private":"Bearer never-publish"}'
+ raw = b'{"content":"Expected","final_url":"https://example.com/final#fragment","private":"Bearer never-publish"}'
result = ProviderResult(
True, "Expected", status_code=200, final_url="https://example.com/final#fragment",
metadata={
+ "native_answer_projection": {'schema_version': 2, 'content_format': 'text',
+ 'content_pointer': ['content'], 'result_fields': {'final_url': {'pointer': ['final_url']}}},
"route": "advanced_extract", "content_type": "text/markdown",
"request_id": "request-123", "exit_code": 0,
"request_method": "POST",
diff --git a/tests/test_scoring.py b/tests/test_scoring.py
index 3b9d3c9eb..b177c578c 100644
--- a/tests/test_scoring.py
+++ b/tests/test_scoring.py
@@ -705,7 +705,8 @@ def setUp(self):
self.provider.validate.return_value = []
def response(c, timeout):
fields = {'images': [{'src': '/image.png'}], 'request_id': 'original-'+c.id}
- raw = json.dumps({'content': texts[c.id], 'metadata': fields}).encode()
+ native_url = 'https://new.example.test/news/' if c.id == 'redirected' else c.url
+ raw = json.dumps({'content': texts[c.id], 'metadata': fields, 'final_url': native_url}).encode()
return ProviderResult(ok=c.id != 'failed', content=texts[c.id],
status_code=200 if c.id != 'failed' else 502,
final_url='https://new.example.test/news/' if c.id == 'redirected' else c.url,
@@ -713,7 +714,8 @@ def response(c, timeout):
metadata={**fields, '_raw_response_b64': base64.b64encode(raw).decode(),
'raw_response_sha256': hashlib.sha256(raw).hexdigest(), 'raw_response_bytes': len(raw),
'native_answer_projection': {'schema_version': 1, 'content_pointer': ['content'],
- 'content_format': 'text', 'answer_fields': {'images': {
+ 'content_format': 'text', 'result_fields': {'final_url': {'pointer': ['final_url']}},
+ 'answer_fields': {'images': {
'pointer': ['metadata', 'images'], 'format': 'identity'}}}})
self.provider.fetch.side_effect = response
diff --git a/tests/test_subprocess_provider.py b/tests/test_subprocess_provider.py
index 375dc8f4d..7bc927896 100644
--- a/tests/test_subprocess_provider.py
+++ b/tests/test_subprocess_provider.py
@@ -121,7 +121,7 @@ def test_worker_receives_shared_limits_and_retains_full_response_envelope(self):
with tempfile.TemporaryDirectory() as directory:
from harness.web_fetch.runner import persist_result_artifacts
target=Path(directory)
- persist_result_artifacts(target, Path('artifacts/worker/control.txt'), result)
+ persist_result_artifacts(target, Path('artifacts/worker/control.txt'), result, request_url='https://example.com')
self.assertEqual(json.loads((target/result.metadata['raw_response_path']).read_bytes()),payload)
def test_persistence_rejects_answer_edit_or_candidate_filtering(self):
From 31a00bad2bbbdd33d50f47ef17fd8beb6dd4900d Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 14:41:27 +0800
Subject: [PATCH 42/50] Bind native context corrections and replayable public
evidence
---
research/native-430-local-report.json | 98 +++++++++++++++++++++------
1 file changed, 79 insertions(+), 19 deletions(-)
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
index 0eccfbdc1..ca73e29cf 100644
--- a/research/native-430-local-report.json
+++ b/research/native-430-local-report.json
@@ -15,20 +15,20 @@
"openclaw-built-in",
"tavily"
],
- "frozen_dataset_sha256": "28e3b3e9ec89ce545aae44355bd28eb0dc383ad6f64aed4dc91f327b14a528ab",
- "results_sha256": "ece94c0911a769c84ce9d896b00d97706fd4435d6d194f03cff77391710934a4",
+ "frozen_dataset_sha256": "e2a5c68fa7c2640414c56c0949a8f344f8091e602e6479f8fd2bb07fec5b9681",
+ "results_sha256": "314a5adb2225fde74cfe0672d25d972c2c99e65b40e337d93c0a1d7418e4fe00",
"measured_scoring_source_sha256": "4ce40cdf2ecb1af4088915a2b869ffbdcb65dce5222d1429578e97d175f13bfc",
"evidence_state": "local_measurement",
"certified": false,
- "evidence_storage": "External data-home snapshot; provider calls, review payloads and generated pages are not stored in Git.",
+ "evidence_storage": "External public saved-input snapshot with source-bound title/final URL corrections. Private originals and the immutable baseline are retained; restore the baseline and correction overlay before rebuilding.",
"evidence_bindings": [
{
"path": "local-run-contract.json",
- "sha256": "9febcc3f51b7a5dbd2dc98e43278df4e4ef47d05c694701eef411fcacb608328"
+ "sha256": "2de0ae1532a51428b2eb19dac44b5e1ba94e67f1433a0420cbeb038dbd602691"
},
{
"path": "result-review.json",
- "sha256": "8bfc48ba7b19ec244e6118e4620cd460d2fe425cd74b878fe1e9003f0f23f1fe"
+ "sha256": "c5cabe370ac2df9c1c6f2b647b4936d9c927de1ae2825f8e9ec937f40c1735c5"
},
{
"path": "archived-reading.json",
@@ -36,15 +36,15 @@
},
{
"path": "execution.json",
- "sha256": "6fdc15530c75676e46e3e44b6213017287d19ddc9c71597493b346072d1ee67c"
+ "sha256": "c519a68dcd812a1e6e41f3fc4efbf50f589f7dd72d90efd4f90bad41a5d64646"
},
{
"path": "local-evidence/result-ledger.json",
- "sha256": "738944823525d424a879572aa894e81a384802692d78ed52ca95400f27f94d6b"
+ "sha256": "2404c76226bb39c5a8820ef926898173e2da88a3c44e56c804c275d99a394d92"
},
{
"path": "frozen-method-options.json",
- "sha256": "04ffbb5a28db653e17d9a7dfe0f6fd517d89b754f6d8fd77a3dd21a4e7de15a7"
+ "sha256": "bbaae307a3cadeb75fee4a7f8f81fde41dca718728dad75b8cdf8c911866a2b3"
},
{
"path": "assets/methodology/annotation.png",
@@ -84,17 +84,17 @@
},
{
"path": "local-evidence/code-block-impact.json",
- "sha256": "bb1bd4f9b3bf0b5704b730d3a84d01dc6f4698947de2fd2d3da2667e2a8b08b9"
+ "sha256": "9d973e63712bff719b58f7ef10bd0202997a8abdd0ed97be584135ab239c0810"
}
],
- "evidence_path": "runs/pr51-native-fairness-20261015/evidence-reviewed",
+ "evidence_path": "runs/pr51-native-fairness-20261015/evidence-context-corrected-public",
"task_replacement": {
"path": "local-evidence/task-replacement/receipt.json",
- "sha256": "c68d3b21c3317538089635f8fb0ca5bf4f874f4a03201b27e42c56491780d6e0"
+ "sha256": "0ef2db8b05a6df206ea293cd8a02601a3d7e71aceb14119d286b8f2e386d100a"
},
"offline_revision": {
- "path": "local-evidence/offline-revisions/246d255fbf52a069-063e7324042d60b4/receipt.json",
- "sha256": "f633998e82f266156902c81dc9d7f33892c588ce2ca2bcab24f8c8b8f30eaa22"
+ "path": "local-evidence/offline-revisions/2d2a35f6b51cd102-6012bbf9c23a9899/receipt.json",
+ "sha256": "3ee08ef92933714b8e990ca469131cc3871e0f41ebacafc86b454569e7a9deaa"
},
"replacement_case": {
"id": "static-sqlite-quickstart",
@@ -149,14 +149,14 @@
"authority": "Frozen pre-call official source; preserve code tokens while allowing whitespace and HTML entities."
}
},
- "current_scoring_source_sha256": "d042394ea0e8c3aa7aed5851663eaf73772fc4357781999e20a87f48e857e00a",
+ "current_scoring_source_sha256": "b17fc68566409232d3840f69aa4511c6dcdfd154c774eb169ea43787659496f7",
"supplemental_review": {
- "path": "local-evidence/native-answer-review/supplement.json",
- "sha256": "5b5a9a22755fe35e8852195d5dfc1b6d1edd4286b50c4e8c5ed8774bc54f7c1f"
+ "path": "local-evidence/native-context-review/supplement.json",
+ "sha256": "9f4d9b577b6a39e7c9dd1b631d02d133ad2f424af0a502729c57ad3245f16741"
},
"retained_review": {
"path": "local-evidence/native-answer-review/retention.json",
- "sha256": "b3275c808947af2312fa48861db1a49b46c62cc8998c6c3afa7965c8f1f4e12f"
+ "sha256": "fd8f769c376e9c3b22bcd8878ae8b8ae32ec536bbbed01d5d20ca417c6c82b75"
},
"review_scope": {
"fresh": 72,
@@ -164,7 +164,7 @@
"total": 4300,
"fresh_independent_blind_review": false
},
- "implementation_commit": "4ef472ff151274e9c526be58bf0e056a1591f420",
+ "implementation_commit": "c6c004f7840f3ed10ed370d2abc2ce621eec429c",
"review_evidence_package": {
"archive_name": "PR-51-public-saved-inputs-20261016.tar.xz",
"url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-public-saved-inputs-20261016.tar.xz",
@@ -194,7 +194,8 @@
"sha256": "2fe714e6e8b410890f8be0329da3cb01bb02796c051b5477de0c2f5e3f4a6b40"
},
"report_renderer_commit": "ffe4a03367dc2b2d2fd168600042c7a3ead10d73",
- "validation_commit": "4ef472ff151274e9c526be58bf0e056a1591f420"
+ "validation_commit": "4ef472ff151274e9c526be58bf0e056a1591f420",
+ "role": "immutable_baseline_for_native_context_correction"
},
"historical_native_route_audit": {
"url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-route-audit-20261016.json",
@@ -217,5 +218,64 @@
"bytes": 27272,
"results_sha256": "ece94c0911a769c84ce9d896b00d97706fd4435d6d194f03cff77391710934a4",
"provider_calls_added": 0
+ },
+ "publication_evidence_views": {
+ "path": "PUBLIC-EVIDENCE-VIEWS.json",
+ "sha256": "c1719bd6694c7fd5c6c93471e82916da4b449ca15ef7f5e0533402ff108af519"
+ },
+ "publication_projection": {
+ "path": "PUBLICATION-PROJECTION.json",
+ "sha256": "87b2c6ab0f11698a92b3037abfea37871f1577d8b203da9108d5621e57b8a27e",
+ "kind": "host_metadata_publication_projection"
+ },
+ "native_answer_context_audit": {
+ "path": "local-evidence/native-context-audit/PR-51-native-context-audit-20261016.json",
+ "sha256": "22fe45a35b43753f1fee5e7cb0180d3b2037533d63e009e3b4fbef37d646544b",
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-native-context-audit-20261016.json",
+ "bytes": 8046875,
+ "result_count": 4300,
+ "original_context_differences": 2,
+ "current_context_differences": 0,
+ "provider_calls_added": 0,
+ "anonymous_download_sha256_verified": true
+ },
+ "original_measured_dataset_sha256": "28e3b3e9ec89ce545aae44355bd28eb0dc383ad6f64aed4dc91f327b14a528ab",
+ "original_measured_results_sha256": "ece94c0911a769c84ce9d896b00d97706fd4435d6d194f03cff77391710934a4",
+ "original_implementation_commit": "4ef472ff151274e9c526be58bf0e056a1591f420",
+ "native_context_correction": {
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-native-context-correction-20261016.tar.xz",
+ "sha256": "93525909c4effac75d3d4a3cece7e9e9cb3bfb2b2b091d25d103cbab570a07e4",
+ "bytes": 43947764,
+ "implementation_commit": "c6c004f7840f3ed10ed370d2abc2ce621eec429c",
+ "baseline_archive_sha256": "6ceb6a8bb35bc0e1d4ba40ff183dbb66e975d37426d7277ec6fca27ef58ad1ea",
+ "input_public_results_sha256": "2d2a35f6b51cd1023ee940f9a310c05610806f36d9c6363410200ffb26556dbe",
+ "output_public_results_sha256": "314a5adb2225fde74cfe0672d25d972c2c99e65b40e337d93c0a1d7418e4fe00",
+ "presentation_binding": {
+ "path": "presentation-binding.json",
+ "sha256": "6f15dd47fef163a361589e1e8e3f54abbd66863e40f3d1c1e0c1650b7935bf38"
+ },
+ "manifest_sha256": "4d260d82f287728fe07236ae1968445f05d5090156af2576d811bf74b595487f",
+ "restore_verified": true,
+ "restored_input_files": 33352,
+ "native_revision_chain_verified": true,
+ "review_bindings_verified": true,
+ "result_ledger_verified": true,
+ "anonymous_download_sha256_verified": true,
+ "provider_calls_added": 0,
+ "resource_route_results_rescored": 16,
+ "retained_identity_reviews_carried": 4,
+ "offline_rescored_results": 20,
+ "original_context_differences": 2,
+ "current_context_differences": 0,
+ "task_verdict_changes": 0,
+ "passes": 1966,
+ "failures": 2334,
+ "report_generated_languages": [
+ "zh",
+ "en"
+ ],
+ "page_layout_checks": 136,
+ "page_layout_issues": 0,
+ "historical_limits_unchanged": true
}
}
From 4384167a764491eed0aa46787b356b520c9233b8 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 14:58:09 +0800
Subject: [PATCH 43/50] Align bilingual context review and pin verified
correction package
---
research/native-430-local-report.json | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/research/native-430-local-report.json b/research/native-430-local-report.json
index ca73e29cf..c7c1b0867 100644
--- a/research/native-430-local-report.json
+++ b/research/native-430-local-report.json
@@ -28,7 +28,7 @@
},
{
"path": "result-review.json",
- "sha256": "c5cabe370ac2df9c1c6f2b647b4936d9c927de1ae2825f8e9ec937f40c1735c5"
+ "sha256": "427b1532a875fb0d196d6f2b529d67a2551e48d84355eaaef1e36bc91081eb49"
},
{
"path": "archived-reading.json",
@@ -243,18 +243,18 @@
"original_measured_results_sha256": "ece94c0911a769c84ce9d896b00d97706fd4435d6d194f03cff77391710934a4",
"original_implementation_commit": "4ef472ff151274e9c526be58bf0e056a1591f420",
"native_context_correction": {
- "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-native-context-correction-20261016.tar.xz",
- "sha256": "93525909c4effac75d3d4a3cece7e9e9cb3bfb2b2b091d25d103cbab570a07e4",
- "bytes": 43947764,
- "implementation_commit": "c6c004f7840f3ed10ed370d2abc2ce621eec429c",
+ "url": "https://github.com/lexmount/browser-eval/releases/download/pr-51-public-evidence-20261016/PR-51-native-context-correction-20261016-v2.tar.xz",
+ "sha256": "f33f5e2f2a8d643ed80bab5ecec57bc7f353924fe4a7ea958b8f0caab6ee3c39",
+ "bytes": 43947732,
+ "implementation_commit": "31a00bad2bbbdd33d50f47ef17fd8beb6dd4900d",
"baseline_archive_sha256": "6ceb6a8bb35bc0e1d4ba40ff183dbb66e975d37426d7277ec6fca27ef58ad1ea",
"input_public_results_sha256": "2d2a35f6b51cd1023ee940f9a310c05610806f36d9c6363410200ffb26556dbe",
"output_public_results_sha256": "314a5adb2225fde74cfe0672d25d972c2c99e65b40e337d93c0a1d7418e4fe00",
"presentation_binding": {
"path": "presentation-binding.json",
- "sha256": "6f15dd47fef163a361589e1e8e3f54abbd66863e40f3d1c1e0c1650b7935bf38"
+ "sha256": "b2dc50c1ae3f75638dc10faea09a39ad5fd451611cf8cc04385abcbbd6d879f2"
},
- "manifest_sha256": "4d260d82f287728fe07236ae1968445f05d5090156af2576d811bf74b595487f",
+ "manifest_sha256": "5186f43a41c66cbbab3c9c49c896e2d6862482039c8e203e6ea4e90520f04d0d",
"restore_verified": true,
"restored_input_files": 33352,
"native_revision_chain_verified": true,
@@ -276,6 +276,7 @@
],
"page_layout_checks": 136,
"page_layout_issues": 0,
- "historical_limits_unchanged": true
+ "historical_limits_unchanged": true,
+ "presentation_revision": "Aligned English numeric copy in the new review round; original opinions and scoring inputs unchanged."
}
}
From fa7d93a2997f022e89eee0aba5902094563e2363 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 15:40:20 +0800
Subject: [PATCH 44/50] Reject mixed native projection schemas and preserve
legacy scope
---
harness/web_fetch/README.md | 2 ++
harness/web_fetch/response_input.py | 18 +++++++---
harness/web_fetch/runner.py | 4 ++-
tests/test_native_route_contract.py | 56 +++++++++++++++++++++++++++++
tests/test_scoring.py | 2 +-
5 files changed, 76 insertions(+), 6 deletions(-)
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index 17b4a9586..b44a4c9d4 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -952,6 +952,8 @@ request-URL fallback applies only when the selected native address is absent.
Resolve relative resource attributes against that verified address. A title absent
from the native response cannot supply required facts.
+Schema-1 declarations must not contain `result_fields`, even an empty object.
+Current persistence rejects mixed declarations, including discarded timeout answers.
Immutable schema-1 acquisitions retain their original text/resource/file scope;
they do not retroactively prove title/address provenance. Bind a separate offline
context audit to historical results, replay the complete affected route when a
diff --git a/harness/web_fetch/response_input.py b/harness/web_fetch/response_input.py
index b4e886f75..9a0a4636a 100644
--- a/harness/web_fetch/response_input.py
+++ b/harness/web_fetch/response_input.py
@@ -138,10 +138,17 @@ def _project_field(payload, spec):
raise ValueError('Unknown native answer field projection')
-def project_native_answer(payload, declaration):
- """Select declared native fields; never inspect references or URL meaning."""
+def require_native_projection_schema(declaration):
+ """Keep historical answer scope separate from schema-2 result context."""
if declaration.get('schema_version') not in {1, 2}:
raise ValueError('Unsupported native answer projection')
+ if declaration['schema_version'] == 1 and 'result_fields' in declaration:
+ raise ValueError('Native result_fields require projection schema 2')
+
+
+def project_native_answer(payload, declaration):
+ """Select declared native fields; never inspect references or URL meaning."""
+ require_native_projection_schema(declaration)
kind = declaration.get('content_format')
if declaration.get('source_format') == 'bytes':
if declaration.get('answer_fields'):
@@ -261,6 +268,7 @@ def identity(value: str) -> tuple | None:
def project_native_result_fields(payload, declaration, *, request_url=None,
diagnostic=None, transport_receipt=None):
"""Replay scored title/address from native fields or same-call evidence."""
+ require_native_projection_schema(declaration)
fields = {'title': None, 'final_url': None}
for name, spec in declaration.get('result_fields', {}).items():
if name not in fields:
@@ -305,6 +313,7 @@ def require_native_answer_projection(content, metadata, raw_response, *, files=(
or (raw_response is not None and content)):
raise ValueError('Scored native answer has undeclared answer fields')
return
+ require_native_projection_schema(declaration)
if raw_response is None:
raise ValueError('Declared native answer lacks its retained raw response')
if present - set(declaration.get('answer_fields', {})):
@@ -365,6 +374,7 @@ def verify_saved_native_answer(root, record):
if metadata.get('native_answer_required'):
raise ValueError('Scored native answer has no projection declaration')
return
+ require_native_projection_schema(declaration)
answer = read_evidence_binding(root, {'path': record['artifact_path'],
'sha256': record['metrics']['content_sha256']}).read_bytes()
# Deadlines discard answers while preserving late raw evidence.
@@ -385,8 +395,8 @@ def verify_saved_native_answer(root, record):
diagnostic = read_evidence_binding(root, {'path': metadata['diagnostic_log_path'],
'sha256': metadata['diagnostic_log_sha256']}).read_bytes()
require_native_answer_projection(answer.decode('utf-8'), metadata, raw, files=files,
- title=record.get('title') if declaration.get('schema_version') == 2 or 'result_fields' in declaration else None,
- final_url=record.get('final_url') if declaration.get('schema_version') == 2 or 'result_fields' in declaration else None,
+ title=record.get('title') if declaration.get('schema_version') == 2 else None,
+ final_url=record.get('final_url') if declaration.get('schema_version') == 2 else None,
request_url=record.get('url'),
diagnostic=diagnostic, transport_receipt=metadata.get('source_transport_receipt'))
if files or 'file_evaluation' in metadata:
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index ba10264e4..634160dfb 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -381,7 +381,9 @@ def persist_result_artifacts(
"""Persist one physical call without changing its scored answer bytes."""
discarded_late_answer = (not result.ok and result.error_type == 'timeout'
and result.content == '')
- from .response_input import require_native_answer_projection
+ from .response_input import require_native_answer_projection, require_native_projection_schema
+ if result.metadata.get('native_answer_projection'):
+ require_native_projection_schema(result.metadata['native_answer_projection'])
raw = result.metadata.get('_raw_response_b64')
if result.metadata.get('native_answer_projection', {}).get('source_format') == 'bytes':
raw = result.metadata.get('_raw_stdout_b64')
diff --git a/tests/test_native_route_contract.py b/tests/test_native_route_contract.py
index a8e697e64..ca4f01574 100644
--- a/tests/test_native_route_contract.py
+++ b/tests/test_native_route_contract.py
@@ -590,3 +590,59 @@ def test_empty_same_call_diagnostic_is_retained_without_inventing_final_url(tmp_
'metrics': {'content_sha256': hashlib.sha256(b'answer').hexdigest()}}
verify_saved_native_answer(tmp_path, record)
assert (tmp_path/result.metadata['diagnostic_log_path']).read_bytes() == b''
+
+
+@pytest.mark.parametrize('result_fields', [{}, {'final_url': {'pointer': ['final_url']}}])
+@pytest.mark.parametrize('discarded_timeout', [False, True])
+def test_current_writer_rejects_schema_one_result_context(tmp_path, result_fields, discarded_timeout):
+ import base64
+ import json
+ from pathlib import Path
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ from harness.web_fetch.scoring import score_attempt
+ content = '' if discarded_timeout else 'answer'
+ raw = json.dumps({'content': content}).encode()
+ declaration = {'schema_version': 1, 'content_pointer': ['content'],
+ 'content_format': 'text', 'result_fields': result_fields}
+ metadata = {'native_answer_projection': declaration,
+ '_raw_response_b64': base64.b64encode(raw).decode()}
+ result = ProviderResult(not discarded_timeout, content,
+ error_type='timeout' if discarded_timeout else None, metadata=metadata)
+ artifact = Path('artifacts/answer.txt')
+ with pytest.raises(ValueError, match='result_fields require projection schema 2'):
+ persist_result_artifacts(tmp_path, artifact, result, request_url='https://example.test/')
+ assert not (tmp_path/artifact).exists()
+ with pytest.raises(ValueError, match='result_fields require projection schema 2'):
+ score_attempt({'url': 'https://example.test/'}, content.encode(), ok=result.ok,
+ metadata=metadata, native_response=raw)
+
+
+def test_saved_schema_one_keeps_original_answer_scope(tmp_path):
+ import base64
+ import hashlib
+ import json
+ from pathlib import Path
+ from harness.web_fetch.models import ProviderResult
+ from harness.web_fetch.runner import persist_result_artifacts
+ from harness.web_fetch.response_input import verify_saved_native_answer
+ images = [{'src': 'https://example.test/image.png'}]
+ raw = json.dumps({'content': 'answer', 'images': images}).encode()
+ declaration = {'schema_version': 1, 'content_pointer': ['content'], 'content_format': 'text',
+ 'answer_fields': {'images': {'pointer': ['images']}}}
+ result = ProviderResult(True, 'answer', metadata={'images': images,
+ 'native_answer_projection': declaration, '_raw_response_b64': base64.b64encode(raw).decode(),
+ 'raw_response_sha256': hashlib.sha256(raw).hexdigest(), 'raw_response_bytes': len(raw)})
+ persist_result_artifacts(tmp_path, Path('artifacts/answer.txt'), result)
+ record = {'ok': True, 'artifact_path': 'artifacts/answer.txt',
+ 'url': 'https://requested.example.test/', 'title': 'historical unaudited title',
+ 'final_url': 'https://historical.example.test/', 'provider_metadata': result.metadata,
+ 'metrics': {'content_sha256': hashlib.sha256(b'answer').hexdigest()}}
+ verify_saved_native_answer(tmp_path, record)
+ result.metadata['images'] = [{'src': 'https://forged.example.test/image.png'}]
+ with pytest.raises(ValueError, match='declared native field projection'):
+ verify_saved_native_answer(tmp_path, record)
+ result.metadata['images'] = images
+ declaration['result_fields'] = {}
+ with pytest.raises(ValueError, match='result_fields require projection schema 2'):
+ verify_saved_native_answer(tmp_path, record)
diff --git a/tests/test_scoring.py b/tests/test_scoring.py
index b177c578c..ec0ef9641 100644
--- a/tests/test_scoring.py
+++ b/tests/test_scoring.py
@@ -713,7 +713,7 @@ def response(c, timeout):
error_type='synthetic_failure' if c.id == 'failed' else None,
metadata={**fields, '_raw_response_b64': base64.b64encode(raw).decode(),
'raw_response_sha256': hashlib.sha256(raw).hexdigest(), 'raw_response_bytes': len(raw),
- 'native_answer_projection': {'schema_version': 1, 'content_pointer': ['content'],
+ 'native_answer_projection': {'schema_version': 2, 'content_pointer': ['content'],
'content_format': 'text', 'result_fields': {'final_url': {'pointer': ['final_url']}},
'answer_fields': {'images': {
'pointer': ['metadata', 'images'], 'format': 'identity'}}}})
From ebb5b537fb42c297d76cf6abf15f70e6ef4a30e5 Mon Sep 17 00:00:00 2001
From: Maisiqi <324751762+Maisiqi@users.noreply.github.com>
Date: Fri, 16 Oct 2026 20:06:26 +0800
Subject: [PATCH 45/50] Render frozen body evidence without applying a newer
text normalizer
---
harness/web_fetch/report.py | 35 ++++++++++++++++++++++++-------
harness/web_fetch/report_build.py | 4 ++--
tests/test_report_body_scope.py | 25 +++++++++++++++++++++-
3 files changed, 54 insertions(+), 10 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 2ddb9aa62..98f341a89 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -1291,21 +1291,38 @@ def _body_scoring_text(record, output):
from .response_input import answer_input
content=answer_input(output.encode('utf-8'),record.get('provider_metadata'),
ok=record.get('ok',False),status_code=record.get('status_code'))[0]
- from .reference_quality import is_html_document_or_fragment,visible_mixed_html_text
+ from .reference_quality import is_html_document_or_fragment
if is_html_document_or_fragment(content):
from .dynamic_lists import visible_html_text
content=visible_html_text(content)
else:
- content=visible_mixed_html_text(content)
+ from .body_coverage import native_markup_body_text
+ fallback=html.unescape(re.sub(r"<(script|style|template)\b[^>]*>.*?","",content,flags=re.I|re.S))
+ content=native_markup_body_text(content,fallback)
return content
-def body_region_rows(case, metrics, output):
+def body_region_rows(case, metrics, output, *, frozen_scores=False):
from .body_coverage import evaluate_body_requirements
contract=case.get('body_requirements') or {}
if not contract:return [],''
saved_regions=metrics.get('body_region_results',[])
recalculated={}
- if saved_regions and all('matched_output_passages' in item for item in saved_regions):
+ if frozen_scores:
+ # The frozen-build boundary validates the saved metric and answer hashes.
+ # A newer normalizer or a redacted public answer cannot reproduce the old
+ # matching-text identity. Render the saved allocation without rescoring it.
+ from .publication_privacy import project_evidence_view
+ evidence={}
+ for item in saved_regions:
+ passages=[]
+ for passage in item.get('matched_output_passages',[]):
+ if passage.get('text'):
+ public,_=project_evidence_view('public-evidence/excerpt.txt',passage['text'].encode(),role='scored_answer',content_format='text')
+ passages.append(public.decode())
+ evidence[item['id']]=passages
+ evidence_counts={item['id']:item.get('matched_units',item.get('matched_tokens',0)) for item in saved_regions}
+ recalculated={item['id']:item for item in saved_regions}
+ elif saved_regions and all('matched_output_passages' in item for item in saved_regions):
from .body_coverage import _compact_with_positions
matching_text=_compact_with_positions(output)[0]
digest=hashlib.sha256(matching_text.encode()).hexdigest()
@@ -1334,8 +1351,10 @@ def body_region_rows(case, metrics, output):
rows.append(('',cell(esc(label))+cell(f'{expected_count:,}')+cell(f'{matched:,}')+
cell(pct(actual['coverage']))+cell(esc(judgement))))
passage='\n\n'.join(evidence.get(item['id'],[]))
+ missing=(model_text('历史评分未保存本段命中文字；请对照完整输出核对。','The historical score has no saved matching passage for this region; inspect the complete output.')
+ if frozen_scores and 'matched_output_passages' not in item else '评分器没有向这一段分配可展示的实际返回文字。')
actual_html=(f'{esc(passage)} ' if passage else
- '评分器没有向这一段分配可展示的实际返回文字。
')
+ ''+missing+'
')
cards.append(f'{index}. {esc(label)} 本段覆盖率为 {pct(actual["coverage"])}。每段至少达到 {pct(actual.get("floor",actual["minimum"]))}；达到 {pct(actual["minimum"])} 时算作主要完整段落。
{esc(judgement)} 这一位置的完整标准正文 {esc(expected)} 参与本段计分的完整实际返回行 {actual_html}
正文判定： 本例分为 {region_count} 段。'
f'每段至少达到 {pct(rules.get("minimum_floor",rules.get("minimum_coverage")))}，'
f'其中至少 {required} 段达到 {pct(rules.get("minimum_coverage"))}。'
'先看下表；需要核对原文时，再展开下面的左右对照。
')
+ if frozen_scores:
+ region_intro+=''+model_text('本节沿用该次评分保存的数值和片段；公开片段已隐藏敏感值。未保存的历史片段不使用当前规则补算，公开副本也不用于重算原件指标。','This section retains the values and passages saved with the score. Sensitive values are hidden in public passages. Missing historical passages are not reconstructed with current rules, and public copies are not used to recalculate original metrics.')+'
'
body+=section('正文是否抓全',region_intro+table(['标准答案位置','标准片段数','命中片段数','覆盖率','判定'],region_rows,metrics=[])+
'查看各段完整标准正文和计分行 '+''.join(region_cards)+'abcdef "):
+ "\\abcdef ",
+ 'abcdef source ',
+ 'abcdef