diff --git a/rpent/benchmarks/docs.css b/rpent/benchmarks/docs.css index df87ece..8c8435f 100644 --- a/rpent/benchmarks/docs.css +++ b/rpent/benchmarks/docs.css @@ -12,7 +12,6 @@ html[data-theme="dark"] .rpent-static-leaderboard{--rpent-fallback-border:#49405 .rpent-static-leaderboard .memory-context{margin:1rem 0} .rpent-static-leaderboard .cost-section .table-wrap table{min-width:340px} .rpent-static-leaderboard .cost-method small{display:block;font-size:12px;margin-top:4px} -.rpent-static-leaderboard .benchmark-method small{display:block;font-size:12px;margin-top:4px} .rpent-static-leaderboard .module-header h3{display:inline;font-size:26px;font-weight:500;line-height:1.3} .rpent-static-leaderboard [data-benchmark] h4{font-size:20px;font-weight:400;line-height:1.55} .rpent-static-leaderboard .module-header{cursor:pointer;padding:12px 0;list-style-position:outside}.rpent-static-leaderboard .module-header::marker{font-size:18px} @@ -42,3 +41,8 @@ html[data-theme="dark"] .rpent-static-leaderboard{--rpent-divider:#be96e0} .rpent-static-leaderboard [data-benchmark],.rpent-static-leaderboard .cost-group{padding-inline:24px} @media(max-width:1200px){.rpent-static-leaderboard .section-layout{display:block}.rpent-static-leaderboard .module-directory{position:static;padding-bottom:16px}.rpent-static-leaderboard .module-directory>summary{display:list-item;list-style-position:inside}} @media(max-width:700px){.rpent-static-leaderboard .page-section-title{font-size:32px}.rpent-static-leaderboard [data-benchmark],.rpent-static-leaderboard .cost-group{padding-inline:0}} + +.rpent-static-leaderboard .scope-note{font-size:14px;line-height:1.6;color:var(--pst-color-text-muted,inherit);margin:14px 0;scroll-margin-top:5rem} +.rpent-static-leaderboard .note-reference{font-size:.75em;margin-inline-start:.2em;white-space:nowrap} +.rpent-static-leaderboard .note-reference a{text-decoration:underline;text-underline-offset:2px} +.rpent-static-leaderboard .note-reference a:focus-visible,.rpent-static-leaderboard .scope-note:focus-visible{outline:2px solid var(--rpent-divider);outline-offset:3px} diff --git a/rpent/benchmarks/embed.css b/rpent/benchmarks/embed.css index 3fde0d7..25674ab 100644 --- a/rpent/benchmarks/embed.css +++ b/rpent/benchmarks/embed.css @@ -106,3 +106,8 @@ noscript:has([data-costs]:target) [data-costs],noscript:has([data-costs] :target :host .page-header{padding-top:0;scroll-margin-top:48px}:host .title-row{display:none} @media(max-width:959.98px){:host{--module-top:52px}} + +.note-reference{font-size:.75em;margin-inline-start:.2em;white-space:nowrap} +.note-reference a{color:inherit;text-decoration:underline;text-underline-offset:2px} +.note-reference a:focus-visible,.scope-note:focus-visible{outline:2px solid var(--best);outline-offset:3px} +.scope-note[tabindex]{scroll-margin-top:calc(var(--module-top) + var(--directory-height) + 88px)} diff --git a/rpent/benchmarks/index.html b/rpent/benchmarks/index.html index 6b4e989..c352311 100644 --- a/rpent/benchmarks/index.html +++ b/rpent/benchmarks/index.html @@ -29,9 +29,9 @@

Performance

+

耗时与 Token 开销

环境

LIBERO-PRO

方法平均每回合耗时（秒）总输出 token
RPent / GPT-6 Astra Codex · low · reasoning412.143,364,938
RPent / GPT-5.6 Sol Codex · xhigh · reasoning529.233,635,426
RPent / GPT-5.6 Sol Codex · no-reasoning344.172,569,676
RPent Flash Mode Molmo2-8B · 视觉定位60.190
RPent / Qwen3.6 27B no-reasoning626.33,270,754

RoboCasa365

方法平均每回合耗时（秒）总输出 token
RPent / GPT-6 Astra Codex · low · reasoning1,168.955,894,436
RPent / GPT-5.6 Sol Codex · xhigh · reasoning1,197.565,258,391
RPent / GPT-5.6 Sol Codex · no-reasoning1,013.214,792,104

RoboTwin

方法平均每回合耗时（秒）总输出 token
RPent / GPT-6 Astra Codex · low · reasoning1,078.82,763,641
RPent / GPT-5.6 Sol Codex · xhigh · reasoning1,828.74,763,915
RPent / GPT-5.6 Sol Codex · no-reasoning957.42,456,401
diff --git a/rpent/benchmarks/leaderboard-en-dark.png b/rpent/benchmarks/leaderboard-en-dark.png index 63d97a2..3f7b439 100644 Binary files a/rpent/benchmarks/leaderboard-en-dark.png and b/rpent/benchmarks/leaderboard-en-dark.png differ diff --git a/rpent/benchmarks/leaderboard-en-light.png b/rpent/benchmarks/leaderboard-en-light.png index 3179f11..75c4a36 100644 Binary files a/rpent/benchmarks/leaderboard-en-light.png and b/rpent/benchmarks/leaderboard-en-light.png differ diff --git a/rpent/benchmarks/leaderboard-zh-dark.png b/rpent/benchmarks/leaderboard-zh-dark.png index 507b386..32af3cc 100644 Binary files a/rpent/benchmarks/leaderboard-zh-dark.png and b/rpent/benchmarks/leaderboard-zh-dark.png differ diff --git a/rpent/benchmarks/leaderboard-zh-light.png b/rpent/benchmarks/leaderboard-zh-light.png index efa0362..a172fa7 100644 Binary files a/rpent/benchmarks/leaderboard-zh-light.png and b/rpent/benchmarks/leaderboard-zh-light.png differ diff --git a/rpent/benchmarks/leaderboard.css b/rpent/benchmarks/leaderboard.css index 50a6436..2a04c2f 100644 --- a/rpent/benchmarks/leaderboard.css +++ b/rpent/benchmarks/leaderboard.css @@ -103,4 +103,8 @@ noscript:has([data-costs]:target) [data-performance],noscript:has([data-costs] : noscript:has([data-costs]:target) [data-costs],noscript:has([data-costs] :target) [data-costs]{display:block} @media(max-width:1000px){noscript .section-layout{display:block}noscript .module-directory{position:static;padding-bottom:16px}noscript .module-directory>summary{display:list-item;list-style-position:inside}} -.benchmark-method small{display:block;font-size:12px;margin-top:4px} + +.note-reference{font-size:.75em;margin-inline-start:.2em;white-space:nowrap} +.note-reference a{color:inherit;text-decoration:underline;text-underline-offset:2px} +.note-reference a:focus-visible,.scope-note:focus-visible{outline:2px solid var(--best);outline-offset:3px} +.scope-note[tabindex]{scroll-margin-top:calc(var(--module-top) + var(--directory-height) + 88px)} diff --git a/rpent/benchmarks/leaderboard.js b/rpent/benchmarks/leaderboard.js index cb6cfcc..7a93427 100644 --- a/rpent/benchmarks/leaderboard.js +++ b/rpent/benchmarks/leaderboard.js @@ -124,17 +124,33 @@ if(id==='robotwin') return lang==='en'?'Clean-to-Randomized Success':'Clean-to-Randomized 成功率'; return tr(views.get(id).label)+' · '+t('rate'); } + const evaluationNotes = new Map([ + ['aspire', 1], ['rpent-flash-mode', 2], + ['gpt-6-astra-low-motor-only', 3], ['gpt-6-astra-low', 4] + ]); + const noteId = number => number===4?'libero-pro-astra-memory':`libero-pro-note-${number}`; + function noteReference(record) { + const section=d.sections.find(s=>s.id==='libero-pro'); + const number=section.views.includes(record.view_id)?evaluationNotes.get(record.configuration_id):null; + if(!number)return ''; + const description=lang==='en'?`Evaluation note ${number} for ${label(record)}`:`${label(record)}的评测说明 ${number}`; + return `[${number}]`; + } + function noteParagraph(paragraph, number) { + const marked=paragraph.startsWith('* ')?`* [${number}] ${h(paragraph.slice(2))}`:h(paragraph); + return `

${marked}

`; + } function chart(view, kind='primary') { const rows=allRows(view).filter(r=>r.rate!==null); return `
${rows.map(r=> - `
${h(label(r))}${h(detailLabel(r))}
${r.rate}%
`).join('')}
`; + `
${h(label(r))}${noteReference(r)}${h(detailLabel(r))}
${r.rate}%
`).join('')}`; } function comparisonMatrix(s) { const columns=s.views.map(id=>views.get(id)); const rows=d.configurations.filter(c=>columns.some(v=>v.record_ids.some(id=>records.get(id).configuration_id===c.id&&records.get(id).rate!==null))); return `
${t('allScores')} ${rows.length} ${lang==='en'?'configurations':'配置'}
${columns.map(v=>``).join('')}${rows.map(c=>{ const sample=columns.flatMap(allRows).find(r=>r.configuration_id===c.id&&r.rate!==null); - return `${columns.map(v=>{ + return `${columns.map(v=>{ const r=allRows(v).find(r=>r.configuration_id===c.id); return ``; }).join('')}`; @@ -147,7 +163,7 @@ function renderSection(s) { const element=document.getElementById('panel-'+s.id); const note=tr(s.notes); - element.innerHTML=sectionBody(s)+(note?note.split('\n\n').map(paragraph=>`

${h(paragraph)}

`).join(''):'')+(s.context?`

${h(tr(s.context))}

`:''); + element.innerHTML=sectionBody(s)+(note?note.split('\n\n').map((paragraph,index)=>noteParagraph(paragraph,index+1)).join(''):'')+(s.context?noteParagraph(tr(s.context),4):''); setupTables(element,s.id); } function setupTables(element,key) { @@ -274,6 +290,7 @@ const anchor=e.target.closest('a[href^="#"]'); if(anchor&&document.getElementById(normalizeAnchor(anchor.hash))){ e.preventDefault();navigate(normalizeAnchor(anchor.hash),true); + if(anchor.closest('.note-reference'))document.getElementById(normalizeAnchor(anchor.hash)).focus({preventScroll:true}); } const summary=e.target.closest('.module-header'); if(summary)markModule(summary.parentElement.dataset.module); diff --git a/rpent/benchmarks/results.json b/rpent/benchmarks/results.json index 6ad8fbd..4f637b5 100644 --- a/rpent/benchmarks/results.json +++ b/rpent/benchmarks/results.json @@ -1,6 +1,6 @@ { "schema_version": 2, - "checked_at": "2026-09-18", + "checked_at": "2026-09-22", "paper_revision": "2607.08448v4", "repository_revision": "43f32aa08cba07bd4d49a4bfa5eba4ef633e9b92", "configurations": [ @@ -843,13 +843,13 @@ "view_id": "long-task", "configuration_id": "gpt-5-5-xhigh", "status": "reported", - "rate": "52.0", - "successes": null, + "rate": "70.00", + "successes": 70, "episodes": 100, "source_ids": [ - "paper-table-3" + "repo-libero-long-reproduction-20260922" ], - "protocol_id": "rpent-pro" + "protocol_id": "rpent-pro-long-reproduction" }, { "id": "long-task--gpt-6-astra-low", @@ -870,13 +870,13 @@ "view_id": "long-swap", "configuration_id": "gpt-5-5-xhigh", "status": "reported", - "rate": "49.0", - "successes": null, + "rate": "55.00", + "successes": 55, "episodes": 100, "source_ids": [ - "paper-table-3" + "repo-libero-long-reproduction-20260922" ], - "protocol_id": "rpent-pro" + "protocol_id": "rpent-pro-long-reproduction" }, { "id": "long-swap--gpt-6-astra-low", @@ -896,14 +896,25 @@ "id": "libero-pro--gpt-5-5-xhigh", "view_id": "libero-pro", "configuration_id": "gpt-5-5-xhigh", - "status": "reported", - "rate": "72.1", + "status": "derived", + "rate": "75.13", "successes": null, "episodes": 800, "source_ids": [ - "paper-table-3" + "paper-table-3", + "repo-libero-long-reproduction-20260922" ], - "protocol_id": "rpent-pro" + "protocol_id": "rpent-pro-updated-long", + "derived_from": [ + "spatial-task--gpt-5-5-xhigh", + "spatial-swap--gpt-5-5-xhigh", + "object-task--gpt-5-5-xhigh", + "object-swap--gpt-5-5-xhigh", + "goal-task--gpt-5-5-xhigh", + "goal-swap--gpt-5-5-xhigh", + "long-task--gpt-5-5-xhigh", + "long-swap--gpt-5-5-xhigh" + ] }, { "id": "libero-pro--gpt-6-astra-low", @@ -3211,6 +3222,17 @@ "id": "contributor-motor-only-long-task-20260920", "type": "contributor_report", "note": "Contributor-reported complete Long Task evaluation on another server: 38 successes in 100 episodes. Codex / GPT-6 Astra / low reasoning; execute_action motor-only control without VLA / primitives or memory. This aggregate is independent of the incomplete local motor-only campaign; no unreported suite, runtime, or token measurements are inferred." + }, + { + "id": "repo-libero-long-reproduction-20260922", + "url": "https://github.com/RLinf/RPent/blob/15420b438e3adbf82e6640799a0668a2609e3bb7/docs/source-en/rst_source/usage/libero.rst#reproducing-results", + "note": "Codex / GPT-5.5 / xhigh reproduction on reproduce/libero reports Long Task 70/100 and Long Swap 55/100. The updated leaderboard combines these two results with the six unchanged Table 3 suites: (81+69+94+91+75+66+70+55)/8 = 75.125%, displayed as 75.13%. This derived Overall is not the paper's original aggregate." + }, + { + "id": "flash-memory-confirmation-20260922", + "kind": "contributor-confirmed", + "url": "https://huggingface.co/datasets/RLinf/RPent-memory", + "note": "The contributor confirms that the displayed Flash Mode evaluation uses directly downloaded, officially released GPT-5.5 exploration memory and the best-performing seed from s0-s9. Molmo2-8B is used for visual localization. No memory snapshot revision is inferred." } ], "astra": { @@ -3351,8 +3373,8 @@ "zh": "Astra · low · reasoning：92.63%（741/800）" }, "context": { - "en": "GPT-6 Astra (memory-enabled configuration): Long Task/Swap and the other six suites use separate memory-file snapshots frozen after their respective exploration phases, with no updates during evaluation. Overall combines two non-overlapping batches: Long 157/200 plus the other suites 584/600, giving 741/800 (92.63%); the 800 episodes do not share a single memory snapshot.", - "zh": "GPT-6 Astra（使用 memory 的配置）：Long Task/Swap 与其余六套件使用各自探索后冻结的 memory 文件快照，评测期间不更新。Overall 合并两个不重叠批次：Long 157/200，加上其余套件 584/600，得到 741/800（92.63%）；并非全部回合共享同一份 memory 快照。" + "en": "* GPT-6 Astra (memory-enabled configuration): Long Task/Swap and the other six suites use separate memory-file snapshots frozen after their respective exploration phases, with no updates during evaluation. Overall combines two non-overlapping batches: Long 157/200 plus the other suites 584/600, giving 741/800 (92.63%); the 800 episodes do not share a single memory snapshot.", + "zh": "* GPT-6 Astra（使用 memory 的配置）：Long Task/Swap 与其余六套件使用各自探索后冻结的 memory 文件快照，评测期间不更新。Overall 合并两个不重叠批次：Long 157/200，加上其余套件 584/600，得到 741/800（92.63%）；并非全部回合共享同一份 memory 快照。" }, "default_view": "libero-pro", "views": [ @@ -3367,8 +3389,8 @@ "long-swap" ], "notes": { - "en": "ASPIRE: Long Task and Long Swap use zero-shot transfer from the LIBERO-90 skill library.\n\nRPent / GPT-6 Motor Only uses motor-only control: it directly outputs end-effector pose increments and gripper commands through execute_action, without invoking VLA / primitives or loading memory.", - "zh": "ASPIRE：Long Task 和 Long Swap 使用 LIBERO-90 技能库进行 zero-shot 迁移。\n\nRPent / GPT-6 Motor Only 为纯电机控制方法，通过 execute_action 直接输出末端位姿增量与夹爪指令，不调用 VLA / primitive，不加载 memory" + "en": "* ASPIRE: Long Task and Long Swap use zero-shot transfer from the LIBERO-90 skill library.\n\n* RPent Flash Mode uses directly downloaded, officially released GPT-5.5 exploration memory; Molmo2-8B is used for visual localization. These results use the best-performing seed from s0-s9.\n\n* RPent / GPT-6 Motor Only uses motor-only control: it directly outputs end-effector pose increments and gripper commands through execute_action, without invoking VLA / primitives or loading memory.", + "zh": "* ASPIRE：Long Task 和 Long Swap 使用 LIBERO-90 技能库进行 zero-shot 迁移。\n\n* RPent Flash Mode 使用直接下载的、官方公开的 GPT-5.5 explore memory；Molmo2-8B 用于视觉定位。该测试结果使用了 s0-s9 中表现最好的 seed。\n\n* RPent / GPT-6 Motor Only 为纯电机控制方法，通过 execute_action 直接输出末端位姿增量与夹爪指令，不调用 VLA / primitive，不加载 memory。" } }, { diff --git a/rpent/benchmarks/table-sort.js b/rpent/benchmarks/table-sort.js index 9b2d182..ce0f7ab 100644 --- a/rpent/benchmarks/table-sort.js +++ b/rpent/benchmarks/table-sort.js @@ -33,10 +33,16 @@ if(state.highlightBest&&column!==state.idColumn&&maximum!==null&&valueOf(row.cells[column])===maximum)row.cells[column].classList.add('metric-best'); } } + function exportText(cell) { + const copy=cell.cloneNode(true); + copy.querySelectorAll('.note-reference').forEach(marker=>marker.remove()); + const text=copy.textContent.trim(); + return text==='—'?'':text; + } function exportTable(table) { const quote=v=>'"'+String(v??'').replaceAll('"','""')+'"'; const state=table._rpentSort; - const rows=[state.headers.map(th=>th.dataset.label),...[...table.tBodies[0].rows].map(row=>[...row.cells].map(cell=>cell.textContent.trim()==='—'?'':cell.textContent.trim()))]; + const rows=[state.headers.map(th=>th.dataset.label),...[...table.tBodies[0].rows].map(row=>[...row.cells].map(exportText))]; const url=URL.createObjectURL(new Blob(['\uFEFF'+rows.map(row=>row.map(quote).join(',')).join('\r\n')],{type:'text/csv;charset=utf-8'})); const a=document.createElement('a');a.href=url;a.download=state.key.replaceAll(/[^a-zA-Z0-9-]/g,'-')+'.csv';a.click(); setTimeout(()=>URL.revokeObjectURL(url),3000);
${t('method')}${h(tr(v.label))}
${h(label(sample))} ${h(detailLabel(sample))}
${h(label(sample))}${noteReference(sample)} ${h(detailLabel(sample))}
${r?.rate!=null?r.rate+'%':`—`}