diff --git a/docs/dashboard-external-records.md b/docs/dashboard-external-records.md new file mode 100644 index 00000000..9a59fcca --- /dev/null +++ b/docs/dashboard-external-records.md @@ -0,0 +1,78 @@ +# External optimization records in the dashboard + +A campaign with no candidate/run records is **not connected**, not a successful +comparison. `stages_declared: []` explicitly declares no applicable stages; +omitting the field leaves unobserved stages **not recorded**. Do not set an empty +list merely because a backend has no built-in collector. + +For existing Ascend experiments, a caller may adapt their artifacts into native +candidate rows without manufacturing a CUDA run or evaluation. Create +`candidates//candidate.json` with `candidate_id`, `title`, and +`external_record_ref: {"path": "records/.json", "sha256": ""}`. +Paths are campaign-relative and hashes cover exact file bytes. These candidates +must not also name a harness run. The external record has this shape (illustrative +values only): + +```json +{ + "schema_version": 1, + "kind": "external_optimization_record", + "candidate_id": "example", + "backend": "ascend", + "protocol": "example-two-service-protocol", + "state": "completed", + "source_refs": [{"path": "sources/result.json", "sha256": ""}], + "performance_verdict": "INCONCLUSIVE", + "quality": {"verdict": "NOT_RUN"}, + "metrics": [{ + "name": "example-tier", "unit": "ms/token", + "baseline": 42, "samples": [40, 38], + "aggregate": 39, "aggregation": "two_round_median", "target": 25 + }], + "events": [{"stage": "validation", "time": "2026-01-01T12:00:00Z", + "summary": "Two service records captured; quality not run"}], + "note": "Recorded intermediate measurements, not final acceptance." +} +``` + +All source references must resolve with matching hashes. No automatic directory +scan or inference of verdicts is performed. Publish immutable content-addressed +source and record files first, then atomically replace candidate metadata. A +partial file, hash mismatch, mismatched candidate or malformed record is withheld; +the UI reports missing evidence rather than an invented success. + +This version represents lower-is-better latency metrics; absolute target gaps +are aggregate minus target. Each metric supports a positive parent, up to two positive samples, a positive +aggregate and target; omitted values remain unrecorded. `two_round_median` +requires two samples and their arithmetic midpoint, accepting floating-point +serialization differences with relative tolerance `1e-9` and absolute tolerance +`1e-12` in the metric unit. This is not a measurement/noise tolerance; coarser +rounding must not be used to change a recorded aggregate. The caller value is +preserved. Records without metrics stay in candidate/event views and are counted +in the chart note; no measurement is fabricated. Keep concurrency tiers and +protocols separate. Absolute plots do not establish cross-run comparability. +Performance and quality verdicts remain separate; quality may include `correct` +and `total`. Neither an external PASS nor a source hash promotes a record into +the generic verified gain ranking. Hashes establish bytes, not the correctness +of the caller's mapping, frozen gates, execution, or experimental conclusions. +Only recorded events populate workflow stages. Summaries are explicitly distinct +from a full campaign optimization log; use `campaign log` for that history. + +Optional `environment` keys `hardware`, `runtime`, `model`, `workload`, `protocol` +and `source` accept SHA-256 fingerprints for caller-declared identity completeness. +They do not certify comparable experiments. For native harness runs the environment +signature additionally accepts `backend: ascend`: verified device selection with +UUID, name and driver; host platform; target Python and package versions including +`torch`, `vllm`, `torch_npu`, `vllm_ascend`; and +`ascend: {cann_version, firmware_version}`. Existing RunSpec, workload, source and +command comparisons still apply. Missing runtime identity remains unverified. + +Collectors and campaign-specific paths, targets and revision pins belong outside +the shared dashboard. Collect existing files read-only, label freshness and +missing history, and preserve failed, partial and running attempts. + +Collectors can atomically update campaign metadata with +`collection: {"state": "live" or "stale", "observed_at": ""}`. +The UI shows this source timestamp separately from the browser's connection to +the dashboard. On collection failure, mark it stale and keep the last complete +source set; a successful HTTP refresh alone does not mean the source is fresh. diff --git a/docs/dashboard.md b/docs/dashboard.md index d405e01a..3f581e52 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -1,5 +1,9 @@ # Offline optimization dashboard +For existing Ascend experiments, see [external optimization records](dashboard-external-records.md) +to connect protocol-specific measurements, verdicts and events without fabricating a harness run. + + The dashboard renders a recorded InferRSI run or an explicitly assembled campaign as a single HTML file. It needs no web service or external assets. It is a read-only view of recorded data; generating it does not run an diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index 7584d597..1b4d3c88 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -160,6 +160,7 @@
metric
+
Overview · all candidates
Workflow Technical events only; select a stage to filter candidates and events
@@ -220,6 +221,34 @@ const liveMode = data.live === true; const byId = id => document.getElementById(id); const chinese = { + 'stages_declared is explicitly empty: undeclared stages mean not applicable, not backend unsupported.':'stages_declared 显式声明为空：未声明的阶段表示不适用，不表示后端不受支持。', + 'Source collection':'数据采集', + 'live':'已连接', + 'stale':'数据陈旧', + 'Performance':'性能判定', + 'Environment identity':'环境身份', + 'protocol_specific':'按协议独立展示', + 'Caller-adapted records; hashes check bytes, not experimental validity.':'调用方适配的记录；哈希验证文件字节，不证明实验结论。', + 'Protocol-specific absolute measurements; recorded verdicts are not recomputed.':'按协议展示绝对测量值；保留原始判定，不重新裁决。', + 'Per metric':'按指标分别查看', + 'Recorded performance failures':'已记录的性能未通过', + 'Native failed / regressed':'原生运行失败 / 下降', + 'Recorded external performance failures':'已记录的外部性能未通过', + 'Native runs only; includes failures and rollbacks':'仅统计原生运行，包含失败和回退', + 'Independent axes; select a candidate for parent and two rounds':'各指标独立展示；选择候选查看父版本和双轮结果', + 'Absolute target gaps appear in candidate details':'候选详情显示各档位距绝对目标的差距', + 'Recorded quality passes; see individual results':'已记录的质量通过数；详见各候选', + 'Caller verdicts; independent of quality':'记录中的性能判定，与质量判定分开', + 'External record summaries; full optimization log not recorded':'外部记录摘要；完整优化日志尚未接入', + 'Protocol measurements':'协议指标', + 'Parent':'父版本', + 'Round 1':'第1轮', + 'Round 2':'第2轮', + 'Aggregate':'聚合值', + 'No comparable measurements':'暂无可比较的测量', + 'Data source not connected: add candidate/run records or external_record_ref. Existing files elsewhere are not imported automatically.':'数据源未接入：请添加候选/运行记录或 external_record_ref。其他目录的实验数据不会自动导入。', + 'External records connected. Protocol-specific measurements and recorded verdicts are shown separately; generic gain ranking does not validate them.':'外部记录已接入。按原协议展示测量和判定，通用提升排名不替代原协议验收。', + 'Language':'语言', 'InferRSI · Optimization dashboard':'InferRSI · 优化看板', 'Dashboards › InferRSI ›':'看板 › InferRSI ›', 'SYNTHETIC DATA':'合成数据', 'READ ONLY · OFFLINE':'只读 · 离线', 'READ ONLY · LIVE':'只读 · 实时', 'Dashboard filters':'看板筛选', @@ -357,6 +386,8 @@ if (typeof text !== 'string') return text; if (language !== 'zh') return text; if (Object.prototype.hasOwnProperty.call(chinese, text)) return chinese[text]; + const protocolPrefix='Protocol-specific absolute measurements; recorded verdicts are not recomputed.'; + if(text.startsWith(protocolPrefix+' '))return chinese[protocolPrefix]+' '+text.slice(protocolPrefix.length+1).replace(/(\d+) external record\(s\) have no recorded metrics; see candidate details and events\./,(_,count)=>`${count} 条外部记录未记录指标；请查看候选详情和事件。`); const temporalPrefix='Terminal events place results; stage dots are events, not durations.'; if(text.startsWith(`${temporalPrefix} `)) { return chinese[temporalPrefix] + ' ' + text.slice(temporalPrefix.length+1) @@ -463,7 +494,8 @@ } const stageNames = {diagnosis:'Diagnosis', knowledge:'Knowledge', search:'Candidate search', execution:'Execution', validation:'Validation', review:'Review'}; const stageSelect = byId('stage-filter'), runSelect = byId('run-filter'), candidateSelect = byId('candidate-filter'); - let selected = null, jumpNotice = ''; + let selected = data.candidates.findIndex(r=>r.external?.metrics?.length), jumpNotice = ''; + if(selected<0)selected=null; const node = (tag, text, cls) => { const el = document.createElement(tag); if (text !== undefined) el.textContent = text; if (cls) el.className = cls; return el; }; const narrativeNode = (tag, text, cls) => {const el=node(tag,text,cls);el.setAttribute('data-narrative','');return el;}; const clear = el => { while (el.firstChild) el.removeChild(el.firstChild); }; @@ -496,17 +528,44 @@ (candidateSelect.value === '' || index === Number(candidateSelect.value)) && (!stageSelect.value || row.events.some(event => event.stage === stageSelect.value))); } + function renderIntegration(integration) { + const root=byId('integration-note');clear(root); + const message=integration.state==='not_connected' + ? 'Data source not connected: add candidate/run records or external_record_ref. Existing files elsewhere are not imported automatically.' + : integration.external_records ? 'External records connected. Protocol-specific measurements and recorded verdicts are shown separately; generic gain ranking does not validate them.' : ''; + if(message)root.appendChild(node('p',message)); + if(integration.collection?.observed_at){ + const line=node('p'); + line.appendChild(node('span','Source collection')); + line.appendChild(node('span',': ')); + line.appendChild(node('span',integration.collection.state)); + line.appendChild(node('span',' · '+integration.collection.observed_at)); + root.appendChild(line); + } + if(integration.stage_declaration==='explicit_empty')root.appendChild(node('p','stages_declared is explicitly empty: undeclared stages mean not applicable, not backend unsupported.')); + root.hidden=!root.firstChild; + } function renderStats() { + renderIntegration(data.integration || {}); const root=byId('stats'); clear(root); const rows=data.candidates, o=data.overview; const missing=rows.flatMap(missingRefs).length; + const external=rows.filter(r=>r.external?.metrics), metrics=external.flatMap(r=>r.external.metrics); + const native=rows.filter(r=>!r.external?.metrics); const figures=[ - ['Best gain',value(o.best_improvement_pct,'%'),o.comparison_state === 'incompatible' ? 'Incompatible records' : o.comparison_state === 'unverified' ? 'Comparison evidence missing or unsupported' : 'Quality and source checks required','green'], + ['Best gain',value(o.best_improvement_pct,'%'),o.comparison_state === 'no_data' ? 'No comparable measurements' : o.comparison_state === 'incompatible' ? 'Incompatible records' : o.comparison_state === 'unverified' ? 'Comparison evidence missing or unsupported' : 'Quality and source checks required','green'], ['Target gap',value(o.target_gap_pct,' pp'),o.target_improvement_pct === null ? 'Target not recorded' : `Target ${value(o.target_improvement_pct,'%')}`,'orange'], ['Candidate / run views',rows.length ? String(rows.length) : 'Not recorded','Recorded candidate or run views',''], ['Within noise',rows.some(r=>['noise','improved','regressed'].includes(r.outcome)) ? String(rows.filter(r => r.outcome === 'noise').length) : 'Not recorded','Uses recorded noise band','orange'], - ['Failed / regressed',rows.some(r=>r.state !== 'not_recorded' || r.quality !== 'not_recorded') ? String(rows.filter(r => ['failed','quality_failed','regressed'].includes(r.outcome) || r.decision==='rolled_back').length) : 'Not recorded','Includes failures and rollbacks','red'], + [external.length ? 'Native failed / regressed' : 'Failed / regressed',native.some(r=>r.state !== 'not_recorded' || r.quality !== 'not_recorded') ? String(native.filter(r => ['failed','quality_failed','regressed'].includes(r.outcome) || r.decision==='rolled_back').length) : 'Not recorded',external.length ? 'Native runs only; includes failures and rollbacks' : 'Includes failures and rollbacks','red'], ['Missing evidence',rows.length ? String(missing) : 'Not recorded','See completeness panel','red'] ]; + if(external.length===rows.length && external.length){ + figures[0]=['Protocol measurements',String(metrics.length),'Independent axes; select a candidate for parent and two rounds','green']; + figures[1]=['Target gap','Per metric','Absolute target gaps appear in candidate details','orange']; + figures[3]=['Quality',String(external.filter(r=>r.external.quality_verdict==='PASS').length)+' / '+external.length,'Recorded quality passes; see individual results','']; + figures[4]=['Recorded performance failures',String(external.filter(r=>r.external.performance_verdict==='FAIL').length),'Caller verdicts; independent of quality','red']; + } + if(external.length && native.length)figures.push(['Recorded external performance failures',String(external.filter(r=>r.external.performance_verdict==='FAIL').length),'Caller verdicts; independent of quality','red']); for(const [label,number,note,colour] of figures){const box=node('div',undefined,'panel stat');box.appendChild(node('div',label,'stat-label'));box.appendChild(node('strong',number,`stat-value ${colour}`));box.appendChild(node('small',note,'hint'));root.appendChild(box);} byId('overview-note').textContent = data.rounds === null ? 'Rounds not recorded' : `${data.rounds.length} recorded rounds${data.corrupt_round_lines.length ? ` · corrupt lines ${data.corrupt_round_lines.join(', ')}` : ''}`; } @@ -551,6 +610,18 @@ heading.appendChild(node('small','Narrative is caller recorded; verdicts and measurements are separate evidence.')); root.appendChild(heading); const grid=node('div',undefined,'insight-grid');root.appendChild(grid); + if(row.external && row.external.protocol){ + const e=row.external, box=insight(grid,'Protocol measurements',`${e.backend} · ${e.protocol}`,true); + detailLine(box,'Performance',e.performance_verdict); + detailLine(box,'Quality',e.quality_verdict+(e.quality_total!==null?' · '+measurement(e.quality_correct)+' / '+measurement(e.quality_total):'')); + const table=node('table'), header=node('tr'); + for(const h of ['Metric','Parent','Round 1','Round 2','Aggregate','Target','Target gap'])header.appendChild(node('th',h));table.appendChild(header); + for(const m of e.metrics){const tr=node('tr');for(const v of [m.name+' ('+m.unit+')',measurement(m.baseline),measurement(m.samples[0]),measurement(m.samples[1]),measurement(m.aggregate),measurement(m.target),m.aggregate!==null&&m.target!==null?measurement(m.aggregate-m.target):'Not recorded'])tr.appendChild(node('td',v));table.appendChild(tr);}box.appendChild(table); + box.appendChild(node('p',e.note));box.appendChild(node('small',e.provenance)); + detailLine(box,'Environment identity',e.identity_state); + for(const ref of e.source_refs)box.appendChild(node('code',refText(ref))); + return; + } const info=row.optimization, context=row.benchmark_context; const optimization=insight(grid,'Change and rationale','Caller recorded',true); detailLine(optimization,'Change type',info.kind || 'Not recorded'); @@ -690,7 +761,7 @@ entry.appendChild(node('pre',Object.keys(event.details).length ? JSON.stringify(event.details,null,2) : translated('No additional structured fields recorded.'))); entry.appendChild(node('code',event.refs.map(refText).join(' | ')));trace.appendChild(entry); } - if(row.artifacts.tool_calls.state==='missing')item(trace,'MISSING · tool calls',row.artifacts.tool_calls.missing_reason).classList.add('missing-item'); + if(row.artifacts.tool_calls?.state==='missing')item(trace,'MISSING · tool calls',row.artifacts.tool_calls.missing_reason).classList.add('missing-item'); if(row.agent_trace_links.unlinked.length)item(trace,'UNLINKED AGENT EVENTS',`Lines ${row.agent_trace_links.unlinked.join(', ')}`).classList.add('missing-item'); if(row.agent_trace_links.mismatched.length)item(trace,'MISMATCHED AGENT EVENTS',`Lines ${row.agent_trace_links.mismatched.join(', ')}`).classList.add('missing-item'); for(const [source,lines] of Object.entries(row.corrupt_event_sources)){if(lines.length)item(trace,`CORRUPT ${source} LINES`,lines.join(', ')).classList.add('missing-item');} @@ -838,6 +909,15 @@ function renderStory(){ const root=byId('story');clear(root);const story=data.story||{state:'not_recorded',versions:[]}; const note=byId('story-note'); + if(story.state!=='present' && data.candidates.some(r=>r.external?.metrics)){ + note.textContent='External record summaries; full optimization log not recorded'; + for(const row of data.candidates.filter(r=>r.external?.metrics)){ + const box=node('section',undefined,'item');box.appendChild(node('h3',row.title)); + box.appendChild(narrativeNode('p',row.external.note || 'Not recorded')); + box.appendChild(node('small',row.external.provenance));root.appendChild(box); + } + return; + } if(story.state!=='present'){ note.textContent=story.state==='not_applicable'?'Single runs have no campaign log':'Not recorded'; root.appendChild(node('p',story.state==='not_applicable'?'A single run has no optimization log. Record a campaign to explain why each optimization was chosen.':`No usable optimization log (${story.state.replaceAll('_',' ')}). Record one with: inferrsi campaign log --directory --file `,'empty')); diff --git a/src/inferrsi/harness/dashboard.py b/src/inferrsi/harness/dashboard.py index 1e98326a..32cbdb36 100644 --- a/src/inferrsi/harness/dashboard.py +++ b/src/inferrsi/harness/dashboard.py @@ -138,10 +138,27 @@ def _round_trend(view: dict) -> dict | None: def _chart(source: Path, view: dict) -> tuple[str, str]: + external = [row for row in view["candidates"] if row.get("external")] + if external: + from .dashboard_external import metric_chart + measured = [row for row in external if row["external"]["metrics"]] + chart = metric_chart(measured) if measured else "" + native = [row for row in view["candidates"] if not row.get("external")] + if native: + native_chart, _ = _chart(source, {**view, "candidates": native}) + chart += native_chart + note = "Protocol-specific absolute measurements; recorded verdicts are not recomputed." + missing = len(external) - len(measured) + if missing: + note += (f" {missing} external record(s) have no recorded metrics; " + "see candidate details and events.") + return chart, note chart_file = source / "summary-chart.json" spec = None recorded_title = False message = "Independent candidate results relative to their paired baselines." + if view["overview"]["comparison_state"] == "no_data": + return "", "No measured candidate results recorded." if view["overview"]["comparison_state"] != "comparable": return "", "Comparison evidence is missing, unsupported, or incompatible." if _inside(source, chart_file) and chart_file.is_file(): diff --git a/src/inferrsi/harness/dashboard_data.py b/src/inferrsi/harness/dashboard_data.py index 0cf9f4c5..7de565b1 100644 --- a/src/inferrsi/harness/dashboard_data.py +++ b/src/inferrsi/harness/dashboard_data.py @@ -489,7 +489,7 @@ def _events(path: Path, root: Path, expected: dict | None = None, unique_candida def _environment_signature(environment: dict, devices: object, backend: object) -> tuple | None: """Compare verified device/runtime identity privately; never expose it in the view.""" - if backend != "cuda" or not isinstance(devices, list) or not devices: + if backend not in ("cuda", "ascend") or not isinstance(devices, list) or not devices: return None device = environment.get("device") host = environment.get("host") @@ -509,6 +509,16 @@ def _environment_signature(environment: dict, devices: object, backend: object) return None if not all(isinstance(packages.get(key), str) and packages[key] for key in ("torch", "vllm")): return None + runtime_identity = () + if backend == "ascend": + ascend = environment.get("ascend") + if not isinstance(ascend, dict) or not all( + isinstance(ascend.get(k), str) and ascend[k] + for k in ("cann_version", "firmware_version") + ) or not all(isinstance(packages.get(k), str) and packages[k] + for k in ("torch_npu", "vllm_ascend")): + return None + runtime_identity = (ascend["cann_version"], ascend["firmware_version"]) selected = [] for identity in devices: matches = [row for row in device["devices"] if isinstance(row, dict) @@ -520,7 +530,8 @@ def _environment_signature(environment: dict, devices: object, backend: object) if not all(isinstance(value, str) and value for value in fields): return None selected.append(fields) - return (host["platform"], target["python"], tuple(sorted(packages.items())), tuple(selected)) + return (backend, host["platform"], target["python"], tuple(sorted(packages.items())), + tuple(selected), runtime_identity) def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: float | None, @@ -839,6 +850,33 @@ def load_dashboard(source: Path) -> dict: for directory in sorted((root / "runs").iterdir()): if directory.is_dir() and not directory.is_symlink(): rows.append(_run(root, directory, {}, directory.name, noise)) + from .dashboard_external import external_record + + for row, entry in zip(rows, entries if not single else []): + reference = entry[1].get("external_record_ref") + if reference is not None: + external, ref = external_record(root, reference, row["candidate_id"]) + row["artifacts"]["external_record"] = ref + row["external"] = external + if row["run_id"] is not None: + external = {} + row["external"] = {} + row["artifacts"]["external_record"] = {**ref, "state": "missing", + "missing_reason": "conflicting_run"} + if external: + # External-only candidates do not promise a native harness run tree. + row["artifacts"] = {key: value for key, value in row["artifacts"].items() + if key in ("candidate_metadata", "external_record")} + for index, source in enumerate(external["source_refs"]): + row["artifacts"][f"external_source_{index}"] = source + row["metric_key"] = external["protocol"] + row["metric_unit"] = "protocol_specific" + row["state"] = external["state"] + row["verdict"] = external["performance_verdict"] + row["events"].extend(external["events"]) + row["quality"] = {"PASS": "passed", "FAIL": "failed"}.get( + external["quality_verdict"], "not_recorded") + row["outcome"] = "unverified" decisions = {(item["round"], item["decision"]["candidate_id"]): item["decision"]["outcome"] for item in (rounds or []) if "decision" in item} for row in rows: @@ -848,7 +886,7 @@ def load_dashboard(source: Path) -> dict: signature = row.pop("_comparison_signature") if row["improvement_pct"] is not None: signatures.append(signature) - comparison_state = "comparable" + comparison_state = "comparable" if signatures else "no_data" if len(signatures) >= 2: if any(signature is None for signature in signatures): comparison_state = "unverified" @@ -892,6 +930,8 @@ def load_dashboard(source: Path) -> dict: from .optimization_story import build_story story = build_story(root, rows, decisions, campaign.get("campaign_id")) + collection = campaign.get("collection") + collection = collection if isinstance(collection, dict) else {} declared = campaign.get("stages_declared") if not isinstance(declared, list): declared = None @@ -901,6 +941,12 @@ def load_dashboard(source: Path) -> dict: "rounds": rounds, "rounds_state": rounds_state, "corrupt_round_lines": corrupt_round_lines, "candidates": rows, + "integration": {"state": "not_connected" if not rows else "connected", + "stage_declaration": "explicit_empty" if declared == [] else + "missing" if declared is None else "declared", + "external_records": sum(bool(row.get("external")) for row in rows), + "collection": {"state": _text(collection.get("state"), ""), + "observed_at": _text(collection.get("observed_at"), "")}}, "overview": {"comparison_state": comparison_state, "best_improvement_pct": best, "target_improvement_pct": target, "target_gap_pct": (_number(target - best) diff --git a/src/inferrsi/harness/dashboard_external.py b/src/inferrsi/harness/dashboard_external.py new file mode 100644 index 00000000..9c86a0ae --- /dev/null +++ b/src/inferrsi/harness/dashboard_external.py @@ -0,0 +1,173 @@ +"""Explicit, hash-bound external records; never reinterpret a campaign verdict.""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from pathlib import Path + + +def external_record(root: Path, reference: object, candidate_id: str) -> tuple[dict, dict]: + from .dashboard_data import STAGES, _explicit_ref, _number, _text + + ref = _explicit_ref(root, reference, "external_record") + if ref["state"] != "present": + return {}, ref + try: + raw = (root / ref["path"]).read_bytes() + if len(raw) > 2 * 1024 * 1024 or hashlib.sha256(raw).hexdigest() != ref["sha256"]: + raise ValueError("changed or oversized") + data = json.loads(raw) + if data.get("schema_version") != 1 or data.get("kind") != "external_optimization_record": + raise ValueError("schema") + if not all(isinstance(data.get(key), str) and data[key] + for key in ("backend", "protocol")): + raise ValueError("backend and protocol required") + if data.get("candidate_id") != candidate_id: + raise ValueError("candidate identity") + sources = data.get("source_refs", []) + if not isinstance(sources, list) or not sources or len(sources) > 100: + raise ValueError("source refs") + sources = [_explicit_ref(root, source, "external_source") for source in sources] + if any(source["state"] != "present" for source in sources): + raise ValueError("missing source") + if not isinstance(data.get("metrics", []), list) or not isinstance( + data.get("events", []), list + ): + raise ValueError("collections") + metrics = [] + for m in data.get("metrics", [])[:100]: + if not isinstance(m, dict): + raise ValueError("metric") + samples = m.get("samples", []) + if not isinstance(samples, list) or len(samples) > 2: + raise ValueError("samples") + samples = [_number(v) for v in samples] + if any(v is None or v <= 0 for v in samples): + raise ValueError("samples") + aggregate = _number(m.get("aggregate")) + for key in ("baseline", "aggregate", "target"): + if m.get(key) is not None and (_number(m[key]) is None or _number(m[key]) <= 0): + raise ValueError("positive metric required") + if m.get("aggregation") == "two_round_median": + if (len(samples) != 2 or aggregate is None or not math.isclose( + aggregate, samples[0] / 2 + samples[1] / 2, + rel_tol=1e-9, abs_tol=1e-12 + )): + raise ValueError("two round aggregate") + metrics.append( + { + "name": _text(m.get("name")), + "unit": _text(m.get("unit")), + "samples": samples, + "aggregate": aggregate, + "aggregation": _text(m.get("aggregation")), + "baseline": _number(m.get("baseline")), + "target": _number(m.get("target")), + } + ) + events = [] + for i, e in enumerate(data.get("events", [])[:1000], 1): + if e.get("stage") not in STAGES: + raise ValueError("stage") + time = e.get("time") + if time is not None and ( + not isinstance(time, str) or not re.fullmatch(r"[0-9TtZz:+.\-]{10,40}", time) + ): + raise ValueError("time") + events.append( + { + "line": i, + "time": time, + "phase": None, + "type": "external.record", + "stage": e["stage"], + "side": None, + "trial": None, + "tool": None, + "details": {"summary": _text(e.get("summary"), "", 1500)}, + "source": "events", + "refs": [ref, *sources], + } + ) + verdict = data.get("performance_verdict") + quality = data.get("quality", {}) + if quality.get("correct") is not None or quality.get("total") is not None: + correct, total = quality.get("correct"), quality.get("total") + if type(correct) is not int or type(total) is not int or not 0 <= correct <= total: + raise ValueError("quality counts") + if total <= 0: + raise ValueError("quality total") + environment = data.get("environment", {}) + required = ("hardware", "runtime", "model", "workload", "protocol", "source") + identity_complete = data.get("backend") == "ascend" and all( + isinstance(environment.get(k), str) and re.fullmatch(r"[0-9a-f]{64}", environment[k]) + for k in required + ) + return { + "backend": _text(data.get("backend")), + "protocol": _text(data.get("protocol")), + "metrics": metrics, + "events": events, + "source_refs": sources, + "performance_verdict": verdict + if verdict in ("PASS", "FAIL", "INCONCLUSIVE") + else "not_recorded", + "quality_verdict": quality.get("verdict") + if quality.get("verdict") in ("PASS", "FAIL", "INCONCLUSIVE", "NOT_RUN") + else "not_recorded", + "quality_correct": _number(quality.get("correct")), + "quality_total": _number(quality.get("total")), + "state": data.get("state") + if data.get("state") in ("running", "completed", "failed", "crashed", "cancelled") + else "not_recorded", + "note": _text(data.get("note"), "", 2000), + "identity_state": "recorded" if identity_complete else "incomplete", + "comparison_state": "protocol_specific", + "provenance": "Caller-adapted records; hashes check bytes, not experimental validity.", + }, ref + except (ValueError, TypeError, AttributeError, KeyError, OSError): + return {}, {**ref, "state": "missing", "missing_reason": "corrupt"} + + +def metric_chart(rows: list[dict]) -> str: + """Plot absolute metrics in separate axes, never rank distinct protocols.""" + from html import escape + + items = [(r, m) for r in rows for m in r["external"]["metrics"]][:100] + parts = [ + '' + ] + for i, (row, metric) in enumerate(items): + y = i * 100 + 25 + samples = metric["samples"] + roles = [("Parent", "#a0aab7", metric["baseline"]), + ("R1", "#5794f2", samples[0] if samples else None), + ("R2", "#ff9830", samples[1] if len(samples) > 1 else None), + ("Aggregate", "#73bf69", metric["aggregate"])] + values = [value for _, _, value in roles] + numbers = [v for v in values + [metric["target"]] if v is not None and v > 0] + if not numbers: + continue + maximum = max(numbers) + label = escape(f"{row['title']} · {metric['name']} ({metric['unit']})") + parts.append(f'{label}') + for j, (label, colour, v) in enumerate(roles): + if v is None or v <= 0: + continue + x = 280 + 650 * v / maximum + parts.append( + f'{label}: {v:.4f}' + f'' + ) + if metric["target"] is not None and metric["target"] > 0: + x = 280 + 650 * metric["target"] / maximum + parts.append( + f'' + ) + return "".join(parts) + "" diff --git a/tests/test_dashboard_external.py b/tests/test_dashboard_external.py new file mode 100644 index 00000000..274db21f --- /dev/null +++ b/tests/test_dashboard_external.py @@ -0,0 +1,278 @@ +import hashlib +import json + +import pytest + +from inferrsi.harness.dashboard import dashboard_snapshot +from inferrsi.harness.dashboard_data import _environment_signature, load_dashboard + + +def assemble(root, record): + (root / "campaign.json").write_text(json.dumps({"campaign_id": "example"})) + source = b'{"synthetic": true}' + (root / "source.json").write_bytes(source) + record = { + **record, + "candidate_id": "example", + "source_refs": [{"path": "source.json", "sha256": hashlib.sha256(source).hexdigest()}], + } + raw = json.dumps(record).encode() + (root / "external.json").write_bytes(raw) + candidate = root / "candidates" / "example" + candidate.mkdir(parents=True, exist_ok=True) + (candidate / "candidate.json").write_text( + json.dumps( + { + "candidate_id": "example", + "title": "Example", + "round": 1, + "external_record_ref": { + "path": "external.json", + "sha256": hashlib.sha256(raw).hexdigest(), + }, + } + ) + ) + + +def record(): + return { + "schema_version": 1, + "kind": "external_optimization_record", + "backend": "ascend", + "protocol": "two-services-v1", + "state": "completed", + "performance_verdict": "FAIL", + "quality": {"verdict": "NOT_RUN"}, + "metrics": [ + { + "name": "C4", + "unit": "ms/token", + "baseline": 42, + "samples": [40, 38], + "aggregate": 39, + "aggregation": "two_round_median", + "target": 25, + } + ], + "events": [ + { + "stage": "validation", + "time": "2026-01-01T12:00:00Z", + "summary": "Recorded comparison failed", + } + ], + } + + +def test_external_records_are_native_rows_without_generic_promotion(tmp_path): + assemble(tmp_path, record()) + view = dashboard_snapshot(tmp_path) + row = view["data"]["candidates"][0] + assert row["verdict"] == "FAIL" + assert row["quality"] == "not_recorded" + assert row["improvement_pct"] is None + assert view["data"]["overview"]["best_improvement_pct"] is None + assert row["external"]["metrics"][0]["aggregate"] == 39 + assert row["external"]["quality_verdict"] == "NOT_RUN" + assert "C4" in view["chart"] + assert view["data"]["stages"][4]["state"] == "recorded" + + +@pytest.mark.parametrize("change", ["hash", "aggregate", "partial", "type"]) +def test_incomplete_or_changed_records_fail_closed(tmp_path, change): + data = record() + if change == "aggregate": + data["metrics"][0]["aggregate"] = 2 + if change == "type": + data["metrics"] = None + assemble(tmp_path, data) + if change in ("hash", "partial"): + (tmp_path / "external.json").write_text("{") + row = load_dashboard(tmp_path)["candidates"][0] + assert not row["external"] + assert row["artifacts"]["external_record"]["state"] == "missing" + + +def test_empty_and_missing_stages_are_distinct(tmp_path): + p = tmp_path / "campaign.json" + p.write_text("{}") + a = load_dashboard(tmp_path) + assert a["overview"]["comparison_state"] == "no_data" + assert a["integration"]["state"] == "not_connected" + assert all(s["state"] == "stage_not_recorded" for s in a["stages"]) + p.write_text('{"stages_declared": []}') + b = load_dashboard(tmp_path) + assert b["integration"]["stage_declaration"] == "explicit_empty" + assert all(s["state"] == "not_applicable" for s in b["stages"]) + + +def test_ascend_identity_requires_runtime_and_firmware(): + env = { + "host": {"platform": "synthetic"}, + "target_python": { + "environment": { + "python": "3.11", + "packages": { + "torch": "example", + "vllm": "example", + "torch_npu": "example", + "vllm_ascend": "example", + }, + } + }, + "device": { + "status": "available", + "selection_verified": True, + "devices": [ + { + "index": "0", + "uuid": "example", + "name": "example-npu", + "driver_version": "example", + } + ], + }, + } + assert _environment_signature(env, ["0"], "ascend") is None + env["ascend"] = {"cann_version": "example", "firmware_version": "example"} + first = _environment_signature(env, ["0"], "ascend") + assert first is not None + env["ascend"]["firmware_version"] = "changed" + assert _environment_signature(env, ["0"], "ascend") != first + + +@pytest.mark.parametrize("change", ["missing_source", "changed_source", "wrong_candidate", + "bad_quality", "too_many_samples", "negative_metric"]) +def test_external_contract_rejects_invalid_provenance_or_values(tmp_path, change): + data = record() + if change == "bad_quality": + data["quality"] = {"verdict": "PASS", "correct": 4, "total": 3} + if change == "too_many_samples": + data["metrics"][0]["samples"] = [40, 38, 36] + if change == "negative_metric": + data["metrics"][0]["baseline"] = -1 + assemble(tmp_path, data) + if change == "missing_source": + (tmp_path / "source.json").unlink() + if change == "changed_source": + (tmp_path / "source.json").write_text("{}") + if change == "wrong_candidate": + p = tmp_path / "candidates/example/candidate.json" + candidate = json.loads(p.read_text()) + candidate["candidate_id"] = "different" + p.write_text(json.dumps(candidate)) + row = load_dashboard(tmp_path)["candidates"][0] + assert not row["external"] + + +def test_running_external_view_refreshes_without_fake_native_missing_files(tmp_path): + data = record() + data.update(state="running", performance_verdict="INCONCLUSIVE", metrics=[]) + assemble(tmp_path, data) + first = dashboard_snapshot(tmp_path)["data"]["candidates"][0] + assert first["state"] == "running" + assert all(ref["state"] == "present" for ref in first["artifacts"].values()) + assert "run_lock" not in first["artifacts"] + data = record() + data["quality"] = {"verdict": "FAIL", "correct": 2, "total": 3} + assemble(tmp_path, data) + second = dashboard_snapshot(tmp_path)["data"]["candidates"][0] + assert second["state"] == "completed" + assert second["quality"] == "failed" + assert second["external"]["quality_correct"] == 2 + assert second["outcome"] == "unverified" + + +def test_external_chart_escapes_markup(tmp_path): + data = record() + data["metrics"][0]["name"] = '' + assemble(tmp_path, data) + chart = dashboard_snapshot(tmp_path)["chart"] + assert "