From f3e1ade56597d1b902ae944f07d87f3f89227864 Mon Sep 17 00:00:00 2001 From: danielholanda <9607530+danielholanda@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:35:54 +0000 Subject: [PATCH] Bump `tracelens-analysis-orchestrator` to `a2575ae` --- .../.federated.json | 6 +- .../tracelens-analysis-orchestrator/SKILL.md | 10 +- .../agents/cpu-idle-analyzer.md | 2 +- .../agents/kernel-fusion-analyzer.md | 135 ++++------- .../agents/model-identification-agent.md | 3 +- .../agents/multi-kernel-analyzer.md | 11 +- .../evals/EVAL_RUBRICS.md | 6 +- .../eval_utils/workflow_scripted_evals.py | 39 ++-- .../evals/partial_tests/README.md | 2 +- .../evals/partial_tests/run_partial_tests.sh | 2 +- .../reference.md | 215 ++++++++---------- .../templates/analysis_template.md | 68 +++--- .../templates/sub_agent_spec.md | 14 +- 13 files changed, 230 insertions(+), 283 deletions(-) diff --git a/skills/tracelens-analysis-orchestrator/.federated.json b/skills/tracelens-analysis-orchestrator/.federated.json index fc2c15e0..0bc58c77 100644 --- a/skills/tracelens-analysis-orchestrator/.federated.json +++ b/skills/tracelens-analysis-orchestrator/.federated.json @@ -2,9 +2,9 @@ "source": "amd-agi-tracelens", "repo": "AMD-AGI/TraceLens", "ref": "main", - "commit": "c5da90ee7d59e49bf75e8602f0211da1f6dcd8b3", + "commit": "a2575ae57960f7c88cd7c5bbbfb51647b1492d18", "path": "TraceLens/Agent/Analysis/skills/analysis-orchestrator", "license": "MIT", - "content_hash": "0e829f9bf572f9dba30436dea555397bca6f60e3501b4be2867ee8803e631850", - "imported_at": "2026-10-08T06:52:06Z" + "content_hash": "bad41c1959d80227a8c3b54e330ae54e99de498b9e667bbbb9a335e5995c6f4d", + "imported_at": "2026-10-28T06:35:54Z" } diff --git a/skills/tracelens-analysis-orchestrator/SKILL.md b/skills/tracelens-analysis-orchestrator/SKILL.md index 8b38da4b..db3be620 100644 --- a/skills/tracelens-analysis-orchestrator/SKILL.md +++ b/skills/tracelens-analysis-orchestrator/SKILL.md @@ -5,8 +5,8 @@ description: >- reports, prepares category data, runs system-level and compute-kernel subagents in parallel, validates outputs, and writes a prioritized stakeholder report (analysis.md). Use when the user asks to follow the analysis orchestrator, run the agentic analysis - workflow, analyze a trace, compare two traces, or mentions standalone or comparative - TraceLens analysis. + workflow, run it from config , analyze a trace, compare two traces, or + mentions standalone or comparative TraceLens analysis. license: MIT --- @@ -22,13 +22,13 @@ Coordinate **system-level** analysis (CPU/idle, kernel fusion, multi-kernel / co ## Full procedure -Follow **[reference.md](reference.md)** for every step (user prompts, `` / `{CMD}` usage, CLI commands, subagent launch text, validation, report `tee` order, plot embedding, and trace diagnostics). +Follow **[reference.md](reference.md)** for every step (user prompts, `` / `{CMD}` usage, CLI commands, subagent launch text, validation, report `tee` order, and plot embedding). ## Workflow index ``` -0. Query User Inputs (Platform, Trace Path(s), Analysis Mode, Environment Setup) -1. Generate Performance Report (branches on analysis mode: training vs inference then, comparison scope) +0. Query User Inputs (interactive answers or a run YAML, Environment Setup) → cache/run_config.yaml +1. Generate Performance Report (one wrapper call on cache/run_config.yaml; semantic pre-processing first) 2. Trace-Quality Gate → Deterministic fallback analysis 3-6. Prepare Category Data (GPU Util, Top Ops, Tree Data, Multi-Kernel Data, Category Filtering) 7. System-Level Analysis (PARALLEL) → system_findings/ diff --git a/skills/tracelens-analysis-orchestrator/agents/cpu-idle-analyzer.md b/skills/tracelens-analysis-orchestrator/agents/cpu-idle-analyzer.md index 502b3e03..263dc973 100644 --- a/skills/tracelens-analysis-orchestrator/agents/cpu-idle-analyzer.md +++ b/skills/tracelens-analysis-orchestrator/agents/cpu-idle-analyzer.md @@ -38,7 +38,7 @@ When invoked by the orchestrator, you will receive the following context: ## Error Handling **If category data files are missing:** -1. Read gpu_utilization directly from category_data/category_manifest.json +1. Read gpu_utilization from metadata/cpu_idle_metadata.json (focus job only on multi-trace) 2. Provide analysis based on available data 3. Note limitations in findings diff --git a/skills/tracelens-analysis-orchestrator/agents/kernel-fusion-analyzer.md b/skills/tracelens-analysis-orchestrator/agents/kernel-fusion-analyzer.md index edab7247..ad0a071a 100644 --- a/skills/tracelens-analysis-orchestrator/agents/kernel-fusion-analyzer.md +++ b/skills/tracelens-analysis-orchestrator/agents/kernel-fusion-analyzer.md @@ -12,7 +12,7 @@ model: claude-opus-4-7-high # Kernel Fusion Analyzer (Experimental) -Analyze GPU kernel fusion opportunities from pre-extracted module-level candidate data. Classify candidates as known patterns, novel patterns, or not fusable. +Analyze GPU kernel fusion opportunities from pre-extracted module-level candidate data. Name each candidate from its fusion kind. --- @@ -25,6 +25,8 @@ When invoked by the orchestrator, you will receive the following context: - `prefix`: Command prefix from `/cache/cmd_prefix.txt` — contains a template with `{CMD}` placeholder; substitute `{CMD}` with the actual command - `comparison_scope`: `standalone` (default) or `comparative` +On multi-trace standalone runs the candidates and metrics are for the focus job only; do not attribute findings to other phases. + **Input files (pre-computed by orchestrator):** 1. `/category_data/fusion_candidates.json` - Candidate summaries with kernel details 2. `/category_data/kernel_fusion_metrics.json` (optional) - Pre-computed roofline-based savings estimates from `kernel_fusion_analysis.py` @@ -50,7 +52,7 @@ Use vendor-agnostic terminology in all narrative text (Insight, Action, Impact): - "compiler fusion" or "graph-level fusion" — not "torch.compile", "Inductor", or other framework-specific names - Focus on operation semantics, not vendor implementation details -**Exception:** When quoting kernel names from the candidates for identification in the Kernels table, use the actual name as-is. +**Exception:** When quoting kernel names from the candidates for identification in the Kernel Name column, use the actual name as-is. --- @@ -72,13 +74,17 @@ Then read `/category_data/kernel_fusion_metrics.json`. The `impact_e - `impact_score`, `impact_score_low`, `impact_score_high`: % of E2E recoverable by fusing (mid / low / high) - `bound_type`: "compute" or "memory" - `fusion_type`: "matrix_compute" or "memory_bound" -- `confidence`: "high" or "medium" -- `time_ms`: Total candidate time across all instances +- `fusion_kind`: The fusion kind (`attention`, `act_mul`, `add_norm`, `elementwise_chain`, `gemm_epilogue`, ...) +- `confidence`: "high", "medium" or "low" +- `time_ms`: Total candidate time across all instances (summed from the real kernels of every instance) - `warning`: Present when some kernels lack perf models +- `rows` (standalone) or `rows_trace1` / `rows_trace2` (comparative): the finished Data-table rows, one per launching op and args, with the keys `operation`, `args`, `kernel_path`, `kernel_name`, `time_ms`, `pct_e2e`, `count`, `flops_per_byte`, `efficiency`, `bound`. Copy them verbatim into the Data table; do not recompute or reformat any cell. + +The file also has `baseline_ms` (Trace 1 E2E GPU time in ms) and, in comparative mode, `baseline2_ms` (Trace 2 E2E); in standalone mode, `overlap_efficiency` (the memory/compute overlap fraction the projections assume). -If `impact_estimates` is empty (or `status` is `NO_DATA`), skip Steps 2-4 entirely. Write **only** the three-line fallback file shown at the end of Step 4 — no P-item cards, no Detailed Analysis blocks, no Impact Summary table. Just the `# heading`, blank line, and the single sentence "No kernel fusion opportunities detected." +If `impact_estimates` is empty (or `status` is `NO_DATA`), skip Steps 2-3 entirely. Write **only** the three-line fallback file shown at the end of Step 3 — no P-item cards, no Detailed Analysis blocks, no Impact Summary table. Just the `# heading`, blank line, and the single sentence "No kernel fusion opportunities detected." -For each entry in `impact_estimates`, look up the matching candidate in `/category_data/fusion_candidates.json` by `base_name == operation` to pull the descriptive fields used in Steps 2-4: +For each entry in `impact_estimates`, look up the matching candidate in `/category_data/fusion_candidates.json` by `base_name == operation` to pull the descriptive fields used in Steps 2-3: **Standalone candidates** (`comparison_scope`: `"standalone"`): - `module_name`: Module or function name from the trace @@ -87,7 +93,6 @@ For each entry in `impact_estimates`, look up the matching candidate in `= 2 GEMM + Softmax, no fused attention kernel | "attention", "sdpa", "self_attn" | -| Unfused RMSNorm | rsqrt + mean or pow + mul | "rmsnorm", "rms_norm" | -| Unfused LayerNorm | rsqrt + mean + sub + mul | "layernorm", "layer_norm" | -| Unfused BatchNorm | mul + add (precomputed scale+shift) | "batchnorm", "batch_norm", "FrozenBatchNorm" | -| Unfused RoPE | neg + cat + mul + add | "rotary", "rope", "apply_rotary" | -| Unfused SiGLU/SwiGLU | SiLU + Mul (may have GEMMs between) | "silu", "swiglu", MLP context | -| Unfused GELU | Multiple GELU component kernels | "gelu" | -| GEMM epilogue | GEMM + 1-2 elementwise as separate kernels | "linear", "conv2d", "addmm" | +### Step 2: Name Each Candidate -Then look for novel patterns: -- Multiple elementwise kernels under one module -- Reduction + elementwise sequences -- Dropout + residual add + normalization under one module -- Repeated small kernels suggesting a decomposed operation +**Specific kinds** (`attention`, `act_mul`, `add_norm`, `add_norm_quant`, `norm_quant`, `rope_kv_store`, `allreduce_norm`, `routing_topk`): the title names that operation. -**Decision 3 -- What recommendation?** Tailor to framework context visible in the parent chain and module names. +**Generic kinds** (`elementwise_chain`, `gemm_epilogue`): name the likely operation when the module and kernel names support it (e.g. "Decomposed RMSNorm in `input_layernorm`", "GEMM + bias + GELU", "Convolution + BatchNorm + ReLU"), otherwise use the generic name. Word a refined name as likely, not certain. -### Step 3: Assign Confidence +Tailor the recommendation to the framework context visible in the parent chain and module names. -Use the `confidence` from `kernel_fusion_metrics.json` when available. Otherwise: - -- **high**: Module name matches a known pattern AND kernel composition confirms it -- **medium**: Module name OR kernel composition suggests a pattern, but not both -- **low**: Speculative - structural analysis suggests fusion is possible - -### Step 4: Write Findings +### Step 3: Write Findings Write `/system_findings/kernel_fusion_findings.md` using the command prefix. **Pay particular attention to § Impact markers (REQUIRED) in [`sub_agent_spec.md`](../templates/sub_agent_spec.md).** Every P-item `**Impact**` line and every Detailed Analysis `**Impact estimate:**` two-bullet block must be wrapped in `` ... `` markers using the `low`/`mid`/`high` impact_score values from `category_data/kernel_fusion_metrics.json::impact_estimates[]`. -Number findings P1, P2, P3... sequentially by impact_score (highest first). The icon is set ONLY by the `confidence` field in `kernel_fusion_metrics.json`: - -| Confidence | Icon | -|------------|------| -| high | 🔴 | -| medium | 🟡 | -| low | 🟢 | +Number findings P1, P2, P3... sequentially by impact_score (highest first). Icons are assigned by PRIORITY NUMBER (not severity or confidence): P1=🔴, P2=🟡, P3+=🟢. -Example: if the highest-savings finding has LOW confidence, write `### 🟢 P1:`. Two HIGH findings in a row are `### 🔴 P1:` and `### 🔴 P2:` (both red). - -**Title format:** `### P: ` +**Title format:** `### P: `, where `` follows Step 2. **Template** (follow the `[standalone]` / `[comparative]` markers): @@ -172,14 +135,14 @@ Example: if the highest-savings finding has LOW confidence, write `### 🟢 P1:` Found N kernel fusion opportunities across M module types. -> **Methodology:** impact_score projections estimate the recoverable fraction of E2E with 85% memory/compute pipeline overlap (i.e. fused kernel time is interpolated between perfect overlap and no overlap). Actual recoverable time may vary with workload and hardware. +> **Methodology:** impact_score projections estimate the recoverable fraction of E2E with <`overlap_efficiency` as a percent> memory/compute pipeline overlap (i.e. fused kernel time is interpolated between perfect overlap and no overlap). Actual recoverable time may vary with workload and hardware. > **Methodology:** Savings are measured as the total GPU time difference between trace1 and trace2, accumulated across all instances. No roofline projection is used. ## Recommendations -### 🔴 P1: ( ms, instances) +### 🔴 P1: **Insight**: @@ -196,40 +159,43 @@ Found N kernel fusion opportunities across M module types. **Impact**: impact_score: X.X -**Confidence**: High/Medium/Low -- +**Confidence**: -- ## Detailed Analysis -#### ( ms, instances) +#### -**Identification:** <1-2 sentences: how this fusion candidate was surfaced> - - +**Identification:** <3-4 sentences: the module and how many times it runs; its parent module and call site (the nearest module in the call stack and its first frame); each distinct launch path in the Data rows (Trace 1 in comparative) with the ops launched from it, skipping `Not found`; then "Surfaced as `` from .", with `fusion_kind` copied from `kernel_fusion_metrics.json`> + + - **Data:** -| Kernel | Type | Duration (us) | Perf model | -|--------|------|--------------|------------| -| | | X.X | Yes/No | + +| Operation | Args | Kernel Path | Kernel Name | Time (ms) | %E2E | Count | FLOPS/Byte | Efficiency | Bound | +|---|---|---|---|---|---|---|---|---|---| +| | | | | | | | | | | + + +**Trace 1** (E2E ms): - -**Trace1 kernels:** +| Operation | Args | Kernel Path | Kernel Name | Time (ms) | %E2E | Count | FLOPS/Byte | Efficiency | Bound | +|---|---|---|---|---|---|---|---|---|---| +| | -| Kernel | Type | Duration (us) | -|--------|------|--------------| -| | | X.X | +**Trace 2** (E2E ms): -**Trace2 kernels:** +| Operation | Args | Kernel Path | Kernel Name | Time (ms) | %E2E | Count | FLOPS/Byte | Efficiency | Bound | +|---|---|---|---|---|---|---|---|---|---| +| | -| Kernel | Type | Duration (us) | -|--------|------|--------------| -| | | X.X | +**Resolution:** + **Impact estimate:** - + - Low end impact_score: X.XX - High end impact_score: X.XX - Coverage: M of N kernels modelled @@ -239,7 +205,7 @@ Found N kernel fusion opportunities across M module types. - + - Low end impact_score: X.XX - High end impact_score: X.XX - Fusion pattern: compute/memory-bound, matrix_compute/memory_bound @@ -259,9 +225,9 @@ Found N kernel fusion opportunities across M module types. No kernel fusion opportunities detected. ``` -Then proceed directly to Step 4.1 validation. +Then proceed directly to Step 3.1 validation. -### Step 4.1: Validate Findings +### Step 3.1: Validate Findings Per [`sub_agent_spec.md`](../templates/sub_agent_spec.md) § Validate findings, run: @@ -287,10 +253,7 @@ If validation fails, fix the findings file and re-run. Max 2 retries. 1. **`kernel_fusion_metrics.json.impact_estimates` is the candidate list.** Every finding maps 1:1 to an entry there. Do not derive findings from candidates absent from that list -- they were dropped by the deterministic threshold gate. 2. **Include pre-computed impact_score** from `kernel_fusion_metrics.json` -- do NOT re-derive impact_score yourself, use the values from the metrics JSON. -3. **Let the data speak** -- classify based on module names AND kernel composition, not just one signal. -4. **Reject confidently** -- not every multi-kernel module is a fusion opportunity; independent operations under a container module are not fusable. Use Step 2's Decision 1 to drop candidates from `impact_estimates` that turn out to be containers, all-GEMM groups, or already-fused subtrees. -5. **Explain reasoning** -- especially for novel patterns, state why you believe the kernels are fusable. -6. Use the **module name** to determine the user-facing operation name. If the module is `aten::conv2d` or `Conv2d`, call it "Convolution" in the finding title, not "GEMM" -- even though convolutions are implemented as GEMMs internally. +3. **Explain reasoning** -- especially for generic kinds, state why you believe the kernels are fusable. --- @@ -300,9 +263,9 @@ If validation fails, fix the findings file and re-run. Max 2 retries. |------------|--------| | Module names | `module_name`, `base_name` fields | | Kernel names, types, durations | `kernels[]` (standalone) or `kernels_trace1[]`/`kernels_trace2[]` (comparative) | +| Per-op shapes, kernel path, efficiency | `rows[]` (standalone) or `rows_trace1[]`/`rows_trace2[]` (comparative) in `kernel_fusion_metrics.json` | | Instance count | `instance_count` field | | Architecture context | `parent_chain` field | -| Already-fused status | `has_fused_kernel` field | | impact_score estimates | `kernel_fusion_metrics.json` `impact_estimates[]` (when available) | | Kernel count delta | `kernel_count_trace1`, `kernel_count_trace2`, `delta` (comparative) | @@ -310,7 +273,7 @@ If validation fails, fix the findings file and re-run. Max 2 retries. | NOT Observable | Why | Instead Say | |----------------|-----|-------------| -| Tensor shapes | Not in candidate JSON | "Cannot assess data flow from candidate data" | +| Data flow between kernels | Shapes are in `rows[].args`, but not which tensor feeds which kernel | "Cannot assess data flow from candidate data" | | Whether kernels share intermediate tensors | Would need data flow analysis | "Likely fusable based on module structure" | | Root cause of decomposition | Could be framework, compiler, or intentional | "Module launches N separate kernels that may be fusable" | | Why trace2 is fused | Architectural difference could be compile flags, library version, etc. | "Trace2 demonstrates a fused path exists; trace1 can adopt the same approach" | diff --git a/skills/tracelens-analysis-orchestrator/agents/model-identification-agent.md b/skills/tracelens-analysis-orchestrator/agents/model-identification-agent.md index 77bd1394..49840525 100644 --- a/skills/tracelens-analysis-orchestrator/agents/model-identification-agent.md +++ b/skills/tracelens-analysis-orchestrator/agents/model-identification-agent.md @@ -22,7 +22,6 @@ When invoked by the orchestrator, you will receive the following context: **Required context provided by orchestrator:** - `output_dir`: Base analysis output directory -- `comparison_scope`: `standalone` or `comparative` - `prefix`: Command prefix from `/cache/cmd_prefix.txt` — contains a template with `{CMD}` placeholder; substitute `{CMD}` with the actual command **Input (produced by script in Step 1):** @@ -57,7 +56,7 @@ Execute the Python script to extract the **name**, **Input type**, and **Input D python3 -c " import sys from TraceLens.Agent.Analysis.utils.report_utils import prepare_model_identification_data -if not prepare_model_identification_data('', ''): +if not prepare_model_identification_data(''): sys.exit(1) " ``` diff --git a/skills/tracelens-analysis-orchestrator/agents/multi-kernel-analyzer.md b/skills/tracelens-analysis-orchestrator/agents/multi-kernel-analyzer.md index bdf7d4cf..07612805 100644 --- a/skills/tracelens-analysis-orchestrator/agents/multi-kernel-analyzer.md +++ b/skills/tracelens-analysis-orchestrator/agents/multi-kernel-analyzer.md @@ -29,10 +29,8 @@ When invoked by the orchestrator, you will receive the following context: - `output_dir`: Base analysis output directory - `prefix`: Command prefix from `/cache/cmd_prefix.txt` — contains a template with `{CMD}` placeholder; substitute `{CMD}` with the actual command -**Input files (pre-computed by orchestrator):** -1. `/category_data/multi_kernel_data.json` - Pre-computed memcpy/communication/overlap data -2. `/metadata/multi_kernel_metadata.json` - Platform specs and GPU utilization -3. `/category_data/category_manifest.json` - Contains gpu_utilization metrics +**Input file (pre-computed by orchestrator):** +- `/category_data/multi_kernel_data.json` - Pre-computed memcpy/communication/overlap data **Output file you must write:** - `/system_findings/multi_kernel_findings.md` @@ -41,11 +39,6 @@ When invoked by the orchestrator, you will receive the following context: ## Error Handling -**If multi_kernel_data.json is missing:** -1. Read gpu_utilization from category_data/category_manifest.json -2. Report based on exposed_memcpy_time_percent and exposed_comm_time_percent -3. Note limitations in findings - **If analysis script fails:** 1. Write a findings file with Status: ERROR 2. Include the error message and traceback diff --git a/skills/tracelens-analysis-orchestrator/evals/EVAL_RUBRICS.md b/skills/tracelens-analysis-orchestrator/evals/EVAL_RUBRICS.md index 9548d432..a6391b8c 100644 --- a/skills/tracelens-analysis-orchestrator/evals/EVAL_RUBRICS.md +++ b/skills/tracelens-analysis-orchestrator/evals/EVAL_RUBRICS.md @@ -327,14 +327,14 @@ For each P-item (`### ...P{N}:`) under `## Compute Kernel Optimizations`: ### marker\_eval\_3: Detail Estimate Markers -**Sub-indices:** `marker_eval_3_P{N}` (one per compute P-item in Detailed Analysis) +**Sub-indices:** `marker_eval_3_P{N}` (one per compute P-item in Detailed Analysis) and `marker_eval_3_fusion_P{N}` (one per P-item under `### Kernel Fusion Insights`) -For each compute P-item section under `## Detailed Analysis`: +For each compute P-item section under `## Detailed Analysis`, and each fusion P-item under `### Kernel Fusion Insights`: | Check | Pass Criteria | |-------|---------------| | `kind=detail_estimate` marker or sentinel | Either a `` / `` pair exists, or the text "not quantifiable from trace data" is present | -| Required attributes | If marker present, it contains `low` and `high` attributes | +| Required attributes | If marker present, it contains `low`, `mid` and `high` attributes (compute and fusion) | | Pairing | Every `impact-begin` has a matching `impact-end` | **Pass:** Marker with correct attributes present, or not-quantifiable sentinel present. diff --git a/skills/tracelens-analysis-orchestrator/evals/eval_utils/workflow_scripted_evals.py b/skills/tracelens-analysis-orchestrator/evals/eval_utils/workflow_scripted_evals.py index 09c97245..36d7375e 100644 --- a/skills/tracelens-analysis-orchestrator/evals/eval_utils/workflow_scripted_evals.py +++ b/skills/tracelens-analysis-orchestrator/evals/eval_utils/workflow_scripted_evals.py @@ -290,7 +290,7 @@ def _check_plot(output_dir: str) -> tuple[str, str]: "Unified perf. report exists", _check_unified_perf_report, "pipeline", - "Re-run TraceLens_generate_perf_report_pytorch (Step 1)", + "Re-run TraceLens_generate_perf_report_from_config (Step 1)", True, ), ( @@ -1067,6 +1067,13 @@ def _check_marker_p_items(output_dir, comparison_scope=None): return rows +def _p_blocks(tier, subsection, matches): + """(tier, P number, block text) for each P-item header in a Detailed Analysis subsection.""" + for i, match in enumerate(matches): + end = matches[i + 1].start() if i + 1 < len(matches) else len(subsection) + yield tier, int(match.group(1)), subsection[match.start() : end] + + def _check_marker_detail_estimates(output_dir, comparison_scope=None): """Marker eval 3: kind=detail_estimate markers in Detailed Analysis.""" content = _read_report(output_dir) @@ -1089,7 +1096,7 @@ def _check_marker_detail_estimates(output_dir, comparison_scope=None): ] m = re.search( - r"^### Compute Kernel Insights[^\n]*\n(.*?)(?=^### (?!.*P\d)|^## |\Z)", + r"^### Compute Kernel Insights[^\n]*\n(.*?)(?=^### (?![^\n]*P\d)|^## |\Z)", detailed_section, re.MULTILINE | re.DOTALL, ) @@ -1099,7 +1106,15 @@ def _check_marker_detail_estimates(output_dir, comparison_scope=None): if not matches: matches = list(_DETAIL_P_HEADER_FALLBACK_RE.finditer(compute_subsection)) - if not matches: + blocks = list(_p_blocks("compute", compute_subsection, matches)) + fusion_subsection = _extract_detailed_analysis_subsection( + content, "### Kernel Fusion Insights" + ) + if fusion_subsection: + fusion_matches = list(_DETAIL_P_HEADER_RE.finditer(fusion_subsection)) + blocks += _p_blocks("fusion", fusion_subsection, fusion_matches) + + if not blocks: intentionally_empty = ( "No compute kernel optimization opportunities identified" in compute_subsection @@ -1124,13 +1139,8 @@ def _check_marker_detail_estimates(output_dir, comparison_scope=None): ] rows = [] - for i, match in enumerate(matches): - pnum = int(match.group(1)) - start = match.start() - end = ( - matches[i + 1].start() if i + 1 < len(matches) else len(compute_subsection) - ) - block = compute_subsection[start:end] + for tier, pnum, block in blocks: + pname = f"P{pnum}" if tier == "compute" else f"{tier} P{pnum}" errors = [] detail_markers = [] @@ -1150,8 +1160,7 @@ def _check_marker_detail_estimates(output_dir, comparison_scope=None): elif detail_markers: for inner in detail_markers: attrs = _marker_attrs_from_inner(inner) - required = ("low", "high") - missing = [a for a in required if a not in attrs] + missing = [a for a in ("low", "mid", "high") if a not in attrs] if missing: errors.append( f"kind=detail_estimate marker missing attributes: " @@ -1167,13 +1176,13 @@ def _check_marker_detail_estimates(output_dir, comparison_scope=None): details = "; ".join(errors) if errors else "" rows.append( _make_marker_row( - f"marker_eval_3_P{pnum}", - f"Detailed Analysis P{pnum} estimate marker (kind=detail_estimate)", + f"marker_eval_3_{pname.replace(' ', '_')}", + f"Detailed Analysis {pname} estimate marker (kind=detail_estimate)", result, details, "template" if result == "FAIL" else "", ( - f"Fix kind=detail_estimate marker for P{pnum} in Detailed Analysis" + f"Fix kind=detail_estimate marker for {pname} in Detailed Analysis" if result == "FAIL" else "" ), diff --git a/skills/tracelens-analysis-orchestrator/evals/partial_tests/README.md b/skills/tracelens-analysis-orchestrator/evals/partial_tests/README.md index 53630888..bb91ae41 100644 --- a/skills/tracelens-analysis-orchestrator/evals/partial_tests/README.md +++ b/skills/tracelens-analysis-orchestrator/evals/partial_tests/README.md @@ -44,7 +44,7 @@ For each model (DeepSeek-R1, Qwen3-30B-A3B) the fixture ships: The runner invokes the **semantic-comparison workflow** (the no-capture, name-first + coherence bucketing method) on the DECODE pair, running only through its "Generate TraceDiff Output" step — equivalent to the main -orchestrator through Step 1.S. That produces +orchestrator through Step 1 pre-processing. That produces `analysis_output/tracediff_output/diff_stats.csv`, whose LCA partition is then compared to gold. diff --git a/skills/tracelens-analysis-orchestrator/evals/partial_tests/run_partial_tests.sh b/skills/tracelens-analysis-orchestrator/evals/partial_tests/run_partial_tests.sh index bcf7c829..9a545e62 100755 --- a/skills/tracelens-analysis-orchestrator/evals/partial_tests/run_partial_tests.sh +++ b/skills/tracelens-analysis-orchestrator/evals/partial_tests/run_partial_tests.sh @@ -15,7 +15,7 @@ set -uo pipefail # far enough to produce the artifact a scripted eval scores against a # pre-baked reference. The first such test is semantic_purity, which runs the # semantic-comparison workflow through its "Generate TraceDiff Output" step -# (== main orchestrator through Step 1.S) to produce +# (== main orchestrator through Step 1 pre-processing) to produce # /tracediff_output/diff_stats.csv # and scores its LCA-partition purity against the committed gold. # diff --git a/skills/tracelens-analysis-orchestrator/reference.md b/skills/tracelens-analysis-orchestrator/reference.md index b68c4f46..96095a1c 100644 --- a/skills/tracelens-analysis-orchestrator/reference.md +++ b/skills/tracelens-analysis-orchestrator/reference.md @@ -6,7 +6,7 @@ See LICENSE for license information. # Analysis orchestrator — reference -This document is the detailed specification for the TraceLens **analysis-orchestrator** skill ([SKILL.md](SKILL.md)). Read it when executing the workflow: step-by-step user prompts, CLI commands, subagent contracts, validation, report assembly, and trace diagnostics. +This document is the detailed specification for the TraceLens **analysis-orchestrator** skill ([SKILL.md](SKILL.md)). Read it when executing the workflow: step-by-step user prompts, CLI commands, subagent contracts, validation, and report assembly. ## Workflow overview @@ -25,8 +25,8 @@ Use vendor-agnostic terminology throughout such as GPU kernels, collective commu ## Workflow Steps ``` -0. Query User Inputs (Platform, Trace Path(s), Analysis Mode, Environment Setup) -1. Generate Performance Report (branches on analysis mode: training vs inference then, comparison scope) +0. Query User Inputs (interactive answers or a run YAML, Environment Setup) → cache/run_config.yaml +1. Generate Performance Report (one wrapper call on cache/run_config.yaml; semantic pre-processing first) 2. Trace-Quality Gate (deterministic fallback branch for graph-under-recorded traces) 3-6. Prepare Category Data (GPU Util, Top Ops, Tree Data, Multi-Kernel Data, Category Filtering) 7. System-Level Analysis (PARALLEL, CPU/Idle + Kernel Fusion + Multi-Kernel) → system_findings/ @@ -39,31 +39,47 @@ Use vendor-agnostic terminology throughout such as GPU kernels, collective commu optionally invoke agent_extension.py (when present), then embed the PNG into the report. ``` -**Subagent usage:** Only invoke Task subagents in steps that explicitly say "subagent" (Step 1.5 semantic diff, Steps 7, 8, 10). All other steps (including Step 8.5) must be performed directly by the orchestrator using the command prefix. +**Subagent usage:** Only invoke Task subagents in steps that explicitly say "subagent" (Step 1 pre-processing semantic diff, Steps 7, 8, 10). All other steps (including Step 8.5) must be performed directly by the orchestrator using the command prefix. --- ## Step 0: Query User Inputs -**When this skill is invoked, immediately ask the user for:** +The prompt selects the input mode: + +- **YAML mode**: the prompt says "from config ``", optionally with a scope (`standalone`, `comparative`, `semantic` or `multi_trace`). Resolve `` to an absolute path. Read `output_dir` from the file and ask only the environment questions (item 5), unless the prompt answers them. Skip the other items. +- **Interactive mode**: the prompt names no config. + +**In Interactive mode, when this skill is invoked, immediately ask the user for:** ### Required Information: -1. **Comparison scope** → `` - - Set from the user’s intent **before** deep-diving on paths: - - **`comparative`** if the skill was triggered by **“comparative analysis”**, **“compare two traces”**, or the user supplies **two** trace paths / explicitly asks to compare trace A vs B. - - **`standalone`** otherwise (including triggers **“standalone analysis”**, **“analyze trace standalone”**, single trace only). +1. **Scope** → `` + + | `` | `` | + |---|---| + | `standalone` | standalone | + | `multi_trace` | standalone | + | `comparative` | comparative | + | `semantic` | comparative | -2. **Trace File Path(s)** + - Set from the user’s intent before asking for paths: + - **`comparative`** if the user asks for a comparative analysis, to compare two traces, or supplies two trace paths. + - **`multi_trace`** (multi-trace standalone) if the user supplies several named jobs of one workload on one platform (for example prefill and decode captures). + - **`standalone`** otherwise, including a request for standalone analysis or a single trace. + +2. **Trace File Path(s)** (PyTorch trace, `.json` or `.json.gz`) - **`standalone`:** **Trace File Path** → `` - - Ask: "Please provide the full path to your PyTorch trace file (.json or .json.gz)" + - Ask: "Please provide the full path to your trace file" - **`comparative`:** ask for both: - **Primary trace (trace1)** → `` - **Comparison trace (trace2)** → `` - - Ask: "Please provide the full path to your primary trace file and your comparison trace file (.json or .json.gz)" + - Ask: "Please provide the full path to your primary trace file and your comparison trace file" + - **`multi_trace`:** ask for each job (2 or more) a name, a phase and **Trace File Path**; jobs that share a phase are combined + - Ask: "Please provide a name, a phase and the full path to the trace file for each job, starting with the job that kernel fusion and system analysis should focus on" 3. **Platform** → `` - **`standalone`**: Ask: "Which platform are you analyzing?" + **`standalone`, `multi_trace`**: Ask: "Which platform are you analyzing?" (`multi_trace`: one platform for all jobs) **`comparative`**: Ask: "Which platform is baseline trace (trace1)?" - Options: 1. **MI300X** @@ -76,8 +92,8 @@ Use vendor-agnostic terminology throughout such as GPU kernels, collective commu 4. **Analysis Mode** → `` - If the user's prompt explicitly specifies an analysis mode or mentions inference/vLLM/SGLang/ATOM, use that. Otherwise, default to `default` without asking. - Options: - 1. **Default (training and non-vLLM/SGLang/ATOM eager inference)** (`` = `default`) — uses `TraceLens_generate_perf_report_pytorch` - 2. **Inference analysis (vLLM/SGLang/ATOM)** (`` = `inference`) — uses `TraceLens_generate_perf_report_pytorch_inference` + 1. **Default (training and non-vLLM/SGLang/ATOM eager inference)** (`` = `default`) + 2. **Inference analysis (vLLM/SGLang/ATOM)** (`` = `inference`) - If **Inference (vLLM/SGLang/ATOM)** is selected, ask **Execution Mode** → ``: 1. **Eager mode** (`` = `eager`) — only the trace file is needed 2. **Graph replay + capture** (`` = `graph_capture`) — also requires a capture folder path @@ -85,16 +101,15 @@ Use vendor-agnostic terminology throughout such as GPU kernels, collective commu Ask for the **Capture Folder Path(s)**: - `standalone`: one folder → ``. Ask: "Please provide the full path to the graph capture traces folder" - `comparative`: one folder per trace → `` (primary/trace1) and `` (comparison/trace2). Ask: "Please provide the graph capture traces folder for the primary trace and for the comparison trace." + - `multi_trace`: one folder per job, optional. Ask: "Please provide the graph capture traces folder for each job that has one." 3. **Graph replay only** (`` = `graph_replay_only`) — graph replay with no capture folder available: - - `standalone`: abort. - - `comparative`: set `` = `semantic`. - - - For `comparative`, `` = `tracediff` unless already set to `semantic`. + - `standalone`, `multi_trace`: continue without a capture folder; the Step 2 gate routes to the deterministic fallback. + - `comparative`: use scope `semantic`. 5. **Environment Setup** - Ask: "Are you running locally or on a cluster?" - - If **local**: No further environment questions — prefix is blank (commands run directly). + - If **local**: No further environment questions,prefix is blank (commands run directly). - If **cluster**: - Ask "Which node should we use?" → `` - Ask "Are you working in a containerized environment (e.g. Docker)?" → if yes, ask for container name → `` @@ -107,7 +122,7 @@ Use vendor-agnostic terminology throughout such as GPU kernels, collective commu 7. **Extension File** (Optional) → `` - Ask: "Do you have a TraceLens extension file to apply? Press Enter to skip." - If provided, resolve to an absolute path and assign to ``. - - If skipped, set `` to empty (no `--extension_file` flag is added to any command). + - If skipped, set `` to empty (no `extension_file` is set in the config). ### Build and Cache Command Prefix @@ -166,110 +181,45 @@ Do NOT proceed to Step 1 until validation passes. **Before executing any command**, read `/cache/cmd_prefix.txt`. It contains a template with a `{CMD}` placeholder. Substitute `{CMD}` with the actual command. All commands below use `` to represent this resolved template. ---- - -## Step 1: Generate Performance Report - -Use **``** to determine which CLI tool to run and then **``** (and, for comparative, **``**) to determine arguments. - -For all of these scripts below, look at the environment variable TL_EXTENSION to recursively search for a file called .json. Do not look for .json; it is not needed. -If it is not found also look in TraceLens/Agent/Analysis/utils/arch/.json. -Use to represent the location of this file +### Resolve the Run Config -**CLI call count:** -- **`standalone`**: one TraceLens CLI call (for ``) -- **`comparative`**: one TraceLens CLI call per trace (for `` and ``) +Run one `config_utils` command. It completes the config, writes `/cache/run_config.yaml` (the input to Step 1), and prints the change lines, then the variables for Steps 1-12 as `key=value` lines. -All commands below append `` and ``, resolved by ``: - -**``** — output paths: - -| scope | value | -|-------|-------| -| `standalone` | `--output_xlsx_path /perf_report.xlsx --output_csvs_dir /perf_report_csvs` | -| `comparative` trace1 | `--output_xlsx_path /perf_report_trace1.xlsx --output_csvs_dir /perf_report_trace1_csvs` | -| `comparative` trace2 | `--profile_json_path --output_xlsx_path /perf_report_trace2.xlsx --output_csvs_dir /perf_report_trace2_csvs` | - -**``** — comparison flags: - -| scope | value | -|-------|-------| -| `standalone` | none | -| `comparative` trace1 | `--comparison_json_path ` | -| `comparative` trace1 if `` provided | `--comparison_json_path --comparison_capture_folder ` | -| `comparative` trace2 | none | - -**``** — graph capture flags: - -| scope | value | -|-------|-------| -| `standalone` if `` provided | `--capture_folder ` | -| `comparative` trace1 if `` provided | `--capture_folder ` | -| `comparative` trace2 if `` provided | `--capture_folder ` | - -**``** — user extension file: - -| condition | value | -|-----------|-------| -| `` provided | `--extension_file ` | -| not provided | none | - ---- - -**Default (training and non-vLLM/SGLang/ATOM eager inference)** (`` = `default`): +**YAML mode:** ```bash - TraceLens_generate_perf_report_pytorch \ - --profile_json_path \ - --gpu_arch_json_path \ - --enable_pseudo_ops \ - --group_by_num_kernels \ - --include_call_stack \ - \ - \ - + python3 -m TraceLens.Agent.Analysis.utils.config_utils check \ + --config \ + [--scope ] ``` -**Inference eager / graph-replay-only mode** (`` = `inference`, `` = `eager` or `graph_replay_only`): +**Interactive mode:** ```bash - TraceLens_generate_perf_report_pytorch_inference \ - --profile_json_path \ - --gpu_arch_json_path \ - --group_by_parent_module \ - --enable_pseudo_ops \ - --group_by_num_kernels \ - --include_call_stack \ - \ - \ - + python3 -m TraceLens.Agent.Analysis.utils.config_utils build \ + --scope \ + --analysis_mode \ + --output_dir \ + --job trace_path=,platform=[,capture_folder=] \ + [--job trace_path=,platform=[,capture_folder=]] \ + [--set extension_file=] ``` -**Inference graph replay + capture mode** (`` = `inference`, `` = `graph_capture`): +Pass the second `--job` for `comparative` and `semantic`, a `capture_folder` only for graph replay + capture, and `--set extension_file` only when an extension file was provided. For `multi_trace`, pass one `--job name=,phase=,...` per job (2 or more, same platform), the focus job first; the variables then describe the focus job and include `focus_job`. -```bash - TraceLens_generate_perf_report_pytorch_inference \ - --profile_json_path \ - --gpu_arch_json_path \ - --group_by_parent_module \ - --enable_pseudo_ops \ - --group_by_num_kernels \ - --include_call_stack \ - \ - \ - \ - -``` +If it exits non-zero, the errors are on stderr. In YAML mode, show them and stop. In Interactive mode, re-ask the field an error names once and rebuild. + +Otherwise, show the change lines to the user and assign each `key=value` line to the variable ``. Assign the variables once, from this run. Never run `check` on `/cache/run_config.yaml`: it is the wrapper's input only, and it no longer holds the original `platform2`. --- -## Step 1.5: Semantic Comparative Ordering (`` = `graph_replay_only`, `` = `comparative`) +## Step 1: Generate Performance Report -Run Step 1 in this order: +### Pre-processing -1. **Trace2 report** — run the analysis-mode CLI above for trace2 using the `comparative` trace2 `` and empty `` (identical to the TraceDiff path). +Only when `` = semantic; otherwise skip to Generate Report. -2. **Semantic diff (subagent)** — launch a Task subagent that reads and follows the FULL instructions in `TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/semantic-comparison-agent.md`. Prompt context: +1. **Semantic diff (subagent)** — launch a Task subagent that reads and follows the FULL instructions in `TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/semantic-comparison-agent.md`. Prompt context: ``` Read and follow the FULL instructions in: @@ -288,16 +238,34 @@ Run the full semantic comparison through "Generate TraceDiff Output" so that Verify `/semantic/tracediff_output/diff_stats.csv` exists before continuing. If it is missing, retry the subagent once; if it still fails, stop and report. -3. **Trace1 report** — run the analysis-mode CLI for trace1 with the `comparative` trace1 `` and `` = `--precomputed_diff_stats /semantic/tracediff_output/diff_stats.csv`. +2. **Record the diff stats**: point trace1's job at that file: + + ```bash + python3 -m TraceLens.Agent.Analysis.utils.config_utils set-diff-stats \ + --config /cache/run_config.yaml \ + --path /semantic/tracediff_output/diff_stats.csv + ``` + + It rewrites `/cache/run_config.yaml` and prints one `added` line; show it. If it fails, stop and report. + +### Generate Report -4. **Confirm** `/perf_report_trace1_csvs/diff_stats.csv` exists (written by the report script). If absent, copy `/semantic/tracediff_output/diff_stats.csv` to that path so the comparative fusion step (Steps 3-6) can read it. +One wrapper call runs every report the config describes, in order: + +```bash + TraceLens_generate_perf_report_from_config --config /cache/run_config.yaml +``` + +This call can run for several minutes, longer for comparative graph-capture runs. Run it with the longest command timeout your runner allows. + +If it exits non-zero or times out, show the tail of its output, emit `[DIAG:pipeline:STEP1_FAIL]`, and stop. --- ## Step 2: Trace-Quality Gate -`` is `/perf_report_csvs/unified_perf_summary.csv` (`standalone`) or `/perf_report_trace1_csvs/unified_perf_summary.csv` (`comparative`, primary trace). +`` is under `/perf_report_csvs/` (`standalone`), `/perf_report_aggregated_csvs/` (`multi_trace`), or `/perf_report_trace1_csvs/` (`comparative`, primary trace), as `unified_perf_summary.csv`. ```bash python3 -c \" @@ -312,7 +280,7 @@ print(f'GRAPH_REPLAY_FRACTION={coverage.graph_replay_fraction}') **`STATUS=OK`:** Proceed **`STATUS=GRAPH_UNDER_RECORDED`:** -If = graph_replay_only and = comparative, i.e. = semantic, skip the rest of this step and proceed with Steps 3-6. +If = semantic, skip the rest of this step and proceed with Steps 3-6. Otherwise, @@ -338,9 +306,13 @@ Execute the TraceLens Agentic Mode orchestrator preparation script: --trace-path \ --platform \ --output-dir \ - --comparison-scope + --comparison-scope \ + [--platform2 ] \ + [--capture-folder ] ``` +Pass `--platform2` only for comparative and `--capture-folder` only if `config_utils` printed one. Use your longest command timeout. + This script performs: - **Step 3:** Assess GPU utilization (computation, idle, communication times) - **Step 4:** Identify top 10 operations by GPU time @@ -391,7 +363,7 @@ Launch system-level sub-agents simultaneously using the Task tool. Do NOT wait b | `kernel_fusion` | `kernel-fusion-analyzer.md` | **Invocation conditions:** -- **CPU/Idle**: Read `category_data/category_manifest.json` and check `gpu_utilization.idle_time_percent`. Only invoke the subagent if `idle_time_percent > 15`. Skip otherwise -- the deterministic script already captured the factual data. +- **CPU/Idle**: Read `category_data/category_manifest.json` and check `gpu_utilization.idle_time_percent`. On multi-trace standalone runs system analysis is scoped to the focus job, so apply this and the Multi-Kernel condition to `gpu_utilization.jobs[]` (what Step 7.1 prints). Only invoke the subagent if `idle_time_percent > 15`. - **Multi-Kernel**: `multi_kernel` category exists in manifest OR `gpu_util['exposed_comm_time_percent'] > 0` OR `gpu_util['exposed_memcpy_time_percent'] > 0` - **Kernel Fusion**: `kernel_fusion` category exists in manifest @@ -616,6 +588,7 @@ If the plot fails (extension-absent branch), retry once. If still failing, proce a. **Initialize** — truncate and write the title line, the `report_mode` + any `kind=warning` markers (see the template title block and the Warnings rule below), then `## Executive Summary` (metrics table, `{{PERF_PLOT}}` placeholder). Use ` tee /analysis.md << 'SECTION_EOF'` (truncating `tee`, not append) for this first write only. - Data sources: `category_data/category_manifest.json` (`gpu_utilization` keys), `priority_data.json` (top bottleneck). + - `multi_trace`: Total Time is the combined time; add one sentence giving each phase's time, summed over its jobs in `gpu_utilization.jobs`. b. **Compute Kernel Optimizations** — append `## Compute Kernel Optimizations` with `### Top Operations` table and P-item cards. Use ` tee -a /analysis.md << 'SECTION_EOF'`. - Data sources: `priority_data.json` — P1 = `findings[0]`, P2 = `findings[1]`, ... ; each card joins its sub-agent's Detailed Analysis block by `(findings[i].category, findings[i].category_rank)`. The Top Operations table materializes `priorities[]` verbatim (one row per entry, array order, no re-sorting). @@ -624,9 +597,11 @@ If the plot fails (extension-absent branch), retry once. If still failing, proce c. **Kernel Fusion** — append `## Kernel Fusion Opportunities (Experimental)`. Use ` tee -a /analysis.md << 'SECTION_EOF'`. - Data source: `system_findings/kernel_fusion_findings.md`. + - `multi_trace`: add the template's scope line. d. **System-Level** — append `## System-Level Optimizations`. Use ` tee -a /analysis.md << 'SECTION_EOF'`. - Data sources: remaining `system_findings/*.md` (cpu_idle, multi_kernel). + - `multi_trace`: add the template's scope line. e. **Detailed Analysis** — append `## Detailed Analysis` with `### Compute Kernel Insights`, `### Kernel Fusion Insights`, `### System-Level Insights` subsections. Use ` tee -a /analysis.md << 'SECTION_EOF'`. - Data sources: copy the `## Detailed Analysis` blocks verbatim from each `*_findings.md` file. Follow the template for formatting. @@ -676,7 +651,7 @@ d. For placeholder values (`X ms`, `Y%`, `Z%`, `W%`) in the Executive Summary me e. For unfilled `` / `` / `` placeholders: substitute the real title/backend/platform from the corresponding findings file or `metadata/*_metadata.json`. f. For Args cell mismatches: copy the matching `operations[].args` value verbatim (preserving `
`) from the corresponding `category_data/_metrics.json` and string-replace the bad cell. g. For marker errors: restore or add the missing/broken marker in place — never delete a card or block to silence an error. Source numeric values from `priority_data.json` (P-items) or `_metrics.json::impact_estimates[]` (detail estimates); use `null` or the sentinel `not quantifiable from trace data` for non-quantifiable items. -h. For priority-consistency errors (R1 P-item count mismatch, R2 P-item category-order mismatch, R3 marker numeric mismatch, R4 Top Ops row-count mismatch): re-render the affected card(s) by re-reading `priority_data.json::findings[N-1]` for `category`, `low` (impact_score_low), `mid` (impact_score), `high` (impact_score_high), and `priorities[]` for the Top Operations table rows (one row per entry, in array order). +h. For priority-consistency errors (R1 P-item count mismatch, R2 P-item category-order mismatch, R3 marker numeric mismatch, R4 Top Ops row-count mismatch, R6 phase mismatch): re-render the affected card(s) by re-reading `priority_data.json::findings[N-1]` for `category`, `phase`, `low` (impact_score_low), `mid` (impact_score), `high` (impact_score_high), and `priorities[]` for the Top Operations table rows (one row per entry, in array order). 2. Run validation again. 3. Maximum 2 retry attempts. If still failing after retry, proceed with a warning. @@ -713,12 +688,4 @@ embed_plot_in_report(sys.argv[1]) \" '' ``` -If the plot is skipped, the `{{PERF_PLOT}}` placeholder is removed so the report remains clean. ---- - -## Trace Feature Detection - -If Steps 1 or many of Steps 3-6 fail or produce unexpected results, check whether the trace uses the following features before retrying: -- **GPU Graph Replay**: raw trace JSON contains `hipGraphLaunch` or `cudaGraphLaunch`. - - **Default mode** (analysis_mode = `default`): Inform the user with `[DIAG:trace_quality:GPU_GRAPH_REPLAY]` that GPU graph replay was detected and that the default analysis mode supports typical PyTorch traces. **Abort** -- do not retry or continue. - - **Inference mode** (analysis_mode = `inference`): Graph launches are expected and supported if graph capture folder is provided, do not abort. If inference_exec_mode is `eager` (no capture folder was provided), continue. +If the plot is skipped, the `{{PERF_PLOT}}` placeholder is removed so the report remains clean. \ No newline at end of file diff --git a/skills/tracelens-analysis-orchestrator/templates/analysis_template.md b/skills/tracelens-analysis-orchestrator/templates/analysis_template.md index 5f0e381e..ff25598b 100644 --- a/skills/tracelens-analysis-orchestrator/templates/analysis_template.md +++ b/skills/tracelens-analysis-orchestrator/templates/analysis_template.md @@ -40,10 +40,7 @@ section that has STANDALONE / COMPARATIVE variants. Delete the unused variant. exists (number sequentially from P1, including CPU/Idle first if invoked). 6. Each section is independently composable -- can be shared standalone. 7. All three tiers (Compute, Kernel Fusion, System) use separate sequential P1/P2/P3 numbering (no gaps). -8. Priority icons are assigned by PRIORITY NUMBER, not severity: - - Compute Kernel: 🔴 P1 → 🟡 P2 → 🟢 P3 → 🟢 P4 ... - - Kernel Fusion: icon by confidence (🔴 high → 🟡 medium → 🟢 low), not priority number - - System-Level: 🔴 P1 → 🟡 P2 → 🟢 P3 → 🟢 P4 ... (only when actionable issues exist) +8. Priority icons are assigned by PRIORITY NUMBER, not severity, in every section (Compute Kernel, Kernel Fusion, System-Level): 🔴 P1 → 🟡 P2 → 🟢 P3 → 🟢 P4 ... (System-Level only when actionable issues exist) 9. Field labels — each section uses EXACTLY these labels: OPTIMIZATION CARDS (§Compute Kernel Optimizations, §Kernel Fusion, §System-Level): @@ -53,7 +50,7 @@ section that has STANDALONE / COMPARATIVE variants. Delete the unused variant. DETAILED ANALYSIS (§Detailed Analysis only): - Compute / System blocks: **Identification:** / **Data:** / **Reasoning for Slowdown:** / **Resolution:** / **Impact estimate:** - - Kernel Fusion blocks: **Identification:** / **Data:** / **Impact estimate:** + - Kernel Fusion blocks: **Identification:** / **Data:** / **Resolution:** / **Impact estimate:** 10. Detailed Analysis: three subsections (`### Compute Kernel Insights`, `### Kernel Fusion Insights`, `### System-Level Insights`) with `#### 🔴/🟡/🟢 Pn: ` blocks matching card titles and order. 11. Model and appendix: Use `model_info["model"]` from `metadata/model_info.json` for the @@ -63,6 +60,9 @@ section that has STANDALONE / COMPARATIVE variants. Delete the unused variant. the library name(s) in parentheses when present in the sub-agent findings. Omit when no library is identified. System-Level and Kernel Fusion titles do NOT include a library parenthetical. +13. Multi-trace standalone (`multi_trace` scope): use the STANDALONE + blocks. Impact scores are a percent of the combined time of all jobs. Compute P-items carry their + phase. Fusion and System sections cover the focus job only and open with a scope line. --> @@ -90,7 +90,7 @@ section that has STANDALONE / COMPARATIVE variants. Delete the unused variant. ## Executive Summary -[1 paragraph overview + key metrics table] +[1 paragraph overview + key metrics table. multi_trace: Total Time is the combined time; add one sentence giving each phase's time, summed over its jobs in `gpu_utilization.jobs`.] + +_Times and percentages are summed across all phases ( + ). See each P-item's Phase field for where the time is spent._ + | Rank | Category | Time (ms) | % of Compute Time | Ops | |------|----------|-----------|-------------------|-----| @@ -152,15 +155,18 @@ One row per entry in `priority_data.json::priorities[]`, in array order (no mani + ### 🔴 P1: () +**Phase**: + **Insight**: [1 sentence - what's wrong] **Action**: [1-2 sentences - category-appropriate: GEMM fusion/tile/library; SDPA tile/backend; elementwise fusion; etc.] - + **Impact**: [impact_score: X.X, OR "Not quantifiable from trace data"] @@ -174,7 +180,7 @@ One row per entry in `priority_data.json::priorities[]`, in array order (no mani **Action**: [1-2 sentences] - + **Impact**: [impact_score: X.X, OR "Not quantifiable from trace data"] @@ -188,7 +194,7 @@ One row per entry in `priority_data.json::priorities[]`, in array order (no mani **Action**: [1-2 sentences] - + **Impact**: [impact_score: X.X, OR "Not quantifiable from trace data"] @@ -200,14 +206,16 @@ One row per entry in `priority_data.json::priorities[]`, in array order (no mani ## Kernel Fusion Opportunities (Experimental) -> **Note:** Kernel fusion analysis is experimental. impact_score projections estimate the recoverable fraction of E2E with 85% memory/compute pipeline overlap. Kernels without perf models use their measured trace time as-is. Candidates where fewer than 75% of kernels have perf models are not reported. Actual recoverable time depends on implementation feasibility and interaction effects. +> **Note:** Kernel fusion analysis is experimental. impact_score projections estimate the recoverable fraction of E2E with memory/compute pipeline overlap. Kernels without perf models use their measured trace time as-is. Candidates are not reported unless at least 50% of their kernels have perf models (75% for candidates of 5 or more kernels). Actual recoverable time depends on implementation feasibility and interaction effects. + +Scope: only (trace ``, capture ``) > **Note:** Kernel fusion analysis is experimental. - - + + ### 🔴 P1: @@ -264,6 +272,7 @@ One row per entry in `priority_data.json::priorities[]`, in array order (no mani > **Note:** System-level analysis is exploratory. The patterns and recommendations below are under active development and may be refined as system-level analysis matures. + @@ -350,6 +359,7 @@ communication/compute overlap). These affect the GPU pipeline as a whole. #### 🔴 P1: () +**Phase**: **Identification:** **Data:** @@ -400,47 +410,47 @@ communication/compute overlap). These affect the GPU pipeline as a whole. ### Kernel Fusion Insights -> **Note:** Kernel fusion analysis is experimental. impact_score projections estimate the recoverable fraction of E2E with 85% memory/compute pipeline overlap. Kernels without perf models use their measured trace time as-is. Actual recoverable time depends on implementation feasibility and interaction effects. +> **Note:** Kernel fusion analysis is experimental. impact_score projections estimate the recoverable fraction of E2E with memory/compute pipeline overlap. Kernels without perf models use their measured trace time as-is. Actual recoverable time depends on implementation feasibility and interaction effects. > **Note:** Kernel fusion analysis is experimental. - - + + -#### 🔴/🟡/🟢 P1: ( ms, instances) +#### 🔴 P1: **Identification:** **Data:** -| Kernel | Type | Duration (us) | Perf model | -|--------|------|--------------|------------| -| | | X.X | Yes/No | +| Operation | Args | Kernel Path | Kernel Name | Time (ms) | %E2E | Count | FLOPS/Byte | Efficiency | Bound | +|---|---|---|---|---|---|---|---|---|---| +| | | | | X.XXX | X.XX | N | X.XX | X.XX% of Y TFLOPS/TB/s | compute/memory-bound | + +**Resolution:** **Impact estimate:** -#### 🔴/🟡/🟢 P2: ( ms, instances) - +#### 🟡 P2: **Identification:** - **Data:** - -| Kernel | Type | Duration (us) | Perf model | -|--------|------|--------------|------------| -| | | X.X | Yes/No | - +**Resolution:** **Impact estimate:** -#### 🔴/🟡/🟢 P3: ( ms, instances) +#### 🟢 P3: +**Identification:** +**Data:** +**Resolution:** +**Impact estimate:** -*Repeat the same Identification + Data + Impact estimate format for each candidate, with anchors `detailed-analysis-fusion-PN`.* + ### System-Level Insights diff --git a/skills/tracelens-analysis-orchestrator/templates/sub_agent_spec.md b/skills/tracelens-analysis-orchestrator/templates/sub_agent_spec.md index 9c52f43e..6c40b128 100644 --- a/skills/tracelens-analysis-orchestrator/templates/sub_agent_spec.md +++ b/skills/tracelens-analysis-orchestrator/templates/sub_agent_spec.md @@ -43,9 +43,10 @@ Each P-item maps 1:1 to a `## Detailed Analysis` reasoning candidate at the same ```markdown ### P1: () +**Phase**: **Insight**: [1 sentence — what's wrong] **Action**: [1-2 sentences — what to do] - + **Impact**: [impact_score: X.X, OR "Not quantifiable from trace data"] ``` @@ -54,6 +55,7 @@ Each P-item maps 1:1 to a `## Detailed Analysis` reasoning candidate at the same - **System tier**: omit the `()` title suffix. Always emit `**Impact**: Not quantifiable from trace data` wrapped in `kind=p_item` markers with `low=null mid=null high=null`. - **Field labels are exact** — `**Insight**`, `**Action**`, `**Impact**`. - **`()` suffix**: the single `category_findings[i].library` for this card (one library per finding by construction). Omit the parenthetical when the value is `Unknown`. +- **Phase**: only when `category_findings[i].phase` is present (multi-trace standalone runs), emit the `**Phase**` line on the card and its Detailed Analysis block and the marker's `phase=`; omit all three otherwise. Do not repeat the phase in the title, insight text, or the Operation column. - **Marker required** — see § Impact markers (REQUIRED). The `low`/`mid`/`high` attributes carry the raw `impact_score_low/impact_score/impact_score_high` values from `category_findings[i]`. For non-quantifiable cards (system tier) use `low=null mid=null high=null`. --- @@ -68,6 +70,8 @@ with an HTML comment and an `####` heading: #### +**Phase**: + **Identification:** … **Data:** … @@ -90,6 +94,7 @@ with an HTML comment and an `####` heading: The five labels below must appear **in this order**, each on its own line with a blank line between them. The validator checks for these as substring matches. +Fusion tier (`tier=fusion`): the same labels except **Reasoning for Slowdown:**, which fusion blocks omit. | Label | Purpose | |-------|---------| @@ -249,6 +254,7 @@ The set of P-items is decided by `category_findings[]` alone — `MIN_PITEM_IMPA | `rank` | Card order within your category (1 = highest impact). Also the `rank=` value in ``. | | `bound_type` | `compute` \| `memory`. Selects the matching Action Prose Guidance row. | | `library` | One per finding. Drives the `()` title suffix. | +| `phase` | Multi-trace standalone runs only. | | `eff_bucket` | Roofline-efficiency band: `"0-30"`, `"30-60"`, `"60-100"`, or `"unknown"` (standalone); `"all"` (comparative). Members within a finding share the same band. | | `impact_score` / `_low` / `_high` | Group-summed % of E2E. Render verbatim into `kind=p_item` and `kind=detail_estimate` markers. | | `estimate_method` | `"quantified"` (impact from a perf model — standalone roofline gap or comparative t2/t1 ratio) or `"heuristic"` (op has no perf model; impact estimated from E2E share — see below). | @@ -282,7 +288,7 @@ Two bullets — low and high. Wrap in `kind=detail_estimate` markers (see § Impact markers). ```markdown - + - Low end impact_score: - High end impact_score: @@ -309,8 +315,8 @@ The block between them is exactly the `impact_score`-based markdown you would ot | `kind` | Where | Required attributes | Optional attributes | |--------|-------|--------------------|---------------------| -| `p_item` | Around every P-item `**Impact**` line in `## Recommendations`. | `low`, `mid`, `high` (all three; use `null` only for system-tier non-quantifiable). | `category` is reserved for the orchestrator template; sub-agents do **not** emit it. | -| `detail_estimate` | Around the two-bullet `Low end ... / High end ...` block under `**Impact estimate:**` in each `## Detailed Analysis` candidate. Skip only for system-tier non-quantifiable estimates. | `low`, `high` (impact_score values, % of E2E). | none | +| `p_item` | Around every P-item `**Impact**` line in `## Recommendations`. | `low`, `mid`, `high` (all three; use `null` only for system-tier non-quantifiable); `phase` when the finding has one. | `category` is reserved for the orchestrator template; sub-agents do **not** emit it. | +| `detail_estimate` | Around the two-bullet `Low end ... / High end ...` block under `**Impact estimate:**` in each `## Detailed Analysis` candidate. Skip only for system-tier non-quantifiable estimates. | `low`, `mid`, `high` (impact_score values, % of E2E). | none | | `op_row` | After the last row of each compute-tier `**Data:**` table (see § Operations Table Schema → Per-row impact marker). | `rank`, `impacts` (CSV of per-row `impact_score`, one per data row in table order; `—` for null). | none | ### Value-source rule