diff --git a/.github/federation.json b/.github/federation.json index 21dbeb8..4b8cfd9 100644 --- a/.github/federation.json +++ b/.github/federation.json @@ -99,6 +99,18 @@ { "path": "projects/rocprofiler-sdk/skills/pc-sampling", "as": "rocprofiler-pc-sampling" + }, + { + "path": "projects/rocprofiler-systems/skills/choosing-a-profiling-preset", + "as": "rocprofiler-systems-choosing-a-profiling-preset" + }, + { + "path": "projects/rocprofiler-systems/skills/discovering-profiling-options", + "as": "rocprofiler-systems-discovering-profiling-options" + }, + { + "path": "projects/rocprofiler-systems/skills/instrumenting-binaries", + "as": "rocprofiler-systems-instrumenting-binaries" } ] } diff --git a/skills/rocprofiler-systems-choosing-a-profiling-preset/.federated.json b/skills/rocprofiler-systems-choosing-a-profiling-preset/.federated.json new file mode 100644 index 0000000..2aedb6c --- /dev/null +++ b/skills/rocprofiler-systems-choosing-a-profiling-preset/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "rocm-rocm-systems", + "repo": "ROCm/rocm-systems", + "ref": "develop", + "commit": "69379f2d8bc59311b4e20aa76ef3375964c87985", + "path": "projects/rocprofiler-systems/skills/choosing-a-profiling-preset", + "license": "MIT", + "content_hash": "0b4d57fa6db34392c87d597ec1cfaaff7860b1d515d91c4fa3bf863bb25ce6fe", + "imported_at": "2026-10-27T18:54:05Z" +} diff --git a/skills/rocprofiler-systems-choosing-a-profiling-preset/SKILL.md b/skills/rocprofiler-systems-choosing-a-profiling-preset/SKILL.md new file mode 100644 index 0000000..c8f9231 --- /dev/null +++ b/skills/rocprofiler-systems-choosing-a-profiling-preset/SKILL.md @@ -0,0 +1,152 @@ +--- +name: rocprofiler-systems-choosing-a-profiling-preset +description: Recommends which rocprofiler-systems runtime profiling preset (balanced, trace-gpu, trace-hpc, sys-trace, workload-trace, etc.) to pass to `rocprof-sys-run --preset=` or `rocprof-sys-sample --preset=` based on the user's workload and profiling goal, and returns the ready-to-run command. Use when the user asks "which preset should I use", describes a workload (GPU kernel, MPI, OpenMP, AI/ML training, production overview) and wants profiling guidance, or invokes `rocprof-sys-run`/`rocprof-sys-sample` without picking a preset. Do NOT use for CMake build presets (`cmake --preset debug|release|ci`) used to configure/compile rocprofiler-systems itself — that is a build-configuration concern, not a profiling-run concern. +--- + +# Choosing a Profiling Preset + +Recommends one of rocprofiler-systems' built-in runtime profiling presets for a +described workload, and returns the exact command to run it. + +## Prerequisites + +None to make a first recommendation from the table below. To confirm what a preset +sets (see "Confirm with `--explain`"), and to run the command, `rocprof-sys-run` or +`rocprof-sys-sample` must be installed or built. + +## Terminology + +"Trace" and "tracing" mean two different things here. Keep them apart in every answer. + +| Term | Meaning | Setting | +| --- | --- | --- | +| Perfetto trace output | Writes a timeline file in Perfetto format | `ROCPROFSYS_TRACE` | +| rocPD output | Writes a rocPD database | `ROCPROFSYS_USE_ROCPD` | +| Profile output | Writes timemory profile files (flat or call-stack based) | `ROCPROFSYS_PROFILE` | +| Call-stack sampling | Samples the CPU call stack at a fixed frequency | `ROCPROFSYS_USE_SAMPLING` | +| API tracing | Intercepts and records the calls of one API | see below | + +API tracing is chosen per API: ROCm (`ROCPROFSYS_ROCM_DOMAINS`), MPI +(`ROCPROFSYS_USE_MPIP`), RCCL (`ROCPROFSYS_USE_RCCLP`), OpenMP (`ROCPROFSYS_USE_OMPT`) +and Kokkos (`ROCPROFSYS_USE_KOKKOSP`). ROCm API tracing is on in every built-in preset, +by default for the HIP runtime API, ROCTx markers, kernel dispatches, memory copies and +scratch memory. A preset may narrow or widen that set, and `--rocm=` overrides it. + +API tracing is separate from output selection. A preset's `tracing.enabled: false` (or +`ROCPROFSYS_TRACE=false`) turns Perfetto trace output off, not API tracing. The collected +data goes to whichever outputs are on: Perfetto, rocPD or profile files. Never say a +preset "has tracing off" without saying which of these you mean. + +## Preset reference + +A routing aid only: it says which preset fits which need, not exactly what each one sets. +The installed binary is the source of truth for that (see "Confirm with `--explain`"). + +| Preset | Category | Use case | +| --- | --- | --- | +| `balanced` | general | Recommended default, moderate overhead | +| `profile-only` | general | Lowest overhead flat profile (production) | +| `detailed` | general | Maximum depth, high overhead | +| `trace-gpu` | gpu | GPU activity timeline (kernel dispatch, memory copies) | +| `trace-hw-counters` | gpu | GPU hardware counters (occupancy, VALU utilization) | +| `workload-trace` | gpu | AI/ML training, long-running GPU/HPC workloads | +| `trace-hpc` | hpc | Compute-intensive MPI/OpenMP/Kokkos/RCCL apps, with CPU counters (PAPI) | +| `trace-openmp` | hpc | OpenMP GPU target-offload apps | +| `profile-mpi` | hpc | MPI communication latency, flat profile, no Perfetto trace | +| `sys-trace` | tracing | Debugging runtime-layer interactions, widest ROCm API tracing (HIP, HSA, ROCTx, RCCL) | +| `runtime-trace` | tracing | Same as `sys-trace` without HSA/compiler APIs | + +## How to choose + +Resolve these in order; the first question that narrows to a single preset wins. Do not +require the user to name a runtime such as MPI before a step applies. + +1. **API tracing or debugging.** Questions about HIP, HSA or runtime-layer behavior (e.g. + "why is my HIP call behaving oddly", "what's HSA doing under the hood") → + `sys-trace` (full ROCm API tracing), or `runtime-trace` when HSA and compiler API + noise should be left out. + +2. **One named measurement.** + - GPU hardware counters such as occupancy or VALU utilization → `trace-hw-counters`. + - MPI communication latency is the *only* concern, no Perfetto trace file needed → + `profile-mpi`. + - OpenMP GPU target-offload kernels → `trace-openmp`. + +3. **Long-running GPU work.** AI/ML training, or a long-running GPU-accelerated HPC job + that needs MPI and RCCL API tracing and a large Perfetto trace buffer → + `workload-trace`. A training job qualifies even if it never mentions MPI, and it wins + over step 4 when the prompt also names MPI or OpenMP. + +4. **HPC runtimes.** A compute-intensive MPI, OpenMP, Kokkos or RCCL application → + `trace-hpc`. It writes a Perfetto trace, traces those runtimes' APIs and sets CPU + hardware-counter events (PAPI). If the user wants neither the trace file nor the + counters, `profile-mpi` or `balanced` write no Perfetto trace. + +5. **GPU activity.** Kernel dispatch or device activity (a GPU activity timeline), with + none of the goals above stated → `trace-gpu`. + +6. **General-purpose profiling.** Pick by overhead tolerance and depth: + - Lowest overhead, quick flat profile (e.g. production) → `profile-only`; it turns + off Perfetto trace output, rocPD output and call-stack sampling, but ROCm API calls + and kernels still appear in the flat profile. + - Maximum depth, willing to accept high overhead → `detailed`. + - Anything else, or a good default for general profiling (profile output, rocPD + output, call-stack sampling and GPU metrics, no Perfetto trace) → `balanced`. + +## Confirm with `--explain` + +The presets are compiled into the installed binary, so `--explain` matches the user's +build exactly and the table above can lag behind it. Once you have a candidate: + +1. Run `rocprof-sys-run --explain=` (or `rocprof-sys-sample --explain=`) as a + single command, with no `command -v` check or `&&` chain in front of it. Use its + `ROCPROFSYS_*` list as the facts about the preset. Quote the settings that matter to + the user's goal, and if they contradict the table or your first choice, follow + `--explain`. +2. If the command is not found or cannot run, use the table and say that the settings + are unconfirmed. + +## Output + +Return the exact invocation. Default to `rocprof-sys-run`: + +```bash +rocprof-sys-run --preset= -- ./myapp +``` + +Both tools take the same `--preset`, and a preset that sets `sampling.enabled` overrides +either one: `balanced` samples at 50 Hz on both, and `profile-only` leaves call-stack +sampling off on both. The tools differ only when the preset says nothing about sampling +(for example `trace-gpu`): `rocprof-sys-sample` turns call-stack sampling on and +`rocprof-sys-run` leaves it off. So use `rocprof-sys-sample` when the user wants call-stack +sampling and the preset does not already enable it: + +```bash +rocprof-sys-sample --preset= -- ./myapp +``` + +State in one line what the recommended preset turns on and what it leaves off, using the +terms above, so the user knows what they gain and give up. For example, `profile-only` +writes a flat profile with no Perfetto trace file, no rocPD database and no call-stack +sampling, while ROCm API calls and kernels still appear in the profile. + +Mention, if relevant: + +- Domain flags (`--gpu`, `--rocm`, `--cpu`, `--parallel`) can layer on top of + any preset to add/override specific metrics, e.g. + `rocprof-sys-run --preset=balanced --gpu=temp,power -- ./myapp`. Use this when the + user's need doesn't fit a preset neatly: start from `balanced` and extend it. +- `--explain=` and `--list-presets` let the user verify the choice themselves. +- `--export-config=.json` freezes the resolved preset+overrides into a + reusable JSON config — this is the supported way to customize a preset, not + hand-editing the shipped JSON files. +- The file format for custom presets is the JSON schema installed at + `/share/rocprofiler-systems/presets/schema.json`. + +## Common Mistakes + +| Mistake | Fix | +| --- | --- | +| Confusing this with the CMake `--preset debug\|release\|ci` used to *build* rocprofiler-systems | Build presets configure compilation; profiling presets configure a profiling *run*. This skill only covers the latter. | +| Looking for the built-in preset JSON files to edit | The built-in presets are compiled into the binary and are not installed as files. Use `--explain=` to inspect one and `--export-config` to produce an editable copy. | +| Picking `detailed` by default for "just get me some data" | Default to `balanced` unless the user explicitly wants maximum depth and can tolerate higher overhead. | diff --git a/skills/rocprofiler-systems-choosing-a-profiling-preset/evals/evals.json b/skills/rocprofiler-systems-choosing-a-profiling-preset/evals/evals.json new file mode 100644 index 0000000..e295650 --- /dev/null +++ b/skills/rocprofiler-systems-choosing-a-profiling-preset/evals/evals.json @@ -0,0 +1,49 @@ +{ + "evaluations": [ + { + "expected_behavior": [ + "recommends the trace-hpc preset", + "gives the exact rocprof-sys-sample --preset=trace-hpc command" + ], + "id": "which-preset-for-hpc-app", + "prompt": "I have an MPI + OpenMP HPC application and want to profile it with rocprof-sys-sample. Which preset should I use?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "recommends the profile-only preset", + "explains it is a flat profile with Perfetto trace output, rocPD output and call-stack sampling disabled" + ], + "id": "lowest-overhead-production-profile", + "prompt": "I want to profile my production service with rocprof-sys-run but need the lowest possible overhead. What preset fits?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "recommends the workload-trace preset", + "mentions it enables MPI/RCCL API tracing with a large Perfetto trace buffer" + ], + "id": "aiml-training-preset-recommendation", + "prompt": "Recommend a rocprofiler-systems preset for profiling a long-running AI/ML training job on GPUs.", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "recommends the workload-trace preset rather than trace-hpc" + ], + "id": "mpi-openmp-training-job-prefers-workload-trace", + "prompt": "My long-running AI/ML training job uses MPI and OpenMP on the host side. Which rocprof-sys-run preset should I use?", + "skill_should_trigger": true + }, + { + "id": "vague-performance-question-no-presets", + "prompt": "How can I make my application run faster?", + "skill_should_trigger": false + }, + { + "id": "cmake-build-preset-question", + "prompt": "Which CMake preset should I use to configure a release build of rocprofiler-systems?", + "skill_should_trigger": false + } + ] +} diff --git a/skills/rocprofiler-systems-choosing-a-profiling-preset/evals/machine.yml b/skills/rocprofiler-systems-choosing-a-profiling-preset/evals/machine.yml new file mode 100644 index 0000000..4d7efd4 --- /dev/null +++ b/skills/rocprofiler-systems-choosing-a-profiling-preset/evals/machine.yml @@ -0,0 +1,2 @@ +os: [Linux] +labels: [mi300x, gpu, rocm] diff --git a/skills/rocprofiler-systems-choosing-a-profiling-preset/skill-card.md b/skills/rocprofiler-systems-choosing-a-profiling-preset/skill-card.md new file mode 100644 index 0000000..09a7368 --- /dev/null +++ b/skills/rocprofiler-systems-choosing-a-profiling-preset/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Recommends the rocprofiler-systems runtime profiling preset that best fits a user's described workload and profiling goal. + +## Owner + +rocprofiler-systems (a project in [rocm-systems](https://github.com/ROCm/rocm-systems/)) + +## License + +MIT diff --git a/skills/rocprofiler-systems-discovering-profiling-options/.federated.json b/skills/rocprofiler-systems-discovering-profiling-options/.federated.json new file mode 100644 index 0000000..84da996 --- /dev/null +++ b/skills/rocprofiler-systems-discovering-profiling-options/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "rocm-rocm-systems", + "repo": "ROCm/rocm-systems", + "ref": "develop", + "commit": "69379f2d8bc59311b4e20aa76ef3375964c87985", + "path": "projects/rocprofiler-systems/skills/discovering-profiling-options", + "license": "MIT", + "content_hash": "ae885e83ffc17f6a39782c076d1d0d6574f086763a6da950fdf892cf3b074eb8", + "imported_at": "2026-10-27T18:54:05Z" +} diff --git a/skills/rocprofiler-systems-discovering-profiling-options/SKILL.md b/skills/rocprofiler-systems-discovering-profiling-options/SKILL.md new file mode 100644 index 0000000..6e14e95 --- /dev/null +++ b/skills/rocprofiler-systems-discovering-profiling-options/SKILL.md @@ -0,0 +1,139 @@ +--- +name: rocprofiler-systems-discovering-profiling-options +description: Answers questions about what rocprofiler-systems configuration options exist and how to use them — e.g. "what hardware counters are available on my system", "what ROCm APIs can I trace", "what settings control sampling/output", "how do I persist my configuration" — by mapping the question to the right `rocprof-sys-avail` or `rocprof-sys-run --help=` command. Use when the user wants to discover or explore ROCPROFSYS_* settings, Timemory components, hardware counters, ROCm API tracing domains, or GPU metrics, rather than apply them. Do NOT use this for picking a starting --preset for a workload — that's `choosing-a-profiling-preset`. +--- + +# Discovering Profiling Options + +Maps a "what's available" or "how do I configure X" question to the exact +`rocprof-sys-avail` / `rocprof-sys-run --help=` command that answers it. +`rocprof-sys-avail` is self-updating and authoritative — prefer running it over +reciting static documentation. + +## Prerequisites + +A working `rocprof-sys-avail` binary, built/installed alongside +`rocprof-sys-run`/`rocprof-sys-sample`. If it is not on `PATH`, run +`source /share/rocprofiler-systems/setup-env.sh` first. + +## Discovery command reference + +| User question | Command | Notes | +| --- | --- | --- | +| What settings/env vars exist, with descriptions? | `rocprof-sys-avail -S -bd` | `-b` suppresses the current-value/availability column, `-d` adds the description column | +| What hardware counters are available on my system? | `rocprof-sys-avail -H -bd -A` | `-A`/`--available` restricts to counters this machine can actually use; add `-c CPU` or `-c GPU` to split by device | +| What ROCm APIs can be traced? | `rocprof-sys-avail -bd -r ROCM_DOMAINS` | Regex-filters settings to the `ROCPROFSYS_ROCM_DOMAINS` row, listing every valid domain (`hip_runtime_api`, `hsa_api`, `kernel_dispatch`, `rccl_api`, `kfd_events`, ...) | +| What operations/API calls exist within one ROCm domain? | `rocprof-sys-avail --list-domains` then `rocprof-sys-avail --list-operations ` | e.g. `--list-operations hip_runtime_api` lists individual HIP calls. Only domains printed by `--list-domains` are accepted: composites must be expanded (`hip_api` → `hip_runtime_api`, `hip_compiler_api`; `hsa_api` → `hsa_core_api`, `hsa_amd_ext_api`, `hsa_image_ext_api`, `hsa_finalize_ext_api`; `kfd_events` → the `kfd_*` domains; `roctx` → `marker_api`), and `kernel_dispatch` has no operation list | +| What Timemory components can I collect? | `rocprof-sys-avail -C -bd` | add `-A` to restrict to components usable on this build | +| What settings exist for one topic (tracing, sampling, output, ...)? | `rocprof-sys-avail --list-categories` then `rocprof-sys-avail -S -bd -c settings::` | category names don't always match plain English — it's `settings::trace`, not `settings::tracing` — so always check `--list-categories` first | +| What GPU power/temp/utilization metrics can I sample? | `rocprof-sys-avail -bd -r AMD_SMI_METRICS` | surfaces `ROCPROFSYS_AMD_SMI_METRICS` choices (`busy, temp, power, mem_usage, ...`) | +| What CLI flags does `rocprof-sys-run` have for topic X? | `rocprof-sys-run --help=` | topics: `preset, general, tracing, profiling, output, sampling, process, counters, backend, execution, debug, misc, gpu, cpu, rocm, parallel` | +| How do I persist chosen settings across runs? | `rocprof-sys-avail -G ~/.rocprof-sys.cfg [--all]` then `export ROCPROFSYS_CONFIG_FILE=~/.rocprof-sys.cfg` | `--all` includes descriptions/categories in the generated file; formats are txt (default), json, xml via `-F` | + +## Terminology + +"Trace" and "tracing" overlap in the output, so map the name you see to what it +controls before answering: + +| Name you see | What it controls | +| --- | --- | +| `ROCPROFSYS_TRACE`, `settings::trace` and `--help=tracing` (its tuning options) | Perfetto trace output | +| `ROCPROFSYS_USE_ROCPD` | rocPD output | +| `ROCPROFSYS_PROFILE`, `--help=profiling` | timemory profile output | +| `ROCPROFSYS_USE_SAMPLING`, `--help=sampling` | call-stack sampling | +| `ROCPROFSYS_ROCM_DOMAINS`, `--help=rocm` | ROCm API tracing | +| `ROCPROFSYS_USE_MPIP`, `_RCCLP`, `_OMPT`, `_KOKKOSP` | MPI, RCCL, OpenMP and Kokkos API tracing | + +"Tracing" means API tracing (recording the calls of one API); "trace output" means the +Perfetto file. A question like "what tracing options are there?" usually means the ROCm +domains, so ask or answer both if it is unclear. + +## How to answer a discovery question + +1. Identify which axis the question is about: settings/env-vars, Timemory + components, hardware counters, ROCm API domains, GPU metrics, or CLI flags. +2. Pick the matching row from the table above. +3. Run the command (or hand it to the user) and report the relevant rows — + don't dump the entire table if the question was narrow; combine with + `-r ` / `-c ` / `-A` to narrow first. +4. If the result is still broad, suggest adding `-A`/`--available` (installed/ + supported only) or a category/regex filter. + +## Persisting configuration + +Once the right settings are identified, freeze them into a config file instead +of re-typing environment variables every run: + +```bash +rocprof-sys-avail -G ~/.rocprof-sys.cfg --all # --all adds descriptions/categories +export ROCPROFSYS_CONFIG_FILE=~/.rocprof-sys.cfg +``` + +Select the format with `-F txt|json|xml`; they load differently: + +- `txt` (default) and `xml`: load with `ROCPROFSYS_CONFIG_FILE`. +- `json`: this is a hierarchical preset file. It does **not** load through + `ROCPROFSYS_CONFIG_FILE` (the run fails with "missing the expected + 'rocprofiler-systems' root object"); load it with + `rocprof-sys-run --preset=.json -- `. + +When the same setting is given in more than one place, the order of priority is: +config file, then environment variable, then command-line parameter, each overriding +the one before. For example, with `ROCPROFSYS_SAMPLING_FREQ = 50` in the file, +`ROCPROFSYS_SAMPLING_FREQ=77` in the environment beats it, and +`rocprof-sys-run --sampling-freq=100 -- ` beats both. + +## Using what you discover + +Names found with the commands above are applied through these settings: + +- **CPU counters** (`-H -c CPU`): set `ROCPROFSYS_PAPI_EVENTS`, `ROCPROFSYS_PROFILE=ON` + **and** add `papi_array` (or `papi_vector`) to `ROCPROFSYS_TIMEMORY_COMPONENTS`, e.g. + `wall_clock,papi_array`; results are written to `papi_array*.txt/json` (or + `papi_vector*`). Without the component the run succeeds but writes no counter files. All + events must come from one namespace — mixing `PAPI_*` presets with `perf::*` + events only prints a "Failure to add named event" warning and drops the + counters, the run still exits 0. + - Host CPU counters read through Linux `perf_event` need `/proc/sys/kernel/perf_event_paranoid` <= 2. + That limit does not apply to metrics read from other sources, such as the PAPI `net:::` + network events, which read `/proc/net/dev`. +- **CPU overflow sampling**: sample on a hardware event instead of a timer. + Set `ROCPROFSYS_USE_SAMPLING=ON`, `ROCPROFSYS_SAMPLING_OVERFLOW=ON`, + `ROCPROFSYS_SAMPLING_OVERFLOW_EVENT=` (list events with + `rocprof-sys-avail -H -c CPU -r overflow`) **and** a large + `ROCPROFSYS_SAMPLING_OVERFLOW_FREQ` (events per sample, e.g. `10000000`). + Setting only the event does nothing (it falls back to CPU-time sampling), and + leaving the interval at its default of 300 events aborted the run when tested. + It also needs `perf_event_paranoid` <= 2. +- **GPU counters** (`-H -c GPU`): names are listed per device, e.g. + `FETCH_SIZE:device=0`. Set `ROCPROFSYS_ROCM_EVENTS` for kernel-dispatch + counters (writes `rocprof-device--.txt/json`), or + `ROCPROFSYS_GPU_PERF_COUNTERS` for polled (PMC) sampling. A name without the + `:device=N` suffix is collected on every device. +- **GPU metrics** (power, temperature, utilization): collected via + AMD SMI. `ROCPROFSYS_AMD_SMI_METRICS` selects which metrics to enable. + - By default, `busy,temp,power,mem_usage` are enabled. + - The value `all` collects everything. + - `ROCPROFSYS_USE_AMD_SMI=OFF` disables this backend entirely. +- **ROCm API tracing**: set `ROCPROFSYS_ROCM_DOMAINS` to a comma-separated list, + e.g. `hip_runtime_api,kernel_dispatch,memory_copy`. + - Copy names exactly from `rocprof-sys-avail -bd -r ROCM_DOMAINS`: one unknown + name will fail produce a failure + - `kfd_events` also needs `HSA_XNACK=1` and an XNACK-capable GPU; check with + `rocminfo | grep -i xnack` (a `xnack-`/`xnack+` suffix on the GPU target means + capable). On a GPU without XNACK it records nothing and prints nothing. + +## Common Mistakes + +| Mistake | Fix | +| --- | --- | +| Guessing a category name for `-c` (e.g. `settings::tracing`) | Run `rocprof-sys-avail --list-categories` first — category names don't always match the obvious English word (it's `settings::trace`) | +| Dumping the full `-S`/`-C`/`-H` table for a narrow question | Combine with `-r ` or `-c `/`-A` to filter before reporting | +| Confusing this skill with picking a profiling preset | Preset selection is `choosing-a-profiling-preset`; this skill is for exploring/fine-tuning individual options, including on top of a chosen preset | +| Typing a ROCm domain name from memory | An unknown `ROCPROFSYS_ROCM_DOMAINS` name silently disables the whole list; copy names from `rocprof-sys-avail -bd -r ROCM_DOMAINS` | +| Forgetting `-A`/`--available` | Without it, `-H`/`-C` list counters/components the current build or hardware doesn't actually support, which can overwhelm the answer | + +`rocprof-sys-avail` is the source of truth for names and descriptions. For +topics beyond option discovery (unified memory profiling, causal profiling), see +the online docs: + diff --git a/skills/rocprofiler-systems-discovering-profiling-options/evals/evals.json b/skills/rocprofiler-systems-discovering-profiling-options/evals/evals.json new file mode 100644 index 0000000..e676cb4 --- /dev/null +++ b/skills/rocprofiler-systems-discovering-profiling-options/evals/evals.json @@ -0,0 +1,41 @@ +{ + "evaluations": [ + { + "expected_behavior": [ + "recommends rocprof-sys-avail -H -bd -A", + "explains -A restricts to counters usable on this machine" + ], + "id": "what-hw-counters-available", + "prompt": "What hardware counters are available on my system for rocprof-sys?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "recommends rocprof-sys-avail -bd -r ROCM_DOMAINS", + "lists example domain choices like hip_runtime_api, hsa_api, kernel_dispatch" + ], + "id": "what-rocm-apis-traceable", + "prompt": "What ROCm APIs can I trace with rocprof-sys-run?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "recommends rocprof-sys-avail -G ~/.rocprof-sys.cfg --all", + "explains setting ROCPROFSYS_CONFIG_FILE" + ], + "id": "persist-configuration", + "prompt": "How do I save my rocprof-sys settings so I don't have to set environment variables every time?", + "skill_should_trigger": true + }, + { + "id": "which-preset-should-i-use", + "prompt": "Which preset should I use to profile my MPI application?", + "skill_should_trigger": false + }, + { + "id": "vague-performance-question-no-tool", + "prompt": "How can I make my application run faster?", + "skill_should_trigger": false + } + ] +} diff --git a/skills/rocprofiler-systems-discovering-profiling-options/evals/machine.yml b/skills/rocprofiler-systems-discovering-profiling-options/evals/machine.yml new file mode 100644 index 0000000..4d7efd4 --- /dev/null +++ b/skills/rocprofiler-systems-discovering-profiling-options/evals/machine.yml @@ -0,0 +1,2 @@ +os: [Linux] +labels: [mi300x, gpu, rocm] diff --git a/skills/rocprofiler-systems-discovering-profiling-options/skill-card.md b/skills/rocprofiler-systems-discovering-profiling-options/skill-card.md new file mode 100644 index 0000000..ba98d25 --- /dev/null +++ b/skills/rocprofiler-systems-discovering-profiling-options/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Maps "what's available" questions about rocprofiler-systems configuration (hardware counters, ROCm tracing domains, components, settings) to the exact `rocprof-sys-avail`/`rocprof-sys-run` command that answers them. + +## Owner + +rocprofiler-systems (a project in [rocm-systems](https://github.com/ROCm/rocm-systems/)) + +## License + +MIT diff --git a/skills/rocprofiler-systems-instrumenting-binaries/.federated.json b/skills/rocprofiler-systems-instrumenting-binaries/.federated.json new file mode 100644 index 0000000..0f1250f --- /dev/null +++ b/skills/rocprofiler-systems-instrumenting-binaries/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "rocm-rocm-systems", + "repo": "ROCm/rocm-systems", + "ref": "develop", + "commit": "69379f2d8bc59311b4e20aa76ef3375964c87985", + "path": "projects/rocprofiler-systems/skills/instrumenting-binaries", + "license": "MIT", + "content_hash": "9d3ae5331652cc27dd99a6f67885d2172b1d2c7dff0fdbc616c746e3b19e5d8c", + "imported_at": "2026-10-27T18:54:05Z" +} diff --git a/skills/rocprofiler-systems-instrumenting-binaries/SKILL.md b/skills/rocprofiler-systems-instrumenting-binaries/SKILL.md new file mode 100644 index 0000000..814611f --- /dev/null +++ b/skills/rocprofiler-systems-instrumenting-binaries/SKILL.md @@ -0,0 +1,262 @@ +--- +name: rocprofiler-systems-instrumenting-binaries +description: Guides instrumentation with `rocprof-sys-instrument` - choosing runtime instrumentation vs binary rewrite (`-o`), including or excluding functions and modules with regex options, previewing what will be instrumented with a dry run (`--simulate`, `--dump-info`, `--print-*`), tuning granularity (`--min-instructions`, `--min-address-range`, `--instrument-loops`), rewriting libraries, and diagnosing why a function is missing from, or too numerous in, a trace. Use when the user asks about rocprof-sys-instrument, binary rewrite, instrumenting a shared library, function or module include/exclude regexes, dry runs, or the difference between a function and a module. Do NOT use to pick a `--preset` (use `choosing-a-profiling-preset`), to list settings, counters or ROCm domains (use `discovering-profiling-options`), or to attach to an already-running process (a separate tool, `rocprof-sys-attach`). +--- + +# Instrumenting Binaries + +`rocprof-sys-instrument` decides which functions of a program get instrumented and +either does it while the program runs or writes a new instrumented binary. For the +full, current flag list run `rocprof-sys-instrument --help` (use +`--help | grep -A3 -- --` for one flag). + +## Prerequisites + +- `rocprof-sys-instrument` and `rocprof-sys-run` on `PATH`; if not, run + `source /share/rocprofiler-systems/setup-env.sh`. +- Build the target with `-g` when you can: modules are then source files and the + tool can report file and line information. Without debug info a whole binary is + one module named after the binary. + +## Runtime instrumentation vs binary rewrite + +| | Runtime instrumentation | Binary rewrite | +| --- | --- | --- | +| Selected by | the default (no `-o`) | `-o []` | +| Command | `rocprof-sys-instrument [opts] -- ./app args` | `rocprof-sys-instrument [opts] -o app.inst -- ./app` | +| What gets instrumented | the executable **and** the shared libraries it loads | only the target's own code; shared libraries are not instrumented | +| Cost (upstream guidance) | more memory, slower to set up, more run-time overhead | much faster to set up, lower overhead | +| Runs the app | yes, immediately, with `args` | no; run `rocprof-sys-run -- ./app.inst args` later | +| Arguments after the target | passed to the app | ignored (give them when you run the result) | + +- Pick **runtime** when the performance of the libraries the app loads matters. +- Pick **binary rewrite** for the lowest overhead, for repeated runs, and for MPI + applications (upstream guidance). +- `--exe-only` (runtime only) leaves every shared library out; + `--max-library-functions N` skips shared libraries with more than `N` functions + (default 20000; `0` disables the check). +- `-o` with no file name writes `.inst` in the current directory (target in the + current directory) or `instrumented/` for a library. + +## Function vs module + +- A **function** is one symbol, for example `big_a` or `Domain::BuildMesh`. +- A **module** is the container a function is defined in: its **source file** when + the binary has debug info (`/path/mod_a.cpp`), otherwise the **binary or library** + itself (`/path/app`). Shared libraries appear as their own modules in runtime mode. + +```bash +rocprof-sys-instrument --simulate -o app.inst --print-available modules -- ./app +rocprof-sys-instrument --simulate -o app.inst --print-available functions -- ./app +rocprof-sys-instrument --simulate -o app.inst --print-available pair -- ./app +``` + +`pair` prints one `[module] --> [function][#instructions]` line per function. +Module options (`-MI`, `-ME`, `-MR`) act on every function in the matching modules; +function options (`-I`, `-E`, `-R`) act on individual functions. + +## Including, excluding and restricting + +| Option | Meaning | +| --- | --- | +| `-I` / `-MI` (include) | Force these in, even if the size heuristics would skip them. Does **not** narrow anything else. | +| `-R` / `-MR` (restrict) | Instrument **only** what matches, ignoring the size heuristics. | +| `-E` / `-ME` (exclude) | Never instrument these. Always applied, and wins over include and restrict. | + +Matching rules (all verified): + +- Patterns are regular expressions matched anywhere in the name (substring, not + anchored). `tiny` matches `tiny_a` and `tiny_b`; write `^tiny_a$` for an exact match. +- Several patterns after one option are OR'd: `-I tiny_a medium_b` is the same as + `-I 'tiny_a|medium_b'`. +- Module patterns match the module name (the source file or binary path). Function + patterns match the function name, and also the signature when `--label args` is set + (so `-I '\(long int\)$'` only works together with `--label args`). +- Exclude beats include at any level: `-ME 'mod_' -I tiny_a` instruments nothing. +- `-MR mod_a` instruments **every** function of `mod_a`, including tiny ones and ones + with loops, regardless of the size thresholds. + +Gotcha: if you give both `--module-restrict` and `--function-restrict`, the function +restriction is silently ignored. `-MR mod_a -R '^medium_b$'` instrumented all of +`mod_a`, not the union the upstream docs describe. Use one of the two, or combine +`-MR` with `-E`. + +Examples: + +```bash +# Only one source file +rocprof-sys-instrument -o app.inst -MR 'solver\.cpp$' -- ./app +# Everything the heuristics pick, minus one noisy library +rocprof-sys-instrument -o app.inst -ME 'libm\.so' -- ./app +# Add a small function the heuristics skip +rocprof-sys-instrument -o app.inst -I '^small_helper$' -- ./app +``` + +## Dry run + +A dry run shows what would be instrumented without instrumenting anything. + +```bash +rocprof-sys-instrument --simulate [options] -- ./app +rocprof-sys-instrument --simulate -o app.inst --print-instrumented pair -- ./app +rocprof-sys-instrument --dump-info --simulate -o app.inst -- ./app +``` + +What `--simulate` does (verified): + +- Exits after the analysis. It does **not** run the application and does **not** + write the rewritten binary, even with `-o`. +- Prints a per-module summary (`N instrumented funcs in `). +- With `--print-available`, `--print-instrumented`, `--print-excluded`, + `--print-coverage` or `--print-overlapping` it also prints those lists. Each takes + `functions`, `modules` or `pair`; the `+` forms (`functions+`, `pair+`) also show + the function signature when `--label args` is used. + +What `--dump-info` does: + +- Writes `available`, `instrumented`, `excluded` and `overlapping` reports as + `.txt` and `.json` to `rocprofsys--output/instrumentation/`, where `` is + the target (runtime) or the `-o` output name (rewrite, e.g. `rocprofsys-app.inst-output`). +- `--print-format json` (or `txt`) limits the formats. Set `ROCPROFSYS_OUTPUT_PATH` + to move the directory. `--print-dir` had no effect in either build tested. +- The `.txt` files list address, address range, instruction count, linkage, + visibility, module and function. The `.json` files add a per-function `heuristics` + block (see "Why is a function missing?"). +- Without `--simulate`, `--dump-info` still produces the rewritten binary as usual. + +Workflow: dry run, read `instrumented` and `excluded`, adjust the regexes or +thresholds, repeat, then drop `--simulate`. + +## Granularity + +A function is instrumented only if it passes **both** size checks (verified): + +- at least `--min-instructions` instructions (default 1024), and +- an address range of at least `--min-address-range` bytes (default 4 x the + instruction minimum, so 4096). + +Interactions that surprise people: + +- Giving only `-i N` sets the address-range minimum to 0 (only instruction count + matters), and giving only `-r N` sets the instruction minimum to 0. Giving both + applies both. `-i 0` or `-r 0` therefore disables both size checks. It does not + instrument every function: a function must still have at least 2 instructions, be + instrumentable by Dyninst, and pass the exclude, overlap, trap, linkage and visibility + checks, and it is never taken from the profiler's own or system modules. +- The environment variable `ROCPROFSYS_DEFAULT_MIN_INSTRUCTIONS` changes the default + instruction minimum, and the default range with it. +- Functions that contain loops use `--min-instructions-loop` and + `--min-address-range-loop`, which follow the non-loop values unless set. These + options decide whether a function with a loop is instrumented at all. +- `-l` / `--instrument-loops` additionally instruments the loops inside instrumented + functions. In a profile each loop shows as a child entry named with its source + range, for example `loop_a [{449,54}-{449,83}]`, and the function's own self time + drops accordingly. + +Example from a small test program (instruction counts: 7, 16, 79, 429, 3657): + +| Options | Instrumented functions | +| --- | --- | +| default | only the 3657-instruction function | +| `-i 100` | the 429 and 3657 functions | +| `-i 20` | 79, 429 and 3657 | +| `-i 0` | all five | +| `-i 50 -r 1000` | 429 and 3657 (the 79-instruction function is only 278 bytes) | + +More detail means more overhead and a bigger trace. Lower the thresholds or add `-l` +to see inside hot code; raise them, restrict to modules, or use `--exe-only` to cut +overhead. Other gates (`--dynamic-callsites`, `--traps`, `--loop-traps`, +`--allow-overlapping`, `--linkage`, `--visibility`) are described in `--help`. + +## Why is a function missing, or why is there too much? + +Missing function: + +1. List what was skipped: `--simulate --print-excluded pair`, or `--dump-info --simulate`. +2. Open `excluded.json`, find the function and read its `heuristics` block + (`should_instrument: false` is the verdict). The fields come in two kinds, and the + cause is a different value in each. + - **Capability fields: false is the cause.** `is_instrumentable`, + `can_instrument_entry` and `can_instrument_exit` false mean Dyninst has no usable + entry or exit point for the function. No option fixes that, so accept the gap or + instrument its callers. + - **Constraint fields: true is the cause.** + - `is_address_range_constrained`, `is_num_instructions_constrained` (and the + `is_loop_*` variants): too small. Lower `-i`/`-r`, or add `-I`. + - `is_user_excluded`: one of your `-E`/`-ME` patterns matches it. + - `is_user_restricted`: a `-R`/`-MR` pattern did not match it. + - `is_overlapping_constrained`: needs `--allow-overlapping`. + - `is_entry_trap_constrained`, `is_exit_trap_constrained`: needs `--traps`. + - `is_linkage_constrained`, `is_visibility_constrained`: see `--linkage`, `--visibility`. + - `is_internal_constrained`: it belongs to the profiler's own libraries. + - `is_module_constrained`: its module is one the profiler never instruments + (system library, Dyninst, or its own libraries). + - `is_routine_constrained`: its name is one the profiler never instruments. + - **No field at all:** a function with fewer than 2 instructions is always skipped. + If every capability field is true and no constraint field is true, check + `num_instructions` (verified: a 1-instruction function stays excluded even with + `-i 0`). +3. If the function is in a shared library, remember binary rewrite never sees it. + +Too much data or too slow: raise `-i`, restrict to the modules you care about +(`-MR`), exclude noisy ones (`-ME`), drop `-l`, or add `--exe-only`. + +## Libraries, RPATH and troubleshooting + +Rewriting a shared library so a normal executable loads the instrumented copy: + +```bash +rocprof-sys-instrument -o ./app.inst -- ./app +mkdir inst && rocprof-sys-instrument -o inst/libfoo.so -- ./libfoo.so +LD_LIBRARY_PATH=$PWD/inst:$LD_LIBRARY_PATH rocprof-sys-run -- ./app.inst +ldd ./app.inst | grep libfoo # must resolve into inst/ +``` + +- Keep the same file name for the rewritten library (`libfoo.so`, or the soname + that `ldd` shows) and write it to a different directory. +- **RPATH vs RUNPATH:** the loader searches `DT_RPATH` directories before + `LD_LIBRARY_PATH`, so if the original library is in an RPATH directory it is loaded + instead of your rewritten copy. + If the RPATH directories do not contain the library, `LD_LIBRARY_PATH` is still used, + and a binary with `RUNPATH` searches `LD_LIBRARY_PATH` first. + Check with `objdump -p | egrep 'RPATH|RUNPATH'` and `ldd`. Only patch the + binary when `ldd` shows the original library. Relink with `-Wl,--enable-new-dtags`, + or change the entry with `patchelf`: + `patchelf --remove-rpath ` makes `LD_LIBRARY_PATH` win, and + `patchelf --set-rpath ` writes a `RUNPATH`, so it does too. Adding + `--force-rpath` keeps a `DT_RPATH`, which brings the problem back. +- **Library as a runtime target:** `rocprof-sys-instrument -- ./libfoo.so` prints + "is not executable ... Switching to binary rewrite mode and assuming + '--simulate --all-functions'", so you get a dry-run approximation and no run. + `-f/--force` is only meant for executables whose name looks like a library; + forcing it on a real `.so` aborted inside Dyninst in testing. +- **Main function not found:** pass it with `-m ` (or `--main-function`). +- **"Failed to transform trace ... in function ''"** (upstream guidance, not + reproduced): add `-E ''`. Functions that share code with it can still pull it + in; exclude those too, or exclude the whole module with `-ME`. + +## Running the result and embedding defaults + +- Run a rewritten binary with `rocprof-sys-run -- ./app.inst args`; the instrumented + functions then appear in the profile or trace like any other. +- `--env VAR=VALUE ...` embeds defaults in the new binary so a later session does not + need to export them. A variable set at run time still overrides the embedded value + (verified with `ROCPROFSYS_PROFILE`). +- `--label args file line return` adds that information to function names (for + example `big_a(long int)` with `args`). +- `-M sampling` instruments only `main` to start and stop the sampler (verified), and + is deprecated upstream; for sampling use `rocprof-sys-sample` or a preset. + +## Common Mistakes + +| Mistake | Fix | +| --- | --- | +| Expecting binary rewrite to instrument shared libraries | It only covers the target's own code. Use runtime mode, or rewrite the library too. | +| Using `-I` to narrow the selection | `-I` only adds. Narrow with `-R`/`-MR`, or exclude with `-E`/`-ME`. | +| Writing `-I tiny` and expecting an exact match | Patterns are unanchored substrings. Use `^tiny_a$`. | +| Combining `-MR` and `-R` | The function restriction is ignored. Use one, or `-MR` plus `-E`. | +| Lowering only `-i` and expecting range to still apply | One option given sets the other to 0. Pass both to apply both. | +| Passing app arguments after the target in a rewrite | They are ignored; pass them to `rocprof-sys-run -- ./app.inst args`. | +| Putting a rewritten library next to a binary with `DT_RPATH` | `LD_LIBRARY_PATH` is ignored. Check with `objdump -p` and relink or patch the rpath. | +| Expecting `--simulate -o` to produce the binary | A dry run writes only reports; remove `--simulate`. | +| Relying on `--print-dir` | It had no effect in tested builds; use `ROCPROFSYS_OUTPUT_PATH`. | diff --git a/skills/rocprofiler-systems-instrumenting-binaries/evals/evals.json b/skills/rocprofiler-systems-instrumenting-binaries/evals/evals.json new file mode 100644 index 0000000..73ae5b2 --- /dev/null +++ b/skills/rocprofiler-systems-instrumenting-binaries/evals/evals.json @@ -0,0 +1,78 @@ +{ + "evaluations": [ + { + "expected_behavior": [ + "explains that runtime mode instruments the executable and the shared libraries it loads while binary rewrite (-o) only covers the target's own code", + "notes that binary rewrite has lower overhead and is preferred for MPI", + "shows that a rewritten binary is run with rocprof-sys-run -- ./app.inst" + ], + "id": "instrumenting-runtime-vs-rewrite", + "prompt": "What is the difference between runtime instrumentation and binary rewrite in rocprof-sys-instrument, and which should I use for my MPI application?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "uses --module-exclude (-ME) to drop libm and --module-restrict (-MR) to keep only solver.cpp", + "explains that patterns are unanchored regular expressions", + "warns that combining --module-restrict with --function-restrict ignores the function restriction" + ], + "id": "instrumenting-exclude-a-library", + "prompt": "How do I stop rocprof-sys-instrument from instrumenting libm, and separately how do I limit it to only the functions in solver.cpp?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "defines a function as one symbol and a module as its source file when debug info exists or the binary or library otherwise", + "shows how to list them with --print-available modules or pair" + ], + "id": "instrumenting-function-vs-module", + "prompt": "In rocprof-sys-instrument what is a module as opposed to a function?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "recommends --simulate, optionally with --print-instrumented or --dump-info", + "states that a dry run neither runs the application nor writes the rewritten binary", + "says --dump-info reports go to rocprofsys--output/instrumentation/" + ], + "id": "instrumenting-dry-run", + "prompt": "Can I preview which functions rocprof-sys-instrument will instrument without actually running my app?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "explains the default minimum of 1024 instructions and the matching address-range minimum", + "recommends lowering --min-instructions (for example -i 0 or a smaller value) and using --instrument-loops", + "notes that more detail means more overhead" + ], + "id": "instrumenting-min-instructions-granularity", + "prompt": "Only a handful of my functions are instrumented. How do I instrument smaller functions and loops with rocprof-sys-instrument?", + "skill_should_trigger": true + }, + { + "expected_behavior": [ + "suggests --print-excluded or --dump-info with --simulate", + "points to the heuristics block in excluded.json to see which constraint applied", + "mentions that binary rewrite does not instrument shared libraries" + ], + "id": "instrumenting-function-missing-from-trace", + "prompt": "My function is missing from the rocprof-sys trace after a binary rewrite. How do I find out why it was excluded?", + "skill_should_trigger": true + }, + { + "id": "instrumenting-attach-to-running-process", + "prompt": "I have a long-running training process already started. How do I attach rocprof-sys to it?", + "skill_should_trigger": false + }, + { + "id": "instrumenting-which-preset-for-training", + "prompt": "Which rocprof-sys preset should I use for an AI training job on GPUs?", + "skill_should_trigger": false + }, + { + "id": "instrumenting-list-available-counters", + "prompt": "What hardware counters can I collect on this machine with rocprof-sys?", + "skill_should_trigger": false + } + ] +} diff --git a/skills/rocprofiler-systems-instrumenting-binaries/evals/machine.yml b/skills/rocprofiler-systems-instrumenting-binaries/evals/machine.yml new file mode 100644 index 0000000..4d7efd4 --- /dev/null +++ b/skills/rocprofiler-systems-instrumenting-binaries/evals/machine.yml @@ -0,0 +1,2 @@ +os: [Linux] +labels: [mi300x, gpu, rocm] diff --git a/skills/rocprofiler-systems-instrumenting-binaries/skill-card.md b/skills/rocprofiler-systems-instrumenting-binaries/skill-card.md new file mode 100644 index 0000000..4b1c85b --- /dev/null +++ b/skills/rocprofiler-systems-instrumenting-binaries/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Guides users of rocprofiler-systems through instrumenting an application with `rocprof-sys-instrument`: runtime vs binary rewrite, including and excluding functions and modules, dry runs, and granularity tuning. + +## Owner + +rocprofiler-systems (a project in [rocm-systems](https://github.com/ROCm/rocm-systems/)) + +## License + +MIT