From 5a70c379dce8543208d41b738e770bb760d8530a Mon Sep 17 00:00:00 2001 From: Douglas McLelland Date: Wed, 21 Oct 2026 20:17:47 +0200 Subject: [PATCH 1/2] feat(brainchip_utils): save benchmark results to file, redraw the plots from it The Akida 1 VWW benchmark now writes the numbers behind its plots to a JSON results file (full-model timing and power per map mode, each mode's mapping, per-layer timing and sparsity), and draws the figures from that file, so they can be redrawn later without hardware: python -m brainchip_utils.benchmark_results docs/ref_benchmark_results.json Power is stored as the plot resamples it (one trace per repeat on a shared grid), not as raw sensor readings. The plot helpers accept a saved mapping or power trace as well as a live model or raw readings, and plot_utils no longer imports akida, so existing callers are unchanged. Refs HUB-40 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DnyD76hgk85WiSBvfGpN6w --- akida1/model_zoo/vww/vww_benchmark.py | 48 +++++--- brainchip_utils/benchmark_results.py | 170 ++++++++++++++++++++++++++ brainchip_utils/plot_utils.py | 163 +++++++++++++++++------- test/test_benchmark_results.py | 139 +++++++++++++++++++++ 4 files changed, 455 insertions(+), 65 deletions(-) create mode 100644 brainchip_utils/benchmark_results.py create mode 100644 test/test_benchmark_results.py diff --git a/akida1/model_zoo/vww/vww_benchmark.py b/akida1/model_zoo/vww/vww_benchmark.py index fd5d7e4..84e56bb 100644 --- a/akida1/model_zoo/vww/vww_benchmark.py +++ b/akida1/model_zoo/vww/vww_benchmark.py @@ -6,6 +6,12 @@ Runs a per-layer timing benchmark on an Akida VWW model, prints a summary table, and generates a summary plot. +The results behind the plots are saved to benchmark_results.json (with +--save-metrics, docs/ref_benchmark_results.json), so the plots can be redrawn +later without hardware: + + python -m brainchip_utils.benchmark_results docs/ref_benchmark_results.json + Example ------- python vww_benchmark.py -l pretrained_models/akidanet_vww.fbz @@ -22,7 +28,8 @@ from vww_data import get_samples from brainchip_utils.hardware_utils import get_mapping_stats, get_akida_device, per_layer_benchmark, full_model_benchmark -from brainchip_utils.plot_utils import plot_full_model_results, plot_per_layer_results, pretty_print_sparsity +from brainchip_utils.plot_utils import mapping_summary, pretty_print_sparsity +from brainchip_utils.benchmark_results import save_benchmark_results, plot_benchmark_results if __name__ == '__main__': @@ -114,6 +121,8 @@ num_nps, num_passes, num_sequences = get_mapping_stats(ak_model) full_results[mm]['num_nps'] = num_nps full_results[mm]['num_passes'] = num_passes + full_results[mm]['num_sequences'] = num_sequences + full_results[mm]['mapping'] = mapping_summary(ak_model) print(f' Mapping: {num_nps} NP(s), {num_passes} pass(es), {num_sequences} sequence(s)') if num_sequences>1: print('WARNING: note, model not completely mapped to hardware') @@ -133,27 +142,28 @@ per_layer_results = per_layer_benchmark(ak_model, device, samples, repeats=NUM_SAMPLES) # ------------------------------------------------------------------------- - # Plots + # Results file and plots # ------------------------------------------------------------------------- - # Map without hw_only so ak_model.sequences is available for plot_mapping + # Map without hw_only so ak_model.sequences is available for the mapping plot ak_model.map(device, mode=akida.MapMode.Minimal) - perlayer_savepath = 'benchmark_results_layers.png' - if args.save_metrics: - perlayer_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_'+perlayer_savepath) - plot_per_layer_results(per_layer_results, ak_model, sparsity_dict, - model_name=args.loadmodel, - example='Visual Wake Words', - savepath=perlayer_savepath) - print('\nPer-layer results plot saved to ' + str(perlayer_savepath)) - - full_savepath = 'benchmark_results_full.png' + + # Save everything the plots need, then plot from what was saved, so the + # figures can be redrawn later from the file alone (see the docstring above). + outdir, prefix = pathlib.Path('.'), '' if args.save_metrics: - full_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_'+full_savepath) - plot_full_model_results(full_results, ak_model, device, - model_name=args.loadmodel, - example='Visual Wake Words', - savepath=full_savepath) - print('Full model results plot saved to ' + str(full_savepath)) + outdir, prefix = pathlib.Path(__file__).parent / 'docs', 'ref_' + results_path = outdir / f'{prefix}benchmark_results.json' + results = save_benchmark_results( + results_path, full_results, per_layer_results, mapping_summary(ak_model), sparsity_dict, + model_name=args.loadmodel, example='Visual Wake Words', + figures={'full': f'{prefix}benchmark_results_full.png', + 'layers': f'{prefix}benchmark_results_layers.png'}, + device=str(device.desc), akida_version=akida.__version__, clock_hz=CLOCK_FREQUENCY) + print(f'\nBenchmark results saved to {results_path}') + + plot_benchmark_results(results, outdir) + for name in results['figures'].values(): + print(f'Plot saved to {outdir / name}') if args.save_metrics: diff --git a/brainchip_utils/benchmark_results.py b/brainchip_utils/benchmark_results.py new file mode 100644 index 0000000..dc2a832 --- /dev/null +++ b/brainchip_utils/benchmark_results.py @@ -0,0 +1,170 @@ +"""Benchmark results files: save what the benchmark plots need, and redraw them from it. + +A hardware benchmark takes from minutes to hours, but its figures only need the +numbers it measured. save_benchmark_results() writes those numbers to a JSON +file next to the figures: full-model timing and power per map mode, the +hardware mapping of each mode, per-layer timing and per-layer sparsity. The +figures can then be redrawn from the file, with no device, model or akida +install, for instance after a change to how they look: + + python -m brainchip_utils.benchmark_results akida1/model_zoo/vww/docs/ref_benchmark_results.json + +Power is stored as plot_utils.power_trace() resamples it for plotting (one trace +per repeat on a shared time grid), not as raw sensor readings. +""" +import argparse +import json +import pathlib + +import numpy as np + +from brainchip_utils.plot_utils import (figure_title, plot_full_model_results, + plot_per_layer_results, power_trace) + +FORMAT_VERSION = 1 + +# Full-model result keys stored as they are (all scalars) +_FULL_MODEL_KEYS = ('mean_inf_clk', 'std_inf_clk', 'mean_clk_ms', 'mean_inf_ms', 'std_inf_ms', + 'n_inferences', 'num_nps', 'num_passes', 'num_sequences', 'projected_clk_ms') +_POWER_KEYS = ('avg_floor_mw', 'avg_total_mw', 'avg_dynamic_mw', 'avg_energy_mj', + 'avg_dynamic_energy_mj') + + +def _round(values, decimals): + """Round a number or nested list of numbers, to keep the file compact.""" + return np.round(np.asarray(values, dtype=float), decimals).tolist() + + +def _ints(values): + """Whole-number arrays (clock counts, nanoseconds) as nested lists of ints.""" + return np.rint(np.asarray(values, dtype=float)).astype(np.int64).tolist() + + +def _full_model_entry(result): + entry = {k: result[k] for k in _FULL_MODEL_KEYS if k in result} + entry['mapping'] = result['mapping'] + power = result.get('power') + if power is not None: + trace = power_trace(power) + if trace is not None: + trace = {'t_s': _round(trace['t_s'], 5), + 'repeats_mw': _round(trace['repeats_mw'], 2), + 'inf_duration_s': _round(trace['inf_duration_s'], 6), + 'inferences_per_repeat': trace['inferences_per_repeat']} + power = {**{k: power[k] for k in _POWER_KEYS}, 'trace': trace} + entry['power'] = power + return entry + + +def _per_layer_entry(per_layer_results, mapping, sparsity_dict): + return { + 'layer_names': list(per_layer_results['layer_names']), + 'per_layer_clocks': np.asarray(per_layer_results['per_layer_clocks']).tolist(), + 'per_layer_ms': np.asarray(per_layer_results['per_layer_ms']).tolist(), + # Raw (num_layers, repeats) arrays, behind the per-layer spread + 'cumulative_clocks': _ints(per_layer_results['cumulative_clocks']), + 'cumulative_times': _ints(per_layer_results['cumulative_times']), + 'mapping': mapping, + 'sparsity': {name: float(s) for name, s in sparsity_dict.items()}, + } + + +def save_benchmark_results(path, full_results, per_layer_results, per_layer_mapping, + sparsity_dict, model_name, example, figures, show_power=True, + **info): + """Write a benchmark's results to a JSON file, and return them as loaded back. + + Plot from the returned dict (plot_benchmark_results()), so the figures drawn + now are exactly the ones the file reproduces later. + + Args: + path: Where to write the file (.json). + full_results: dict mapping mode name to the full_model_benchmark() + result, each with 'num_nps', 'num_passes' and a + 'mapping' (from plot_utils.mapping_summary()) added. + per_layer_results: Result dict from per_layer_benchmark(). + per_layer_mapping: mapping_summary() of the model as the per-layer plot + shows it. + sparsity_dict: Per-layer output sparsity, from compute_sparsity(). + model_name: Model name or path; a path is stored as its file stem. + example: Example name, shown in the figure titles. + figures: dict with the file names of the 'full' and 'layers' + figures, which plot_benchmark_results() writes next to + the results file. + show_power: Whether the full-model figure has a power row. + **info: Extra provenance stored as it is (e.g. device, + akida_version, clock_hz). + """ + results = { + 'format_version': FORMAT_VERSION, + 'example': example, + 'model_name': figure_title(model_name), + **info, + 'figures': dict(figures), + 'show_power': show_power, + 'full_model': {mode: _full_model_entry(res) for mode, res in full_results.items()}, + 'per_layer': _per_layer_entry(per_layer_results, per_layer_mapping, sparsity_dict), + } + path = pathlib.Path(path) + path.write_text(json.dumps(results, separators=(',', ':')) + '\n') + return load_benchmark_results(path) + + +def load_benchmark_results(path): + """Read a file written by save_benchmark_results().""" + results = json.loads(pathlib.Path(path).read_text()) + if results.get('format_version') != FORMAT_VERSION: + raise ValueError(f'{path}: unsupported benchmark results format ' + f'{results.get("format_version")!r} (expected {FORMAT_VERSION})') + return results + + +def plot_benchmark_results(results, outdir): + """Draw the full-model and per-layer figures from saved results. + + Args: + results: dict from save_benchmark_results() or load_benchmark_results(). + outdir: Directory to write the figures to, under the file names stored + in results['figures']. + + Returns: + (full-model figure, per-layer figure) + """ + outdir = pathlib.Path(outdir) + full_fig = plot_full_model_results(results['full_model'], + model_name=results['model_name'], + example=results['example'], + show_power=results['show_power'], + savepath=outdir / results['figures']['full']) + per_layer = results['per_layer'] + layers_fig = plot_per_layer_results(per_layer, per_layer['mapping'], per_layer['sparsity'], + model_name=results['model_name'], + example=results['example'], + savepath=outdir / results['figures']['layers']) + return full_fig, layers_fig + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Redraw benchmark figures from a saved benchmark results file') + parser.add_argument('results', nargs='+', help='Benchmark results file(s) (.json)') + parser.add_argument('-o', '--outdir', + help='Where to write the figures (default: next to each results file)') + args = parser.parse_args(argv) + + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + for path in map(pathlib.Path, args.results): + results = load_benchmark_results(path) + outdir = pathlib.Path(args.outdir) if args.outdir else path.parent + figs = plot_benchmark_results(results, outdir) + for fig in figs: + plt.close(fig) + for name in results['figures'].values(): + print(f'Saved {outdir / name}') + + +if __name__ == '__main__': + main() diff --git a/brainchip_utils/plot_utils.py b/brainchip_utils/plot_utils.py index 1863d61..48b4c1e 100644 --- a/brainchip_utils/plot_utils.py +++ b/brainchip_utils/plot_utils.py @@ -3,7 +3,6 @@ import numpy as np import matplotlib.pyplot as plt from matplotlib.lines import Line2D -import akida MAPPING_COLOR = 'steelblue' @@ -15,6 +14,9 @@ INF_END_STYLE = dict(color='firebrick', ls='--', lw=1.2) FLOOR_STYLE = dict(color='gray', ls=':', lw=1.0) +# Points per repeat in a power trace, on a grid shared by every repeat of a map mode +POWER_TRACE_POINTS = 500 + def pretty_print_sparsity(sparsity_dict): col_w = max(len(k) for k in sparsity_dict) + 2 @@ -81,7 +83,7 @@ def plot_per_layer_timing(results, ax, colors=None): if colors is None: colors = ['steelblue'] * num_layers - raw_times = results['cumulative_times'] # (num_layers, repeats) in ns + raw_times = np.asarray(results['cumulative_times']) # (num_layers, repeats) in ns per_layer_raw = np.copy(raw_times) per_layer_raw[1:] = raw_times[1:] - raw_times[:-1] per_layer_ms_std = np.std(per_layer_raw / 1e6, axis=1) @@ -102,7 +104,7 @@ def plot_cumulative_timing(results, ax, colors=None): if colors is None: colors = ['seagreen'] * num_layers - cumulative_ms = np.mean(results['cumulative_times'], axis=1) / 1e6 + cumulative_ms = np.mean(np.asarray(results['cumulative_times']), axis=1) / 1e6 x = np.arange(num_layers) ax.bar(x, cumulative_ms, color=colors, edgecolor='white') @@ -112,16 +114,26 @@ def plot_cumulative_timing(results, ax, colors=None): ax.set_xticklabels(layer_names, rotation=45, ha='right', fontsize=8) -def plot_mapping(ak_model, ax, colors=None, legend=True): - """Bar chart of NPs per layer, with pass and sequence boundaries. +def mapping_summary(ak_model): + """NPs per layer in mapping order, with the pass and sequence boundaries. - Returns (has_pass_boundary, has_sequence_boundary), so a caller drawing - several panels can build one shared legend (pass legend=False). + ak_model must be mapped without hw_only=True, so that ak_model.sequences is + populated. The result is a plain dict that plot_mapping() draws and that a + benchmark results file stores, so the mapping can be plotted again without + a device. + + Returns: + dict with keys: + layer_names - list of layer names, in mapping order + layer_nps - list of NPs used by each layer + pass_ends - indices of layers that end a pass (within a sequence) + sequence_ends - indices of layers that end a sequence + The last layer of the model ends neither. """ layer_names = [] layer_nps = [] pass_ends = [] - seq_ends = [] + sequence_ends = [] for si, seq in enumerate(ak_model.sequences): for pi, hwpass in enumerate(seq.passes): @@ -131,17 +143,35 @@ def plot_mapping(ak_model, ax, colors=None, legend=True): is_last_pass = (pi == len(seq.passes) - 1) is_last_seq = (si == len(ak_model.sequences) - 1) if not (is_last_pass and is_last_seq): - pass_end_idx = len(layer_names) - 0.5 if is_last_pass: - seq_ends.append(pass_end_idx) + sequence_ends.append(len(layer_names) - 1) else: - pass_ends.append(pass_end_idx) + pass_ends.append(len(layer_names) - 1) + + return {'layer_names': layer_names, 'layer_nps': layer_nps, + 'pass_ends': pass_ends, 'sequence_ends': sequence_ends} + + +def plot_mapping(mapping, ax, colors=None, legend=True): + """Bar chart of NPs per layer, with pass and sequence boundaries. + + mapping is either a mapped akida.Model (see mapping_summary()) or the dict + mapping_summary() returns. + + Returns (has_pass_boundary, has_sequence_boundary), so a caller drawing + several panels can build one shared legend (pass legend=False). + """ + if hasattr(mapping, 'sequences'): + mapping = mapping_summary(mapping) + layer_names = mapping['layer_names'] + pass_ends = [i + 0.5 for i in mapping['pass_ends']] + seq_ends = [i + 0.5 for i in mapping['sequence_ends']] if colors is None: colors = MAPPING_COLOR x = np.arange(len(layer_names)) - ax.bar(x, layer_nps, color=colors, edgecolor='white') + ax.bar(x, mapping['layer_nps'], color=colors, edgecolor='white') for pos in pass_ends: ax.axvline(x=pos, **PASS_STYLE) @@ -170,28 +200,34 @@ def _fit_layer_labels(ax, max_slanted=20): plt.setp(labels, fontsize=11) -def plot_power_trace(power_data, ax, legend=True, stats=True): - """Plot per-repeat power traces with the mean overlaid. +def power_trace(power_data, num_points=POWER_TRACE_POINTS): + """Per-repeat power traces on a common time grid, t=0 at inference start. - Each repeat is time-normalised so that t=0 coincides with the start of the - inference window. Individual repeat traces are drawn with transparency; - the mean trace is drawn on top. + This is the data plot_power_trace() draws, and what a benchmark results + file stores instead of the raw sensor readings (which run to hundreds of + thousands of points per map mode). Args: - power_data: The 'power' dict returned by full_model_benchmark, which - must contain 'readings' and 'repeat_meta'. - legend: Draw a legend on the axes. Pass False when the figure has a - shared legend. - stats: Draw the time/power/energy summary box on the axes. + power_data: The 'power' dict returned by full_model_benchmark (with + 'readings' and 'repeat_meta'), or a dict that already holds + a 'trace', which is returned as it is. + num_points: Points per repeat on the common time grid. + + Returns: + dict with keys: + t_s - common time grid (s), t=0 at inference start + repeats_mw - one power trace (mW) per repeat on that grid + inf_duration_s - length of each repeat's inference run (s) + inferences_per_repeat - forward calls in each inference run + or None if no repeat has enough readings to draw. """ + if 'trace' in power_data: + return power_data['trace'] + readings_arr = np.array(power_data['readings']) abs_times = readings_arr[:, 0] powers = readings_arr[:, 1] * readings_arr[:, 2] # V × mA = mW - repeat_meta = power_data['repeat_meta'] - avg_floor = power_data['avg_floor_mw'] - avg_dyn = power_data['avg_dynamic_mw'] - avg_dyn_e = power_data['avg_dynamic_energy_mj'] # Build one time-normalised power trace per repeat repeat_traces = [] # list of (t_rel, p) arrays, t=0 at inference start @@ -206,29 +242,56 @@ def plot_power_trace(power_data, ax, legend=True, stats=True): repeat_traces.append((t_rel, p_sel)) if not repeat_traces: - return + return None # Common time grid spanning all repeats t_grid = np.linspace( min(tr[0][0] for tr in repeat_traces), max(tr[0][-1] for tr in repeat_traces), - 500, + num_points, ) + return { + 't_s': t_grid.tolist(), + 'repeats_mw': [np.interp(t_grid, t_rel, p_sel).tolist() + for t_rel, p_sel in repeat_traces], + 'inf_duration_s': [meta['inf_timestamps'][-1] - meta['inf_timestamps'][0] + for meta in repeat_meta], + 'inferences_per_repeat': len(repeat_meta[0]['inf_timestamps']) - 1, + } + + +def plot_power_trace(power_data, ax, legend=True, stats=True): + """Plot per-repeat power traces with the mean overlaid. + + Each repeat is time-normalised so that t=0 coincides with the start of the + inference window. Individual repeat traces are drawn with transparency; + the mean trace is drawn on top. + + Args: + power_data: The 'power' dict returned by full_model_benchmark (with + 'readings' and 'repeat_meta'), or its saved form, with a + 'trace' from power_trace() in their place. + legend: Draw a legend on the axes. Pass False when the figure has a + shared legend. + stats: Draw the time/power/energy summary box on the axes. + """ + trace = power_trace(power_data) + if trace is None: + return + + avg_floor = power_data['avg_floor_mw'] + avg_dyn = power_data['avg_dynamic_mw'] + avg_dyn_e = power_data['avg_dynamic_energy_mj'] - interp_powers = [] - for t_rel, p_sel in repeat_traces: - p_interp = np.interp(t_grid, t_rel, p_sel) - interp_powers.append(p_interp) + t_grid = np.asarray(trace['t_s']) + interp_powers = np.asarray(trace['repeats_mw']) + for p_interp in interp_powers: ax.plot(t_grid, p_interp, **REPEAT_STYLE) ax.plot(t_grid, np.mean(interp_powers, axis=0), label='Mean power', **MEAN_STYLE) - mean_inf_duration = np.mean([ - meta['inf_timestamps'][-1] - meta['inf_timestamps'][0] - for meta in repeat_meta - ]) - n_inf = len(repeat_meta[0]['inf_timestamps']) - 1 - avg_inf_time_ms = mean_inf_duration / n_inf * 1000 + mean_inf_duration = np.mean(trace['inf_duration_s']) + avg_inf_time_ms = mean_inf_duration / trace['inferences_per_repeat'] * 1000 ax.axvline(0, label='Inference start', **INF_START_STYLE) ax.axvline(mean_inf_duration, label='Inference end', **INF_END_STYLE) @@ -277,8 +340,8 @@ def _mode_summary(result): return ' · '.join(parts) -def plot_full_model_results(full_results, ak_model, device, model_name=None, savepath=None, - show_power=True, example=None): +def plot_full_model_results(full_results, ak_model=None, device=None, model_name=None, + savepath=None, show_power=True, example=None): """Plot power trace and hardware mapping for each map mode. Layout: rows = plot type (power, mapping); cols = map mode. If show_power is @@ -290,9 +353,12 @@ def plot_full_model_results(full_results, ak_model, device, model_name=None, sav Args: full_results: dict mapping mode name (e.g. 'Minimal', 'AllNps') to the result dict returned by full_model_benchmark(), optionally - with 'num_nps' and 'num_passes' added. - ak_model: akida.Model — re-mapped internally for each mode. Left in - the state of the last mode after this call returns. + with 'num_nps' and 'num_passes' added. A result that holds + a 'mapping' (from mapping_summary()) is plotted from it, as + results loaded from a benchmark results file are. + ak_model: akida.Model, re-mapped for each mode without a 'mapping'. + Left in the state of the last mode re-mapped. Not needed + when every result holds a 'mapping'. device: Akida device passed to ak_model.map(). model_name: Model name or path; a path is shown as its file stem. savepath: If provided, save the figure to this path (PNG). @@ -314,7 +380,11 @@ def plot_full_model_results(full_results, ak_model, device, model_name=None, sav has_pass = has_seq = False for i, mode_name in enumerate(modes): result = full_results[mode_name] - ak_model.map(device, mode=getattr(akida.MapMode, mode_name)) + mapping = result.get('mapping') + if mapping is None: + import akida + ak_model.map(device, mode=getattr(akida.MapMode, mode_name)) + mapping = ak_model ax_top = axs[0, i] ax_map = axs[nrows - 1, i] @@ -330,7 +400,7 @@ def plot_full_model_results(full_results, ak_model, device, model_name=None, sav else: _power_placeholder(axs[0, i]) - p, q = plot_mapping(ak_model, ax_map, legend=False) + p, q = plot_mapping(mapping, ax_map, legend=False) has_pass, has_seq = has_pass or p, has_seq or q ax_map.yaxis.label.set_fontsize(12) ax_map.tick_params(axis='y', labelsize=11) @@ -384,7 +454,8 @@ def plot_per_layer_results(per_layer_results, ak_model, sparsity_dict, Args: per_layer_results: Result dict from per_layer_benchmark(). - ak_model: akida.Model with sequences populated by a prior map() call. + ak_model: akida.Model with sequences populated by a prior map() call, + or the dict mapping_summary() returns for it. sparsity_dict: dict[layer_name, float] of output sparsity per layer, as returned by compute_sparsity(). Shifted by one position to derive per-layer input sparsity (first layer = 0). diff --git a/test/test_benchmark_results.py b/test/test_benchmark_results.py new file mode 100644 index 0000000..e98b66c --- /dev/null +++ b/test/test_benchmark_results.py @@ -0,0 +1,139 @@ +"""Unit tests for brainchip_utils.benchmark_results (no hardware or model files required).""" +import json +import subprocess +import sys +from types import SimpleNamespace + +import matplotlib +import numpy as np +import pytest + +matplotlib.use('Agg') + +from brainchip_utils.benchmark_results import (load_benchmark_results, main, # noqa: E402 + plot_benchmark_results, save_benchmark_results) +from brainchip_utils.plot_utils import (mapping_summary, plot_full_model_results, # noqa: E402 + plot_per_layer_results) + + +def _fake_power(level_mw, rng, repeats=3): + """Raw power data in the shape full_model_benchmark() returns: floor, inference, floor.""" + readings, repeat_meta = [], [] + for rr in range(repeats): + t0 = 10.0 * rr + t = t0 + np.arange(0.0, 3.0, 0.001) + power = 110 + level_mw * ((t >= t0 + 1.0) & (t <= t0 + 2.0)) + rng.normal(0, 0.5, t.size) + readings += [(ti, 0.8, pi / 0.8) for ti, pi in zip(t, power)] + repeat_meta.append({'floor_pre_start': t0, 'floor_post_end': t0 + 3.0, + 'inf_timestamps': list(np.linspace(t0 + 1.0, t0 + 2.0, 101))}) + return {'readings': readings, 'repeat_meta': repeat_meta, + 'avg_floor_mw': 110.0, 'avg_total_mw': 110.0 + level_mw, 'avg_dynamic_mw': level_mw, + 'avg_energy_mj': 1.0, 'avg_dynamic_energy_mj': level_mw / 100} + + +def _fake_model(nps_per_pass_per_seq): + """A mapped-model stand-in: one list of passes (lists of NPs per layer) per sequence.""" + count = iter(range(1000)) + layer = lambda n: SimpleNamespace(name=f'layer_{next(count)}', # noqa: E731 + mapping=SimpleNamespace(nps=[0] * n)) + return SimpleNamespace(sequences=[ + SimpleNamespace(passes=[SimpleNamespace(layers=[layer(n) for n in p]) for p in seq]) + for seq in nps_per_pass_per_seq]) + + +def _fake_benchmark(rng, with_power=True): + model = _fake_model([[[1, 2], [3]], [[1]]]) + full_results = {} + for mode, level in [('Minimal', 20.0), ('AllNps', 40.0)]: + full_results[mode] = { + 'map_mode': object(), # an akida.MapMode in a real run; not stored + 'mean_inf_clk': 1.2e6, 'std_inf_clk': 1e3, 'mean_clk_ms': 3.0, 'mean_inf_ms': 3.1, + 'std_inf_ms': 0.1, 'n_inferences': 300, 'num_nps': 7, 'num_passes': 3, + 'num_sequences': 2, 'mapping': mapping_summary(model), + 'power': _fake_power(level, rng) if with_power else None} + clocks = np.cumsum(rng.integers(1000, 2000, size=(4, 50)), axis=0).astype(float) + per_layer_clocks = np.copy(clocks) + per_layer_clocks[1:] = clocks[1:] - clocks[:-1] + per_layer = {'layer_names': [f'layer_{i}' for i in range(4)], + 'per_layer_clocks': per_layer_clocks.mean(axis=1), + 'per_layer_ms': per_layer_clocks.mean(axis=1) / 400e6 * 1000, + 'cumulative_clocks': clocks, 'cumulative_times': clocks * 2.5} + sparsity = {f'layer_{i}': 0.1 * i for i in range(4)} + return model, full_results, per_layer, sparsity + + +def _save(tmp_path, rng, **kwargs): + model, full_results, per_layer, sparsity = _fake_benchmark(rng, **kwargs) + results = save_benchmark_results( + tmp_path / 'results.json', full_results, per_layer, mapping_summary(model), sparsity, + model_name='./models/x.fbz', example='Example', + figures={'full': 'full.png', 'layers': 'layers.png'}, device='PCIe/AKD1500/16MB/0') + return model, full_results, per_layer, sparsity, results + + +def test_mapping_summary_marks_pass_and_sequence_ends(): + summary = mapping_summary(_fake_model([[[1, 2], [3]], [[1]]])) + assert summary == {'layer_names': ['layer_0', 'layer_1', 'layer_2', 'layer_3'], + 'layer_nps': [1, 2, 3, 1], 'pass_ends': [1], 'sequence_ends': [2]} + + +def test_saved_file_is_plain_json_without_raw_power_readings(tmp_path): + *_, results = _save(tmp_path, np.random.default_rng(0)) + stored = json.loads((tmp_path / 'results.json').read_text()) + assert stored == results + assert stored['model_name'] == 'x' and stored['device'] == 'PCIe/AKD1500/16MB/0' + power = stored['full_model']['Minimal']['power'] + assert 'readings' not in power and 'repeat_meta' not in power + assert len(power['trace']['repeats_mw']) == 3 + assert 'map_mode' not in stored['full_model']['Minimal'] + assert np.shape(stored['per_layer']['cumulative_times']) == (4, 50) + + +def test_plots_from_saved_results_match_plots_from_live_results(tmp_path): + model, full_results, per_layer, sparsity, results = _save(tmp_path, np.random.default_rng(0)) + full_fig, layers_fig = plot_benchmark_results(results, tmp_path) + assert (tmp_path / 'full.png').exists() and (tmp_path / 'layers.png').exists() + + live_full = plot_full_model_results(full_results, model_name='./models/x.fbz', + example='Example') + live_layers = plot_per_layer_results(per_layer, model, sparsity, + model_name='./models/x.fbz', example='Example') + for saved_fig, live_fig in [(full_fig, live_full), (layers_fig, live_layers)]: + assert saved_fig._suptitle.get_text() == live_fig._suptitle.get_text() + for saved_ax, live_ax in zip(saved_fig.axes, live_fig.axes, strict=True): + for saved_line, live_line in zip(saved_ax.get_lines(), live_ax.get_lines(), strict=True): + np.testing.assert_allclose(np.asarray(saved_line.get_ydata(), dtype=float), + np.asarray(live_line.get_ydata(), dtype=float), atol=0.01) + assert ([p.get_height() for p in saved_ax.patches] == + pytest.approx([p.get_height() for p in live_ax.patches])) + assert ([t.get_text() for t in saved_ax.get_xticklabels()] == + [t.get_text() for t in live_ax.get_xticklabels()]) + + +def test_results_without_power_redraw_without_a_power_row(tmp_path): + *_, results = _save(tmp_path, np.random.default_rng(0), with_power=False) + results['show_power'] = False + full_fig, _ = plot_benchmark_results(results, tmp_path) + assert len(full_fig.axes) == 2 # one mapping panel per mode + + +def test_command_line_redraws_next_to_the_results_file(tmp_path): + _save(tmp_path, np.random.default_rng(0)) + for name in ('full.png', 'layers.png'): + (tmp_path / name).unlink(missing_ok=True) + main([str(tmp_path / 'results.json')]) + assert (tmp_path / 'full.png').exists() and (tmp_path / 'layers.png').exists() + + +def test_unknown_format_version_is_rejected(tmp_path): + path = tmp_path / 'results.json' + path.write_text(json.dumps({'format_version': 99})) + with pytest.raises(ValueError, match='unsupported'): + load_benchmark_results(path) + + +def test_redrawing_does_not_need_akida(): + # Block the akida import, as on a machine without it installed + code = ("import sys; sys.modules['akida'] = None; " + "import brainchip_utils.benchmark_results") + subprocess.run([sys.executable, '-c', code], check=True) From fa84b55921e33bf05282bbc7c9cb7dc2b0208166 Mon Sep 17 00:00:00 2001 From: Douglas McLelland Date: Wed, 21 Oct 2026 20:26:16 +0200 Subject: [PATCH 2/2] fix(brainchip_utils): store skipped per-layer timings as null in benchmark results NaN timings (a layer the per-layer benchmark can't measure on its own, as for models with skip connections) are written as JSON null and read back as NaN, so the file stays strict JSON. Refs HUB-40 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DnyD76hgk85WiSBvfGpN6w --- brainchip_utils/benchmark_results.py | 34 ++++++++++++++++++++-------- brainchip_utils/plot_utils.py | 6 ++--- test/test_benchmark_results.py | 19 ++++++++++++++++ 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/brainchip_utils/benchmark_results.py b/brainchip_utils/benchmark_results.py index dc2a832..504a64b 100644 --- a/brainchip_utils/benchmark_results.py +++ b/brainchip_utils/benchmark_results.py @@ -30,14 +30,28 @@ 'avg_dynamic_energy_mj') -def _round(values, decimals): - """Round a number or nested list of numbers, to keep the file compact.""" - return np.round(np.asarray(values, dtype=float), decimals).tolist() +def _nan_to_none(values): + """Nested lists with NaN (e.g. a layer the per-layer benchmark skipped) as None, + which JSON stores as null. The plot helpers read null back as NaN.""" + if isinstance(values, list): + return [_nan_to_none(v) for v in values] + return None if isinstance(values, float) and np.isnan(values) else values + + +def _floats(values, decimals=None): + """A number or nested list of numbers, optionally rounded to keep the file compact.""" + values = np.asarray(values, dtype=float) + if decimals is not None: + values = np.round(values, decimals) + return _nan_to_none(values.tolist()) def _ints(values): """Whole-number arrays (clock counts, nanoseconds) as nested lists of ints.""" - return np.rint(np.asarray(values, dtype=float)).astype(np.int64).tolist() + values = np.asarray(values, dtype=float) + ints = np.rint(np.nan_to_num(values)).astype(np.int64).astype(object) + ints[np.isnan(values)] = None + return ints.tolist() def _full_model_entry(result): @@ -47,9 +61,9 @@ def _full_model_entry(result): if power is not None: trace = power_trace(power) if trace is not None: - trace = {'t_s': _round(trace['t_s'], 5), - 'repeats_mw': _round(trace['repeats_mw'], 2), - 'inf_duration_s': _round(trace['inf_duration_s'], 6), + trace = {'t_s': _floats(trace['t_s'], 5), + 'repeats_mw': _floats(trace['repeats_mw'], 2), + 'inf_duration_s': _floats(trace['inf_duration_s'], 6), 'inferences_per_repeat': trace['inferences_per_repeat']} power = {**{k: power[k] for k in _POWER_KEYS}, 'trace': trace} entry['power'] = power @@ -59,8 +73,8 @@ def _full_model_entry(result): def _per_layer_entry(per_layer_results, mapping, sparsity_dict): return { 'layer_names': list(per_layer_results['layer_names']), - 'per_layer_clocks': np.asarray(per_layer_results['per_layer_clocks']).tolist(), - 'per_layer_ms': np.asarray(per_layer_results['per_layer_ms']).tolist(), + 'per_layer_clocks': _floats(per_layer_results['per_layer_clocks']), + 'per_layer_ms': _floats(per_layer_results['per_layer_ms']), # Raw (num_layers, repeats) arrays, behind the per-layer spread 'cumulative_clocks': _ints(per_layer_results['cumulative_clocks']), 'cumulative_times': _ints(per_layer_results['cumulative_times']), @@ -106,7 +120,7 @@ def save_benchmark_results(path, full_results, per_layer_results, per_layer_mapp 'per_layer': _per_layer_entry(per_layer_results, per_layer_mapping, sparsity_dict), } path = pathlib.Path(path) - path.write_text(json.dumps(results, separators=(',', ':')) + '\n') + path.write_text(json.dumps(results, separators=(',', ':'), allow_nan=False) + '\n') return load_benchmark_results(path) diff --git a/brainchip_utils/plot_utils.py b/brainchip_utils/plot_utils.py index 48b4c1e..b51226d 100644 --- a/brainchip_utils/plot_utils.py +++ b/brainchip_utils/plot_utils.py @@ -77,13 +77,13 @@ def _mapping_legend_handles(has_pass, has_seq): def plot_per_layer_timing(results, ax, colors=None): layer_names = results['layer_names'] - per_layer_ms = results['per_layer_ms'] + per_layer_ms = np.asarray(results['per_layer_ms'], dtype=float) # null (saved NaN) -> NaN num_layers = len(layer_names) if colors is None: colors = ['steelblue'] * num_layers - raw_times = np.asarray(results['cumulative_times']) # (num_layers, repeats) in ns + raw_times = np.asarray(results['cumulative_times'], dtype=float) # (num_layers, repeats) in ns per_layer_raw = np.copy(raw_times) per_layer_raw[1:] = raw_times[1:] - raw_times[:-1] per_layer_ms_std = np.std(per_layer_raw / 1e6, axis=1) @@ -104,7 +104,7 @@ def plot_cumulative_timing(results, ax, colors=None): if colors is None: colors = ['seagreen'] * num_layers - cumulative_ms = np.mean(np.asarray(results['cumulative_times']), axis=1) / 1e6 + cumulative_ms = np.mean(np.asarray(results['cumulative_times'], dtype=float), axis=1) / 1e6 x = np.arange(num_layers) ax.bar(x, cumulative_ms, color=colors, edgecolor='white') diff --git a/test/test_benchmark_results.py b/test/test_benchmark_results.py index e98b66c..cf69388 100644 --- a/test/test_benchmark_results.py +++ b/test/test_benchmark_results.py @@ -110,6 +110,25 @@ def test_plots_from_saved_results_match_plots_from_live_results(tmp_path): [t.get_text() for t in live_ax.get_xticklabels()]) +def test_skipped_layers_are_stored_as_null_and_read_back_as_nan(tmp_path): + # A layer the per-layer benchmark couldn't measure on its own (e.g. an Add in + # a model with skip connections) has NaN timings + model, full_results, per_layer, sparsity = _fake_benchmark(np.random.default_rng(0)) + for key in ('per_layer_clocks', 'per_layer_ms'): + per_layer[key][2] = np.nan + for key in ('cumulative_clocks', 'cumulative_times'): + per_layer[key][2, :] = np.nan + results = save_benchmark_results( + tmp_path / 'results.json', full_results, per_layer, mapping_summary(model), sparsity, + model_name='x', example='Example', figures={'full': 'full.png', 'layers': 'layers.png'}) + assert 'NaN' not in (tmp_path / 'results.json').read_text() # strict JSON + assert results['per_layer']['per_layer_ms'][2] is None + assert set(results['per_layer']['cumulative_times'][2]) == {None} + assert results['per_layer']['cumulative_times'][1][0] == round(per_layer['cumulative_times'][1, 0]) + _, layers_fig = plot_benchmark_results(results, tmp_path) + assert np.isnan(layers_fig.axes[0].patches[2].get_height()) + + def test_results_without_power_redraw_without_a_power_row(tmp_path): *_, results = _save(tmp_path, np.random.default_rng(0), with_power=False) results['show_power'] = False