diff --git a/akida1/model_zoo/vww/vww_benchmark.py b/akida1/model_zoo/vww/vww_benchmark.py index fd5d7e4..84e56bb 100644 --- a/akida1/model_zoo/vww/vww_benchmark.py +++ b/akida1/model_zoo/vww/vww_benchmark.py @@ -6,6 +6,12 @@ Runs a per-layer timing benchmark on an Akida VWW model, prints a summary table, and generates a summary plot. +The results behind the plots are saved to benchmark_results.json (with +--save-metrics, docs/ref_benchmark_results.json), so the plots can be redrawn +later without hardware: + + python -m brainchip_utils.benchmark_results docs/ref_benchmark_results.json + Example ------- python vww_benchmark.py -l pretrained_models/akidanet_vww.fbz @@ -22,7 +28,8 @@ from vww_data import get_samples from brainchip_utils.hardware_utils import get_mapping_stats, get_akida_device, per_layer_benchmark, full_model_benchmark -from brainchip_utils.plot_utils import plot_full_model_results, plot_per_layer_results, pretty_print_sparsity +from brainchip_utils.plot_utils import mapping_summary, pretty_print_sparsity +from brainchip_utils.benchmark_results import save_benchmark_results, plot_benchmark_results if __name__ == '__main__': @@ -114,6 +121,8 @@ num_nps, num_passes, num_sequences = get_mapping_stats(ak_model) full_results[mm]['num_nps'] = num_nps full_results[mm]['num_passes'] = num_passes + full_results[mm]['num_sequences'] = num_sequences + full_results[mm]['mapping'] = mapping_summary(ak_model) print(f' Mapping: {num_nps} NP(s), {num_passes} pass(es), {num_sequences} sequence(s)') if num_sequences>1: print('WARNING: note, model not completely mapped to hardware') @@ -133,27 +142,28 @@ per_layer_results = per_layer_benchmark(ak_model, device, samples, repeats=NUM_SAMPLES) # ------------------------------------------------------------------------- - # Plots + # Results file and plots # ------------------------------------------------------------------------- - # Map without hw_only so ak_model.sequences is available for plot_mapping + # Map without hw_only so ak_model.sequences is available for the mapping plot ak_model.map(device, mode=akida.MapMode.Minimal) - perlayer_savepath = 'benchmark_results_layers.png' - if args.save_metrics: - perlayer_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_'+perlayer_savepath) - plot_per_layer_results(per_layer_results, ak_model, sparsity_dict, - model_name=args.loadmodel, - example='Visual Wake Words', - savepath=perlayer_savepath) - print('\nPer-layer results plot saved to ' + str(perlayer_savepath)) - - full_savepath = 'benchmark_results_full.png' + + # Save everything the plots need, then plot from what was saved, so the + # figures can be redrawn later from the file alone (see the docstring above). + outdir, prefix = pathlib.Path('.'), '' if args.save_metrics: - full_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_'+full_savepath) - plot_full_model_results(full_results, ak_model, device, - model_name=args.loadmodel, - example='Visual Wake Words', - savepath=full_savepath) - print('Full model results plot saved to ' + str(full_savepath)) + outdir, prefix = pathlib.Path(__file__).parent / 'docs', 'ref_' + results_path = outdir / f'{prefix}benchmark_results.json' + results = save_benchmark_results( + results_path, full_results, per_layer_results, mapping_summary(ak_model), sparsity_dict, + model_name=args.loadmodel, example='Visual Wake Words', + figures={'full': f'{prefix}benchmark_results_full.png', + 'layers': f'{prefix}benchmark_results_layers.png'}, + device=str(device.desc), akida_version=akida.__version__, clock_hz=CLOCK_FREQUENCY) + print(f'\nBenchmark results saved to {results_path}') + + plot_benchmark_results(results, outdir) + for name in results['figures'].values(): + print(f'Plot saved to {outdir / name}') if args.save_metrics: diff --git a/brainchip_utils/benchmark_results.py b/brainchip_utils/benchmark_results.py new file mode 100644 index 0000000..504a64b --- /dev/null +++ b/brainchip_utils/benchmark_results.py @@ -0,0 +1,184 @@ +"""Benchmark results files: save what the benchmark plots need, and redraw them from it. + +A hardware benchmark takes from minutes to hours, but its figures only need the +numbers it measured. save_benchmark_results() writes those numbers to a JSON +file next to the figures: full-model timing and power per map mode, the +hardware mapping of each mode, per-layer timing and per-layer sparsity. The +figures can then be redrawn from the file, with no device, model or akida +install, for instance after a change to how they look: + + python -m brainchip_utils.benchmark_results akida1/model_zoo/vww/docs/ref_benchmark_results.json + +Power is stored as plot_utils.power_trace() resamples it for plotting (one trace +per repeat on a shared time grid), not as raw sensor readings. +""" +import argparse +import json +import pathlib + +import numpy as np + +from brainchip_utils.plot_utils import (figure_title, plot_full_model_results, + plot_per_layer_results, power_trace) + +FORMAT_VERSION = 1 + +# Full-model result keys stored as they are (all scalars) +_FULL_MODEL_KEYS = ('mean_inf_clk', 'std_inf_clk', 'mean_clk_ms', 'mean_inf_ms', 'std_inf_ms', + 'n_inferences', 'num_nps', 'num_passes', 'num_sequences', 'projected_clk_ms') +_POWER_KEYS = ('avg_floor_mw', 'avg_total_mw', 'avg_dynamic_mw', 'avg_energy_mj', + 'avg_dynamic_energy_mj') + + +def _nan_to_none(values): + """Nested lists with NaN (e.g. a layer the per-layer benchmark skipped) as None, + which JSON stores as null. The plot helpers read null back as NaN.""" + if isinstance(values, list): + return [_nan_to_none(v) for v in values] + return None if isinstance(values, float) and np.isnan(values) else values + + +def _floats(values, decimals=None): + """A number or nested list of numbers, optionally rounded to keep the file compact.""" + values = np.asarray(values, dtype=float) + if decimals is not None: + values = np.round(values, decimals) + return _nan_to_none(values.tolist()) + + +def _ints(values): + """Whole-number arrays (clock counts, nanoseconds) as nested lists of ints.""" + values = np.asarray(values, dtype=float) + ints = np.rint(np.nan_to_num(values)).astype(np.int64).astype(object) + ints[np.isnan(values)] = None + return ints.tolist() + + +def _full_model_entry(result): + entry = {k: result[k] for k in _FULL_MODEL_KEYS if k in result} + entry['mapping'] = result['mapping'] + power = result.get('power') + if power is not None: + trace = power_trace(power) + if trace is not None: + trace = {'t_s': _floats(trace['t_s'], 5), + 'repeats_mw': _floats(trace['repeats_mw'], 2), + 'inf_duration_s': _floats(trace['inf_duration_s'], 6), + 'inferences_per_repeat': trace['inferences_per_repeat']} + power = {**{k: power[k] for k in _POWER_KEYS}, 'trace': trace} + entry['power'] = power + return entry + + +def _per_layer_entry(per_layer_results, mapping, sparsity_dict): + return { + 'layer_names': list(per_layer_results['layer_names']), + 'per_layer_clocks': _floats(per_layer_results['per_layer_clocks']), + 'per_layer_ms': _floats(per_layer_results['per_layer_ms']), + # Raw (num_layers, repeats) arrays, behind the per-layer spread + 'cumulative_clocks': _ints(per_layer_results['cumulative_clocks']), + 'cumulative_times': _ints(per_layer_results['cumulative_times']), + 'mapping': mapping, + 'sparsity': {name: float(s) for name, s in sparsity_dict.items()}, + } + + +def save_benchmark_results(path, full_results, per_layer_results, per_layer_mapping, + sparsity_dict, model_name, example, figures, show_power=True, + **info): + """Write a benchmark's results to a JSON file, and return them as loaded back. + + Plot from the returned dict (plot_benchmark_results()), so the figures drawn + now are exactly the ones the file reproduces later. + + Args: + path: Where to write the file (.json). + full_results: dict mapping mode name to the full_model_benchmark() + result, each with 'num_nps', 'num_passes' and a + 'mapping' (from plot_utils.mapping_summary()) added. + per_layer_results: Result dict from per_layer_benchmark(). + per_layer_mapping: mapping_summary() of the model as the per-layer plot + shows it. + sparsity_dict: Per-layer output sparsity, from compute_sparsity(). + model_name: Model name or path; a path is stored as its file stem. + example: Example name, shown in the figure titles. + figures: dict with the file names of the 'full' and 'layers' + figures, which plot_benchmark_results() writes next to + the results file. + show_power: Whether the full-model figure has a power row. + **info: Extra provenance stored as it is (e.g. device, + akida_version, clock_hz). + """ + results = { + 'format_version': FORMAT_VERSION, + 'example': example, + 'model_name': figure_title(model_name), + **info, + 'figures': dict(figures), + 'show_power': show_power, + 'full_model': {mode: _full_model_entry(res) for mode, res in full_results.items()}, + 'per_layer': _per_layer_entry(per_layer_results, per_layer_mapping, sparsity_dict), + } + path = pathlib.Path(path) + path.write_text(json.dumps(results, separators=(',', ':'), allow_nan=False) + '\n') + return load_benchmark_results(path) + + +def load_benchmark_results(path): + """Read a file written by save_benchmark_results().""" + results = json.loads(pathlib.Path(path).read_text()) + if results.get('format_version') != FORMAT_VERSION: + raise ValueError(f'{path}: unsupported benchmark results format ' + f'{results.get("format_version")!r} (expected {FORMAT_VERSION})') + return results + + +def plot_benchmark_results(results, outdir): + """Draw the full-model and per-layer figures from saved results. + + Args: + results: dict from save_benchmark_results() or load_benchmark_results(). + outdir: Directory to write the figures to, under the file names stored + in results['figures']. + + Returns: + (full-model figure, per-layer figure) + """ + outdir = pathlib.Path(outdir) + full_fig = plot_full_model_results(results['full_model'], + model_name=results['model_name'], + example=results['example'], + show_power=results['show_power'], + savepath=outdir / results['figures']['full']) + per_layer = results['per_layer'] + layers_fig = plot_per_layer_results(per_layer, per_layer['mapping'], per_layer['sparsity'], + model_name=results['model_name'], + example=results['example'], + savepath=outdir / results['figures']['layers']) + return full_fig, layers_fig + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Redraw benchmark figures from a saved benchmark results file') + parser.add_argument('results', nargs='+', help='Benchmark results file(s) (.json)') + parser.add_argument('-o', '--outdir', + help='Where to write the figures (default: next to each results file)') + args = parser.parse_args(argv) + + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + for path in map(pathlib.Path, args.results): + results = load_benchmark_results(path) + outdir = pathlib.Path(args.outdir) if args.outdir else path.parent + figs = plot_benchmark_results(results, outdir) + for fig in figs: + plt.close(fig) + for name in results['figures'].values(): + print(f'Saved {outdir / name}') + + +if __name__ == '__main__': + main() diff --git a/brainchip_utils/plot_utils.py b/brainchip_utils/plot_utils.py index 1863d61..b51226d 100644 --- a/brainchip_utils/plot_utils.py +++ b/brainchip_utils/plot_utils.py @@ -3,7 +3,6 @@ import numpy as np import matplotlib.pyplot as plt from matplotlib.lines import Line2D -import akida MAPPING_COLOR = 'steelblue' @@ -15,6 +14,9 @@ INF_END_STYLE = dict(color='firebrick', ls='--', lw=1.2) FLOOR_STYLE = dict(color='gray', ls=':', lw=1.0) +# Points per repeat in a power trace, on a grid shared by every repeat of a map mode +POWER_TRACE_POINTS = 500 + def pretty_print_sparsity(sparsity_dict): col_w = max(len(k) for k in sparsity_dict) + 2 @@ -75,13 +77,13 @@ def _mapping_legend_handles(has_pass, has_seq): def plot_per_layer_timing(results, ax, colors=None): layer_names = results['layer_names'] - per_layer_ms = results['per_layer_ms'] + per_layer_ms = np.asarray(results['per_layer_ms'], dtype=float) # null (saved NaN) -> NaN num_layers = len(layer_names) if colors is None: colors = ['steelblue'] * num_layers - raw_times = results['cumulative_times'] # (num_layers, repeats) in ns + raw_times = np.asarray(results['cumulative_times'], dtype=float) # (num_layers, repeats) in ns per_layer_raw = np.copy(raw_times) per_layer_raw[1:] = raw_times[1:] - raw_times[:-1] per_layer_ms_std = np.std(per_layer_raw / 1e6, axis=1) @@ -102,7 +104,7 @@ def plot_cumulative_timing(results, ax, colors=None): if colors is None: colors = ['seagreen'] * num_layers - cumulative_ms = np.mean(results['cumulative_times'], axis=1) / 1e6 + cumulative_ms = np.mean(np.asarray(results['cumulative_times'], dtype=float), axis=1) / 1e6 x = np.arange(num_layers) ax.bar(x, cumulative_ms, color=colors, edgecolor='white') @@ -112,16 +114,26 @@ def plot_cumulative_timing(results, ax, colors=None): ax.set_xticklabels(layer_names, rotation=45, ha='right', fontsize=8) -def plot_mapping(ak_model, ax, colors=None, legend=True): - """Bar chart of NPs per layer, with pass and sequence boundaries. +def mapping_summary(ak_model): + """NPs per layer in mapping order, with the pass and sequence boundaries. - Returns (has_pass_boundary, has_sequence_boundary), so a caller drawing - several panels can build one shared legend (pass legend=False). + ak_model must be mapped without hw_only=True, so that ak_model.sequences is + populated. The result is a plain dict that plot_mapping() draws and that a + benchmark results file stores, so the mapping can be plotted again without + a device. + + Returns: + dict with keys: + layer_names - list of layer names, in mapping order + layer_nps - list of NPs used by each layer + pass_ends - indices of layers that end a pass (within a sequence) + sequence_ends - indices of layers that end a sequence + The last layer of the model ends neither. """ layer_names = [] layer_nps = [] pass_ends = [] - seq_ends = [] + sequence_ends = [] for si, seq in enumerate(ak_model.sequences): for pi, hwpass in enumerate(seq.passes): @@ -131,17 +143,35 @@ def plot_mapping(ak_model, ax, colors=None, legend=True): is_last_pass = (pi == len(seq.passes) - 1) is_last_seq = (si == len(ak_model.sequences) - 1) if not (is_last_pass and is_last_seq): - pass_end_idx = len(layer_names) - 0.5 if is_last_pass: - seq_ends.append(pass_end_idx) + sequence_ends.append(len(layer_names) - 1) else: - pass_ends.append(pass_end_idx) + pass_ends.append(len(layer_names) - 1) + + return {'layer_names': layer_names, 'layer_nps': layer_nps, + 'pass_ends': pass_ends, 'sequence_ends': sequence_ends} + + +def plot_mapping(mapping, ax, colors=None, legend=True): + """Bar chart of NPs per layer, with pass and sequence boundaries. + + mapping is either a mapped akida.Model (see mapping_summary()) or the dict + mapping_summary() returns. + + Returns (has_pass_boundary, has_sequence_boundary), so a caller drawing + several panels can build one shared legend (pass legend=False). + """ + if hasattr(mapping, 'sequences'): + mapping = mapping_summary(mapping) + layer_names = mapping['layer_names'] + pass_ends = [i + 0.5 for i in mapping['pass_ends']] + seq_ends = [i + 0.5 for i in mapping['sequence_ends']] if colors is None: colors = MAPPING_COLOR x = np.arange(len(layer_names)) - ax.bar(x, layer_nps, color=colors, edgecolor='white') + ax.bar(x, mapping['layer_nps'], color=colors, edgecolor='white') for pos in pass_ends: ax.axvline(x=pos, **PASS_STYLE) @@ -170,28 +200,34 @@ def _fit_layer_labels(ax, max_slanted=20): plt.setp(labels, fontsize=11) -def plot_power_trace(power_data, ax, legend=True, stats=True): - """Plot per-repeat power traces with the mean overlaid. +def power_trace(power_data, num_points=POWER_TRACE_POINTS): + """Per-repeat power traces on a common time grid, t=0 at inference start. - Each repeat is time-normalised so that t=0 coincides with the start of the - inference window. Individual repeat traces are drawn with transparency; - the mean trace is drawn on top. + This is the data plot_power_trace() draws, and what a benchmark results + file stores instead of the raw sensor readings (which run to hundreds of + thousands of points per map mode). Args: - power_data: The 'power' dict returned by full_model_benchmark, which - must contain 'readings' and 'repeat_meta'. - legend: Draw a legend on the axes. Pass False when the figure has a - shared legend. - stats: Draw the time/power/energy summary box on the axes. + power_data: The 'power' dict returned by full_model_benchmark (with + 'readings' and 'repeat_meta'), or a dict that already holds + a 'trace', which is returned as it is. + num_points: Points per repeat on the common time grid. + + Returns: + dict with keys: + t_s - common time grid (s), t=0 at inference start + repeats_mw - one power trace (mW) per repeat on that grid + inf_duration_s - length of each repeat's inference run (s) + inferences_per_repeat - forward calls in each inference run + or None if no repeat has enough readings to draw. """ + if 'trace' in power_data: + return power_data['trace'] + readings_arr = np.array(power_data['readings']) abs_times = readings_arr[:, 0] powers = readings_arr[:, 1] * readings_arr[:, 2] # V × mA = mW - repeat_meta = power_data['repeat_meta'] - avg_floor = power_data['avg_floor_mw'] - avg_dyn = power_data['avg_dynamic_mw'] - avg_dyn_e = power_data['avg_dynamic_energy_mj'] # Build one time-normalised power trace per repeat repeat_traces = [] # list of (t_rel, p) arrays, t=0 at inference start @@ -206,29 +242,56 @@ def plot_power_trace(power_data, ax, legend=True, stats=True): repeat_traces.append((t_rel, p_sel)) if not repeat_traces: - return + return None # Common time grid spanning all repeats t_grid = np.linspace( min(tr[0][0] for tr in repeat_traces), max(tr[0][-1] for tr in repeat_traces), - 500, + num_points, ) + return { + 't_s': t_grid.tolist(), + 'repeats_mw': [np.interp(t_grid, t_rel, p_sel).tolist() + for t_rel, p_sel in repeat_traces], + 'inf_duration_s': [meta['inf_timestamps'][-1] - meta['inf_timestamps'][0] + for meta in repeat_meta], + 'inferences_per_repeat': len(repeat_meta[0]['inf_timestamps']) - 1, + } + + +def plot_power_trace(power_data, ax, legend=True, stats=True): + """Plot per-repeat power traces with the mean overlaid. + + Each repeat is time-normalised so that t=0 coincides with the start of the + inference window. Individual repeat traces are drawn with transparency; + the mean trace is drawn on top. + + Args: + power_data: The 'power' dict returned by full_model_benchmark (with + 'readings' and 'repeat_meta'), or its saved form, with a + 'trace' from power_trace() in their place. + legend: Draw a legend on the axes. Pass False when the figure has a + shared legend. + stats: Draw the time/power/energy summary box on the axes. + """ + trace = power_trace(power_data) + if trace is None: + return + + avg_floor = power_data['avg_floor_mw'] + avg_dyn = power_data['avg_dynamic_mw'] + avg_dyn_e = power_data['avg_dynamic_energy_mj'] - interp_powers = [] - for t_rel, p_sel in repeat_traces: - p_interp = np.interp(t_grid, t_rel, p_sel) - interp_powers.append(p_interp) + t_grid = np.asarray(trace['t_s']) + interp_powers = np.asarray(trace['repeats_mw']) + for p_interp in interp_powers: ax.plot(t_grid, p_interp, **REPEAT_STYLE) ax.plot(t_grid, np.mean(interp_powers, axis=0), label='Mean power', **MEAN_STYLE) - mean_inf_duration = np.mean([ - meta['inf_timestamps'][-1] - meta['inf_timestamps'][0] - for meta in repeat_meta - ]) - n_inf = len(repeat_meta[0]['inf_timestamps']) - 1 - avg_inf_time_ms = mean_inf_duration / n_inf * 1000 + mean_inf_duration = np.mean(trace['inf_duration_s']) + avg_inf_time_ms = mean_inf_duration / trace['inferences_per_repeat'] * 1000 ax.axvline(0, label='Inference start', **INF_START_STYLE) ax.axvline(mean_inf_duration, label='Inference end', **INF_END_STYLE) @@ -277,8 +340,8 @@ def _mode_summary(result): return ' · '.join(parts) -def plot_full_model_results(full_results, ak_model, device, model_name=None, savepath=None, - show_power=True, example=None): +def plot_full_model_results(full_results, ak_model=None, device=None, model_name=None, + savepath=None, show_power=True, example=None): """Plot power trace and hardware mapping for each map mode. Layout: rows = plot type (power, mapping); cols = map mode. If show_power is @@ -290,9 +353,12 @@ def plot_full_model_results(full_results, ak_model, device, model_name=None, sav Args: full_results: dict mapping mode name (e.g. 'Minimal', 'AllNps') to the result dict returned by full_model_benchmark(), optionally - with 'num_nps' and 'num_passes' added. - ak_model: akida.Model — re-mapped internally for each mode. Left in - the state of the last mode after this call returns. + with 'num_nps' and 'num_passes' added. A result that holds + a 'mapping' (from mapping_summary()) is plotted from it, as + results loaded from a benchmark results file are. + ak_model: akida.Model, re-mapped for each mode without a 'mapping'. + Left in the state of the last mode re-mapped. Not needed + when every result holds a 'mapping'. device: Akida device passed to ak_model.map(). model_name: Model name or path; a path is shown as its file stem. savepath: If provided, save the figure to this path (PNG). @@ -314,7 +380,11 @@ def plot_full_model_results(full_results, ak_model, device, model_name=None, sav has_pass = has_seq = False for i, mode_name in enumerate(modes): result = full_results[mode_name] - ak_model.map(device, mode=getattr(akida.MapMode, mode_name)) + mapping = result.get('mapping') + if mapping is None: + import akida + ak_model.map(device, mode=getattr(akida.MapMode, mode_name)) + mapping = ak_model ax_top = axs[0, i] ax_map = axs[nrows - 1, i] @@ -330,7 +400,7 @@ def plot_full_model_results(full_results, ak_model, device, model_name=None, sav else: _power_placeholder(axs[0, i]) - p, q = plot_mapping(ak_model, ax_map, legend=False) + p, q = plot_mapping(mapping, ax_map, legend=False) has_pass, has_seq = has_pass or p, has_seq or q ax_map.yaxis.label.set_fontsize(12) ax_map.tick_params(axis='y', labelsize=11) @@ -384,7 +454,8 @@ def plot_per_layer_results(per_layer_results, ak_model, sparsity_dict, Args: per_layer_results: Result dict from per_layer_benchmark(). - ak_model: akida.Model with sequences populated by a prior map() call. + ak_model: akida.Model with sequences populated by a prior map() call, + or the dict mapping_summary() returns for it. sparsity_dict: dict[layer_name, float] of output sparsity per layer, as returned by compute_sparsity(). Shifted by one position to derive per-layer input sparsity (first layer = 0). diff --git a/test/test_benchmark_results.py b/test/test_benchmark_results.py new file mode 100644 index 0000000..cf69388 --- /dev/null +++ b/test/test_benchmark_results.py @@ -0,0 +1,158 @@ +"""Unit tests for brainchip_utils.benchmark_results (no hardware or model files required).""" +import json +import subprocess +import sys +from types import SimpleNamespace + +import matplotlib +import numpy as np +import pytest + +matplotlib.use('Agg') + +from brainchip_utils.benchmark_results import (load_benchmark_results, main, # noqa: E402 + plot_benchmark_results, save_benchmark_results) +from brainchip_utils.plot_utils import (mapping_summary, plot_full_model_results, # noqa: E402 + plot_per_layer_results) + + +def _fake_power(level_mw, rng, repeats=3): + """Raw power data in the shape full_model_benchmark() returns: floor, inference, floor.""" + readings, repeat_meta = [], [] + for rr in range(repeats): + t0 = 10.0 * rr + t = t0 + np.arange(0.0, 3.0, 0.001) + power = 110 + level_mw * ((t >= t0 + 1.0) & (t <= t0 + 2.0)) + rng.normal(0, 0.5, t.size) + readings += [(ti, 0.8, pi / 0.8) for ti, pi in zip(t, power)] + repeat_meta.append({'floor_pre_start': t0, 'floor_post_end': t0 + 3.0, + 'inf_timestamps': list(np.linspace(t0 + 1.0, t0 + 2.0, 101))}) + return {'readings': readings, 'repeat_meta': repeat_meta, + 'avg_floor_mw': 110.0, 'avg_total_mw': 110.0 + level_mw, 'avg_dynamic_mw': level_mw, + 'avg_energy_mj': 1.0, 'avg_dynamic_energy_mj': level_mw / 100} + + +def _fake_model(nps_per_pass_per_seq): + """A mapped-model stand-in: one list of passes (lists of NPs per layer) per sequence.""" + count = iter(range(1000)) + layer = lambda n: SimpleNamespace(name=f'layer_{next(count)}', # noqa: E731 + mapping=SimpleNamespace(nps=[0] * n)) + return SimpleNamespace(sequences=[ + SimpleNamespace(passes=[SimpleNamespace(layers=[layer(n) for n in p]) for p in seq]) + for seq in nps_per_pass_per_seq]) + + +def _fake_benchmark(rng, with_power=True): + model = _fake_model([[[1, 2], [3]], [[1]]]) + full_results = {} + for mode, level in [('Minimal', 20.0), ('AllNps', 40.0)]: + full_results[mode] = { + 'map_mode': object(), # an akida.MapMode in a real run; not stored + 'mean_inf_clk': 1.2e6, 'std_inf_clk': 1e3, 'mean_clk_ms': 3.0, 'mean_inf_ms': 3.1, + 'std_inf_ms': 0.1, 'n_inferences': 300, 'num_nps': 7, 'num_passes': 3, + 'num_sequences': 2, 'mapping': mapping_summary(model), + 'power': _fake_power(level, rng) if with_power else None} + clocks = np.cumsum(rng.integers(1000, 2000, size=(4, 50)), axis=0).astype(float) + per_layer_clocks = np.copy(clocks) + per_layer_clocks[1:] = clocks[1:] - clocks[:-1] + per_layer = {'layer_names': [f'layer_{i}' for i in range(4)], + 'per_layer_clocks': per_layer_clocks.mean(axis=1), + 'per_layer_ms': per_layer_clocks.mean(axis=1) / 400e6 * 1000, + 'cumulative_clocks': clocks, 'cumulative_times': clocks * 2.5} + sparsity = {f'layer_{i}': 0.1 * i for i in range(4)} + return model, full_results, per_layer, sparsity + + +def _save(tmp_path, rng, **kwargs): + model, full_results, per_layer, sparsity = _fake_benchmark(rng, **kwargs) + results = save_benchmark_results( + tmp_path / 'results.json', full_results, per_layer, mapping_summary(model), sparsity, + model_name='./models/x.fbz', example='Example', + figures={'full': 'full.png', 'layers': 'layers.png'}, device='PCIe/AKD1500/16MB/0') + return model, full_results, per_layer, sparsity, results + + +def test_mapping_summary_marks_pass_and_sequence_ends(): + summary = mapping_summary(_fake_model([[[1, 2], [3]], [[1]]])) + assert summary == {'layer_names': ['layer_0', 'layer_1', 'layer_2', 'layer_3'], + 'layer_nps': [1, 2, 3, 1], 'pass_ends': [1], 'sequence_ends': [2]} + + +def test_saved_file_is_plain_json_without_raw_power_readings(tmp_path): + *_, results = _save(tmp_path, np.random.default_rng(0)) + stored = json.loads((tmp_path / 'results.json').read_text()) + assert stored == results + assert stored['model_name'] == 'x' and stored['device'] == 'PCIe/AKD1500/16MB/0' + power = stored['full_model']['Minimal']['power'] + assert 'readings' not in power and 'repeat_meta' not in power + assert len(power['trace']['repeats_mw']) == 3 + assert 'map_mode' not in stored['full_model']['Minimal'] + assert np.shape(stored['per_layer']['cumulative_times']) == (4, 50) + + +def test_plots_from_saved_results_match_plots_from_live_results(tmp_path): + model, full_results, per_layer, sparsity, results = _save(tmp_path, np.random.default_rng(0)) + full_fig, layers_fig = plot_benchmark_results(results, tmp_path) + assert (tmp_path / 'full.png').exists() and (tmp_path / 'layers.png').exists() + + live_full = plot_full_model_results(full_results, model_name='./models/x.fbz', + example='Example') + live_layers = plot_per_layer_results(per_layer, model, sparsity, + model_name='./models/x.fbz', example='Example') + for saved_fig, live_fig in [(full_fig, live_full), (layers_fig, live_layers)]: + assert saved_fig._suptitle.get_text() == live_fig._suptitle.get_text() + for saved_ax, live_ax in zip(saved_fig.axes, live_fig.axes, strict=True): + for saved_line, live_line in zip(saved_ax.get_lines(), live_ax.get_lines(), strict=True): + np.testing.assert_allclose(np.asarray(saved_line.get_ydata(), dtype=float), + np.asarray(live_line.get_ydata(), dtype=float), atol=0.01) + assert ([p.get_height() for p in saved_ax.patches] == + pytest.approx([p.get_height() for p in live_ax.patches])) + assert ([t.get_text() for t in saved_ax.get_xticklabels()] == + [t.get_text() for t in live_ax.get_xticklabels()]) + + +def test_skipped_layers_are_stored_as_null_and_read_back_as_nan(tmp_path): + # A layer the per-layer benchmark couldn't measure on its own (e.g. an Add in + # a model with skip connections) has NaN timings + model, full_results, per_layer, sparsity = _fake_benchmark(np.random.default_rng(0)) + for key in ('per_layer_clocks', 'per_layer_ms'): + per_layer[key][2] = np.nan + for key in ('cumulative_clocks', 'cumulative_times'): + per_layer[key][2, :] = np.nan + results = save_benchmark_results( + tmp_path / 'results.json', full_results, per_layer, mapping_summary(model), sparsity, + model_name='x', example='Example', figures={'full': 'full.png', 'layers': 'layers.png'}) + assert 'NaN' not in (tmp_path / 'results.json').read_text() # strict JSON + assert results['per_layer']['per_layer_ms'][2] is None + assert set(results['per_layer']['cumulative_times'][2]) == {None} + assert results['per_layer']['cumulative_times'][1][0] == round(per_layer['cumulative_times'][1, 0]) + _, layers_fig = plot_benchmark_results(results, tmp_path) + assert np.isnan(layers_fig.axes[0].patches[2].get_height()) + + +def test_results_without_power_redraw_without_a_power_row(tmp_path): + *_, results = _save(tmp_path, np.random.default_rng(0), with_power=False) + results['show_power'] = False + full_fig, _ = plot_benchmark_results(results, tmp_path) + assert len(full_fig.axes) == 2 # one mapping panel per mode + + +def test_command_line_redraws_next_to_the_results_file(tmp_path): + _save(tmp_path, np.random.default_rng(0)) + for name in ('full.png', 'layers.png'): + (tmp_path / name).unlink(missing_ok=True) + main([str(tmp_path / 'results.json')]) + assert (tmp_path / 'full.png').exists() and (tmp_path / 'layers.png').exists() + + +def test_unknown_format_version_is_rejected(tmp_path): + path = tmp_path / 'results.json' + path.write_text(json.dumps({'format_version': 99})) + with pytest.raises(ValueError, match='unsupported'): + load_benchmark_results(path) + + +def test_redrawing_does_not_need_akida(): + # Block the akida import, as on a machine without it installed + code = ("import sys; sys.modules['akida'] = None; " + "import brainchip_utils.benchmark_results") + subprocess.run([sys.executable, '-c', code], check=True)