Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 29 additions & 19 deletions akida1/model_zoo/vww/vww_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@
Runs a per-layer timing benchmark on an Akida VWW model, prints a summary
table, and generates a summary plot.

The results behind the plots are saved to benchmark_results.json (with
--save-metrics, docs/ref_benchmark_results.json), so the plots can be redrawn
later without hardware:

python -m brainchip_utils.benchmark_results docs/ref_benchmark_results.json

Example
-------
python vww_benchmark.py -l pretrained_models/akidanet_vww.fbz
Expand All @@ -22,7 +28,8 @@

from vww_data import get_samples
from brainchip_utils.hardware_utils import get_mapping_stats, get_akida_device, per_layer_benchmark, full_model_benchmark
from brainchip_utils.plot_utils import plot_full_model_results, plot_per_layer_results, pretty_print_sparsity
from brainchip_utils.plot_utils import mapping_summary, pretty_print_sparsity
from brainchip_utils.benchmark_results import save_benchmark_results, plot_benchmark_results


if __name__ == '__main__':
Expand Down Expand Up @@ -114,6 +121,8 @@
num_nps, num_passes, num_sequences = get_mapping_stats(ak_model)
full_results[mm]['num_nps'] = num_nps
full_results[mm]['num_passes'] = num_passes
full_results[mm]['num_sequences'] = num_sequences
full_results[mm]['mapping'] = mapping_summary(ak_model)
print(f' Mapping: {num_nps} NP(s), {num_passes} pass(es), {num_sequences} sequence(s)')
if num_sequences>1:
print('WARNING: note, model not completely mapped to hardware')
Expand All @@ -133,27 +142,28 @@
per_layer_results = per_layer_benchmark(ak_model, device, samples, repeats=NUM_SAMPLES)

# -------------------------------------------------------------------------
# Plots
# Results file and plots
# -------------------------------------------------------------------------
# Map without hw_only so ak_model.sequences is available for plot_mapping
# Map without hw_only so ak_model.sequences is available for the mapping plot
ak_model.map(device, mode=akida.MapMode.Minimal)
perlayer_savepath = 'benchmark_results_layers.png'
if args.save_metrics:
perlayer_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_'+perlayer_savepath)
plot_per_layer_results(per_layer_results, ak_model, sparsity_dict,
model_name=args.loadmodel,
example='Visual Wake Words',
savepath=perlayer_savepath)
print('\nPer-layer results plot saved to ' + str(perlayer_savepath))

full_savepath = 'benchmark_results_full.png'

# Save everything the plots need, then plot from what was saved, so the
# figures can be redrawn later from the file alone (see the docstring above).
outdir, prefix = pathlib.Path('.'), ''
if args.save_metrics:
full_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_'+full_savepath)
plot_full_model_results(full_results, ak_model, device,
model_name=args.loadmodel,
example='Visual Wake Words',
savepath=full_savepath)
print('Full model results plot saved to ' + str(full_savepath))
outdir, prefix = pathlib.Path(__file__).parent / 'docs', 'ref_'
results_path = outdir / f'{prefix}benchmark_results.json'
results = save_benchmark_results(
results_path, full_results, per_layer_results, mapping_summary(ak_model), sparsity_dict,
model_name=args.loadmodel, example='Visual Wake Words',
figures={'full': f'{prefix}benchmark_results_full.png',
'layers': f'{prefix}benchmark_results_layers.png'},
device=str(device.desc), akida_version=akida.__version__, clock_hz=CLOCK_FREQUENCY)
print(f'\nBenchmark results saved to {results_path}')

plot_benchmark_results(results, outdir)
for name in results['figures'].values():
print(f'Plot saved to {outdir / name}')


if args.save_metrics:
Expand Down
184 changes: 184 additions & 0 deletions brainchip_utils/benchmark_results.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
"""Benchmark results files: save what the benchmark plots need, and redraw them from it.

A hardware benchmark takes from minutes to hours, but its figures only need the
numbers it measured. save_benchmark_results() writes those numbers to a JSON
file next to the figures: full-model timing and power per map mode, the
hardware mapping of each mode, per-layer timing and per-layer sparsity. The
figures can then be redrawn from the file, with no device, model or akida
install, for instance after a change to how they look:

python -m brainchip_utils.benchmark_results akida1/model_zoo/vww/docs/ref_benchmark_results.json

Power is stored as plot_utils.power_trace() resamples it for plotting (one trace
per repeat on a shared time grid), not as raw sensor readings.
"""
import argparse
import json
import pathlib

import numpy as np

from brainchip_utils.plot_utils import (figure_title, plot_full_model_results,
plot_per_layer_results, power_trace)

FORMAT_VERSION = 1

# Full-model result keys stored as they are (all scalars)
_FULL_MODEL_KEYS = ('mean_inf_clk', 'std_inf_clk', 'mean_clk_ms', 'mean_inf_ms', 'std_inf_ms',
'n_inferences', 'num_nps', 'num_passes', 'num_sequences', 'projected_clk_ms')
_POWER_KEYS = ('avg_floor_mw', 'avg_total_mw', 'avg_dynamic_mw', 'avg_energy_mj',
'avg_dynamic_energy_mj')


def _nan_to_none(values):
"""Nested lists with NaN (e.g. a layer the per-layer benchmark skipped) as None,
which JSON stores as null. The plot helpers read null back as NaN."""
if isinstance(values, list):
return [_nan_to_none(v) for v in values]
return None if isinstance(values, float) and np.isnan(values) else values


def _floats(values, decimals=None):
"""A number or nested list of numbers, optionally rounded to keep the file compact."""
values = np.asarray(values, dtype=float)
if decimals is not None:
values = np.round(values, decimals)
return _nan_to_none(values.tolist())


def _ints(values):
"""Whole-number arrays (clock counts, nanoseconds) as nested lists of ints."""
values = np.asarray(values, dtype=float)
ints = np.rint(np.nan_to_num(values)).astype(np.int64).astype(object)
ints[np.isnan(values)] = None
return ints.tolist()


def _full_model_entry(result):
entry = {k: result[k] for k in _FULL_MODEL_KEYS if k in result}
entry['mapping'] = result['mapping']
power = result.get('power')
if power is not None:
trace = power_trace(power)
if trace is not None:
trace = {'t_s': _floats(trace['t_s'], 5),
'repeats_mw': _floats(trace['repeats_mw'], 2),
'inf_duration_s': _floats(trace['inf_duration_s'], 6),
'inferences_per_repeat': trace['inferences_per_repeat']}
power = {**{k: power[k] for k in _POWER_KEYS}, 'trace': trace}
entry['power'] = power
return entry


def _per_layer_entry(per_layer_results, mapping, sparsity_dict):
return {
'layer_names': list(per_layer_results['layer_names']),
'per_layer_clocks': _floats(per_layer_results['per_layer_clocks']),
'per_layer_ms': _floats(per_layer_results['per_layer_ms']),
# Raw (num_layers, repeats) arrays, behind the per-layer spread
'cumulative_clocks': _ints(per_layer_results['cumulative_clocks']),
'cumulative_times': _ints(per_layer_results['cumulative_times']),
'mapping': mapping,
'sparsity': {name: float(s) for name, s in sparsity_dict.items()},
}


def save_benchmark_results(path, full_results, per_layer_results, per_layer_mapping,
sparsity_dict, model_name, example, figures, show_power=True,
**info):
"""Write a benchmark's results to a JSON file, and return them as loaded back.

Plot from the returned dict (plot_benchmark_results()), so the figures drawn
now are exactly the ones the file reproduces later.

Args:
path: Where to write the file (.json).
full_results: dict mapping mode name to the full_model_benchmark()
result, each with 'num_nps', 'num_passes' and a
'mapping' (from plot_utils.mapping_summary()) added.
per_layer_results: Result dict from per_layer_benchmark().
per_layer_mapping: mapping_summary() of the model as the per-layer plot
shows it.
sparsity_dict: Per-layer output sparsity, from compute_sparsity().
model_name: Model name or path; a path is stored as its file stem.
example: Example name, shown in the figure titles.
figures: dict with the file names of the 'full' and 'layers'
figures, which plot_benchmark_results() writes next to
the results file.
show_power: Whether the full-model figure has a power row.
**info: Extra provenance stored as it is (e.g. device,
akida_version, clock_hz).
"""
results = {
'format_version': FORMAT_VERSION,
'example': example,
'model_name': figure_title(model_name),
**info,
'figures': dict(figures),
'show_power': show_power,
'full_model': {mode: _full_model_entry(res) for mode, res in full_results.items()},
'per_layer': _per_layer_entry(per_layer_results, per_layer_mapping, sparsity_dict),
}
path = pathlib.Path(path)
path.write_text(json.dumps(results, separators=(',', ':'), allow_nan=False) + '\n')
return load_benchmark_results(path)


def load_benchmark_results(path):
"""Read a file written by save_benchmark_results()."""
results = json.loads(pathlib.Path(path).read_text())
if results.get('format_version') != FORMAT_VERSION:
raise ValueError(f'{path}: unsupported benchmark results format '
f'{results.get("format_version")!r} (expected {FORMAT_VERSION})')
return results


def plot_benchmark_results(results, outdir):
"""Draw the full-model and per-layer figures from saved results.

Args:
results: dict from save_benchmark_results() or load_benchmark_results().
outdir: Directory to write the figures to, under the file names stored
in results['figures'].

Returns:
(full-model figure, per-layer figure)
"""
outdir = pathlib.Path(outdir)
full_fig = plot_full_model_results(results['full_model'],
model_name=results['model_name'],
example=results['example'],
show_power=results['show_power'],
savepath=outdir / results['figures']['full'])
per_layer = results['per_layer']
layers_fig = plot_per_layer_results(per_layer, per_layer['mapping'], per_layer['sparsity'],
model_name=results['model_name'],
example=results['example'],
savepath=outdir / results['figures']['layers'])
return full_fig, layers_fig


def main(argv=None):
parser = argparse.ArgumentParser(
description='Redraw benchmark figures from a saved benchmark results file')
parser.add_argument('results', nargs='+', help='Benchmark results file(s) (.json)')
parser.add_argument('-o', '--outdir',
help='Where to write the figures (default: next to each results file)')
args = parser.parse_args(argv)

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

for path in map(pathlib.Path, args.results):
results = load_benchmark_results(path)
outdir = pathlib.Path(args.outdir) if args.outdir else path.parent
figs = plot_benchmark_results(results, outdir)
for fig in figs:
plt.close(fig)
for name in results['figures'].values():
print(f'Saved {outdir / name}')


if __name__ == '__main__':
main()
Loading