diff --git a/.claude/knowledge/akida.md b/.claude/knowledge/akida.md index 554cf2d..40dc866 100644 --- a/.claude/knowledge/akida.md +++ b/.claude/knowledge/akida.md @@ -161,6 +161,24 @@ pipeline instead, so the deployed model receives an already-shaped tensor. how much pooling can come before it. *Source: `akida1/model_zoo/uored_vafcls/uored_vafcls_model.py` docstring.* +**[Akida 2] `DepthwiseConv2DTranspose` maps to hardware only with 8-bit inputs and 8-bit +weights.** At 4 bits the layer falls back to software. So a model with transposed +depthwise convolutions, quantized to 4 bits throughout (`-w 4 -a 4`), no longer runs +fully on hardware: the mapper splits it into hardware and software sequences, and +mapping with `hw_only=True` fails with "Input should be 8-bit. Weights should be 8-bit." +Regular `DepthwiseConv2D` and pointwise `Conv2D` layers map at 4 bits. +- That's why `akida2/model_zoo/portrait128`, an AkidaUNet with five transposed depthwise + layers in its decoder, has no 4-bit variant. BrainChip's model zoo also lists that model + at 8 bits only. +- A mixed-precision 4-bit model should be possible: `quantizeml.models.quantize` takes a + per-layer `q_config`, which could keep these layers, and the activations feeding them, + at 8 bits. **(unconfirmed)**: not yet built or benchmarked; mapping such a model with + `hw_only=True` would confirm it. + +*Source: measured with akida 2.19.3, mapping onto `akida.SixNodesIPv2()` and on the +Akida 2 FPGA, 2026-10-22 (`akida2/model_zoo/portrait128`, HUB-43); +[model zoo performance](https://doc.brainchipinc.com/model_zoo_performance.html#segmentation).* + ## Backbones and blocks **[both] Separable convolutions: fused on Akida 1, two layers on Akida 2.** diff --git a/.claude/skills/review-content/check_content.py b/.claude/skills/review-content/check_content.py index d536f56..c0b76c9 100644 --- a/.claude/skills/review-content/check_content.py +++ b/.claude/skills/review-content/check_content.py @@ -118,7 +118,11 @@ def check_text(files, added): where = rel if path.suffix == ".ipynb" else f"{rel}:{lineno}" yield "machine-path", where, PATH_RE.search(line).group(0) is_new_zoo_script = rel in added and "model_zoo" in rel and path.suffix == ".py" - if is_new_zoo_script and not any(COPYRIGHT in l for l in lines[:3]): + # A Jupytext notebook mirror (`*_notebook.py`) must start with its YAML header, + # so it can't carry the copyright line; its .ipynb is the notebook itself. + is_jupytext_mirror = bool(lines) and lines[0].strip() == "# ---" and any( + "jupytext:" in l for l in lines[:6]) + if is_new_zoo_script and not is_jupytext_mirror and not any(COPYRIGHT in l for l in lines[:3]): yield "copyright", rel, "missing copyright header in first 3 lines" diff --git a/.gitignore b/.gitignore index ffa49f6..6c3e2db 100644 --- a/.gitignore +++ b/.gitignore @@ -220,3 +220,7 @@ __marimo__/ # Claude Code: personal, never shared CLAUDE.local.md .claude/settings.local.json + +# Logs written by wget / nohup when run without a terminal +wget-log* +nohup.out diff --git a/README.md b/README.md index 9fbd7d1..4450600 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Akida 1 and Akida 2 examples are available today; more Akida 2 models can be fou | --- | --- | --- | --- | | **Chip** | AKD1500 | AKD2500 | — | | **Typical use cases** | Image classification, keyword spotting, object detection | Larger models, higher accuracy targets | Always-on sensing, edge inference | -| **Examples in this repo** | [Image Classification (PlantVillage)](akida1/model_zoo/plant_village) · [ImageNet / AkidaNet](akida1/model_zoo/imagenet_akidanet) · [Visual Wake Words](akida1/model_zoo/vww) · [Keyword Spotting](akida1/model_zoo/speech_commands) · [ECG Arrhythmia](akida1/model_zoo/arrhythmia_classification) | [Visual Wake Words](akida2/model_zoo/vww) · [Image Classification (PlantVillage)](akida2/model_zoo/plant_village) · [Keyword Spotting](akida2/model_zoo/speech_commands) | 🔜 Coming soon | +| **Examples in this repo** | [Image Classification (PlantVillage)](akida1/model_zoo/plant_village) · [ImageNet / AkidaNet](akida1/model_zoo/imagenet_akidanet) · [Visual Wake Words](akida1/model_zoo/vww) · [Keyword Spotting](akida1/model_zoo/speech_commands) · [ECG Arrhythmia](akida1/model_zoo/arrhythmia_classification) | [Visual Wake Words](akida2/model_zoo/vww) · [Image Classification (PlantVillage)](akida2/model_zoo/plant_village) · [Keyword Spotting](akida2/model_zoo/speech_commands) · [Portrait Segmentation (Portrait128)](akida2/model_zoo/portrait128) | 🔜 Coming soon | --- diff --git a/akida2/README.md b/akida2/README.md index e5d0bd1..d8fe5f8 100644 --- a/akida2/README.md +++ b/akida2/README.md @@ -162,6 +162,17 @@ benchmarks for every mapping, and the steps to reproduce them. 3.991 + + Portrait segmentation + Segmentation + Portrait128 + 8-bit + 0.9185 binary IoU + HwPr + 254.603 + 6.365 + + diff --git a/akida2/model_zoo/portrait128/README.md b/akida2/model_zoo/portrait128/README.md new file mode 100644 index 0000000..7c5a9e0 --- /dev/null +++ b/akida2/model_zoo/portrait128/README.md @@ -0,0 +1,307 @@ +BrainChip Dev Hub + +# Portrait Segmentation (Portrait128) — Akida 2 + +## Model Card + +Float binary IoU: **0.8883**  |  Float pixel accuracy: **94.31%**  |  Parameters: **1,058,865** + +The model labels every pixel of a 128 × 128 portrait as person or background. Two +metrics are reported, both on the validation split and with each pixel's logit +thresholded at 0 (a probability of 0.5): + +- **Binary IoU**: the intersection over union of the predicted and true masks, + averaged over the two classes (person and background). This is the headline + metric: it penalizes a mask that is the wrong shape even when most pixels are right. +- **Pixel accuracy**: the fraction of pixels labeled correctly. Background covers + much of each image, so this is always high and says less about mask quality. + +On Akida 2 the model is quantized with **`quantizeml`** to 8-bit weights and +activations. Post-training quantization (PTQ) is followed by one epoch of +quantization-aware training (QAT), which fine-tunes the model with the quantization in +place, as for this model in BrainChip's model zoo. + +There is no 4-bit variant. The decoder upsamples with transposed depthwise +convolutions, and on Akida 2 those layers need 8-bit inputs and 8-bit weights, so a +model quantized to 4 bits throughout can't run fully on the hardware. BrainChip's +[model zoo](https://doc.brainchipinc.com/model_zoo_performance.html#segmentation) also +publishes this model at 8 bits only. + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariantWeights / ActsQATQuantized IoUQuantized pixel acc.Akida IoUAkida pixel acc.Sparsity
8-bitw8 / a81 epoch0.918595.96%0.918595.96%40.95%
+ +**Akida 2 hardware benchmark (FPGA @ 25 MHz)** + +Latency is measured on the Akida 2 FPGA reference platform, which runs at +**25 MHz**. A projected latency at a higher target clock is also shown to +indicate expected performance on faster silicon. The cycle count is fixed for a +given model and mapping regardless of clock rate, so the projection is an exact +rescale of the measured cycles. + +> **Note:** the projected clock is BrainChip's current **target** for AKD2500 production +> silicon — that silicon does not exist yet, so treat this as a target rather than a +> measured value; it may still change before production. Power measurement on the FPGA +> platform is still under development, so only latency is reported at this time. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariantMappingNPsPassesCyclesLatency @ 25 MHz (ms)Projected @ 1000 MHz (ms) (target)
8-bitMinimal6739900830396.0339.901
AllNPs6939891115395.6459.891
HwPr18786365085254.6036.365
+ +### Benchmark plots + +Two plots are shown below; click to open them. The **full-model** plot shows the +hardware mapping (NPs used per layer, with pass boundaries) under each mapping mode +(Minimal, AllNPs and HwPr): Minimal mapping schedules the model onto the fewest NPs +required, while AllNPs and HwPr spread it across more NPs. The **per-layer** plot shows +per-layer latency, input sparsity and NP mapping under Minimal mapping. Akida is +event-driven, so a layer's cost depends on how many non-zero activations it receives; +sparsity is measured on real dataset samples. + +
+8-bit (w8 / a8, QAT) + +8-bit full-model benchmark results + +8-bit per-layer benchmark results + +
+ +The model is an **Akida U-Net** from `akida_models`: an **AkidaNet** encoder with +width multiplier **alpha = 0.5**, initialized from ImageNet-trained weights, takes the +**128 × 128** input down to a 4 × 4 feature map. A decoder of five separable +transposed-convolution blocks upsamples it back to 128 × 128, and a 1 × 1 +convolution produces one logit per pixel. There are no skip connections between +encoder and decoder. Transposed convolutions are an Akida 2 feature, so this model +has no Akida 1 version. + +## Requirements + +For environment requirements and setup, see the [Requirements](../../../README.md#requirements) +section of the top-level README. + +## Dataset + +Portrait128 is a binary person-segmentation dataset of **18,698** RGB portraits at +**128 × 128**, each with a person/background mask. It was assembled by Anil Sathyan +for the [Portrait-Segmentation](https://github.com/anilsathyan7/Portrait-Segmentation) +project from the PFCN portrait dataset, hand-picked portraits from the Supervisely +person dataset, and selfies collected from the web whose masks were generated with a +DeepLab model. It also includes augmented and synthetic-background variants. + +The dataset ships as a single pool. The scripts split it 80 % / 20 % into training and +validation, with a fixed shuffle seed, so every run sees the same validation images. + + + +Reference: Shen et al., *Automatic Portrait Segmentation for Image Stylization*, +Computer Graphics Forum (Eurographics), 2016 (the PFCN dataset). + +## Dataset setup + +The dataset is distributed by the Portrait-Segmentation project as +[Portseg_128](https://drive.google.com/file/d/1UBLzvcqvt_fin9Y-48I_-lWQYfYpt_6J/view) +(Google Drive). Download the archive and extract it: the scripts need its two arrays, +`img_uint8.npy` (images) and `msk_uint8.npy` (masks, 0 or 255), directly in one folder, +not in a subfolder: + +``` +data/portrait128/ +├── img_uint8.npy +└── msk_uint8.npy +``` + +To check the arrays before training: + +```bash +python -c "import numpy as np; x = np.load('data/portrait128/img_uint8.npy', mmap_mode='r'); y = np.load('data/portrait128/msk_uint8.npy', mmap_mode='r'); print(x.shape, x.dtype, y.shape, y.dtype, np.unique(y[:20]))" +``` + +You should see `(18698, 128, 128, 3) uint8 (18698, 128, 128, 1) uint8 [ 0 255]`. Masks +stored as 0/1 instead of 0/255 would be read as all background. + +The scripts default to looking for the data at `./data/portrait128`. If you +want to store the dataset on a dedicated data drive, you can pass the path +explicitly to each script (see `--data` / `-d` in the individual scripts). +Alternatively, it may be more convenient to keep the dataset in its preferred +location and create a symbolic link from the default path (one-off step): + +```bash +ln -s /path/to/your/data/portrait128 ./data/portrait128 +``` + +Here `/path/to/your/data/portrait128` is the folder that holds the two arrays. + +This way the scripts work out of the box without any extra arguments. + +### Calibration samples + +Quantization with `quantizeml` is calibrated on real data, so a batch of 1024 +Portrait128 samples is needed in addition to the dataset itself. A prepared batch is +published on the BrainChip mirror, and `portrait128_train.sh` fetches it into `data/` +automatically: + +```bash +wget -N https://data.brainchip.com/dataset-mirror/samples/portrait128/portrait128_batch1024.npz -P data/ +``` + +The training notebook does not use this file. It builds its samples directly +from the training split with `get_samples()` from +[portrait128_data.py](portrait128_data.py), so that the process is visible and easy +to adapt to your own data. The same module can also write a pack from your local +copy of the dataset: + +```bash +python portrait128_data.py --extract data/portrait128_batch1024.npz +``` + +## Pipeline + +Training produces a float model, quantizes it to 8 bits with `quantizeml`, fine-tunes +it for one epoch of QAT, and converts the result to Akida format: + +| Stage | Description | +|---|---| +| Full-precision | Float32 training of the full model, starting from an ImageNet-trained AkidaNet encoder | +| 8-bit quantization | `quantizeml quantize` to 8-bit weights and activations (8-bit input), calibrated on real samples, then one epoch of QAT at a tenth of the float learning rate; the PTQ model itself is not kept | +| Conversion to Akida | Automated conversion of the quantized model to Akida 2 format with `cnn2snn convert` | + +## Reference Models + +Pretrained models are made available here, within the `pretrained_models/` +folder. However, those are handled using the `git-lfs` package (git large +file storage). For those to be downloaded with the repo, you will need to +set up `git-lfs`. For further instructions, see the +[Trained models](../../../README.md#trained-models) section of the top-level README. + +## Usage + +### Notebook + +Two notebooks are provided that walk through a) preparation of a trained Akida-compatible model and +b) evaluation and benchmarking of that model on Akida. + +[portrait128_notebook_training.ipynb](portrait128_notebook_training.ipynb) walks through the +complete training pipeline end-to-end. It is written to expose and explain the Akida-specific +aspects of the workflow: how a segmentation model is built for Akida 2, what the +quantization constraints mean in practice, and what the conversion step does. Start +here if you want to understand *why* the pipeline is structured the way it is. + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Brainchip-Inc/brainchip_devhub/blob/main/akida2/model_zoo/portrait128/portrait128_notebook_training.ipynb) + +On Colab, the setup cell installs the toolchain but can't download the dataset, which is +only distributed through Google Drive. Copy `img_uint8.npy` and `msk_uint8.npy` into the +notebook's `data/portrait128` folder before running the data cells; the setup cell prints +where, and how to mount your Drive. + +[portrait128_notebook_benchmark.ipynb](portrait128_notebook_benchmark.ipynb) walks through +evaluation of model accuracy on Akida and, if a hardware device is available, covers benchmarking +of model latency. + +> **Note:** the hardware benchmark section requires a physical Akida 2 FPGA +> platform with a connected board. + +### Script + +For straightforward reproduction of the training and evaluation results, run +the full pipeline in one shot: + +```bash +bash portrait128_train.sh [DATADIR] +``` + +The optional `DATADIR` argument overrides the default dataset location +(`./data/portrait128`). + +## Contributing and Maintenance + +This README is autogenerated from `docs/README.md.template` +so that the accuracy and hardware benchmark values are written directly +by the code (via the `metrics.json` file, also in the docs folder). + +When the associated model or training pipeline is modified to improve +performance, you should rerun the evaluations of the float and quantized +model versions, plus the hardware benchmark, including the +`--save-metrics` argument, and then regenerate the README from the template +using `update_readme.py`: +```bash +# Float model +python portrait128_eval.py -l pretrained_models/akida_unet_portrait128.h5 --save-metrics + +# 8-bit model (QAT) +python portrait128_eval.py -l pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.h5 --save-metrics +python portrait128_eval.py -l pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz --save-metrics +python portrait128_benchmark.py -l pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz --save-metrics + +python update_readme.py +``` +Then commit the changed files (template, metrics and updated README). + +Likewise, if you want to edit the contents of this README, you should +not edit it directly, but instead edit `docs/README.md.template` and +then regenerate the README using +``` bash +python update_readme.py +``` diff --git a/akida2/model_zoo/portrait128/colab_setup.py b/akida2/model_zoo/portrait128/colab_setup.py new file mode 100644 index 0000000..7936d03 --- /dev/null +++ b/akida2/model_zoo/portrait128/colab_setup.py @@ -0,0 +1,82 @@ +# Copyright 2026 Brainchip Holdings Ltd. Apache 2.0 License +"""Colab setup for the Portrait128 (Akida 2) training notebook. + +IF YOU ARE RUNNING THIS LOCALLY: you can ignore this file completely. +It exists solely to make the "Open in Colab" badge work, and does nothing +on a normal local run. It is not part of the Portrait128 model/training code +(see portrait128_data.py, portrait128_model.py, portrait128_train.py for that). + +If you ARE on Colab, this is the file that gets you running: + - Clones this repo (skipping Git LFS smudge, since pretrained weights + aren't needed for the default training path) + - Points Python at the right folders (repo root for brainchip_utils, + this example's folder for portrait128_data / portrait128_model / + portrait128_train) + - Installs akida_models, tf_keras, quantizeml + - Checks for the Portrait128 arrays. The dataset is distributed through + Google Drive rather than a direct download link, so it is not fetched + automatically: copy img_uint8.npy and msk_uint8.npy into DATA_DIR (for + example from your mounted Google Drive) before running the data cells. + +Called from the notebook's first cell like: + import colab_setup + colab_setup.setup() +""" +import os +import subprocess +import sys + +REPO_URL = 'https://github.com/Brainchip-Inc/brainchip_devhub.git' +REPO_DIR = 'brainchip_devhub' +EXAMPLE_SUBDIR = 'akida2/model_zoo/portrait128' + +DATA_DIR = './data/portrait128' +DATA_FILES = ('img_uint8.npy', 'msk_uint8.npy') +DATASET_PAGE = 'https://drive.google.com/file/d/1UBLzvcqvt_fin9Y-48I_-lWQYfYpt_6J/view' + + +def _run(cmd): + print(f'$ {cmd}') + subprocess.run(cmd, shell=True, check=True) + + +def setup(): + """Set up a fresh Colab session to run this notebook. No-op if not on Colab.""" + if 'google.colab' not in sys.modules: + print('Not running on Colab \u2014 nothing to do. (This step only matters ' + 'for Colab; local runs already have everything they need.)') + return + + if not os.path.exists(REPO_DIR): + # Skip Git LFS smudge: pretrained weights aren't needed for the default + # (RUN_FLOAT_TRAINING = True) path, so avoid downloading them here. + os.environ['GIT_LFS_SKIP_SMUDGE'] = '1' + _run(f'git clone --depth 1 {REPO_URL} {REPO_DIR}') + + os.chdir(os.path.join(REPO_DIR, EXAMPLE_SUBDIR)) + + # Repo root on sys.path for `brainchip_utils`; example folder for the + # local portrait128_data / portrait128_model / portrait128_train modules + # imported later in the notebook. + repo_root = os.path.abspath(os.path.join(os.getcwd(), '..', '..', '..')) + sys.path.insert(0, repo_root) + sys.path.insert(0, os.getcwd()) + + # Akida 2 quantization uses quantizeml (installed as a dependency of + # akida_models / cnn2snn, but pinned here for a clean Colab environment). + _run('pip install -q akida_models==1.14.0 tf_keras quantizeml') + + os.makedirs(DATA_DIR, exist_ok=True) + missing = [f for f in DATA_FILES if not os.path.exists(os.path.join(DATA_DIR, f))] + if missing: + print(f'Dataset not found: copy {", ".join(missing)} into ' + f'{os.path.abspath(DATA_DIR)}.') + print(f'The arrays come from the Portseg_128 archive: {DATASET_PAGE}') + print('To copy them from Google Drive, mount it first:\n' + ' from google.colab import drive; drive.mount("/content/drive")') + else: + print('Dataset already present at', DATA_DIR) + + print(f'Colab setup complete. Working directory: {os.getcwd()}') + print('If TensorFlow was just installed/upgraded, restart the runtime ' + '(Runtime > Restart session) and re-run this cell before continuing.') diff --git a/akida2/model_zoo/portrait128/data/.gitignore b/akida2/model_zoo/portrait128/data/.gitignore new file mode 100644 index 0000000..647613e --- /dev/null +++ b/akida2/model_zoo/portrait128/data/.gitignore @@ -0,0 +1,4 @@ +# Git to Ignore everything in this directory +* +# Except this .gitignore file +!.gitignore \ No newline at end of file diff --git a/akida2/model_zoo/portrait128/docs/README.md.template b/akida2/model_zoo/portrait128/docs/README.md.template new file mode 100644 index 0000000..b705adf --- /dev/null +++ b/akida2/model_zoo/portrait128/docs/README.md.template @@ -0,0 +1,307 @@ +BrainChip Dev Hub + +# Portrait Segmentation (Portrait128) — Akida 2 + +## Model Card + +Float binary IoU: **{float_iou}**  |  Float pixel accuracy: **{float_acc}**  |  Parameters: **{params}** + +The model labels every pixel of a 128 × 128 portrait as person or background. Two +metrics are reported, both on the validation split and with each pixel's logit +thresholded at 0 (a probability of 0.5): + +- **Binary IoU**: the intersection over union of the predicted and true masks, + averaged over the two classes (person and background). This is the headline + metric: it penalizes a mask that is the wrong shape even when most pixels are right. +- **Pixel accuracy**: the fraction of pixels labeled correctly. Background covers + much of each image, so this is always high and says less about mask quality. + +On Akida 2 the model is quantized with **`quantizeml`** to 8-bit weights and +activations. Post-training quantization (PTQ) is followed by one epoch of +quantization-aware training (QAT), which fine-tunes the model with the quantization in +place, as for this model in BrainChip's model zoo. + +There is no 4-bit variant. The decoder upsamples with transposed depthwise +convolutions, and on Akida 2 those layers need 8-bit inputs and 8-bit weights, so a +model quantized to 4 bits throughout can't run fully on the hardware. BrainChip's +[model zoo](https://doc.brainchipinc.com/model_zoo_performance.html#segmentation) also +publishes this model at 8 bits only. + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariantWeights / ActsQATQuantized IoUQuantized pixel acc.Akida IoUAkida pixel acc.Sparsity
8-bitw8 / a81 epoch{w8a8_quant_iou}{w8a8_quant_acc}{w8a8_akida_iou}{w8a8_akida_acc}{w8a8_sparsity}
+ +**Akida 2 hardware benchmark (FPGA @ 25 MHz)** + +Latency is measured on the Akida 2 FPGA reference platform, which runs at +**25 MHz**. A projected latency at a higher target clock is also shown to +indicate expected performance on faster silicon. The cycle count is fixed for a +given model and mapping regardless of clock rate, so the projection is an exact +rescale of the measured cycles. + +> **Note:** the projected clock is BrainChip's current **target** for AKD2500 production +> silicon — that silicon does not exist yet, so treat this as a target rather than a +> measured value; it may still change before production. Power measurement on the FPGA +> platform is still under development, so only latency is reported at this time. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariantMappingNPsPassesCyclesLatency @ 25 MHz (ms)Projected @ 1000 MHz (ms) (target)
8-bitMinimal{w8a8_minimal_nps}{w8a8_minimal_passes}{w8a8_minimal_cycles}{w8a8_minimal_latency_ms}{w8a8_minimal_projected_ms}
AllNPs{w8a8_allnps_nps}{w8a8_allnps_passes}{w8a8_allnps_cycles}{w8a8_allnps_latency_ms}{w8a8_allnps_projected_ms}
HwPr{w8a8_hwpr_nps}{w8a8_hwpr_passes}{w8a8_hwpr_cycles}{w8a8_hwpr_latency_ms}{w8a8_hwpr_projected_ms}
+ +### Benchmark plots + +Two plots are shown below; click to open them. The **full-model** plot shows the +hardware mapping (NPs used per layer, with pass boundaries) under each mapping mode +(Minimal, AllNPs and HwPr): Minimal mapping schedules the model onto the fewest NPs +required, while AllNPs and HwPr spread it across more NPs. The **per-layer** plot shows +per-layer latency, input sparsity and NP mapping under Minimal mapping. Akida is +event-driven, so a layer's cost depends on how many non-zero activations it receives; +sparsity is measured on real dataset samples. + +
+8-bit (w8 / a8, QAT) + +8-bit full-model benchmark results + +8-bit per-layer benchmark results + +
+ +The model is an **Akida U-Net** from `akida_models`: an **AkidaNet** encoder with +width multiplier **alpha = 0.5**, initialized from ImageNet-trained weights, takes the +**128 × 128** input down to a 4 × 4 feature map. A decoder of five separable +transposed-convolution blocks upsamples it back to 128 × 128, and a 1 × 1 +convolution produces one logit per pixel. There are no skip connections between +encoder and decoder. Transposed convolutions are an Akida 2 feature, so this model +has no Akida 1 version. + +## Requirements + +For environment requirements and setup, see the [Requirements](../../../README.md#requirements) +section of the top-level README. + +## Dataset + +Portrait128 is a binary person-segmentation dataset of **18,698** RGB portraits at +**128 × 128**, each with a person/background mask. It was assembled by Anil Sathyan +for the [Portrait-Segmentation](https://github.com/anilsathyan7/Portrait-Segmentation) +project from the PFCN portrait dataset, hand-picked portraits from the Supervisely +person dataset, and selfies collected from the web whose masks were generated with a +DeepLab model. It also includes augmented and synthetic-background variants. + +The dataset ships as a single pool. The scripts split it 80 % / 20 % into training and +validation, with a fixed shuffle seed, so every run sees the same validation images. + + + +Reference: Shen et al., *Automatic Portrait Segmentation for Image Stylization*, +Computer Graphics Forum (Eurographics), 2016 (the PFCN dataset). + +## Dataset setup + +The dataset is distributed by the Portrait-Segmentation project as +[Portseg_128](https://drive.google.com/file/d/1UBLzvcqvt_fin9Y-48I_-lWQYfYpt_6J/view) +(Google Drive). Download the archive and extract it: the scripts need its two arrays, +`img_uint8.npy` (images) and `msk_uint8.npy` (masks, 0 or 255), directly in one folder, +not in a subfolder: + +``` +data/portrait128/ +├── img_uint8.npy +└── msk_uint8.npy +``` + +To check the arrays before training: + +```bash +python -c "import numpy as np; x = np.load('data/portrait128/img_uint8.npy', mmap_mode='r'); y = np.load('data/portrait128/msk_uint8.npy', mmap_mode='r'); print(x.shape, x.dtype, y.shape, y.dtype, np.unique(y[:20]))" +``` + +You should see `(18698, 128, 128, 3) uint8 (18698, 128, 128, 1) uint8 [ 0 255]`. Masks +stored as 0/1 instead of 0/255 would be read as all background. + +The scripts default to looking for the data at `./data/portrait128`. If you +want to store the dataset on a dedicated data drive, you can pass the path +explicitly to each script (see `--data` / `-d` in the individual scripts). +Alternatively, it may be more convenient to keep the dataset in its preferred +location and create a symbolic link from the default path (one-off step): + +```bash +ln -s /path/to/your/data/portrait128 ./data/portrait128 +``` + +Here `/path/to/your/data/portrait128` is the folder that holds the two arrays. + +This way the scripts work out of the box without any extra arguments. + +### Calibration samples + +Quantization with `quantizeml` is calibrated on real data, so a batch of 1024 +Portrait128 samples is needed in addition to the dataset itself. A prepared batch is +published on the BrainChip mirror, and `portrait128_train.sh` fetches it into `data/` +automatically: + +```bash +wget -N https://data.brainchip.com/dataset-mirror/samples/portrait128/portrait128_batch1024.npz -P data/ +``` + +The training notebook does not use this file. It builds its samples directly +from the training split with `get_samples()` from +[portrait128_data.py](portrait128_data.py), so that the process is visible and easy +to adapt to your own data. The same module can also write a pack from your local +copy of the dataset: + +```bash +python portrait128_data.py --extract data/portrait128_batch1024.npz +``` + +## Pipeline + +Training produces a float model, quantizes it to 8 bits with `quantizeml`, fine-tunes +it for one epoch of QAT, and converts the result to Akida format: + +| Stage | Description | +|---|---| +| Full-precision | Float32 training of the full model, starting from an ImageNet-trained AkidaNet encoder | +| 8-bit quantization | `quantizeml quantize` to 8-bit weights and activations (8-bit input), calibrated on real samples, then one epoch of QAT at a tenth of the float learning rate; the PTQ model itself is not kept | +| Conversion to Akida | Automated conversion of the quantized model to Akida 2 format with `cnn2snn convert` | + +## Reference Models + +Pretrained models are made available here, within the `pretrained_models/` +folder. However, those are handled using the `git-lfs` package (git large +file storage). For those to be downloaded with the repo, you will need to +set up `git-lfs`. For further instructions, see the +[Trained models](../../../README.md#trained-models) section of the top-level README. + +## Usage + +### Notebook + +Two notebooks are provided that walk through a) preparation of a trained Akida-compatible model and +b) evaluation and benchmarking of that model on Akida. + +[portrait128_notebook_training.ipynb](portrait128_notebook_training.ipynb) walks through the +complete training pipeline end-to-end. It is written to expose and explain the Akida-specific +aspects of the workflow: how a segmentation model is built for Akida 2, what the +quantization constraints mean in practice, and what the conversion step does. Start +here if you want to understand *why* the pipeline is structured the way it is. + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Brainchip-Inc/brainchip_devhub/blob/main/akida2/model_zoo/portrait128/portrait128_notebook_training.ipynb) + +On Colab, the setup cell installs the toolchain but can't download the dataset, which is +only distributed through Google Drive. Copy `img_uint8.npy` and `msk_uint8.npy` into the +notebook's `data/portrait128` folder before running the data cells; the setup cell prints +where, and how to mount your Drive. + +[portrait128_notebook_benchmark.ipynb](portrait128_notebook_benchmark.ipynb) walks through +evaluation of model accuracy on Akida and, if a hardware device is available, covers benchmarking +of model latency. + +> **Note:** the hardware benchmark section requires a physical Akida 2 FPGA +> platform with a connected board. + +### Script + +For straightforward reproduction of the training and evaluation results, run +the full pipeline in one shot: + +```bash +bash portrait128_train.sh [DATADIR] +``` + +The optional `DATADIR` argument overrides the default dataset location +(`./data/portrait128`). + +## Contributing and Maintenance + +This README is autogenerated from `docs/README.md.template` +so that the accuracy and hardware benchmark values are written directly +by the code (via the `metrics.json` file, also in the docs folder). + +When the associated model or training pipeline is modified to improve +performance, you should rerun the evaluations of the float and quantized +model versions, plus the hardware benchmark, including the +`--save-metrics` argument, and then regenerate the README from the template +using `update_readme.py`: +```bash +# Float model +python portrait128_eval.py -l pretrained_models/akida_unet_portrait128.h5 --save-metrics + +# 8-bit model (QAT) +python portrait128_eval.py -l pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.h5 --save-metrics +python portrait128_eval.py -l pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz --save-metrics +python portrait128_benchmark.py -l pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz --save-metrics + +python update_readme.py +``` +Then commit the changed files (template, metrics and updated README). + +Likewise, if you want to edit the contents of this README, you should +not edit it directly, but instead edit `docs/README.md.template` and +then regenerate the README using +``` bash +python update_readme.py +``` diff --git a/akida2/model_zoo/portrait128/docs/metrics.json b/akida2/model_zoo/portrait128/docs/metrics.json new file mode 100644 index 0000000..14005cf --- /dev/null +++ b/akida2/model_zoo/portrait128/docs/metrics.json @@ -0,0 +1,25 @@ +{ + "float_acc": "94.31%", + "float_iou": "0.8883", + "params": "1,058,865", + "w8a8_quant_iou": "0.9185", + "w8a8_quant_acc": "95.96%", + "w8a8_akida_iou": "0.9185", + "w8a8_akida_acc": "95.96%", + "w8a8_allnps_cycles": "9891115", + "w8a8_allnps_latency_ms": "395.645", + "w8a8_allnps_nps": "69", + "w8a8_allnps_passes": "3", + "w8a8_allnps_projected_ms": "9.891", + "w8a8_hwpr_cycles": "6365085", + "w8a8_hwpr_latency_ms": "254.603", + "w8a8_hwpr_nps": "187", + "w8a8_hwpr_passes": "8", + "w8a8_hwpr_projected_ms": "6.365", + "w8a8_minimal_cycles": "9900830", + "w8a8_minimal_latency_ms": "396.033", + "w8a8_minimal_nps": "67", + "w8a8_minimal_passes": "3", + "w8a8_minimal_projected_ms": "9.901", + "w8a8_sparsity": "40.95%" +} diff --git a/akida2/model_zoo/portrait128/docs/ref_benchmark_results_full_akida_unet_portrait128_i8_w8_a8_qat.png b/akida2/model_zoo/portrait128/docs/ref_benchmark_results_full_akida_unet_portrait128_i8_w8_a8_qat.png new file mode 100644 index 0000000..c27b2c1 Binary files /dev/null and b/akida2/model_zoo/portrait128/docs/ref_benchmark_results_full_akida_unet_portrait128_i8_w8_a8_qat.png differ diff --git a/akida2/model_zoo/portrait128/docs/ref_benchmark_results_layers_akida_unet_portrait128_i8_w8_a8_qat.png b/akida2/model_zoo/portrait128/docs/ref_benchmark_results_layers_akida_unet_portrait128_i8_w8_a8_qat.png new file mode 100644 index 0000000..9a77372 Binary files /dev/null and b/akida2/model_zoo/portrait128/docs/ref_benchmark_results_layers_akida_unet_portrait128_i8_w8_a8_qat.png differ diff --git a/akida2/model_zoo/portrait128/docs/zoo_card.json b/akida2/model_zoo/portrait128/docs/zoo_card.json new file mode 100644 index 0000000..ecd0077 --- /dev/null +++ b/akida2/model_zoo/portrait128/docs/zoo_card.json @@ -0,0 +1,16 @@ +{ + "domain": "Image", + "order": 30, + "rows": [ + { + "task": "Portrait segmentation", + "category": "Segmentation", + "dataset": "Portrait128", + "variant": "8-bit", + "metric_key": "w8a8_akida_iou", + "metric_label": "binary IoU", + "bench_prefix": "w8a8_", + "notes": "" + } + ] +} diff --git a/akida2/model_zoo/portrait128/models/.gitignore b/akida2/model_zoo/portrait128/models/.gitignore new file mode 100644 index 0000000..647613e --- /dev/null +++ b/akida2/model_zoo/portrait128/models/.gitignore @@ -0,0 +1,4 @@ +# Git to Ignore everything in this directory +* +# Except this .gitignore file +!.gitignore \ No newline at end of file diff --git a/akida2/model_zoo/portrait128/portrait128_benchmark.py b/akida2/model_zoo/portrait128/portrait128_benchmark.py new file mode 100644 index 0000000..ef0e8dc --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_benchmark.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python +# Copyright 2026 Brainchip Holdings Ltd. Apache 2.0 License +""" +Portrait128 hardware benchmark for Akida 2. + +Runs a latency benchmark on an Akida Portrait128 segmentation model, prints a +summary, checks per-layer activation sparsity, and generates summary plots. + +Benchmark details: + * The Akida 2 reference hardware is an FPGA running at 25 MHz, so latency is + measured at that clock. + * Because the FPGA runs at a low clock, a PROJECTED latency at the AKD2500 + target clock is also reported. Cycle count is fixed for a given model and + mapping regardless of clock speed, so projected_latency_ms = + cycles / target_clock. + * Power is not measured. The FPGA power-measurement path is still under + development, so this script reports latency only. + +Example +------- + python portrait128_benchmark.py -l models/akida_unet_portrait128_i8_w8_a8.fbz +""" +import argparse +import json +import pathlib +import sys +import time + +import numpy as np +import akida +from akida_models.sparsity import compute_sparsity + +from portrait128_data import get_samples +from brainchip_utils.hardware_utils import (get_mapping_stats, get_akida_device, + per_layer_benchmark, full_model_benchmark, + AKIDA_CLOCKS_HZ) +from brainchip_utils.plot_utils import (plot_full_model_results, plot_per_layer_results, + pretty_print_sparsity) + +# Measured clock: Akida 2 reference hardware is an FPGA clocked at 25 MHz. +MEASURED_CLOCK = AKIDA_CLOCKS_HZ['AKIDA2_FPGA'] + +# Projected clock: latency is also projected to the AKD2500 target silicon clock, to +# indicate expected performance on production hardware. Cycle count is clock-independent, +# so the projection is exact given the target frequency. AKD2500 production silicon does +# not exist yet, so this is BrainChip's current internal target (1 GHz), not a measured +# value -- it may still change before production. See brainchip_utils.hardware_utils.AKIDA_CLOCKS_HZ. +PROJECTED_CLOCK = AKIDA_CLOCKS_HZ['AKD2500'] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Hardware latency benchmark for an Akida 2 Portrait128 model') + parser.add_argument('-l', '--loadmodel', required=True, + help='Model to load (.fbz akida model)') + parser.add_argument('-d', '--data', default='./data/portrait128', + help='Portrait128 folder (contains img_uint8.npy and msk_uint8.npy)') + parser.add_argument('--save-metrics', action='store_true', + help='Write benchmark values to metrics.json') + args = parser.parse_args() + + NUM_SAMPLES = 100 + + # ------------------------------------------------------------------------- + # Model + # ------------------------------------------------------------------------- + ak_model = akida.Model(args.loadmodel) + imsize = tuple(ak_model.input_shape) + + # ------------------------------------------------------------------------- + # Device + # ------------------------------------------------------------------------- + device = get_akida_device(target_version=ak_model.ip_version) + if device is None: + print("No compatible Akida hardware device found. Skipping benchmarking") + sys.exit(0) + + # ------------------------------------------------------------------------- + # Sample + # ------------------------------------------------------------------------- + # Processing in Akida is activity dependent (because it exploits sparsity) + # and that activity is dependent on the input. That makes it imperative to + # use real inputs when benchmarking Akida, rather than synthetic random + # samples. + samples = get_samples(args.data, imsize, num_samples=NUM_SAMPLES) + + # ------------------------------------------------------------------------- + # Full-model benchmark (latency only) + # ------------------------------------------------------------------------- + # NOTE: power measurement is intentionally not requested here -- the FPGA + # power path is still under development, so we report latency only. When a + # power path exists, this is where full_model_benchmark's power fields would + # be surfaced. + map_modes = ['Minimal', 'AllNps', 'HwPr'] + full_results = dict() + for mm in map_modes: + map_mode = getattr(akida.MapMode, mm) + print(f'\nRunning full-model benchmark (MapMode={mm})...') + res = full_model_benchmark(ak_model, device, samples, + map_mode=map_mode, + clock_freq=MEASURED_CLOCK) + + # Projected latency at a higher target clock. Cycle count (mean_inf_clk) + # is clock-independent, so this is an exact rescale, not an estimate of + # host overhead. + res['projected_clk_ms'] = res['mean_inf_clk'] / PROJECTED_CLOCK * 1000 + full_results[mm] = res + + # Re-map without hw_only to populate ak_model.sequences for stats + ak_model.map(device, mode=map_mode) + num_nps, num_passes, num_sequences = get_mapping_stats(ak_model) + full_results[mm]['num_nps'] = num_nps + full_results[mm]['num_passes'] = num_passes + print(f' Mapping: {num_nps} NP(s), {num_passes} pass(es), {num_sequences} sequence(s)') + print(f' Measured latency @ {MEASURED_CLOCK/1e6:.0f} MHz: ' + f'{res["mean_clk_ms"]:.3f} ms') + print(f' Projected latency @ {PROJECTED_CLOCK/1e6:.0f} MHz: ' + f'{res["projected_clk_ms"]:.3f} ms') + if num_sequences > 1: + print('WARNING: note, model not completely mapped to hardware') + + # ------------------------------------------------------------------------- + # Per-layer benchmark. Minimal mapping mode, batch-size 1 + # ------------------------------------------------------------------------- + ak_model.map(device, mode=akida.MapMode.Minimal, hw_only=True) + ak_model.summary() + + # Check sparsity per-layer + sparsity_dict = compute_sparsity(ak_model, samples=samples) + pretty_print_sparsity(sparsity_dict) + + print(f'Running per-layer benchmark ({NUM_SAMPLES} samples)...') + per_layer_results = per_layer_benchmark(ak_model, device, samples, + repeats=NUM_SAMPLES, + clock_freq=MEASURED_CLOCK) + + # ------------------------------------------------------------------------- + # Plots + # ------------------------------------------------------------------------- + # Map without hw_only so ak_model.sequences is available for plot_mapping + ak_model.map(device, mode=akida.MapMode.Minimal) + # Plot filenames include the model stem, so each benchmarked model keeps + # its own reference figures. + model_stem = pathlib.Path(args.loadmodel).stem + perlayer_savepath = f'benchmark_results_layers_{model_stem}.png' + if args.save_metrics: + perlayer_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_' + perlayer_savepath) + plot_per_layer_results(per_layer_results, ak_model, sparsity_dict, + model_name=args.loadmodel, + example='Portrait128 segmentation (Akida 2)', + savepath=perlayer_savepath) + print('\nPer-layer results plot saved to ' + str(perlayer_savepath)) + + full_savepath = f'benchmark_results_full_{model_stem}.png' + if args.save_metrics: + full_savepath = pathlib.Path(__file__).parent / 'docs' / ('ref_' + full_savepath) + plot_full_model_results(full_results, ak_model, device, + model_name=args.loadmodel, + example='Portrait128 segmentation (Akida 2)', + savepath=full_savepath, + show_power=False) # power measurement not yet available + print('Full model results plot saved to ' + str(full_savepath)) + + if args.save_metrics: + # Updates the stored metrics used to generate the README performance + # tables. For code maintenance only, run against the pretrained models. + # + # Keys are variant-prefixed to match the multi-variant v2 README table. + # The variant is inferred from the loaded .fbz filename. Power keys are + # intentionally absent: benchmarking is latency-only until the FPGA power + # path exists. + stem = pathlib.Path(args.loadmodel).stem + variant = 'w8a8' # the only quantized model; benchmarking needs its .fbz + + metrics_path = pathlib.Path(__file__).parent / 'docs' / 'metrics.json' + metrics = json.loads(metrics_path.read_text()) if metrics_path.exists() else {} + metrics[f'{variant}_sparsity'] = f'{np.mean(list(sparsity_dict.values())) * 100:.2f}%' + for mm, res in full_results.items(): + mode = mm.lower() + metrics[f'{variant}_{mode}_nps'] = str(res['num_nps']) + metrics[f'{variant}_{mode}_passes'] = str(res['num_passes']) + metrics[f'{variant}_{mode}_cycles'] = f'{res["mean_inf_clk"]:.0f}' + metrics[f'{variant}_{mode}_latency_ms'] = f'{res["mean_clk_ms"]:.3f}' + metrics[f'{variant}_{mode}_projected_ms'] = f'{res["projected_clk_ms"]:.3f}' + metrics_path.write_text(json.dumps(metrics, indent=4) + '\n') + print(f'Metrics saved to {metrics_path}') diff --git a/akida2/model_zoo/portrait128/portrait128_data.py b/akida2/model_zoo/portrait128/portrait128_data.py new file mode 100644 index 0000000..3983585 --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_data.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# Copyright 2026 Brainchip Holdings Ltd. Apache 2.0 License +""" +Data loading for the Portrait128 person-segmentation dataset. + +The dataset is two NumPy arrays in one folder: + * img_uint8.npy: RGB portraits, shape (N, 128, 128, 3), uint8 + * msk_uint8.npy: person masks, shape (N, 128, 128, 1), uint8, 255 = person, + 0 = background + +For training, both arrays are loaded into RAM (about 1.2 GB); calibration +samples are read straight from disk. + +The dataset ships as a single pool, so it is split here: 80 % training and 20 % +validation, by a shuffle with a fixed seed (SPLIT_SEED). That seed is +independent of the training seed, so every script and notebook sees the same +validation images whatever --seed is used for training. + +Images are delivered as uint8 in [0, 255]: the model rescales them itself. Masks +are delivered as int32 0/1 labels with the same spatial shape as the image. + +Run as a script, it writes a calibration-sample pack for `quantizeml quantize`: + + python portrait128_data.py --extract data/portrait128_batch1024.npz +""" + +import argparse +import os + +import numpy as np +import tensorflow as tf +import tf_keras as keras +from tf_keras.utils import set_random_seed + +IMAGES_FILE = 'img_uint8.npy' +MASKS_FILE = 'msk_uint8.npy' + +VALIDATION_SPLIT = 0.2 +# Fixed seed for the train/validation split, independent of the training seed +SPLIT_SEED = 1 + + +def _load_arrays(data_path): + """Memory-map the image and mask arrays.""" + x = np.load(os.path.join(data_path, IMAGES_FILE), mmap_mode='r') + y = np.load(os.path.join(data_path, MASKS_FILE), mmap_mode='r') + return x, y + + +def _check_shape(x, input_shape): + if tuple(x.shape[1:]) != tuple(input_shape): + raise ValueError(f'Portrait128 images have shape {tuple(x.shape[1:])}, ' + f'but the model expects {tuple(input_shape)}.') + + +def _split_indices(num_samples, validation_split=VALIDATION_SPLIT, random_state=SPLIT_SEED): + """Shuffle sample indices and split them into training and validation sets. + + Args: + num_samples (int): number of samples in the dataset. + validation_split (float, optional): fraction used for validation. Defaults to 0.2. + random_state (int, optional): seed of the shuffle. Defaults to SPLIT_SEED. + + Returns: + np.ndarray, np.ndarray: training indices, validation indices + """ + rng = np.random.default_rng(random_state) + indices = np.arange(num_samples) + rng.shuffle(indices) + + validation_size = int(validation_split * num_samples) + return indices[validation_size:], indices[:validation_size] + + +def get_data(data_path, input_shape, batch_size, dtype=tf.uint8, seed=42): + """ Loads Portrait128 data. + + Training images are augmented with random horizontal flips, translations + (up to 10 % in each direction) and zooms (up to 20 %). Each image and its + mask are stacked into one 4-channel tensor before augmentation, so both + receive exactly the same geometric transform. + + Args: + data_path (str): folder containing img_uint8.npy and msk_uint8.npy + input_shape (tuple): input image shape (height, width, channels) + batch_size (int): the batch size + dtype (tf.dtypes.DType, optional): input data type. Defaults to tf.uint8. + seed (int, optional): seed for shuffling and augmentation. Defaults to 42. + + Returns: + tf.data.Dataset, tf.data.Dataset: training dataset, validation dataset + """ + set_random_seed(seed) + + # Load both arrays into RAM. Masks are stored as 0/255; integer division + # turns them into 0/1 labels once, here. + x, y = _load_arrays(data_path) + _check_shape(x, input_shape) + x = np.asarray(x) + y = np.asarray(y) // 255 + train_indices, val_indices = _split_indices(len(x)) + + data_augmentation = keras.Sequential([ + keras.layers.RandomFlip("horizontal", seed=seed), + keras.layers.RandomTranslation(height_factor=0.1, width_factor=0.1, seed=seed), + keras.layers.RandomZoom(0.2, seed=seed), + ]) + + def cast_data(image, label): + image = tf.cast(image, dtype) + label = tf.cast(label, tf.int32) + return image, label + + def data_generator(indices): + for i in indices: + yield x[i], y[i] + + def apply_data_augmentation(image, mask): + # Concatenate image and mask along the last axis + combined = tf.concat([image, mask], axis=-1) + # Apply the same augmentation to both image and mask + augmented_combined = data_augmentation(combined) + # Split the augmented combined tensor back into image and mask + augmented_image, augmented_mask = tf.split(augmented_combined, [3, 1], axis=-1) + + return augmented_image, augmented_mask + + output_signature = (tf.TensorSpec(shape=x.shape[1:], dtype=tf.uint8), + tf.TensorSpec(shape=y.shape[1:], dtype=tf.uint8)) + + train_dataset = tf.data.Dataset.from_generator( + generator=lambda: data_generator(train_indices), + output_signature=output_signature) + + validation_dataset = tf.data.Dataset.from_generator( + generator=lambda: data_generator(val_indices), + output_signature=output_signature) + + SHUFFLE_BUFFER_SIZE = 1000 + train_batches = (train_dataset + .shuffle(SHUFFLE_BUFFER_SIZE, seed=seed) + .map(apply_data_augmentation) + .map(cast_data, num_parallel_calls=tf.data.AUTOTUNE, deterministic=True) + .batch(batch_size) + .prefetch(buffer_size=tf.data.AUTOTUNE)) + + validation_batches = (validation_dataset + .map(cast_data, num_parallel_calls=tf.data.AUTOTUNE, deterministic=True) + .batch(batch_size) + .prefetch(buffer_size=tf.data.AUTOTUNE)) + + # The datasets come from Python generators, so their length is unknown to + # tf.data. Declaring it lets Keras show per-epoch progress and lets callers + # use len(). + train_batches = train_batches.apply( + tf.data.experimental.assert_cardinality(-(-len(train_indices) // batch_size))) + validation_batches = validation_batches.apply( + tf.data.experimental.assert_cardinality(-(-len(val_indices) // batch_size))) + + return train_batches, validation_batches + + +def get_samples(data_path, input_shape, num_samples=1024): + """ Loads image samples from the training split as a numpy array. + + No augmentation is applied. Suitable for model calibration and testing. + + Args: + data_path (str): folder containing img_uint8.npy + input_shape (tuple): input image shape (height, width, channels) + num_samples (int): number of samples to return. Defaults to 1024. + + Returns: + np.ndarray: array of shape (num_samples, height, width, channels), dtype uint8 + """ + x = np.load(os.path.join(data_path, IMAGES_FILE), mmap_mode='r') + _check_shape(x, input_shape) + train_indices, _ = _split_indices(len(x)) + + # The training indices are already in shuffled order; sort the selection so + # the memory-mapped reads are sequential. + selection = np.sort(train_indices[:num_samples]) + return np.asarray(x[selection], dtype=np.uint8) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Write Portrait128 calibration samples to an .npz file') + parser.add_argument('--extract', required=True, + help='Output .npz path') + parser.add_argument('-d', '--data', default='./data/portrait128', + help='Portrait128 folder (contains img_uint8.npy and msk_uint8.npy)') + parser.add_argument('-n', '--num_samples', type=int, default=1024, + help='Number of training-split images to save') + args = parser.parse_args() + + samples = get_samples(args.data, (128, 128, 3), num_samples=args.num_samples) + # Saved as a single 'data' array, the layout `quantizeml quantize --samples` reads + np.savez(args.extract, data=samples) + print(f'{len(samples)} samples saved to {args.extract}') diff --git a/akida2/model_zoo/portrait128/portrait128_eval.py b/akida2/model_zoo/portrait128/portrait128_eval.py new file mode 100644 index 0000000..6f39423 --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_eval.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright 2026 Brainchip Holdings Ltd. Apache 2.0 License +""" +Portrait128 evaluation for tf_keras or akida models. + +Reports the binary IoU (mean of the person and background IoUs) and the pixel +accuracy on the validation split, both thresholded at a logit of 0. + +Example +------- + python portrait128_eval.py -d ./data/portrait128 -l models/akida_unet_portrait128.h5 +""" +import argparse +import json +import pathlib +import numpy as np +import tensorflow as tf + +from tqdm import tqdm + +import akida + +from cnn2snn import load_quantized_model + +from portrait128_data import get_data +from portrait128_train import get_metrics +from brainchip_utils.hardware_utils import get_akida_device + +tf.config.experimental.enable_op_determinism() + + +# --------------------------------------------------------------------------- +# Evaluation on Akida +# --------------------------------------------------------------------------- +def evaluate_akida_model(akida_model, val_dataset): + """Run inference with an Akida model and return (binary IoU, pixel accuracy).""" + device = get_akida_device(target_version=akida_model.ip_version) + if device is not None: + akida_model.map(device, mode=akida.MapMode.Minimal) + print('Running inference on Akida hardware device') + akida_model.summary() + + iou_metric, acc_metric = get_metrics() + + # Akida can't directly digest the tensorflow dataset, we need to + # manually iterate over the dataset to deliver inputs as numpy arrays. + # The Akida output already has the mask's (B, H, W, 1) shape, so it can be + # compared with the labels directly: no squeeze is needed. + for batch, label_batch in tqdm(val_dataset, desc="Evaluating on Akida"): + logits_batch = akida_model.predict(batch.numpy().astype(np.uint8)) + iou_metric.update_state(label_batch, logits_batch) + acc_metric.update_state(label_batch, logits_batch) + + binary_iou = float(iou_metric.result()) + accuracy = float(acc_metric.result()) + print(f'Akida binary IoU: {binary_iou:.4f}, pixel accuracy: {accuracy:.4f}') + return binary_iou, accuracy + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-l', '--loadmodel', required=True, + help='Model to load (.h5 tf_keras or .fbz akida model)') + parser.add_argument('-d', '--data', default='./data/portrait128', + help='Portrait128 folder (contains img_uint8.npy and msk_uint8.npy)') + parser.add_argument('--save-metrics', action='store_true', + help='Write IoU and accuracy (and param count for .h5) to metrics.json') + args = parser.parse_args() + + # --------------------------------------------------------------------------- + # Model + # --------------------------------------------------------------------------- + if args.loadmodel.endswith('.h5'): + model = load_quantized_model(args.loadmodel) + model.compile(metrics=get_metrics()) + isakida = False + imsize = model.input_shape[1:] + elif args.loadmodel.endswith('.fbz'): + model = akida.Model(args.loadmodel) + isakida = True + imsize = tuple(model.input_shape) + + # --------------------------------------------------------------------------- + # Data loading + # --------------------------------------------------------------------------- + train_ds, val_ds = get_data(args.data, imsize, batch_size=32) + + # --------------------------------------------------------------------------- + # Evaluation + # --------------------------------------------------------------------------- + if isakida: + binary_iou, accuracy = evaluate_akida_model(model, val_ds) + else: + results = model.evaluate(val_ds, verbose=0, return_dict=True) + binary_iou, accuracy = results['binary_iou'], results['accuracy'] + print(f'Validation binary IoU: {binary_iou:.4f}, pixel accuracy: {accuracy:.4f}') + + # --------------------------------------------------------------------------- + # Persist metrics + # --------------------------------------------------------------------------- + if args.save_metrics: + # Updates the stored metrics used to generate the README performance + # tables. For code maintenance only, run against the pretrained models. + # + # Models are told apart by filename. The 8-bit model contributes + # "quantized" metrics (from its .h5) and "akida" metrics (from its .fbz): + # akida_unet_portrait128.h5 -> float_iou, float_acc, params + # akida_unet_portrait128_i8_w8_a8_qat.{h5,fbz} -> w8a8_quant_* / w8a8_akida_* + # (The PTQ model is a throwaway on the way to QAT: it is never evaluated + # or stored, so it has no metrics.) + metrics_path = pathlib.Path(__file__).parent / 'docs' / 'metrics.json' + metrics = json.loads(metrics_path.read_text()) if metrics_path.exists() else {} + iou_str = f'{binary_iou:.4f}' + acc_str = f'{accuracy * 100:.2f}%' + stem = pathlib.Path(args.loadmodel).stem + + # Determine the variant prefix from the filename. + if 'i8_w8_a8' in stem: + variant = 'w8a8' + else: + variant = None # float model + + if variant is None: + # Float model: record float metrics and parameter count. + metrics['float_iou'] = iou_str + metrics['float_acc'] = acc_str + metrics['params'] = f'{model.count_params():,}' + elif isakida: + metrics[f'{variant}_akida_iou'] = iou_str + metrics[f'{variant}_akida_acc'] = acc_str + else: + metrics[f'{variant}_quant_iou'] = iou_str + metrics[f'{variant}_quant_acc'] = acc_str + metrics_path.write_text(json.dumps(metrics, indent=4) + '\n') + print(f'Metrics saved to {metrics_path}') diff --git a/akida2/model_zoo/portrait128/portrait128_model.py b/akida2/model_zoo/portrait128/portrait128_model.py new file mode 100644 index 0000000..aa80533 --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_model.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# Copyright 2026 Brainchip Holdings Ltd. Apache 2.0 License +""" +Create an Akida U-Net model for Portrait128 person segmentation, targeting the +Akida 2 platform. + +The model is a binary segmentation network on 128 x 128 RGB input. Its output is +a 128 x 128 x 1 map of logits: one value per pixel, positive for "person" and +negative for "background". + +Architecture: + * Encoder: AkidaNet (`akida_models.akidanet_imagenet`) with width multiplier + alpha=0.5 and no classification top. Five stride-2 stages take the 128 x 128 + input down to a 4 x 4 x 512 feature map. + * Decoder: five separable transposed-convolution blocks + (`sepconv_transpose_block`), each doubling the spatial size, back up to + 128 x 128, with Dropout(0.5) after each block. Pointwise weights in the + decoder carry an L2 penalty of 4e-5. + * Head: a 1 x 1 convolution to one channel, with no activation (logits). + +There are no skip connections between encoder and decoder. Upsampling with +transposed convolutions is an Akida 2 feature: this model cannot be built for +Akida 1. + +Inputs are uint8 in [0, 255]. The encoder's `input_scaling=(128, -1)` adds a +Rescaling layer that maps them to [-1, 1) inside the model, so the same uint8 +data pipeline feeds the float, quantized and Akida models. At conversion the +rescaling is folded into the Akida input layer. + +Encoder initialization: with `base_weights='imagenet'` the encoder starts from +BrainChip's ImageNet-trained AkidaNet 0.5 (Akida 2) float weights, loaded by +layer name; the decoder and head start from random weights. That is how the +training pipeline builds the model. Convolution weights don't depend on the +input size, so weights trained at 224 x 224 load directly into this +128 x 128 model. Without `base_weights` the whole model starts from random +weights. + +Usage: + python portrait128_model.py [-bw imagenet] [-s OUTPUT_PATH] +""" + +import argparse + +from tf_keras import Model, regularizers +from tf_keras.layers import Dropout +from tf_keras.utils import set_random_seed + +from cnn2snn import set_akida_version, AkidaVersion +from akida_models import akidanet_imagenet +from akida_models.layer_blocks import sepconv_transpose_block, conv_block +from akida_models.model_io import get_model_path +from akida_models.utils import fetch_file + +# ImageNet-trained AkidaNet 0.5 float weights (Akida 2) from the akida_models model store +IMAGENET_WEIGHTS = 'akidanet_imagenet_224_alpha_0.5.h5' +IMAGENET_WEIGHTS_HASH = 'd818dec7c924757de2e6970b996da9c807ccdf9a9044262091ce526678c0640c' + + +def _fetch_imagenet_weights(): + """Download (or reuse the cached copy of) the ImageNet AkidaNet 0.5 weights.""" + with set_akida_version(AkidaVersion.v2): + url, fname, file_hash = get_model_path('akidanet', model_name_v2=IMAGENET_WEIGHTS, + file_hash_v2=IMAGENET_WEIGHTS_HASH) + return fetch_file(url, fname=fname, file_hash=file_hash, cache_subdir='models') + + +def build_portrait128_model(input_shape=(128, 128, 3), + alpha=0.5, + input_scaling=(128, -1), + base_weights=None, + seed=42): + """Instantiates the Akida U-Net architecture for Portrait128. + + Args: + input_shape (tuple, optional): input shape tuple. Defaults to (128, 128, 3). + alpha (float, optional): controls the width (number of filters) of the model. + Defaults to 0.5. + input_scaling (tuple, optional): scale factor and offset to apply to inputs. + Defaults to (128, -1). Following Akida convention, the scale factor is + used as a divisor. + base_weights (str, optional): weights loaded by layer name after the model + is built: 'imagenet' for the ImageNet AkidaNet 0.5 encoder weights, or the + path to an .h5 file. Defaults to None (random initialization). + seed (int, optional): random seed for weight initialization. Defaults to 42. + + Returns: + tf_keras.Model: a Keras Model instance. + """ + set_random_seed(seed) + + # Transposed convolutions are only available on Akida 2 + with set_akida_version(AkidaVersion.v2): + # Define weight regularization, will apply to pointwise weights of sepconv transposed layers + weight_regularizer = regularizers.l2(4e-5) + + # Create an AkidaNet network without top layers + encoder = akidanet_imagenet(input_shape=input_shape, + alpha=alpha, + include_top=False, + input_scaling=input_scaling) + + # Add the decoder layers + x = encoder.layers[-1].output + x = sepconv_transpose_block(x, + filters=int(512 * alpha), + kernel_size=(3, 3), + strides=2, + padding='same', + kernel_initializer='he_normal', + add_batchnorm=True, + relu_activation='ReLU7.5', + name='sepconv_t_0', + pointwise_regularizer=weight_regularizer) + x = Dropout(0.5)(x) + x = sepconv_transpose_block(x, + filters=int(256 * alpha), + kernel_size=(3, 3), + strides=2, + padding='same', + kernel_initializer='he_normal', + add_batchnorm=True, + relu_activation='ReLU7.5', + name='sepconv_t_1', + pointwise_regularizer=weight_regularizer) + x = Dropout(0.5)(x) + x = sepconv_transpose_block(x, + filters=int(128 * alpha), + kernel_size=(3, 3), + strides=2, + padding='same', + kernel_initializer='he_normal', + add_batchnorm=True, + relu_activation='ReLU7.5', + name='sepconv_t_2', + pointwise_regularizer=weight_regularizer) + x = Dropout(0.5)(x) + x = sepconv_transpose_block(x, + filters=int(64 * alpha), + kernel_size=(3, 3), + strides=2, + padding='same', + kernel_initializer='he_normal', + add_batchnorm=True, + relu_activation='ReLU7.5', + name='sepconv_t_3', + pointwise_regularizer=weight_regularizer) + x = Dropout(0.5)(x) + x = sepconv_transpose_block(x, + filters=int(32 * alpha), + kernel_size=(3, 3), + strides=2, + padding='same', + kernel_initializer='he_normal', + add_batchnorm=True, + relu_activation='ReLU7.5', + name='sepconv_t_4', + pointwise_regularizer=weight_regularizer) + x = Dropout(0.5)(x) + x = conv_block(x, filters=1, kernel_size=(1, 1), relu_activation=False, name='head') + + # Build the whole model: encoder followed by decoder + model = Model(inputs=encoder.input, outputs=x, name='akida_unet') + + if base_weights is not None: + if base_weights == 'imagenet': + base_weights = _fetch_imagenet_weights() + model.load_weights(base_weights, by_name=True) + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Build the Akida U-Net Portrait128 model for Akida 2') + parser.add_argument("-s", + "--savepath", + type=str, + default='./models/akida_unet_portrait128.h5', + help="Save model with the specified path + name") + parser.add_argument("-bw", + "--base_weights", + type=str, + default=None, + help="Weights to load by layer name: 'imagenet' or an .h5 path") + parser.add_argument('--seed', type=int, default=42, + help='Random seed for reproducibility') + args = parser.parse_args() + + model = build_portrait128_model(base_weights=args.base_weights, seed=args.seed) + model.summary() + model.save(args.savepath, include_optimizer=False) + print(f'Model saved to {args.savepath}') diff --git a/akida2/model_zoo/portrait128/portrait128_notebook.py b/akida2/model_zoo/portrait128/portrait128_notebook.py new file mode 100644 index 0000000..fef08ad --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_notebook.py @@ -0,0 +1,369 @@ +# --- +# jupyter: +# jupytext: +# formats: ipynb,py:percent +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.19.6 +# kernelspec: +# display_name: Python 3 +# language: python +# name: python3 +# --- + +# %% [markdown] +# BrainChip Dev Hub +# +# # Portrait Segmentation (Portrait128) — Akida 2 Training +# +# Run Time: ~7 hours with training included (NVIDIA RTX 2080 Ti) / TBD with training skipped +# +# This notebook walks through the complete pipeline to train, quantize, convert, and evaluate an Akida U-Net model on the **Portrait128** dataset for Akida 2 hardware. +# +# Portrait128 is a binary segmentation task: for each 128 × 128 portrait, the model labels every pixel as **person** or **background**. +# +# The pipeline follows the standard Akida workflow: +# 1. Train a float model +# 2. 8-bit quantization (PTQ), then one epoch of QAT +# 3. Conversion to Akida `.fbz` format +# 4. Evaluation on Akida +# +# The focus is on the **Akida-specific** aspects of the pipeline. The data loading, +# model definition and training loop live in the accompanying Python files: they are +# standard tf_keras code and are imported here rather than repeated. +# +# Segmentation differs from the classification examples in this model zoo in two ways +# that matter on Akida. The model has to **upsample** back to full resolution, which +# needs transposed convolutions, an Akida 2 feature. And its output is a full-size map +# rather than a handful of class scores, so the last layers of the network run at the +# highest resolution. Both come up below. + +# %% +# Colab-only setup. Local users: ignore this cell — it does nothing for you. +import sys, os + +if 'google.colab' in sys.modules: + if not os.path.exists('colab_setup.py'): + !wget -q https://raw.githubusercontent.com/Brainchip-Inc/brainchip_devhub/main/akida2/model_zoo/portrait128/colab_setup.py + import colab_setup; colab_setup.setup() + +# %% [markdown] +# ## Setup +# +# The default dataset path is `./data/portrait128`, a folder holding the two arrays +# `img_uint8.npy` and `msk_uint8.npy`. See the [README](README.md) for where to get +# them and for symlink setup. Update `DATA_PATH` below if you keep them elsewhere. +# +# Setting `RUN_FLOAT_TRAINING` to `False` skips the training run and loads the float +# model saved in `MODELS_DIR` by an earlier run of this notebook or of +# `portrait128_train.sh`. + +# %% +import os +import numpy as np +import tensorflow as tf +from tqdm import tqdm + +DATA_PATH = './data/portrait128' +MODELS_DIR = './models' +os.makedirs(MODELS_DIR, exist_ok=True) + +RUN_FLOAT_TRAINING = True + +SEED = 42 + +# Must be called before any TF ops to make GPU ops deterministic. +tf.config.experimental.enable_op_determinism() + +# %% [markdown] +# ## Dataset +# +# `get_data` returns a training and a validation `tf.data.Dataset`. The full loading +# and augmentation code is in [portrait128_data.py](portrait128_data.py). +# +# A few points that matter for what follows: +# +# * **Images stay uint8**, in their original `[0, 255]` range. The model rescales them +# itself (see Model below), so the same data feed the float, quantized and Akida +# models without any change, and a deployed Akida model can take camera pixels +# directly. +# * **Labels are masks**, of shape `(128, 128, 1)`: 1 for person, 0 for background. +# * **Augmentation is geometric** (flips, shifts and zooms), and each image is stacked +# with its mask before it is transformed. That way the mask moves with the image; +# transforming them separately would misalign the labels. +# * **The split is fixed.** The dataset ships as one pool, so it is split 80/20 with a +# fixed shuffle seed, independent of `SEED`. Every run is evaluated on the same +# validation images. + +# %% +from portrait128_data import get_data + +BATCH_SIZE = 32 +INPUT_SHAPE = (128, 128, 3) + +train_ds, val_ds = get_data(DATA_PATH, INPUT_SHAPE, BATCH_SIZE, seed=SEED) + +# %% [markdown] +# ## Model +# +# The model is the **Akida U-Net** from `akida_models`, an encoder–decoder: +# +# * The **encoder** is AkidaNet with width multiplier alpha = 0.5. Its five stride-2 +# stages reduce the 128 × 128 input to a 4 × 4 × 512 feature map: coarse in space +# but rich in features. +# * The **decoder** is five separable *transposed*-convolution blocks. Each doubles the +# spatial size, back up to 128 × 128. A transposed convolution with stride 2 is a +# learned upsampling, and it is the reason this model needs Akida 2: Akida 1 has no +# upsampling layer. Dropout after each block regularizes the decoder during +# training; it does nothing at inference and is dropped at conversion. +# * A final **1 × 1 convolution** gives one logit per pixel: positive for person, +# negative for background. There is no sigmoid in the model. The loss applies it, +# and a threshold at 0 gives the same mask as a threshold at probability 0.5. +# +# Unlike the classic U-Net there are **no skip connections** from encoder to decoder, +# so all the detail in the output has to pass through the 4 × 4 bottleneck. That keeps +# the graph a simple chain. The cost is softer mask edges than a U-Net with skips +# would give. +# +# The input rescaling is part of the model. The encoder's `input_scaling=(128, -1)` +# adds a layer that maps `[0, 255]` to `[-1, 1)`. Keeping it in the graph means one data +# pipeline serves every version of the model. It also means the conversion step can +# fold the rescaling into the Akida input layer's parameters, so the deployed model +# still takes plain uint8 pixels. +# +# **Encoder initialization.** `base_weights='imagenet'` loads BrainChip's +# ImageNet-trained AkidaNet 0.5 weights into the encoder, by layer name; the decoder +# starts from random weights. The ImageNet model was trained at 224 × 224, but +# convolution weights don't depend on the input size, so they load directly into this +# 128 × 128 encoder. Starting from features that already recognize edges, textures and +# shapes is why the learning rate below can be small. + +# %% +from portrait128_model import build_portrait128_model + +model = build_portrait128_model(base_weights='imagenet', seed=SEED) +model.summary() + +# %% [markdown] +# ## Float Training +# +# The full training code is in [portrait128_train.py](portrait128_train.py). The +# choices that matter here: +# +# * **Loss**: binary cross-entropy on the logits, averaged over every pixel. Each pixel +# is its own binary classification. +# * **Metrics**: *binary IoU*, the overlap between predicted and true masks averaged +# over person and background, and *pixel accuracy*. Background covers a large share of +# every image, so pixel accuracy is high even for poor masks; IoU penalizes a mask of +# the wrong shape, so it is the one to watch. +# * **Learning rate**: Adam with a one-cycle cosine schedule, stepped every batch. It +# warms up from a 25th of the peak to the peak `LEARNING_RATE` over the first 30 % of +# training, then decays. The peak, 3e-5, is low because the encoder starts +# pretrained: a high rate early on would destroy the ImageNet features before the +# decoder has learned to use them. + +# %% +from portrait128_train import train_portrait128, get_metrics + +if RUN_FLOAT_TRAINING: + LEARNING_RATE = 3e-5 + EPOCHS = 50 + # Freshly build the datasets for reproducibility + train_ds, val_ds = get_data(DATA_PATH, INPUT_SHAPE, BATCH_SIZE, seed=SEED) + + train_portrait128(model, train_ds, val_ds, EPOCHS, LEARNING_RATE, seed=SEED) + + float_model_path = os.path.join(MODELS_DIR, 'akida_unet_portrait128.h5') + model.save(float_model_path, include_optimizer=False) + print(f'Float model saved to {float_model_path}') +else: + from tf_keras.models import load_model + print('Training skipped. Loading an existing float model...') + model = load_model(os.path.join(MODELS_DIR, 'akida_unet_portrait128.h5')) + +# %% [markdown] +# ### Evaluate float model + +# %% +model.compile(metrics=get_metrics()) +float_results = model.evaluate(val_ds, verbose=0, return_dict=True) +float_iou, float_accuracy = float_results['binary_iou'], float_results['accuracy'] +print(f'Float binary IoU: {float_iou:.4f}, pixel accuracy: {float_accuracy:.4f}') + +# %% [markdown] +# ## Quantization (quantizeml) +# +# Akida runs on integers, so the weights and activations have to be quantized. Akida 2 +# quantizes with **quantizeml** (not `cnn2snn.quantize`). We quantize to **8 bits** +# (i8/w8/a8) by post-training quantization (PTQ) only: 256 levels per weight and +# activation track the float values closely enough that no retraining is needed. +# `input_weight_bits` sets the input layer separately, because it runs on Akida's +# dedicated input processor; here it is 8-bit like everything else. +# +# **Why no 4-bit variant?** Other Akida 2 examples also offer a 4-bit model, fine-tuned +# with quantization-aware training (QAT). This one can't. The decoder upsamples with +# transposed depthwise convolutions, and on Akida 2 those layers need 8-bit inputs and +# 8-bit weights. In a model quantized to 4 bits throughout, all five of them would fall +# back to software, so the model would no longer run fully on the hardware. +# +# Quantization with `quantizeml` requires **calibration samples**. Calibration runs real +# inputs through the model to measure the range of every layer's activations, and those +# ranges set the quantization scales. Random data would give the wrong ranges, so the +# samples should be representative images for the task. They are drawn from the +# **training** split, to avoid leaking validation data into the model. + +# %% +from quantizeml.models import quantize, QuantizationParams + +from portrait128_data import get_samples + +NUM_SAMPLES = 1024 +samples = get_samples(DATA_PATH, INPUT_SHAPE, num_samples=NUM_SAMPLES) + +# --- 8-bit (i8 / w8 / a8), PTQ --- +qparams_8bit = QuantizationParams(input_weight_bits=8, weight_bits=8, activation_bits=8) +model_8bit = quantize(model, qparams=qparams_8bit, samples=samples) + +model_8bit.compile(metrics=get_metrics()) +ptq_results = model_8bit.evaluate(val_ds, verbose=0, return_dict=True) +iou_ptq, acc_ptq = ptq_results['binary_iou'], ptq_results['accuracy'] +print(f'8-bit PTQ binary IoU: {iou_ptq:.4f}, pixel accuracy: {acc_ptq:.4f}') + +# %% [markdown] +# ### One epoch of QAT +# +# The 8-bit model then gets one epoch of quantization-aware training (QAT), as the +# model in BrainChip's model zoo does. During QAT the forward pass uses the quantized +# weights and activations, while the gradients update an underlying float copy of the +# weights, so the network adjusts to the rounding that PTQ introduced. +# +# QAT starts from a model that is already close to a good solution, so it uses a lower +# learning rate: a tenth of the float rate here. A higher rate would undo the float +# training rather than refine it. +# +# A model quantized with `quantizeml` is a standard Keras model, so it goes straight +# back into the same training function used for float training. The PTQ model itself +# is not kept; compare its result above with the one after QAT. + +# %% +QAT_EPOCHS = 1 +QAT_LR = 3e-6 +train_ds, val_ds = get_data(DATA_PATH, INPUT_SHAPE, BATCH_SIZE, seed=SEED) +train_portrait128(model_8bit, train_ds, val_ds, QAT_EPOCHS, QAT_LR, seed=SEED) + +q8_path = os.path.join(MODELS_DIR, 'akida_unet_portrait128_i8_w8_a8_qat.h5') +model_8bit.save(q8_path, include_optimizer=False) + +model_8bit.compile(metrics=get_metrics()) +q8_results = model_8bit.evaluate(val_ds, verbose=0, return_dict=True) +iou_8bit, acc_8bit = q8_results['binary_iou'], q8_results['accuracy'] +print(f'8-bit QAT binary IoU: {iou_8bit:.4f}, pixel accuracy: {acc_8bit:.4f}') + +# %% [markdown] +# Note: the quantized model can be saved using the standard method. However, +# for later reloading, because of the custom quantized layers in the model +# we have to use the `load_model` function from `quantizeml.model_io` (a wrapper +# around the standard tf_keras loading function). + +# %% +# Just to illustrate how to load a quantized model +del model_8bit + +from quantizeml.model_io import load_model +model_8bit = load_model(q8_path) + +# %% [markdown] +# ## Conversion to Akida Format +# +# `cnn2snn.convert` compiles the quantized Keras model into an Akida `.fbz` +# model that can be loaded and executed directly on an Akida 2 hardware device. +# The converter verifies hardware compatibility and maps each layer to its +# corresponding Akida primitive. +# +# In the summary below, note the following: +# +# * The input rescaling has no layer of its own. It was folded into the first +# convolution. +# * Each decoder block becomes a `DepthwiseConv2DTranspose` followed by a pointwise +# `Conv2D`. On Akida 2 the depthwise and pointwise parts of a separable convolution +# are two separate layers. The `DepthwiseConv2DTranspose` layers are the ones that +# need 8 bits to run on the hardware. +# * The last layer is a `Dequantizer`, so the model outputs float logits of shape +# `(128, 128, 1)`, which can be thresholded at 0 just like the Keras model's. + +# %% +from cnn2snn import convert + +akida_8bit = convert(model_8bit) +akida_8bit.save(os.path.join(MODELS_DIR, 'akida_unet_portrait128_i8_w8_a8_qat.fbz')) + +akida_8bit.summary() + + +# %% [markdown] +# ## Evaluation on Akida (software backend) +# +# We now run evaluation through the Akida model, to check that its results match +# those of the quantized tf_keras model. Here, we deliberately use the software +# backend (the default, since we do not check for and map to a connected hardware +# device): this delivers a bit-accurate simulation of the results that will be +# obtained when running the model on hardware. + +# %% [markdown] +# ### Run Evaluation on Akida +# +# The Akida runtime cannot consume `tf.data.Dataset` objects directly; it expects a +# 4D numpy array `(n, h, w, c)` of uint8. So we iterate over validation batches +# manually and convert each one. +# +# The output already has the mask's shape, `(B, 128, 128, 1)`, so it goes straight +# into the same IoU and accuracy metrics as the labels, with no reshaping. (Classifiers +# return `(B, 1, 1, C)` and need a squeeze first.) + +# %% +def evaluate_akida(akida_model, ds): + iou_metric, acc_metric = get_metrics() + for batch, label_batch in tqdm(ds, desc='Evaluating on Akida'): + logits = akida_model.predict(batch.numpy().astype(np.uint8)) + iou_metric.update_state(label_batch, logits) + acc_metric.update_state(label_batch, logits) + return float(iou_metric.result()), float(acc_metric.result()) + +akida_iou_8bit, akida_acc_8bit = evaluate_akida(akida_8bit, val_ds) +print(f'Akida 8-bit QAT binary IoU: {akida_iou_8bit:.4f}, pixel accuracy: {akida_acc_8bit:.4f}') + +# %% [markdown] +# ## Activation Sparsity +# +# Akida is event-based: its standard neural processors only do work for non-zero +# activations. The more zeros a layer receives (its input *sparsity*), the less time and +# energy it costs. Sparsity depends on the data, so it has to be measured by running real +# images through the model. Here, we reuse the calibration samples built earlier. +# +# The input layer is the exception. It runs on the dedicated input processor, a +# conventional convolution whose cost doesn't depend on how many of its inputs are zero. +# +# For this model, the per-layer benchmark in `portrait128_notebook_benchmark.ipynb` shows +# where the time goes. The slowest layers are the encoder's first full convolutions +# (`conv_1` to `conv_3`), which apply 3 × 3 kernels across every channel of the 64 × 64 and +# 32 × 32 feature maps. The decoder's upsampling layers, which run at up to 128 × 128, come +# next, and the deep encoder layers at 8 × 8 and 4 × 4 cost little. So sparsity in those +# early convolutions has the most room to reduce latency. + +# %% +from akida_models.sparsity import compute_sparsity +from brainchip_utils.plot_utils import pretty_print_sparsity + +# Reuse the calibration samples built earlier (sparsity needs real activations too). +pretty_print_sparsity(compute_sparsity(akida_8bit, samples=samples)) + +# %% [markdown] +# ## Summary + +# %% +print(f'{"Variant":<16}{"Keras IoU":<12}{"Keras acc":<12}{"Akida IoU":<12}{"Akida acc":<12}') +print(f'{"float":<16}{float_iou:<12.4f}{float_accuracy:<12.4f}{"-":<12}{"-":<12}') +print(f'{"8-bit PTQ":<16}{iou_ptq:<12.4f}{acc_ptq:<12.4f}{"-":<12}{"-":<12}') +print(f'{"8-bit QAT":<16}{iou_8bit:<12.4f}{acc_8bit:<12.4f}{akida_iou_8bit:<12.4f}{akida_acc_8bit:<12.4f}') diff --git a/akida2/model_zoo/portrait128/portrait128_notebook_benchmark.ipynb b/akida2/model_zoo/portrait128/portrait128_notebook_benchmark.ipynb new file mode 100644 index 0000000..4657e68 --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_notebook_benchmark.ipynb @@ -0,0 +1,485 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "\"BrainChip\n", + "\n", + "# Portrait Segmentation (Portrait128) — Akida 2 Benchmark\n", + "\n", + "Run Time: ~35 minutes on the Akida 2 FPGA\n", + "\n", + "This notebook walks through evaluating and benchmarking a model on the **Portrait128** dataset, targeting Akida 2 hardware.\n", + "\n", + "For details on the dataset and preparation of the model, see the neighbouring README.md and `portrait128_notebook_training.ipynb`.\n", + "\n", + "> **Note:** the hardware benchmark needs a physical Akida 2 FPGA device. Every hardware step is guarded, so the notebook runs end to end without a board: the latency cells are skipped, and sparsity is still computed on the software backend. It will not produce hardware numbers on Colab.\n", + "\n", + "The Akida 2 reference platform is an FPGA running at 25 MHz. Alongside the measured latency, the notebook reports a projected latency at the AKD2500 target clock. Power is not measured yet, so the benchmark is latency-only." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cell-02", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import akida\n", + "\n", + "from brainchip_utils.hardware_utils import AKIDA_CLOCKS_HZ\n", + "\n", + "DATA_PATH = './data/portrait128'\n", + "INPUT_SHAPE = (128, 128, 3)\n", + "\n", + "# Pretrained models committed to the repo. If you trained your own with the\n", + "# training notebook or portrait128_train.sh, set MODELS_DIR = './models/'.\n", + "MODELS_DIR = './pretrained_models/'\n", + "\n", + "# The converted 8-bit (QAT) model produced by the training pipeline\n", + "MODEL_FILENAME = 'akida_unet_portrait128_i8_w8_a8_qat.fbz'\n", + "\n", + "# Clocks (see portrait128_benchmark.py and brainchip_utils.hardware_utils.AKIDA_CLOCKS_HZ).\n", + "MEASURED_CLOCK = AKIDA_CLOCKS_HZ['AKIDA2_FPGA'] # 25 MHz FPGA\n", + "PROJECTED_CLOCK = AKIDA_CLOCKS_HZ['AKD2500'] # 1 GHz target (pre-production)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## Model\n", + "\n", + "The notebook loads a converted Akida model (`.fbz`) from `pretrained_models/`, for example\n", + "`pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz`. Pretrained models are committed\n", + "through `git-lfs`; see the [Trained models](../../../README.md#trained-models) section of the\n", + "top-level README for set-up.\n", + "\n", + "The model takes a uint8 RGB image of 128 × 128 and returns one logit per pixel, as a\n", + "`(128, 128, 1)` map. Its first layer, `InputConv2D`, runs on Akida's dedicated input\n", + "processor, which takes the uint8 image directly." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cell-04", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Model Summary \n", + "_________________________________________________\n", + "Input shape Output shape Sequences Layers\n", + "=================================================\n", + "[128, 128, 3] [128, 128, 1] 1 36 \n", + "_________________________________________________\n", + "\n", + "_____________________________________________________________________________\n", + "Layer (type) Output shape Kernel shape \n", + "\n", + "====================== SW/conv_0-dequantizer (Software) =====================\n", + "\n", + "conv_0 (InputConv2D) [64, 64, 16] (3, 3, 3, 16) \n", + "_____________________________________________________________________________\n", + "conv_1 (Conv2D) [64, 64, 32] (3, 3, 16, 32) \n", + "_____________________________________________________________________________\n", + "conv_2 (Conv2D) [32, 32, 64] (3, 3, 32, 64) \n", + "_____________________________________________________________________________\n", + "conv_3 (Conv2D) [32, 32, 64] (3, 3, 64, 64) \n", + "_____________________________________________________________________________\n", + "dw_separable_4 (DepthwiseConv2D) [16, 16, 64] (3, 3, 64, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_4 (Conv2D) [16, 16, 128] (1, 1, 64, 128) \n", + "_____________________________________________________________________________\n", + "dw_separable_5 (DepthwiseConv2D) [16, 16, 128] (3, 3, 128, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_5 (Conv2D) [16, 16, 128] (1, 1, 128, 128)\n", + "_____________________________________________________________________________\n", + "dw_separable_6 (DepthwiseConv2D) [8, 8, 128] (3, 3, 128, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_6 (Conv2D) [8, 8, 256] (1, 1, 128, 256)\n", + "_____________________________________________________________________________\n", + "dw_separable_7 (DepthwiseConv2D) [8, 8, 256] (3, 3, 256, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_7 (Conv2D) [8, 8, 256] (1, 1, 256, 256)\n", + "_____________________________________________________________________________\n", + "dw_separable_8 (DepthwiseConv2D) [8, 8, 256] (3, 3, 256, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_8 (Conv2D) [8, 8, 256] (1, 1, 256, 256)\n", + "_____________________________________________________________________________\n", + "dw_separable_9 (DepthwiseConv2D) [8, 8, 256] (3, 3, 256, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_9 (Conv2D) [8, 8, 256] (1, 1, 256, 256)\n", + "_____________________________________________________________________________\n", + "dw_separable_10 (DepthwiseConv2D) [8, 8, 256] (3, 3, 256, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_10 (Conv2D) [8, 8, 256] (1, 1, 256, 256)\n", + "_____________________________________________________________________________\n", + "dw_separable_11 (DepthwiseConv2D) [8, 8, 256] (3, 3, 256, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_11 (Conv2D) [8, 8, 256] (1, 1, 256, 256)\n", + "_____________________________________________________________________________\n", + "dw_separable_12 (DepthwiseConv2D) [4, 4, 256] (3, 3, 256, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_12 (Conv2D) [4, 4, 512] (1, 1, 256, 512)\n", + "_____________________________________________________________________________\n", + "dw_separable_13 (DepthwiseConv2D) [4, 4, 512] (3, 3, 512, 1) \n", + "_____________________________________________________________________________\n", + "pw_separable_13 (Conv2D) [4, 4, 512] (1, 1, 512, 512)\n", + "_____________________________________________________________________________\n", + "dw_sepconv_t_0 (DepthwiseConv2DTranspose) [8, 8, 512] (3, 3, 512, 1) \n", + "_____________________________________________________________________________\n", + "pw_sepconv_t_0 (Conv2D) [8, 8, 256] (1, 1, 512, 256)\n", + "_____________________________________________________________________________\n", + "dw_sepconv_t_1 (DepthwiseConv2DTranspose) [16, 16, 256] (3, 3, 256, 1) \n", + "_____________________________________________________________________________\n", + "pw_sepconv_t_1 (Conv2D) [16, 16, 128] (1, 1, 256, 128)\n", + "_____________________________________________________________________________\n", + "dw_sepconv_t_2 (DepthwiseConv2DTranspose) [32, 32, 128] (3, 3, 128, 1) \n", + "_____________________________________________________________________________\n", + "pw_sepconv_t_2 (Conv2D) [32, 32, 64] (1, 1, 128, 64) \n", + "_____________________________________________________________________________\n", + "dw_sepconv_t_3 (DepthwiseConv2DTranspose) [64, 64, 64] (3, 3, 64, 1) \n", + "_____________________________________________________________________________\n", + "pw_sepconv_t_3 (Conv2D) [64, 64, 32] (1, 1, 64, 32) \n", + "_____________________________________________________________________________\n", + "dw_sepconv_t_4 (DepthwiseConv2DTranspose) [128, 128, 32] (3, 3, 32, 1) \n", + "_____________________________________________________________________________\n", + "pw_sepconv_t_4 (Conv2D) [128, 128, 16] (1, 1, 32, 16) \n", + "_____________________________________________________________________________\n", + "head (Conv2D) [128, 128, 1] (1, 1, 16, 1) \n", + "_____________________________________________________________________________\n", + "dequantizer (Dequantizer) [128, 128, 1] N/A \n", + "_____________________________________________________________________________\n" + ] + } + ], + "source": [ + "ak_model = akida.Model(os.path.join(MODELS_DIR, MODEL_FILENAME))\n", + "ak_model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-05", + "metadata": {}, + "source": [ + "## Device\n", + "\n", + "`get_akida_device` returns `None` when no compatible hardware is present, in which case the latency benchmark below is skipped." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cell-06", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Akida device found\n", + "Akida device found: \n" + ] + } + ], + "source": [ + "from brainchip_utils.hardware_utils import get_akida_device\n", + "\n", + "device = get_akida_device(target_version=ak_model.ip_version)\n", + "if device is None:\n", + " print('No compatible Akida hardware device found — benchmark will be skipped.')\n", + "else:\n", + " print('Akida device found:', device)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-07", + "metadata": {}, + "source": [ + "## Samples\n", + "\n", + "Akida's standard neural processors only do work for non-zero activations, so latency depends on the input. Random or synthetic data drives unrepresentative activity and gives misleading timings, so the benchmark runs on real images from the training split." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cell-08", + "metadata": {}, + "outputs": [], + "source": [ + "from portrait128_data import get_samples\n", + "\n", + "NUM_SAMPLES = 100\n", + "samples = get_samples(DATA_PATH, INPUT_SHAPE, num_samples=NUM_SAMPLES)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-09", + "metadata": {}, + "source": [ + "## Latency benchmark\n", + "\n", + "The model is benchmarked in each of the three mapping modes, which trade the number of neural processors (NPs) used against the number of passes:\n", + "\n", + "* **Minimal** maps the model onto the fewest NPs that can hold it.\n", + "* **AllNps** spreads it over as many NPs as possible, while keeping the minimum number of passes.\n", + "* **HwPr** (hardware partial reconfiguration) splits the model into several passes so each layer can use more NPs, reloading weights between passes on every inference.\n", + "\n", + "Which mode is fastest depends on the model, so all three are measured.\n", + "\n", + "The FPGA runs at 25 MHz, far below production silicon. The number of clock cycles per inference is fixed for a given model and mapping, whatever the clock, so the latency at the AKD2500 target clock is an exact rescale: `projected_ms = mean_inf_clk / PROJECTED_CLOCK * 1000`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cell-10", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Power measurement not available: only supported on AKD1500 (IpVersion.v1).\n", + " Repeat 1/10 done.\n", + " Repeat 2/10 done.\n", + " Repeat 3/10 done.\n", + " Repeat 4/10 done.\n", + " Repeat 5/10 done.\n", + " Repeat 6/10 done.\n", + " Repeat 7/10 done.\n", + " Repeat 8/10 done.\n", + " Repeat 9/10 done.\n", + " Repeat 10/10 done.\n", + "\n", + " Mean inference time: 405.299 ms (σ=7.030 ms)\n", + " Mean on-chip time: 396.036 ms (9900888 clocks)\n", + " Total inferences run: 1000\n", + "[Minimal] NPs=67 passes=3 latency@25MHz=396.036 ms projected@1000MHz=9.901 ms\n", + "Power measurement not available: only supported on AKD1500 (IpVersion.v1).\n", + " Repeat 1/10 done.\n", + " Repeat 2/10 done.\n", + " Repeat 3/10 done.\n", + " Repeat 4/10 done.\n", + " Repeat 5/10 done.\n", + " Repeat 6/10 done.\n", + " Repeat 7/10 done.\n", + " Repeat 8/10 done.\n", + " Repeat 9/10 done.\n", + " Repeat 10/10 done.\n", + "\n", + " Mean inference time: 404.907 ms (σ=7.040 ms)\n", + " Mean on-chip time: 395.646 ms (9891155 clocks)\n", + " Total inferences run: 1000\n", + "[AllNps] NPs=69 passes=3 latency@25MHz=395.646 ms projected@1000MHz=9.891 ms\n", + "Power measurement not available: only supported on AKD1500 (IpVersion.v1).\n", + " Repeat 1/10 done.\n", + " Repeat 2/10 done.\n", + " Repeat 3/10 done.\n", + " Repeat 4/10 done.\n", + " Repeat 5/10 done.\n", + " Repeat 6/10 done.\n", + " Repeat 7/10 done.\n", + " Repeat 8/10 done.\n", + " Repeat 9/10 done.\n", + " Repeat 10/10 done.\n", + "\n", + " Mean inference time: 257.312 ms (σ=5.065 ms)\n", + " Mean on-chip time: 254.591 ms (6364781 clocks)\n", + " Total inferences run: 1000\n", + "[HwPr] NPs=187 passes=8 latency@25MHz=254.591 ms projected@1000MHz=6.365 ms\n" + ] + } + ], + "source": [ + "from brainchip_utils.hardware_utils import full_model_benchmark, get_mapping_stats\n", + "\n", + "if device is not None:\n", + " for mm in ['Minimal', 'AllNps', 'HwPr']:\n", + " map_mode = getattr(akida.MapMode, mm)\n", + " res = full_model_benchmark(ak_model, device, samples,\n", + " map_mode=map_mode, clock_freq=MEASURED_CLOCK)\n", + " projected_ms = res['mean_inf_clk'] / PROJECTED_CLOCK * 1000\n", + " ak_model.map(device, mode=map_mode)\n", + " num_nps, num_passes, num_sequences = get_mapping_stats(ak_model)\n", + " print(f'[{mm}] NPs={num_nps} passes={num_passes} '\n", + " f'latency@25MHz={res[\"mean_clk_ms\"]:.3f} ms '\n", + " f'projected@1000MHz={projected_ms:.3f} ms')\n", + "else:\n", + " print('Hardware not available — skipping latency benchmark.')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## Per-layer benchmark & sparsity\n", + "\n", + "The per-layer benchmark (Minimal mapping) shows where the time goes, and the sparsity printout shows why. A layer's cost grows with the number of non-zero activations it receives, so on Akida, per-layer latency and input sparsity are linked: a layer with dense input and a large feature map is slow.\n", + "\n", + "In this model, the slowest layers are the encoder's first full convolutions (`conv_1` to `conv_3`), which apply 3 × 3 kernels across every channel of the 64 × 64 and 32 × 32 feature maps. The decoder's upsampling layers, at up to 128 × 128, come next, and the deep encoder layers at 8 × 8 and 4 × 4 cost little.\n", + "\n", + "The input layer (`conv_0`) is the exception to the sparsity rule: it runs on the dedicated input processor, whose cost doesn't depend on how many of its inputs are zero.\n", + "\n", + "Sparsity doesn't need hardware, so it is also computed on the software backend when no device is present." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cell-12", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Layer Sparsity\n", + "----------------------------\n", + "conv_0 26.02%\n", + "conv_1 41.78%\n", + "conv_2 49.98%\n", + "conv_3 35.52%\n", + "dw_separable_4 11.18%\n", + "pw_separable_4 27.09%\n", + "dw_separable_5 9.95%\n", + "pw_separable_5 59.14%\n", + "dw_separable_6 13.68%\n", + "pw_separable_6 42.10%\n", + "dw_separable_7 13.74%\n", + "pw_separable_7 49.42%\n", + "dw_separable_8 19.40%\n", + "pw_separable_8 54.57%\n", + "dw_separable_9 20.78%\n", + "pw_separable_9 62.35%\n", + "dw_separable_10 25.13%\n", + "pw_separable_10 69.91%\n", + "dw_separable_11 32.66%\n", + "pw_separable_11 48.60%\n", + "dw_separable_12 24.16%\n", + "pw_separable_12 78.75%\n", + "dw_separable_13 39.64%\n", + "pw_separable_13 70.33%\n", + "dw_sepconv_t_0 66.37%\n", + "pw_sepconv_t_0 55.17%\n", + "dw_sepconv_t_1 54.85%\n", + "pw_sepconv_t_1 53.14%\n", + "dw_sepconv_t_2 46.27%\n", + "pw_sepconv_t_2 49.41%\n", + "dw_sepconv_t_3 41.03%\n", + "pw_sepconv_t_3 49.47%\n", + "dw_sepconv_t_4 41.18%\n", + "pw_sepconv_t_4 50.55%\n", + "head 0.00%\n", + "----------------------------\n", + "Mean 40.95%\n", + "\n", + "Layer Latency (ms) Clocks\n", + "-------------------------------------\n", + "conv_0 2.6448\n", + "conv_1 57.7118\n", + "conv_2 68.1212\n", + "conv_3 75.7012\n", + "dw_separable_4 5.3596\n", + "pw_separable_4 5.6954\n", + "dw_separable_5 4.3391\n", + "pw_separable_5 10.9067\n", + "dw_separable_6 1.7087\n", + "pw_separable_6 6.5842\n", + "dw_separable_7 1.6388\n", + "pw_separable_7 6.2570\n", + "dw_separable_8 1.4526\n", + "pw_separable_8 6.1165\n", + "dw_separable_9 1.3138\n", + "pw_separable_9 6.0867\n", + "dw_separable_10 1.1077\n", + "pw_separable_10 6.0284\n", + "dw_separable_11 0.9170\n", + "pw_separable_11 7.3058\n", + "dw_separable_12 1.2139\n", + "pw_separable_12 5.5492\n", + "dw_separable_13 0.6104\n", + "pw_separable_13 8.4229\n", + "dw_sepconv_t_0 1.3844\n", + "pw_sepconv_t_0 10.1121\n", + "dw_sepconv_t_1 3.2150\n", + "pw_sepconv_t_1 20.7983\n", + "dw_sepconv_t_2 6.7933\n", + "pw_sepconv_t_2 11.4291\n", + "dw_sepconv_t_3 13.8141\n", + "pw_sepconv_t_3 7.1611\n", + "dw_sepconv_t_4 9.3013\n", + "pw_sepconv_t_4 9.5852\n", + "head 3.6043\n", + "dequantizer 6.0524\n", + "-------------------------------------\n", + "Total 396.0441\n", + "Per-layer benchmark complete.\n" + ] + } + ], + "source": [ + "from brainchip_utils.hardware_utils import per_layer_benchmark\n", + "from akida_models.sparsity import compute_sparsity\n", + "from brainchip_utils.plot_utils import pretty_print_sparsity\n", + "\n", + "if device is not None:\n", + " ak_model.map(device, mode=akida.MapMode.Minimal, hw_only=True)\n", + " sparsity_dict = compute_sparsity(ak_model, samples=samples)\n", + " pretty_print_sparsity(sparsity_dict)\n", + " per_layer_results = per_layer_benchmark(ak_model, device, samples,\n", + " repeats=NUM_SAMPLES, clock_freq=MEASURED_CLOCK)\n", + " print('Per-layer benchmark complete.')\n", + "else:\n", + " # Sparsity can still be computed on the software backend without hardware.\n", + " sparsity_dict = compute_sparsity(ak_model, samples=samples)\n", + " pretty_print_sparsity(sparsity_dict)\n", + " print('Hardware not available — skipped latency; sparsity computed on software backend.')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/akida2/model_zoo/portrait128/portrait128_notebook_training.ipynb b/akida2/model_zoo/portrait128/portrait128_notebook_training.ipynb new file mode 100644 index 0000000..10025fd --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_notebook_training.ipynb @@ -0,0 +1,534 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "\"BrainChip\n", + "\n", + "# Portrait Segmentation (Portrait128) — Akida 2 Training\n", + "\n", + "Run Time: ~7 hours with training included (NVIDIA RTX 2080 Ti) / TBD with training skipped\n", + "\n", + "This notebook walks through the complete pipeline to train, quantize, convert, and evaluate an Akida U-Net model on the **Portrait128** dataset for Akida 2 hardware.\n", + "\n", + "Portrait128 is a binary segmentation task: for each 128 × 128 portrait, the model labels every pixel as **person** or **background**.\n", + "\n", + "The pipeline follows the standard Akida workflow:\n", + "1. Train a float model\n", + "2. 8-bit quantization (PTQ), then one epoch of QAT\n", + "3. Conversion to Akida `.fbz` format\n", + "4. Evaluation on Akida\n", + "\n", + "The focus is on the **Akida-specific** aspects of the pipeline. The data loading,\n", + "model definition and training loop live in the accompanying Python files: they are\n", + "standard tf_keras code and are imported here rather than repeated.\n", + "\n", + "Segmentation differs from the classification examples in this model zoo in two ways\n", + "that matter on Akida. The model has to **upsample** back to full resolution, which\n", + "needs transposed convolutions, an Akida 2 feature. And its output is a full-size map\n", + "rather than a handful of class scores, so the last layers of the network run at the\n", + "highest resolution. Both come up below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-01", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab-only setup. Local users: ignore this cell — it does nothing for you.\n", + "import sys, os\n", + "\n", + "if 'google.colab' in sys.modules:\n", + " if not os.path.exists('colab_setup.py'):\n", + " !wget -q https://raw.githubusercontent.com/Brainchip-Inc/brainchip_devhub/main/akida2/model_zoo/portrait128/colab_setup.py\n", + " import colab_setup; colab_setup.setup()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-02", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "The default dataset path is `./data/portrait128`, a folder holding the two arrays\n", + "`img_uint8.npy` and `msk_uint8.npy`. See the [README](README.md) for where to get\n", + "them and for symlink setup. Update `DATA_PATH` below if you keep them elsewhere.\n", + "\n", + "Setting `RUN_FLOAT_TRAINING` to `False` skips the training run and loads the float\n", + "model saved in `MODELS_DIR` by an earlier run of this notebook or of\n", + "`portrait128_train.sh`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-03", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "from tqdm import tqdm\n", + "\n", + "DATA_PATH = './data/portrait128'\n", + "MODELS_DIR = './models'\n", + "os.makedirs(MODELS_DIR, exist_ok=True)\n", + "\n", + "RUN_FLOAT_TRAINING = True\n", + "\n", + "SEED = 42\n", + "\n", + "# Must be called before any TF ops to make GPU ops deterministic.\n", + "tf.config.experimental.enable_op_determinism()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-04", + "metadata": {}, + "source": [ + "## Dataset\n", + "\n", + "`get_data` returns a training and a validation `tf.data.Dataset`. The full loading\n", + "and augmentation code is in [portrait128_data.py](portrait128_data.py).\n", + "\n", + "A few points that matter for what follows:\n", + "\n", + "* **Images stay uint8**, in their original `[0, 255]` range. The model rescales them\n", + " itself (see Model below), so the same data feed the float, quantized and Akida\n", + " models without any change, and a deployed Akida model can take camera pixels\n", + " directly.\n", + "* **Labels are masks**, of shape `(128, 128, 1)`: 1 for person, 0 for background.\n", + "* **Augmentation is geometric** (flips, shifts and zooms), and each image is stacked\n", + " with its mask before it is transformed. That way the mask moves with the image;\n", + " transforming them separately would misalign the labels.\n", + "* **The split is fixed.** The dataset ships as one pool, so it is split 80/20 with a\n", + " fixed shuffle seed, independent of `SEED`. Every run is evaluated on the same\n", + " validation images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-05", + "metadata": {}, + "outputs": [], + "source": [ + "from portrait128_data import get_data\n", + "\n", + "BATCH_SIZE = 32\n", + "INPUT_SHAPE = (128, 128, 3)\n", + "\n", + "train_ds, val_ds = get_data(DATA_PATH, INPUT_SHAPE, BATCH_SIZE, seed=SEED)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-06", + "metadata": {}, + "source": [ + "## Model\n", + "\n", + "The model is the **Akida U-Net** from `akida_models`, an encoder–decoder:\n", + "\n", + "* The **encoder** is AkidaNet with width multiplier alpha = 0.5. Its five stride-2\n", + " stages reduce the 128 × 128 input to a 4 × 4 × 512 feature map: coarse in space\n", + " but rich in features.\n", + "* The **decoder** is five separable *transposed*-convolution blocks. Each doubles the\n", + " spatial size, back up to 128 × 128. A transposed convolution with stride 2 is a\n", + " learned upsampling, and it is the reason this model needs Akida 2: Akida 1 has no\n", + " upsampling layer. Dropout after each block regularizes the decoder during\n", + " training; it does nothing at inference and is dropped at conversion.\n", + "* A final **1 × 1 convolution** gives one logit per pixel: positive for person,\n", + " negative for background. There is no sigmoid in the model. The loss applies it,\n", + " and a threshold at 0 gives the same mask as a threshold at probability 0.5.\n", + "\n", + "Unlike the classic U-Net there are **no skip connections** from encoder to decoder,\n", + "so all the detail in the output has to pass through the 4 × 4 bottleneck. That keeps\n", + "the graph a simple chain. The cost is softer mask edges than a U-Net with skips\n", + "would give.\n", + "\n", + "The input rescaling is part of the model. The encoder's `input_scaling=(128, -1)`\n", + "adds a layer that maps `[0, 255]` to `[-1, 1)`. Keeping it in the graph means one data\n", + "pipeline serves every version of the model. It also means the conversion step can\n", + "fold the rescaling into the Akida input layer's parameters, so the deployed model\n", + "still takes plain uint8 pixels.\n", + "\n", + "**Encoder initialization.** `base_weights='imagenet'` loads BrainChip's\n", + "ImageNet-trained AkidaNet 0.5 weights into the encoder, by layer name; the decoder\n", + "starts from random weights. The ImageNet model was trained at 224 × 224, but\n", + "convolution weights don't depend on the input size, so they load directly into this\n", + "128 × 128 encoder. Starting from features that already recognize edges, textures and\n", + "shapes is why the learning rate below can be small." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-07", + "metadata": {}, + "outputs": [], + "source": [ + "from portrait128_model import build_portrait128_model\n", + "\n", + "model = build_portrait128_model(base_weights='imagenet', seed=SEED)\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-08", + "metadata": {}, + "source": [ + "## Float Training\n", + "\n", + "The full training code is in [portrait128_train.py](portrait128_train.py). The\n", + "choices that matter here:\n", + "\n", + "* **Loss**: binary cross-entropy on the logits, averaged over every pixel. Each pixel\n", + " is its own binary classification.\n", + "* **Metrics**: *binary IoU*, the overlap between predicted and true masks averaged\n", + " over person and background, and *pixel accuracy*. Background covers a large share of\n", + " every image, so pixel accuracy is high even for poor masks; IoU penalizes a mask of\n", + " the wrong shape, so it is the one to watch.\n", + "* **Learning rate**: Adam with a one-cycle cosine schedule, stepped every batch. It\n", + " warms up from a 25th of the peak to the peak `LEARNING_RATE` over the first 30 % of\n", + " training, then decays. The peak, 3e-5, is low because the encoder starts\n", + " pretrained: a high rate early on would destroy the ImageNet features before the\n", + " decoder has learned to use them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-09", + "metadata": {}, + "outputs": [], + "source": [ + "from portrait128_train import train_portrait128, get_metrics\n", + "\n", + "if RUN_FLOAT_TRAINING:\n", + " LEARNING_RATE = 3e-5\n", + " EPOCHS = 50\n", + " # Freshly build the datasets for reproducibility\n", + " train_ds, val_ds = get_data(DATA_PATH, INPUT_SHAPE, BATCH_SIZE, seed=SEED)\n", + "\n", + " train_portrait128(model, train_ds, val_ds, EPOCHS, LEARNING_RATE, seed=SEED)\n", + "\n", + " float_model_path = os.path.join(MODELS_DIR, 'akida_unet_portrait128.h5')\n", + " model.save(float_model_path, include_optimizer=False)\n", + " print(f'Float model saved to {float_model_path}')\n", + "else:\n", + " from tf_keras.models import load_model\n", + " print('Training skipped. Loading an existing float model...')\n", + " model = load_model(os.path.join(MODELS_DIR, 'akida_unet_portrait128.h5'))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "### Evaluate float model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-11", + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(metrics=get_metrics())\n", + "float_results = model.evaluate(val_ds, verbose=0, return_dict=True)\n", + "float_iou, float_accuracy = float_results['binary_iou'], float_results['accuracy']\n", + "print(f'Float binary IoU: {float_iou:.4f}, pixel accuracy: {float_accuracy:.4f}')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "## Quantization (quantizeml)\n", + "\n", + "Akida runs on integers, so the weights and activations have to be quantized. Akida 2\n", + "quantizes with **quantizeml** (not `cnn2snn.quantize`). We quantize to **8 bits**\n", + "(i8/w8/a8) by post-training quantization (PTQ) only: 256 levels per weight and\n", + "activation track the float values closely enough that no retraining is needed.\n", + "`input_weight_bits` sets the input layer separately, because it runs on Akida's\n", + "dedicated input processor; here it is 8-bit like everything else.\n", + "\n", + "**Why no 4-bit variant?** Other Akida 2 examples also offer a 4-bit model, fine-tuned\n", + "with quantization-aware training (QAT). This one can't. The decoder upsamples with\n", + "transposed depthwise convolutions, and on Akida 2 those layers need 8-bit inputs and\n", + "8-bit weights. In a model quantized to 4 bits throughout, all five of them would fall\n", + "back to software, so the model would no longer run fully on the hardware.\n", + "\n", + "Quantization with `quantizeml` requires **calibration samples**. Calibration runs real\n", + "inputs through the model to measure the range of every layer's activations, and those\n", + "ranges set the quantization scales. Random data would give the wrong ranges, so the\n", + "samples should be representative images for the task. They are drawn from the\n", + "**training** split, to avoid leaking validation data into the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-13", + "metadata": {}, + "outputs": [], + "source": [ + "from quantizeml.models import quantize, QuantizationParams\n", + "\n", + "from portrait128_data import get_samples\n", + "\n", + "NUM_SAMPLES = 1024\n", + "samples = get_samples(DATA_PATH, INPUT_SHAPE, num_samples=NUM_SAMPLES)\n", + "\n", + "# --- 8-bit (i8 / w8 / a8), PTQ ---\n", + "qparams_8bit = QuantizationParams(input_weight_bits=8, weight_bits=8, activation_bits=8)\n", + "model_8bit = quantize(model, qparams=qparams_8bit, samples=samples)\n", + "\n", + "model_8bit.compile(metrics=get_metrics())\n", + "ptq_results = model_8bit.evaluate(val_ds, verbose=0, return_dict=True)\n", + "iou_ptq, acc_ptq = ptq_results['binary_iou'], ptq_results['accuracy']\n", + "print(f'8-bit PTQ binary IoU: {iou_ptq:.4f}, pixel accuracy: {acc_ptq:.4f}')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "### One epoch of QAT\n", + "\n", + "The 8-bit model then gets one epoch of quantization-aware training (QAT), as the\n", + "model in BrainChip's model zoo does. During QAT the forward pass uses the quantized\n", + "weights and activations, while the gradients update an underlying float copy of the\n", + "weights, so the network adjusts to the rounding that PTQ introduced.\n", + "\n", + "QAT starts from a model that is already close to a good solution, so it uses a lower\n", + "learning rate: a tenth of the float rate here. A higher rate would undo the float\n", + "training rather than refine it.\n", + "\n", + "A model quantized with `quantizeml` is a standard Keras model, so it goes straight\n", + "back into the same training function used for float training. The PTQ model itself\n", + "is not kept; compare its result above with the one after QAT." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-15", + "metadata": {}, + "outputs": [], + "source": [ + "QAT_EPOCHS = 1\n", + "QAT_LR = 3e-6\n", + "train_ds, val_ds = get_data(DATA_PATH, INPUT_SHAPE, BATCH_SIZE, seed=SEED)\n", + "train_portrait128(model_8bit, train_ds, val_ds, QAT_EPOCHS, QAT_LR, seed=SEED)\n", + "\n", + "q8_path = os.path.join(MODELS_DIR, 'akida_unet_portrait128_i8_w8_a8_qat.h5')\n", + "model_8bit.save(q8_path, include_optimizer=False)\n", + "\n", + "model_8bit.compile(metrics=get_metrics())\n", + "q8_results = model_8bit.evaluate(val_ds, verbose=0, return_dict=True)\n", + "iou_8bit, acc_8bit = q8_results['binary_iou'], q8_results['accuracy']\n", + "print(f'8-bit QAT binary IoU: {iou_8bit:.4f}, pixel accuracy: {acc_8bit:.4f}')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-16", + "metadata": {}, + "source": [ + "Note: the quantized model can be saved using the standard method. However,\n", + "for later reloading, because of the custom quantized layers in the model\n", + "we have to use the `load_model` function from `quantizeml.model_io` (a wrapper\n", + "around the standard tf_keras loading function)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-17", + "metadata": {}, + "outputs": [], + "source": [ + "# Just to illustrate how to load a quantized model\n", + "del model_8bit\n", + "\n", + "from quantizeml.model_io import load_model\n", + "model_8bit = load_model(q8_path)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-18", + "metadata": {}, + "source": [ + "## Conversion to Akida Format\n", + "\n", + "`cnn2snn.convert` compiles the quantized Keras model into an Akida `.fbz`\n", + "model that can be loaded and executed directly on an Akida 2 hardware device.\n", + "The converter verifies hardware compatibility and maps each layer to its\n", + "corresponding Akida primitive.\n", + "\n", + "In the summary below, note the following:\n", + "\n", + "* The input rescaling has no layer of its own. It was folded into the first\n", + " convolution.\n", + "* Each decoder block becomes a `DepthwiseConv2DTranspose` followed by a pointwise\n", + " `Conv2D`. On Akida 2 the depthwise and pointwise parts of a separable convolution\n", + " are two separate layers. The `DepthwiseConv2DTranspose` layers are the ones that\n", + " need 8 bits to run on the hardware.\n", + "* The last layer is a `Dequantizer`, so the model outputs float logits of shape\n", + " `(128, 128, 1)`, which can be thresholded at 0 just like the Keras model's." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-19", + "metadata": {}, + "outputs": [], + "source": [ + "from cnn2snn import convert\n", + "\n", + "akida_8bit = convert(model_8bit)\n", + "akida_8bit.save(os.path.join(MODELS_DIR, 'akida_unet_portrait128_i8_w8_a8_qat.fbz'))\n", + "\n", + "akida_8bit.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-20", + "metadata": {}, + "source": [ + "## Evaluation on Akida (software backend)\n", + "\n", + "We now run evaluation through the Akida model, to check that its results match\n", + "those of the quantized tf_keras model. Here, we deliberately use the software\n", + "backend (the default, since we do not check for and map to a connected hardware\n", + "device): this delivers a bit-accurate simulation of the results that will be\n", + "obtained when running the model on hardware." + ] + }, + { + "cell_type": "markdown", + "id": "cell-21", + "metadata": {}, + "source": [ + "### Run Evaluation on Akida\n", + "\n", + "The Akida runtime cannot consume `tf.data.Dataset` objects directly; it expects a\n", + "4D numpy array `(n, h, w, c)` of uint8. So we iterate over validation batches\n", + "manually and convert each one.\n", + "\n", + "The output already has the mask's shape, `(B, 128, 128, 1)`, so it goes straight\n", + "into the same IoU and accuracy metrics as the labels, with no reshaping. (Classifiers\n", + "return `(B, 1, 1, C)` and need a squeeze first.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-22", + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_akida(akida_model, ds):\n", + " iou_metric, acc_metric = get_metrics()\n", + " for batch, label_batch in tqdm(ds, desc='Evaluating on Akida'):\n", + " logits = akida_model.predict(batch.numpy().astype(np.uint8))\n", + " iou_metric.update_state(label_batch, logits)\n", + " acc_metric.update_state(label_batch, logits)\n", + " return float(iou_metric.result()), float(acc_metric.result())\n", + "\n", + "akida_iou_8bit, akida_acc_8bit = evaluate_akida(akida_8bit, val_ds)\n", + "print(f'Akida 8-bit QAT binary IoU: {akida_iou_8bit:.4f}, pixel accuracy: {akida_acc_8bit:.4f}')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-23", + "metadata": {}, + "source": [ + "## Activation Sparsity\n", + "\n", + "Akida is event-based: its standard neural processors only do work for non-zero\n", + "activations. The more zeros a layer receives (its input *sparsity*), the less time and\n", + "energy it costs. Sparsity depends on the data, so it has to be measured by running real\n", + "images through the model. Here, we reuse the calibration samples built earlier.\n", + "\n", + "The input layer is the exception. It runs on the dedicated input processor, a\n", + "conventional convolution whose cost doesn't depend on how many of its inputs are zero.\n", + "\n", + "For this model, the per-layer benchmark in `portrait128_notebook_benchmark.ipynb` shows\n", + "where the time goes. The slowest layers are the encoder's first full convolutions\n", + "(`conv_1` to `conv_3`), which apply 3 × 3 kernels across every channel of the 64 × 64 and\n", + "32 × 32 feature maps. The decoder's upsampling layers, which run at up to 128 × 128, come\n", + "next, and the deep encoder layers at 8 × 8 and 4 × 4 cost little. So sparsity in those\n", + "early convolutions has the most room to reduce latency." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-24", + "metadata": {}, + "outputs": [], + "source": [ + "from akida_models.sparsity import compute_sparsity\n", + "from brainchip_utils.plot_utils import pretty_print_sparsity\n", + "\n", + "# Reuse the calibration samples built earlier (sparsity needs real activations too).\n", + "pretty_print_sparsity(compute_sparsity(akida_8bit, samples=samples))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-25", + "metadata": {}, + "source": [ + "## Summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-26", + "metadata": {}, + "outputs": [], + "source": [ + "print(f'{\"Variant\":<16}{\"Keras IoU\":<12}{\"Keras acc\":<12}{\"Akida IoU\":<12}{\"Akida acc\":<12}')\n", + "print(f'{\"float\":<16}{float_iou:<12.4f}{float_accuracy:<12.4f}{\"-\":<12}{\"-\":<12}')\n", + "print(f'{\"8-bit PTQ\":<16}{iou_ptq:<12.4f}{acc_ptq:<12.4f}{\"-\":<12}{\"-\":<12}')\n", + "print(f'{\"8-bit QAT\":<16}{iou_8bit:<12.4f}{acc_8bit:<12.4f}{akida_iou_8bit:<12.4f}{akida_acc_8bit:<12.4f}')" + ] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,py:percent" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/akida2/model_zoo/portrait128/portrait128_train.py b/akida2/model_zoo/portrait128/portrait128_train.py new file mode 100644 index 0000000..c74137a --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_train.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# Copyright 2026 Brainchip Holdings Ltd. Apache 2.0 License +""" +Portrait128 training: float training, and the quantization-aware training (QAT) +epoch of the 8-bit model. + +The model outputs one logit per pixel, so the loss is binary cross-entropy on +logits. Training reports the binary IoU (mean of the person and background +IoUs) and pixel accuracy. For both, a pixel counts as "person" when its logit +is above 0, i.e. its sigmoid probability is above 0.5. + +The learning rate follows a one-cycle cosine schedule, stepped every batch: it +rises from learning_rate / 25 to learning_rate over the first 30 % of training, +then decays to learning_rate / 250000. + +Example +------- + python portrait128_train.py -d ./data/portrait128 -e 50 -lr 3e-5 \\ + -l models/akida_unet_portrait128.h5 -s models/akida_unet_portrait128.h5 +""" +import argparse +import os + +import tensorflow as tf +from tf_keras.losses import BinaryCrossentropy +from tf_keras.metrics import BinaryIoU, BinaryAccuracy +from tf_keras.optimizers.legacy import Adam +from tf_keras import regularizers +from tf_keras.layers import ReLU +from tf_keras.utils import set_random_seed + +from akida_models.param_scheduler import get_cosine_lr_scheduler +from cnn2snn import load_quantized_model + +from portrait128_data import get_data + +# Must be called before any TF ops to make GPU ops (conv backward passes, +# bilinear resize, etc.) deterministic. Has a small throughput cost. +tf.config.experimental.enable_op_determinism() + +# Disable quantizeml's run-time assertions (checks on quantized tensor values +# inside every quantized op) for training, QAT and evaluation. Set before any +# quantized model runs. +os.environ["ASSERT_ENABLED"] = "0" + + +class LogitBinaryIoU(BinaryIoU): + """Binary IoU over person and background, on logits. + + A pixel is "person" when its logit is strictly positive, the same rule as + BinaryAccuracy(threshold=0). BinaryIoU on its own counts a logit of exactly 0 + as "person", and quantized and Akida models can output exact zeros, so + without this the two metrics would disagree on the same mask. + """ + + def __init__(self, name='binary_iou', **kwargs): + super().__init__(target_class_ids=[0, 1], threshold=0.5, name=name, **kwargs) + + def update_state(self, y_true, y_pred, sample_weight=None): + y_pred = tf.cast(tf.cast(y_pred, self._dtype) > 0, self._dtype) + return super().update_state(y_true, y_pred, sample_weight) + + def get_config(self): + return {'name': self.name, 'dtype': self.dtype} + + +def get_metrics(): + """Binary IoU and pixel accuracy on logits (a pixel is "person" if its logit > 0).""" + return [LogitBinaryIoU(), BinaryAccuracy(threshold=0.0, name='accuracy')] + + +def train_portrait128(model, train_ds, val_ds, epochs, learning_rate, + regularization=None, seed=42): + set_random_seed(seed) + + if regularization is not None: + print('Adding Activity Regularization to ReLU layers') + regularizer = regularizers.L1L2(regularization, regularization) + for layer in model.layers: + if isinstance(layer, ReLU): + layer.activity_regularizer = regularizer + + # The model outputs logits (no sigmoid), so the loss applies the sigmoid + # itself (from_logits=True). + model.compile(optimizer=Adam(learning_rate=learning_rate), + loss=BinaryCrossentropy(from_logits=True), + metrics=get_metrics()) + + # One-cycle cosine schedule over every training step: warm-up for the + # first 30 % of steps, then decay. + total_steps = epochs * len(train_ds) + callbacks = [get_cosine_lr_scheduler(learning_rate, total_steps, pct_start=0.3)] + + model.fit( + train_ds, + epochs=epochs, + callbacks=callbacks, + validation_data=val_ds, + ) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-l', '--loadmodel', required=True, + help='Model to load (.h5 tf_keras or quantized model)') + parser.add_argument('-s', '--savemodel', required=True, + help='Model save path') + parser.add_argument('-d', '--data', default='./data/portrait128', + help='Portrait128 folder (contains img_uint8.npy and msk_uint8.npy)') + parser.add_argument('-b', '--batch_size', type=int, default=32) + parser.add_argument('-e', '--epochs', type=int, default=50) + parser.add_argument('-lr', '--learning_rate', type=float, default=3e-5, + help='Peak learning rate') + parser.add_argument('-reg', '--regularization', type=float, default=None, + help='Activity Regularization to increase sparsity') + parser.add_argument('--seed', type=int, default=42, + help='Random seed for reproducibility') + args = parser.parse_args() + + # --------------------------------------------------------------------------- + # Model + # --------------------------------------------------------------------------- + model = load_quantized_model(args.loadmodel) + + # --------------------------------------------------------------------------- + # Data loading + # --------------------------------------------------------------------------- + train_ds, val_ds = get_data(args.data, model.input_shape[1:], args.batch_size, + seed=args.seed) + + train_portrait128(model=model, + train_ds=train_ds, + val_ds=val_ds, + epochs=args.epochs, + learning_rate=args.learning_rate, + regularization=args.regularization, + seed=args.seed) + + model.save(args.savemodel, include_optimizer=False) + print(f'Model saved as {args.savemodel}.') diff --git a/akida2/model_zoo/portrait128/portrait128_train.sh b/akida2/model_zoo/portrait128/portrait128_train.sh new file mode 100644 index 0000000..d6fc6c9 --- /dev/null +++ b/akida2/model_zoo/portrait128/portrait128_train.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Copyright 2026 Brainchip Holdings Ltd. Apache 2.0 License +# +# End-to-end Portrait128 segmentation pipeline for Akida 2. +# +# The model's encoder starts from ImageNet-trained AkidaNet 0.5 weights; the +# decoder is trained from random initialization. +# +# Quantization uses `quantizeml`: 8-bit (i8/w8/a8) post-training quantization, +# then one epoch of quantization-aware training (QAT) at a tenth of the float +# learning rate, as for the model in BrainChip's model zoo. The PTQ model is a +# throwaway on the way to QAT (_pretmp). There is no 4-bit variant: on Akida 2 +# the decoder's transposed depthwise convolutions need 8-bit inputs and weights, +# so a 4-bit model can't run fully on hardware. +# Conversion to .fbz is done with `cnn2snn convert` (accepts quantizeml models). +# +# Usage: +# bash portrait128_train.sh [DATADIR] +# where the optional DATADIR overrides the default dataset location +# (./data/portrait128). + +set -e + +DATADIR="${1:-}" +DATA_ARG=${DATADIR:+-d "$DATADIR"} + +# Download batch of samples for calibration +wget -N https://data.brainchip.com/dataset-mirror/samples/portrait128/portrait128_batch1024.npz \ + -P data/ + +# 1. Build the model (ImageNet-pretrained AkidaNet encoder + untrained decoder) +python portrait128_model.py -bw imagenet -s models/akida_unet_portrait128.h5 + +# 2. Train the model +python portrait128_train.py -l models/akida_unet_portrait128.h5 -s models/akida_unet_portrait128.h5 -e 50 -lr 3e-5 $DATA_ARG +python portrait128_eval.py -l models/akida_unet_portrait128.h5 $DATA_ARG + +# 8-bit quantization (i8/w8/a8) -- PTQ, then one epoch of QAT. PTQ is a throwaway (_pretmp). +quantizeml quantize -m models/akida_unet_portrait128.h5 -i 8 -w 8 -a 8 \ + -s models/akida_unet_portrait128_i8_w8_a8_pretmp.h5 \ + --samples data/portrait128_batch1024.npz + +python portrait128_train.py -l models/akida_unet_portrait128_i8_w8_a8_pretmp.h5 -s models/akida_unet_portrait128_i8_w8_a8_qat.h5 -e 1 -lr 3e-6 $DATA_ARG +python portrait128_eval.py -l models/akida_unet_portrait128_i8_w8_a8_qat.h5 $DATA_ARG + +cnn2snn convert -m models/akida_unet_portrait128_i8_w8_a8_qat.h5 +python portrait128_eval.py -l models/akida_unet_portrait128_i8_w8_a8_qat.fbz $DATA_ARG + +python portrait128_benchmark.py -l models/akida_unet_portrait128_i8_w8_a8_qat.fbz $DATA_ARG || true + +rm -f models/akida_unet_portrait128_i8_w8_a8_pretmp.h5 diff --git a/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128.h5 b/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128.h5 new file mode 100644 index 0000000..6db74ec --- /dev/null +++ b/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4728add11ebd0bb752c8c6e391968330bcf53be9508e0c91207b0ba7ab50ce71 +size 4501976 diff --git a/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz b/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz new file mode 100644 index 0000000..0b361df --- /dev/null +++ b/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.fbz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:868ade91f8e1d5c620ff116669fdf3a6a1a6b0626492e1e5eea505489054f5ef +size 1302167 diff --git a/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.h5 b/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.h5 new file mode 100644 index 0000000..514576b --- /dev/null +++ b/akida2/model_zoo/portrait128/pretrained_models/akida_unet_portrait128_i8_w8_a8_qat.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0f5624b97fac7014ae1c76845edab4dac2897f3ef781653e791ea52f82ec015 +size 4525096 diff --git a/akida2/model_zoo/portrait128/update_readme.py b/akida2/model_zoo/portrait128/update_readme.py new file mode 100644 index 0000000..152a422 --- /dev/null +++ b/akida2/model_zoo/portrait128/update_readme.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Copyright 2025 Brainchip Holdings Ltd. Apache 2.0 License +"""Regenerate README.md from README.md.template + metrics.json.""" +import json +import pathlib +import runpy + +here = pathlib.Path(__file__).parent +metrics = json.loads((here / "docs" / "metrics.json").read_text()) +template = (here / "docs" / "README.md.template").read_text() +(here / "README.md").write_text(template.format_map(metrics)) +print("README.md updated.") + +# Refresh the Akida 2 landing README, whose model zoo table reads this metrics.json. +runpy.run_path(str(here.parents[1] / "update_readme.py"), run_name="__main__")