Add native Bark TTS model family

CMakeLists.txt

@@ -1055,6 +1055,26 @@ audiocpp_add_model(magpie_tts
         engine::models::magpie_tts::make_magpie_tts_loader
 )
 
+audiocpp_add_model(bark_tts
+    SOURCES
+        src/models/bark_tts/assets.cpp
+        src/models/bark_tts/codec.cpp
+        src/models/bark_tts/generator.cpp
+        src/models/bark_tts/session.cpp
+        src/models/bark_tts/tokenizer.cpp
+        src/models/bark_tts/transformer.cpp
+    INCLUDES
+        engine/models/bark_tts/assets.h
+        engine/models/bark_tts/codec.h
+        engine/models/bark_tts/generator.h
+        engine/models/bark_tts/session.h
+        engine/models/bark_tts/tokenizer.h
+        engine/models/bark_tts/transformer.h
+        engine/models/bark_tts/types.h
+    LOADERS
+        engine::models::bark_tts::make_bark_tts_loader
+)
+
 audiocpp_add_model(confucius4_tts
     SOURCES
         src/models/confucius4_tts/assets.cpp
@@ -2279,6 +2299,9 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
         add_engine_unittest(hf_tokenizer_vocab_bounds_test tests/unittests/test_hf_tokenizer_vocab_bounds.cpp)
         add_test(NAME hf_tokenizer_vocab_bounds_test COMMAND hf_tokenizer_vocab_bounds_test)
 
+        add_engine_unittest(bark_tokenizer_test tests/unittests/test_bark_tokenizer.cpp)
+        add_test(NAME bark_tokenizer_test COMMAND bark_tokenizer_test)
+
         add_engine_unittest(safetensors_offsets_test tests/unittests/test_safetensors_offsets.cpp)
         add_test(NAME safetensors_offsets_test COMMAND safetensors_offsets_test)
 
@@ -2716,6 +2739,17 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
         )
         target_link_libraries(miocodec_wavlm_parity PRIVATE engine_runtime ggml)
 
+        add_executable(bark_codec_parity EXCLUDE_FROM_ALL tests/bark_tts/bark_codec_parity.cpp)
+        target_link_libraries(bark_codec_parity PRIVATE engine_runtime ggml)
+        if (ENGINE_ENABLE_OPENMP)
+            target_link_libraries(bark_codec_parity PRIVATE OpenMP::OpenMP_CXX)
+        endif()
+        add_executable(bark_transformer_parity EXCLUDE_FROM_ALL tests/bark_tts/bark_transformer_parity.cpp)
+        target_link_libraries(bark_transformer_parity PRIVATE engine_runtime ggml)
+        if (ENGINE_ENABLE_OPENMP)
+            target_link_libraries(bark_transformer_parity PRIVATE OpenMP::OpenMP_CXX)
+        endif()
+
         add_engine_unittest(dots_tts_vocoder_parity tests/dots_tts/dots_tts_vocoder_parity.cpp)
 
         # Needs the GGUF and a PyTorch reference dump, so it is driven by hand

README.md

@@ -78,6 +78,7 @@ Runtime tags summarize the supported loading paths. GGUF package precision varie
 | **confucius4_tts** | Clone | zh, en, ja, ko, de, fr, es, id, it, th, pt, ru, ms, vi | Confucius4-TTS multilingual voice cloning | GGUF F32, Stream |
 | **cosyvoice3** | TTS, Clone | zh, en, ja, ko, de, es, fr, it, ru, yue | Fun-CosyVoice3 zero-shot, cross-lingual, and instruction-conditioned TTS | GGUF F32/Q8 |
 | **dots_tts** | TTS, Clone, Edit, Ctrl | multilingual | DotTTS SOAR, MeanFlow, and Edit | GGUF 16/Q8, Stream |
+| **bark_tts** | TTS | 13 languages | Bark Small, 261 speaker presets | GGUF 16/Q8 |
 | **dramabox** | TTS, Clone | en | DramaBox expressive TTS and voice cloning | GGUF Q8 |
 | **fish_audio** | TTS, Clone, Ctrl | auto, en, zh | Fish Audio S2 Pro | GGUF 16/Q8 |
 | **firered_audio** | ASR, TTS, Clone, Design, Ctrl | zh, en | FireRedAudio multimodal speech/audio model with ASR, understanding, cloning, design, and edit paths | GGUF original/Q8 |

docs/models/bark_tts.md

@@ -0,0 +1,28 @@
+# Bark Small
+
+Bark is a multilingual, expressive text-to-audio model. The audio.cpp package
+contains the complete Bark Small semantic, coarse, fine, and EnCodec pipeline,
+plus all 261 upstream speaker histories.
+
+```sh
+python3 tools/model_manager_v2.py install bark_small_f16
+
+audiocpp_cli --task tts --family bark_tts \
+  --model Bark-Small-GGUF/bark-small-f16.gguf \
+  --text "Hello from Bark. [laughs] This model can be quite expressive." \
+  --option history_prompt=v2/en_speaker_6 --output bark.wav
+```
+
+Use `--option history_prompt=<preset>` to select another packaged history. Presets
+follow upstream names such as `v2/de_speaker_3`, `v2/ja_speaker_0`, or
+`announcer`. `temperature`, `top_k`, `max_tokens`, and `seed` are available as
+request options. Bark supports non-speech cues written in brackets, although
+the result remains probabilistic.
+
+F16 is the recommended package. The hybrid Q8 package keeps both autoregressive
+semantic/coarse transformers and the complete EnCodec path in F16, and only
+quantizes the non-causal fine transformer's dense matrices. Quantizing Bark's
+autoregressive stages causes token errors to compound and is not quality-safe.
+
+Source model: [suno/bark-small](https://huggingface.co/suno/bark-small), pinned
+to revision `1dbd7a128513b8ae4a4e2130fed57b7ac9da5bcd`. Bark is MIT licensed.

include/engine/models/bark_tts/assets.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "engine/framework/assets/resource_bundle.h"
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/models/bark_tts/types.h"
+
+#include <filesystem>
+#include <memory>
+#include <unordered_map>
+
+namespace engine::models::bark_tts {
+
+struct BarkAssets {
+    engine::assets::ResourceBundle resources;
+    std::shared_ptr<const engine::assets::TensorSource> weights;
+    BarkConfig config;
+    std::unordered_map<std::string, BarkSpeakerPreset> presets;
+};
+
+std::shared_ptr<const BarkAssets> load_bark_assets(const std::filesystem::path & model_path);
+
+}  // namespace engine::models::bark_tts

include/engine/models/bark_tts/codec.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+namespace engine::core { class ExecutionContext; }
+namespace engine::models::bark_tts {
+
+struct BarkAssets;
+
+class BarkCodecDecoder {
+public:
+    BarkCodecDecoder(std::shared_ptr<const BarkAssets> assets,
+                     engine::core::ExecutionContext & execution,
+                     engine::assets::TensorStorageType storage_type);
+    ~BarkCodecDecoder();
+    std::vector<float> decode(const std::vector<std::vector<int32_t>> & codes) const;
+private:
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::models::bark_tts

include/engine/models/bark_tts/generator.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "engine/models/bark_tts/codec.h"
+#include "engine/models/bark_tts/tokenizer.h"
+#include "engine/models/bark_tts/transformer.h"
+#include "engine/models/bark_tts/types.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace engine::core { class ExecutionContext; }
+namespace engine::models::bark_tts {
+
+class BarkGenerator {
+public:
+    BarkGenerator(std::shared_ptr<const BarkAssets> assets,
+                  engine::core::ExecutionContext & execution,
+                  engine::assets::TensorStorageType transformer_storage,
+                  engine::assets::TensorStorageType codec_storage);
+    std::vector<float> synthesize(const std::string & text, const BarkGenerationOptions & options) const;
+private:
+    std::shared_ptr<const BarkAssets> assets_;
+    BarkTokenizer tokenizer_;
+    BarkTransformer semantic_;
+    BarkTransformer coarse_;
+    BarkTransformer fine_;
+    BarkCodecDecoder codec_;
+};
+
+}  // namespace engine::models::bark_tts

include/engine/models/bark_tts/session.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "engine/framework/model_spec/metadata.h"
+#include "engine/framework/runtime/session_base.h"
+#include "engine/models/bark_tts/assets.h"
+
+#include <memory>
+
+namespace engine::core { class ExecutionContext; }
+namespace engine::models::bark_tts {
+class BarkGenerator;
+std::shared_ptr<engine::runtime::IVoiceModelLoader> make_bark_tts_loader();
+
+class BarkSession final : public engine::runtime::RuntimeSessionBase,
+                          public engine::runtime::IOfflineVoiceTaskSession {
+public:
+    BarkSession(engine::runtime::TaskSpec task, engine::runtime::SessionOptions options,
+                std::shared_ptr<const BarkAssets> assets,
+                std::shared_ptr<const engine::model_spec::ModelContract> contract);
+    ~BarkSession() override;
+    std::string family() const override;
+    engine::runtime::VoiceTaskKind task_kind() const override;
+    engine::runtime::RunMode run_mode() const override;
+    void prepare(const engine::runtime::SessionPreparationRequest & request) override;
+    engine::runtime::TaskResult run(const engine::runtime::TaskRequest & request) override;
+private:
+    engine::runtime::TaskSpec task_;
+    engine::runtime::SessionOptions options_;
+    std::shared_ptr<const BarkAssets> assets_;
+    std::shared_ptr<const engine::model_spec::ModelContract> contract_;
+    std::unique_ptr<engine::core::ExecutionContext> execution_;
+    std::unique_ptr<BarkGenerator> generator_;
+};
+}  // namespace engine::models::bark_tts

include/engine/models/bark_tts/tokenizer.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace engine::models::bark_tts {
+
+class BarkTokenizer {
+public:
+    explicit BarkTokenizer(const std::string & tokenizer_json);
+    std::vector<int32_t> encode(const std::string & text) const;
+
+private:
+    std::unordered_map<std::string, int32_t> vocab_;
+    int32_t unknown_ = 100;
+};
+
+}  // namespace engine::models::bark_tts

include/engine/models/bark_tts/transformer.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/models/bark_tts/assets.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace engine::core { class ExecutionContext; }
+
+namespace engine::models::bark_tts {
+
+class BarkTransformer {
+public:
+    BarkTransformer(std::shared_ptr<const BarkAssets> assets,
+                    engine::core::ExecutionContext & execution,
+                    std::string prefix,
+                    BarkTransformerConfig config,
+                    bool causal,
+                    engine::assets::TensorStorageType storage_type);
+    ~BarkTransformer();
+
+    // For causal models, embedding_channels may contain one or more token rows
+    // whose embeddings are summed. Returns logits for the final position.
+    std::vector<float> causal_logits(const std::vector<std::vector<int32_t>> & embedding_channels) const;
+
+    // Fine Bark sums codebook embeddings 0..codebook_idx and returns one logit
+    // row per sequence position.
+    std::vector<float> fine_logits(const std::vector<std::vector<int32_t>> & codes,
+                                   int codebook_idx) const;
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::models::bark_tts

include/engine/models/bark_tts/types.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace engine::models::bark_tts {
+
+struct BarkTransformerConfig {
+    int64_t hidden = 768;
+    int64_t layers = 12;
+    int64_t heads = 12;
+    int64_t block_size = 1024;
+    int64_t input_vocab = 0;
+    int64_t output_vocab = 0;
+    bool bias = false;
+};
+
+struct BarkConfig {
+    BarkTransformerConfig semantic;
+    BarkTransformerConfig coarse;
+    BarkTransformerConfig fine;
+    int64_t codebook_size = 1024;
+    int64_t codebook_dim = 128;
+    int64_t sample_rate = 24000;
+};
+
+struct BarkSpeakerPreset {
+    std::vector<int32_t> semantic;
+    std::vector<std::vector<int32_t>> coarse;
+    std::vector<std::vector<int32_t>> fine;
+};
+
+struct BarkGenerationOptions {
+    std::string voice_id = "v2/en_speaker_6";
+    float temperature = 0.7F;
+    int top_k = 50;
+    int64_t max_tokens = 768;
+    uint64_t seed = 0;
+};
+
+}  // namespace engine::models::bark_tts

model_specs/bark_tts.json

@@ -0,0 +1,78 @@
+{
+  "schema_version": 1,
+  "family": "bark_tts",
+  "display_name": "Bark Small",
+  "description": "Suno Bark is a multilingual text-to-audio model with semantic, coarse-acoustic, and fine-acoustic generation followed by a 24 kHz EnCodec decoder. This package includes Bark Small and its downloaded built-in speaker presets.",
+  "category": "tts",
+  "status": "supported",
+  "tasks": ["tts"],
+  "modes": ["offline"],
+  "languages": ["de", "en", "es", "fr", "hi", "it", "ja", "ko", "pl", "pt", "ru", "tr", "zh"],
+  "runtime": {"tags": ["gguf"]},
+  "capabilities": {"tts": ["built_in_voices"]},
+  "options": {
+    "request": [
+      {"name": "history_prompt", "type": "string", "description": "Packaged Bark speaker history, for example v2/en_speaker_6.", "required": false, "default": "v2/en_speaker_6"},
+      {"name": "temperature", "type": "float", "description": "Semantic and coarse token sampling temperature.", "required": false, "min": 0.01, "default": 0.7},
+      {"name": "top_k", "type": "int", "description": "Semantic and coarse token top-k sampling limit.", "required": false, "min": 1, "default": 50},
+      {"name": "max_tokens", "type": "int", "description": "Maximum generated semantic token count.", "required": false, "min": 1, "default": 768},
+      {"name": "seed", "type": "int", "description": "Sampling seed.", "required": false, "min": 0, "default": 0}
+    ],
+    "session": [
+      {"name": "weight_type", "type": "enum", "description": "Transformer matmul storage type.", "preset": "weight_type_full", "required": false, "default": "native"},
+      {"name": "codec_weight_type", "type": "enum", "description": "EnCodec decoder storage type; native is recommended for audio quality.", "preset": "weight_type_full", "required": false, "default": "native"},
+      {"name": "graph_arena_mb", "type": "int", "description": "Reusable graph arena size in MiB.", "required": false, "min": 64, "default": 768}
+    ],
+    "load": []
+  },
+  "package_defaults": {
+    "download": {"kind": "huggingface_snapshot", "repo": "audio-cpp/audio.cpp-gguf", "revision": "main", "gated": false}
+  },
+  "packages": [
+    {
+      "id": "bark_small_f16",
+      "display_name": "Bark Small F16 GGUF",
+      "default": true,
+      "format": "gguf",
+      "precision": "f16",
+      "target_directory": "Bark-Small-GGUF",
+      "files": ["Bark-Small-GGUF/bark-small-f16.gguf"],
+      "strip_prefix": "Bark-Small-GGUF"
+    },
+    {
+      "id": "bark_small_q8_0",
+      "display_name": "Bark Small Q8_0 GGUF",
+      "default": false,
+      "format": "gguf",
+      "precision": "q8_0",
+      "target_directory": "Bark-Small-GGUF",
+      "files": ["Bark-Small-GGUF/bark-small-q8_0.gguf"],
+      "strip_prefix": "Bark-Small-GGUF"
+    }
+  ],
+  "dependencies": [],
+  "ui": {
+    "recommended_package": "bark_small_f16",
+    "tags": ["TTS", "GGUF"],
+    "docs": ["docs/models/bark_tts.md", "docs/tts.md", "docs/gguf.md"]
+  },
+  "sources": [
+    {
+      "format": "gguf",
+      "roots": {"model": ".", "weights": "$gguf"},
+      "files": {
+        "config_json": "model:config.json",
+        "generation_config_json": "model:generation_config.json",
+        "tokenizer_json": "model:tokenizer.json",
+        "speaker_presets_json": "model:speaker_presets.json"
+      },
+      "tensors": {"weights": {"source": "weights:", "prefix": "bark"}}
+    }
+  ],
+  "provenance": {
+    "model": "suno/bark-small",
+    "model_revision": "1dbd7a128513b8ae4a4e2130fed57b7ac9da5bcd",
+    "license": "MIT",
+    "upstream": "https://github.com/suno-ai/bark"
+  }
+}