Add native Coqui XTTS v2 voice cloning

CMakeLists.txt

@@ -1236,6 +1236,23 @@ audiocpp_add_model(index_tts2
         engine::models::index_tts2::make_index_tts2_loader
 )
 
+audiocpp_add_model(xtts_v2
+    SOURCES
+        src/models/xtts_v2/assets.cpp
+        src/models/xtts_v2/audio_features.cpp
+        src/models/xtts_v2/conditioning.cpp
+        src/models/xtts_v2/decoder.cpp
+        src/models/xtts_v2/gpt.cpp
+        src/models/xtts_v2/request.cpp
+        src/models/xtts_v2/session.cpp
+        src/models/xtts_v2/speaker_encoder.cpp
+        src/models/xtts_v2/tokenizer.cpp
+    INCLUDES
+        engine/models/xtts_v2/session.h
+    LOADERS
+        engine::models::xtts_v2::make_xtts_v2_loader
+)
+
 audiocpp_add_model(nemotron_asr
     SOURCES
         src/models/nemotron_asr/assets.cpp
@@ -2156,6 +2173,15 @@ if (ENGINE_BUILD_WARMBENCH)
     add_engine_warmbench(higgs_audio_tts_warm_bench tests/higgs_audio_tts/higgs_audio_tts_warm_bench.cpp)
     add_engine_warmbench(hviske_asr_warm_bench tests/hviske_asr/hviske_asr_warm_bench.cpp)
     add_engine_warmbench(index_tts2_warm_bench tests/index_tts2/index_tts2_warm_bench.cpp)
+    add_engine_warmbench(xtts_v2_conditioning_probe tests/xtts_v2/xtts_v2_conditioning_probe.cpp)
+    target_sources(xtts_v2_conditioning_probe PRIVATE
+        src/models/xtts_v2/assets.cpp
+        src/models/xtts_v2/audio_features.cpp
+        src/models/xtts_v2/conditioning.cpp
+        src/models/xtts_v2/decoder.cpp
+        src/models/xtts_v2/gpt.cpp
+        src/models/xtts_v2/speaker_encoder.cpp
+        src/models/xtts_v2/tokenizer.cpp)
     add_engine_warmbench(irodori_tts_warm_bench tests/irodori_tts/irodori_tts_warm_bench.cpp)
     add_engine_warmbench(marblenet_vad_warm_bench tests/marblenet_vad/marblenet_vad_warm_bench.cpp)
     add_engine_warmbench(miocodec_warm_bench tests/miocodec/miocodec_warm_bench.cpp)

docs/community_models/xtts_v2.md

@@ -0,0 +1,51 @@
+# Coqui XTTS v2
+
+XTTS v2 is a multilingual, zero-shot voice-cloning model. The audio.cpp port
+runs the conditioning encoder, Perceiver resampler, 30-layer GPT-2 acoustic
+token generator, ResNet speaker encoder, and speaker-conditioned HiFiGAN
+decoder natively through ggml. Python is needed only to convert the original
+PyTorch checkpoint.
+
+The model supports `en`, `es`, `fr`, `de`, `it`, `pt`, `pl`, `tr`, `ru`, `nl`,
+`cs`, `ar`, `zh-cn`, `ja`, `hu`, `ko`, and `hi`. Output is mono 24 kHz audio.
+A clean reference recording of at least three seconds is required.
+
+```sh
+audio.cpp -m models/XTTS-v2-GGUF/xtts-v2-q8_0.gguf \
+  --task clone \
+  --voice-ref reference.wav \
+  --language en \
+  -p "This voice was synthesized locally by audio.cpp." \
+  --seed 42 \
+  -o xtts-v2.wav
+```
+
+The default generation settings match Coqui's published XTTS v2 configuration:
+temperature `0.75`, top-k `50`, top-p `0.85`, and repetition penalty `5.0`.
+Use `--speed` to adjust speaking rate without changing pitch.
+
+## Conversion
+
+Download the official `coqui/XTTS-v2` snapshot, then run:
+
+```sh
+python tools/community_models/convert_xtts_v2.py \
+  --model-dir /path/to/XTTS-v2 \
+  --output-dir /tmp/xtts-v2-staging \
+  --run-converter build/bin/audiocpp_gguf \
+  --type q8_0
+```
+
+The converter excludes optimizer, scaler, and other training-only state. It
+also materializes the effective weights of the PyTorch weight-normalized
+HiFiGAN layers. Convolution tensors, token embeddings, and the sampling head
+remain F16 in the mixed Q8 package to protect synthesis quality.
+
+## License
+
+The XTTS v2 weights and their outputs are licensed under the Coqui Public Model
+License 1.0.0 for non-commercial use only. The converted package includes the
+original `LICENSE.txt`; downstream redistribution must keep the license or its
+URL with the model and its outputs. The audio.cpp source changes retain the
+repository's source-code license.
+

include/engine/models/xtts_v2/assets.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "engine/framework/assets/resource_bundle.h"
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/models/xtts_v2/types.h"
+
+#include <filesystem>
+#include <memory>
+
+namespace engine::models::xtts_v2 {
+
+struct XttsV2Assets {
+    assets::ResourceBundle resources;
+    XttsV2Config config;
+    std::shared_ptr<const assets::TensorSource> gpt;
+    std::shared_ptr<const assets::TensorSource> decoder;
+    std::shared_ptr<const assets::TensorSource> speaker_encoder;
+    std::filesystem::path tokenizer_path;
+};
+
+std::shared_ptr<const XttsV2Assets> load_xtts_v2_assets(const std::filesystem::path & model_path);
+
+}  // namespace engine::models::xtts_v2

include/engine/models/xtts_v2/audio_features.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "engine/framework/runtime/model.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace engine::models::xtts_v2 {
+
+struct XttsV2MelFeatures {
+    std::vector<float> values;  // channel-major [channels, frames]
+    int64_t channels = 0;
+    int64_t frames = 0;
+};
+
+struct XttsV2PreparedReference {
+    std::vector<float> waveform_22050;
+    std::vector<float> waveform_16000;
+};
+
+XttsV2PreparedReference prepare_xtts_v2_reference(const runtime::AudioBuffer & audio);
+
+XttsV2MelFeatures compute_xtts_v2_conditioning_mel(
+    const std::vector<float> & waveform_22050,
+    const std::vector<float> & mel_stats,
+    size_t threads = 0);
+
+XttsV2MelFeatures compute_xtts_v2_speaker_mel(
+    const std::vector<float> & waveform_16000,
+    const std::vector<float> & window,
+    const std::vector<float> & mel_filterbank,
+    size_t threads = 0);
+
+}  // namespace engine::models::xtts_v2

include/engine/models/xtts_v2/conditioning.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/framework/core/execution_context.h"
+#include "engine/models/xtts_v2/assets.h"
+#include "engine/models/xtts_v2/audio_features.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+namespace engine::models::xtts_v2 {
+
+struct XttsV2ConditioningLatent {
+    std::vector<float> values;  // frame-major [32, 1024]
+    int64_t frames = 32;
+    int64_t dims = 1024;
+};
+
+class XttsV2ConditioningRuntime {
+public:
+    XttsV2ConditioningRuntime(
+        const XttsV2Assets & assets,
+        core::ExecutionContext & execution,
+        size_t weight_context_bytes,
+        size_t graph_context_bytes,
+        assets::TensorStorageType matmul_storage_type,
+        assets::TensorStorageType conv_storage_type);
+    ~XttsV2ConditioningRuntime();
+
+    XttsV2ConditioningLatent encode(const XttsV2MelFeatures & mel);
+    const std::vector<float> & mel_stats() const noexcept;
+
+private:
+    struct Weights;
+    class Graph;
+    core::ExecutionContext & execution_;
+    size_t graph_context_bytes_ = 0;
+    std::shared_ptr<const Weights> weights_;
+    std::unique_ptr<Graph> graph_;
+};
+
+}  // namespace engine::models::xtts_v2

include/engine/models/xtts_v2/decoder.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "engine/framework/core/execution_context.h"
+#include "engine/models/xtts_v2/assets.h"
+#include "engine/models/xtts_v2/gpt.h"
+#include "engine/models/xtts_v2/speaker_encoder.h"
+
+#include <memory>
+#include <vector>
+
+namespace engine::models::xtts_v2 {
+
+class XttsV2DecoderRuntime {
+public:
+    XttsV2DecoderRuntime(const XttsV2Assets & assets, core::ExecutionContext & execution,
+        size_t weight_context_bytes, size_t graph_context_bytes,
+        assets::TensorStorageType conv_storage_type);
+    ~XttsV2DecoderRuntime();
+
+    std::vector<float> decode(const std::vector<float> & latents, int64_t frames,
+                              const XttsV2SpeakerEmbedding & speaker);
+
+private:
+    struct Weights;
+    class Graph;
+    core::ExecutionContext & execution_;
+    size_t graph_context_bytes_;
+    std::shared_ptr<const Weights> weights_;
+    std::unique_ptr<Graph> graph_;
+};
+
+}  // namespace engine::models::xtts_v2

include/engine/models/xtts_v2/gpt.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include "engine/framework/core/backend_weight_store.h"
+#include "engine/framework/core/execution_context.h"
+#include "engine/framework/modules/conv_modules.h"
+#include "engine/framework/modules/linear_module.h"
+#include "engine/framework/modules/norm_modules.h"
+#include "engine/models/xtts_v2/assets.h"
+
+#include <memory>
+#include <vector>
+
+namespace engine::models::xtts_v2 {
+
+struct XttsV2GptLayerWeights {
+    modules::NormWeights attn_norm;
+    modules::LinearWeights qkv;
+    modules::LinearWeights attn_out;
+    modules::NormWeights mlp_norm;
+    modules::LinearWeights mlp_in;
+    modules::LinearWeights mlp_out;
+};
+
+struct XttsV2GptWeights {
+    std::shared_ptr<core::BackendWeightStore> store;
+    core::TensorValue text_embedding;
+    core::TensorValue audio_embedding;
+    core::TensorValue text_positions;
+    core::TensorValue audio_positions;
+    std::vector<XttsV2GptLayerWeights> layers;
+    modules::NormWeights transformer_norm;
+    modules::NormWeights output_norm;
+    modules::LinearWeights audio_head;
+};
+
+struct XttsV2GptPrefillResult {
+    std::vector<float> logits;
+    std::vector<float> latent;
+};
+
+struct XttsV2GptGeneration {
+    std::vector<int32_t> codes;
+    std::vector<float> latents;  // frame-major [codes, 1024]
+};
+
+std::shared_ptr<const XttsV2GptWeights> load_xtts_v2_gpt_weights(
+    const XttsV2Assets & assets,
+    core::ExecutionContext & execution,
+    size_t weight_context_bytes,
+    assets::TensorStorageType storage_type);
+
+class XttsV2GptRuntime {
+public:
+    XttsV2GptRuntime(
+        const XttsV2Assets & assets,
+        core::ExecutionContext & execution,
+        size_t weight_context_bytes,
+        size_t graph_context_bytes,
+        assets::TensorStorageType storage_type);
+    ~XttsV2GptRuntime();
+
+    XttsV2GptPrefillResult prefill(
+        const std::vector<float> & conditioning_latent,
+        const std::vector<int32_t> & text_tokens);
+    XttsV2GptGeneration generate(
+        const std::vector<float> & conditioning_latent,
+        const std::vector<int32_t> & text_tokens,
+        const XttsV2GenerationOptions & options);
+
+private:
+    class PrefillGraph;
+    core::ExecutionContext & execution_;
+    size_t graph_context_bytes_;
+    std::shared_ptr<const XttsV2GptWeights> weights_;
+    std::unique_ptr<PrefillGraph> prefill_;
+};
+
+}  // namespace engine::models::xtts_v2

include/engine/models/xtts_v2/request.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "engine/framework/runtime/model.h"
+#include "engine/models/xtts_v2/types.h"
+
+namespace engine::models::xtts_v2 {
+
+XttsV2Request parse_xtts_v2_request(const runtime::TaskRequest & request);
+
+}  // namespace engine::models::xtts_v2

include/engine/models/xtts_v2/session.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include "engine/framework/model_spec/metadata.h"
+#include "engine/framework/runtime/session_base.h"
+#include "engine/models/xtts_v2/assets.h"
+#include "engine/models/xtts_v2/conditioning.h"
+#include "engine/models/xtts_v2/decoder.h"
+#include "engine/models/xtts_v2/gpt.h"
+#include "engine/models/xtts_v2/speaker_encoder.h"
+#include "engine/models/xtts_v2/tokenizer.h"
+
+#include <memory>
+
+namespace engine::models::xtts_v2 {
+
+std::shared_ptr<runtime::IVoiceModelLoader> make_xtts_v2_loader();
+
+class XttsV2Session final : public runtime::RuntimeSessionBase,
+                            public runtime::IOfflineVoiceTaskSession {
+public:
+    XttsV2Session(runtime::TaskSpec task, runtime::SessionOptions options,
+        std::shared_ptr<const XttsV2Assets> assets,
+        std::shared_ptr<const engine::model_spec::ModelContract> contract);
+    ~XttsV2Session() override;
+
+    std::string family() const override;
+    runtime::VoiceTaskKind task_kind() const override;
+    runtime::RunMode run_mode() const override;
+    void prepare(const runtime::SessionPreparationRequest & request) override;
+    runtime::TaskResult run(const runtime::TaskRequest & request) override;
+
+private:
+    runtime::TaskSpec task_;
+    std::shared_ptr<const XttsV2Assets> assets_;
+    std::shared_ptr<const engine::model_spec::ModelContract> contract_;
+    XttsV2Tokenizer tokenizer_;
+    std::unique_ptr<XttsV2ConditioningRuntime> conditioning_;
+    std::unique_ptr<XttsV2SpeakerEncoderRuntime> speaker_;
+    std::unique_ptr<XttsV2GptRuntime> gpt_;
+    std::unique_ptr<XttsV2DecoderRuntime> decoder_;
+};
+
+}  // namespace engine::models::xtts_v2