From 721ed4bedc0679f7e2008e192d85803e280ccd05 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 01:04:22 -0400 Subject: [PATCH 01/11] xtts_v2: add checkpoint conversion contract --- docs/community_models/xtts_v2.md | 51 +++++++ model_specs/xtts_v2.json | 79 ++++++++++ tools/community_models/convert_xtts_v2.py | 168 ++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 docs/community_models/xtts_v2.md create mode 100644 model_specs/xtts_v2.json create mode 100644 tools/community_models/convert_xtts_v2.py diff --git a/docs/community_models/xtts_v2.md b/docs/community_models/xtts_v2.md new file mode 100644 index 00000000..b013779a --- /dev/null +++ b/docs/community_models/xtts_v2.md @@ -0,0 +1,51 @@ +# Coqui XTTS v2 + +XTTS v2 is a multilingual, zero-shot voice-cloning model. The audio.cpp port +runs the conditioning encoder, Perceiver resampler, 30-layer GPT-2 acoustic +token generator, ResNet speaker encoder, and speaker-conditioned HiFiGAN +decoder natively through ggml. Python is needed only to convert the original +PyTorch checkpoint. + +The model supports `en`, `es`, `fr`, `de`, `it`, `pt`, `pl`, `tr`, `ru`, `nl`, +`cs`, `ar`, `zh-cn`, `ja`, `hu`, `ko`, and `hi`. Output is mono 24 kHz audio. +A clean reference recording of at least three seconds is required. + +```sh +audio.cpp -m models/XTTS-v2-GGUF/xtts-v2-q8_0.gguf \ + --task clone \ + --voice-ref reference.wav \ + --language en \ + -p "This voice was synthesized locally by audio.cpp." \ + --seed 42 \ + -o xtts-v2.wav +``` + +The default generation settings match Coqui's published XTTS v2 configuration: +temperature `0.75`, top-k `50`, top-p `0.85`, and repetition penalty `5.0`. +Use `--speed` to adjust speaking rate without changing pitch. + +## Conversion + +Download the official `coqui/XTTS-v2` snapshot, then run: + +```sh +python tools/community_models/convert_xtts_v2.py \ + --model-dir /path/to/XTTS-v2 \ + --output-dir /tmp/xtts-v2-staging \ + --run-converter build/bin/audiocpp_gguf \ + --type q8_0 +``` + +The converter excludes optimizer, scaler, and other training-only state. It +also materializes the effective weights of the PyTorch weight-normalized +HiFiGAN layers. Convolution tensors, token embeddings, and the sampling head +remain F16 in the mixed Q8 package to protect synthesis quality. + +## License + +The XTTS v2 weights and their outputs are licensed under the Coqui Public Model +License 1.0.0 for non-commercial use only. The converted package includes the +original `LICENSE.txt`; downstream redistribution must keep the license or its +URL with the model and its outputs. The audio.cpp source changes retain the +repository's source-code license. + diff --git a/model_specs/xtts_v2.json b/model_specs/xtts_v2.json new file mode 100644 index 00000000..a24df8ae --- /dev/null +++ b/model_specs/xtts_v2.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "family": "xtts_v2", + "display_name": "Coqui XTTS v2", + "description": "Multilingual zero-shot voice-cloning TTS with GPT acoustic-token generation and a speaker-conditioned HiFiGAN decoder at 24 kHz.", + "category": "tts", + "status": "community", + "tasks": ["tts", "clone"], + "modes": ["offline"], + "languages": ["en", "es", "fr", "de", "it", "pt", "pl", "tr", "ru", "nl", "cs", "ar", "zh-cn", "ja", "hu", "ko", "hi"], + "runtime": {"tags": ["gguf"]}, + "capabilities": {"clone": ["speaker_reference", "cross_lingual"]}, + "options": { + "request": [ + {"name": "language", "type": "string", "description": "Required XTTS language code.", "required": true, "default": "en"}, + {"name": "temperature", "type": "float", "description": "GPT sampling temperature.", "required": false, "min": 0.01, "max": 2.0, "default": 0.75}, + {"name": "top_k", "type": "int", "description": "GPT top-k sampling cutoff.", "required": false, "min": 1, "default": 50}, + {"name": "top_p", "type": "float", "description": "GPT nucleus-sampling cutoff.", "required": false, "min": 0.01, "max": 1.0, "default": 0.85}, + {"name": "repetition_penalty", "type": "float", "description": "GPT repetition penalty.", "required": false, "min": 1.0, "default": 5.0}, + {"name": "speed", "type": "float", "description": "Output speaking-rate multiplier.", "required": false, "min": 0.05, "max": 4.0, "default": 1.0}, + {"name": "seed", "type": "int", "description": "Non-negative deterministic sampling seed.", "required": false, "min": 0} + ], + "load": [ + {"name": "xtts_v2.weight_type", "type": "enum", "preset": "weight_type_full", "description": "GPT matrix storage type.", "required": false, "default": "native"}, + {"name": "xtts_v2.conv_weight_type", "type": "enum", "preset": "weight_type_conv", "description": "Conditioning, decoder, and speaker-encoder convolution storage type.", "required": false, "default": "native"}, + {"name": "xtts_v2.gpt_graph_arena_mb", "type": "int", "description": "GPT graph arena size.", "required": false, "min": 64, "default": 512}, + {"name": "xtts_v2.reference_graph_arena_mb", "type": "int", "description": "Reference-conditioning graph arena size.", "required": false, "min": 32, "default": 256}, + {"name": "xtts_v2.decoder_graph_arena_mb", "type": "int", "description": "HiFiGAN decoder graph arena size.", "required": false, "min": 32, "default": 256} + ] + }, + "package_defaults": { + "download": {"kind": "huggingface_snapshot", "repo": "audio-cpp/audio.cpp-gguf", "revision": "refs/pr/REPLACE_WITH_HF_PR"} + }, + "packages": [ + { + "id": "xtts_v2_q8_0", + "display_name": "Coqui XTTS v2 Q8_0 GGUF", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "XTTS-v2-GGUF", + "files": ["XTTS-v2-GGUF/xtts-v2-q8_0.gguf", "XTTS-v2-GGUF/LICENSE.txt"], + "strip_prefix": "XTTS-v2-GGUF" + }, + { + "id": "xtts_v2_f16", + "display_name": "Coqui XTTS v2 F16 GGUF", + "format": "gguf", + "precision": "f16", + "target_directory": "XTTS-v2-GGUF", + "files": ["XTTS-v2-GGUF/xtts-v2-f16.gguf", "XTTS-v2-GGUF/LICENSE.txt"], + "strip_prefix": "XTTS-v2-GGUF" + } + ], + "sources": [ + { + "format": "gguf", + "roots": {"model": ".", "weights": "$gguf"}, + "files": {"config": "model:config.json", "tokenizer": "model:vocab.json", "model_license": "model:LICENSE.txt"}, + "tensors": { + "gpt": {"source": "weights:", "prefix": "gpt"}, + "decoder": {"source": "weights:", "prefix": "decoder"}, + "speaker_encoder": {"source": "weights:", "prefix": "speaker_encoder"} + } + }, + { + "format": "safetensors", + "roots": {"model": "."}, + "files": {"config": "model:config.json", "tokenizer": "model:vocab.json", "model_license": "model:LICENSE.txt"}, + "tensors": {"gpt": "model:gpt.safetensors", "decoder": "model:decoder.safetensors", "speaker_encoder": "model:speaker_encoder.safetensors"} + } + ], + "dependencies": [], + "ui": { + "recommended_package": "xtts_v2_q8_0", + "tags": ["TTS", "Clone", "Multilingual", "Non-commercial", "GGUF"], + "docs": ["docs/community_models/xtts_v2.md"] + } +} diff --git a/tools/community_models/convert_xtts_v2.py b/tools/community_models/convert_xtts_v2.py new file mode 100644 index 00000000..7891d5f1 --- /dev/null +++ b/tools/community_models/convert_xtts_v2.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Convert Coqui XTTS v2 into the tensor layout consumed by audio.cpp. + +The official ``model.pth`` also contains optimizer, scaler, and trainer state. This +tool exports only the inference graph, resolves PyTorch weight parametrizations, +and stages the tokenizer/config/license sidecars without modifying the source +snapshot. + +Example: + python tools/community_models/convert_xtts_v2.py \ + --model-dir /path/to/coqui-XTTS-v2 \ + --output-dir /tmp/xtts-v2-staging \ + --run-converter build/bin/audiocpp_gguf --type f16 +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +from pathlib import Path + +import torch +from safetensors.torch import save_file + + +PREFIXES = { + "gpt": "gpt.", + "decoder": "hifigan_decoder.waveform_decoder.", + "speaker_encoder": "hifigan_decoder.speaker_encoder.", +} + + +def require_file(path: Path, label: str) -> Path: + if not path.is_file(): + raise FileNotFoundError(f"missing {label}: {path}") + return path + + +def materialize_weight_norm(tensors: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + """Replace parametrizations.weight.original{0,1} with the effective weight. + + torch.nn.utils.parametrizations.weight_norm uses ``g * v / ||v||`` and, for + Conv1d/ConvTranspose1d, normalizes over every axis except output channel 0. + Coqui removes these parametrizations before inference, so the native graph must + bind the materialized value rather than either checkpoint component. + """ + out = dict(tensors) + suffix = ".parametrizations.weight.original0" + for key in tuple(out): + if not key.endswith(suffix): + continue + stem = key[: -len(suffix)] + v_key = stem + ".parametrizations.weight.original1" + if v_key not in out: + raise KeyError(f"weight-norm tensor {key} has no matching {v_key}") + g = out.pop(key).float() + v = out.pop(v_key).float() + dims = tuple(range(1, v.ndim)) + out[stem + ".weight"] = (v * (g / torch.linalg.vector_norm(v, dim=dims, keepdim=True))).contiguous() + return out + + +def extract_group(state: dict[str, torch.Tensor], prefix: str) -> dict[str, torch.Tensor]: + group: dict[str, torch.Tensor] = {} + for name, value in state.items(): + if not name.startswith(prefix) or not isinstance(value, torch.Tensor): + continue + short = name[len(prefix) :] + # BatchNorm's counter is training-only and GGUF does not support scalar i64 + # tensors. Running statistics and frontend buffers remain essential. + if short.endswith(".num_batches_tracked"): + continue + group[short] = value.detach().cpu().contiguous() + if not group: + raise RuntimeError(f"checkpoint contains no tensors under {prefix!r}") + return materialize_weight_norm(group) + + +def validate_config(path: Path) -> None: + config = json.loads(path.read_text(encoding="utf-8")) + args = config.get("model_args", {}) + expected = { + "gpt_layers": 30, + "gpt_n_model_channels": 1024, + "gpt_n_heads": 16, + "gpt_number_text_tokens": 6681, + "gpt_num_audio_tokens": 1026, + "gpt_use_perceiver_resampler": True, + "output_sample_rate": 24000, + } + mismatches = {key: (args.get(key), value) for key, value in expected.items() if args.get(key) != value} + if mismatches: + raise ValueError(f"unsupported XTTS checkpoint configuration: {mismatches}") + + +def converter_command(output_dir: Path, converter: Path, quant_type: str) -> list[str]: + command = [str(converter)] + for namespace in PREFIXES: + command += ["--input", f"{namespace}={output_dir / (namespace + '.safetensors')}"] + command += [ + "--root", str(output_dir / "root"), + "--family", "xtts_v2", + "--type", quant_type, + "--output", str(output_dir / f"xtts-v2-{quant_type}.gguf"), + ] + # Convolutions have no quantized execution path. Keeping embeddings and the + # small output head in F16 also avoids sampling regressions from Q8 logits. + if quant_type.startswith("q"): + command += [ + "--keep-type", "decoder/*=f16", + "--keep-type", "speaker_encoder/*=f16", + "--keep-type", "gpt/conditioning_encoder.*=f16", + "--keep-type", "gpt/text_embedding.weight=f16", + "--keep-type", "gpt/mel_embedding.weight=f16", + "--keep-type", "gpt/mel_head.*=f16", + ] + return command + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--run-converter", type=Path) + parser.add_argument("--type", default="f16", choices=("f32", "f16", "q8_0")) + args = parser.parse_args() + + model_dir = args.model_dir.resolve() + output_dir = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + root = output_dir / "root" + root.mkdir(parents=True, exist_ok=True) + + config = require_file(model_dir / "config.json", "config.json") + validate_config(config) + checkpoint = torch.load( + require_file(model_dir / "model.pth", "model.pth"), + map_location="cpu", + weights_only=False, + ) + state = checkpoint.get("model") if isinstance(checkpoint, dict) else None + if not isinstance(state, dict): + raise TypeError("XTTS model.pth does not contain a model state dictionary") + + for namespace, prefix in PREFIXES.items(): + tensors = extract_group(state, prefix) + destination = output_dir / f"{namespace}.safetensors" + save_file(tensors, str(destination)) + parameters = sum(t.numel() for t in tensors.values()) + print(f"wrote {destination} ({len(tensors)} tensors, {parameters:,} values)") + + for filename in ("config.json", "vocab.json", "LICENSE.txt"): + shutil.copyfile(require_file(model_dir / filename, filename), root / filename) + print(f"staged config, tokenizer, and CPML license in {root}") + + command = converter_command(output_dir, args.run_converter or Path("audiocpp_gguf"), args.type) + if args.run_converter: + subprocess.run(command, check=True) + else: + print("next:") + print(" ".join(command)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 17f3b585ecd19613e6d1407dee27ba49c13381d6 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 01:07:18 -0400 Subject: [PATCH 02/11] xtts_v2: add assets requests and BPE tokenizer --- include/engine/models/xtts_v2/assets.h | 23 +++ include/engine/models/xtts_v2/request.h | 10 ++ include/engine/models/xtts_v2/tokenizer.h | 27 ++++ include/engine/models/xtts_v2/types.h | 48 ++++++ src/models/xtts_v2/assets.cpp | 50 +++++++ src/models/xtts_v2/request.cpp | 76 ++++++++++ src/models/xtts_v2/tokenizer.cpp | 169 ++++++++++++++++++++++ 7 files changed, 403 insertions(+) create mode 100644 include/engine/models/xtts_v2/assets.h create mode 100644 include/engine/models/xtts_v2/request.h create mode 100644 include/engine/models/xtts_v2/tokenizer.h create mode 100644 include/engine/models/xtts_v2/types.h create mode 100644 src/models/xtts_v2/assets.cpp create mode 100644 src/models/xtts_v2/request.cpp create mode 100644 src/models/xtts_v2/tokenizer.cpp diff --git a/include/engine/models/xtts_v2/assets.h b/include/engine/models/xtts_v2/assets.h new file mode 100644 index 00000000..78d1ec79 --- /dev/null +++ b/include/engine/models/xtts_v2/assets.h @@ -0,0 +1,23 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/models/xtts_v2/types.h" + +#include +#include + +namespace engine::models::xtts_v2 { + +struct XttsV2Assets { + assets::ResourceBundle resources; + XttsV2Config config; + std::shared_ptr gpt; + std::shared_ptr decoder; + std::shared_ptr speaker_encoder; + std::filesystem::path tokenizer_path; +}; + +std::shared_ptr load_xtts_v2_assets(const std::filesystem::path & model_path); + +} // namespace engine::models::xtts_v2 diff --git a/include/engine/models/xtts_v2/request.h b/include/engine/models/xtts_v2/request.h new file mode 100644 index 00000000..8ad94583 --- /dev/null +++ b/include/engine/models/xtts_v2/request.h @@ -0,0 +1,10 @@ +#pragma once + +#include "engine/framework/runtime/model.h" +#include "engine/models/xtts_v2/types.h" + +namespace engine::models::xtts_v2 { + +XttsV2Request parse_xtts_v2_request(const runtime::TaskRequest & request); + +} // namespace engine::models::xtts_v2 diff --git a/include/engine/models/xtts_v2/tokenizer.h b/include/engine/models/xtts_v2/tokenizer.h new file mode 100644 index 00000000..06d24746 --- /dev/null +++ b/include/engine/models/xtts_v2/tokenizer.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { + +class XttsV2Tokenizer { +public: + explicit XttsV2Tokenizer(const std::filesystem::path & path); + ~XttsV2Tokenizer(); + XttsV2Tokenizer(XttsV2Tokenizer &&) noexcept; + XttsV2Tokenizer & operator=(XttsV2Tokenizer &&) noexcept; + + std::vector encode(const std::string & text, const std::string & language) const; + int32_t start_token() const noexcept; + int32_t stop_token() const noexcept; + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::xtts_v2 diff --git a/include/engine/models/xtts_v2/types.h b/include/engine/models/xtts_v2/types.h new file mode 100644 index 00000000..802f0602 --- /dev/null +++ b/include/engine/models/xtts_v2/types.h @@ -0,0 +1,48 @@ +#pragma once + +#include "engine/framework/runtime/model.h" + +#include +#include + +namespace engine::models::xtts_v2 { + +struct XttsV2Config { + int64_t sample_rate = 24000; + int64_t conditioning_sample_rate = 22050; + int64_t speaker_sample_rate = 16000; + int64_t text_vocab = 6681; + int64_t audio_vocab = 1026; + int64_t model_dim = 1024; + int64_t gpt_layers = 30; + int64_t gpt_heads = 16; + int64_t max_text_tokens = 402; + int64_t max_audio_tokens = 605; + int64_t start_text_token = 261; + int64_t stop_text_token = 0; + int64_t start_audio_token = 1024; + int64_t stop_audio_token = 1025; + int64_t conditioning_tokens = 32; + int64_t speaker_dim = 512; + int64_t code_stride = 1024; + int64_t output_hop = 256; +}; + +struct XttsV2GenerationOptions { + float temperature = 0.75F; + float top_p = 0.85F; + int64_t top_k = 50; + float repetition_penalty = 5.0F; + float speed = 1.0F; + int64_t max_tokens = 603; + uint32_t seed = 0; +}; + +struct XttsV2Request { + std::string text; + std::string language = "en"; + runtime::AudioBuffer speaker_audio; + XttsV2GenerationOptions generation; +}; + +} // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/assets.cpp b/src/models/xtts_v2/assets.cpp new file mode 100644 index 00000000..21a877ef --- /dev/null +++ b/src/models/xtts_v2/assets.cpp @@ -0,0 +1,50 @@ +#include "engine/models/xtts_v2/assets.h" + +#include "engine/framework/io/json.h" +#include "engine/framework/model_spec/package.h" + +#include +#include + +namespace engine::models::xtts_v2 { +namespace json = engine::io::json; + +std::shared_ptr load_xtts_v2_assets(const std::filesystem::path & model_path) { + auto resources = engine::model_spec::load_resource_bundle( + model_path, engine::model_spec::default_spec_path("xtts_v2")); + const auto root = resources.parse_json("config"); + const auto & args = root.require("model_args"); + + auto assets = std::make_shared(); + assets->resources = std::move(resources); + auto & config = assets->config; + config.sample_rate = json::require_i64(root.require("audio"), "output_sample_rate"); + config.conditioning_sample_rate = json::require_i64(args, "input_sample_rate"); + config.text_vocab = json::require_i64(args, "gpt_number_text_tokens"); + config.audio_vocab = json::require_i64(args, "gpt_num_audio_tokens"); + config.model_dim = json::require_i64(args, "gpt_n_model_channels"); + config.gpt_layers = json::require_i64(args, "gpt_layers"); + config.gpt_heads = json::require_i64(args, "gpt_n_heads"); + config.max_text_tokens = json::require_i64(args, "gpt_max_text_tokens"); + config.max_audio_tokens = json::require_i64(args, "gpt_max_audio_tokens"); + config.start_audio_token = json::require_i64(args, "gpt_start_audio_token"); + config.stop_audio_token = json::require_i64(args, "gpt_stop_audio_token"); + config.code_stride = json::require_i64(args, "gpt_code_stride_len"); + config.output_hop = json::require_i64(args, "output_hop_length"); + config.speaker_dim = json::require_i64(args, "d_vector_dim"); + + if (config.sample_rate != 24000 || config.conditioning_sample_rate != 22050 || + config.text_vocab != 6681 || config.audio_vocab != 1026 || + config.model_dim != 1024 || config.gpt_layers != 30 || config.gpt_heads != 16 || + !json::optional_bool(args, "gpt_use_perceiver_resampler", false)) { + throw std::runtime_error("unsupported Coqui XTTS v2 checkpoint architecture"); + } + assets->gpt = assets->resources.open_tensor_source("gpt"); + assets->decoder = assets->resources.open_tensor_source("decoder"); + assets->speaker_encoder = assets->resources.open_tensor_source("speaker_encoder"); + assets->tokenizer_path = assets->resources.require_file("tokenizer"); + (void) assets->resources.require_file("model_license"); + return assets; +} + +} // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/request.cpp b/src/models/xtts_v2/request.cpp new file mode 100644 index 00000000..1fc89489 --- /dev/null +++ b/src/models/xtts_v2/request.cpp @@ -0,0 +1,76 @@ +#include "engine/models/xtts_v2/request.h" + +#include "engine/framework/io/text.h" +#include "engine/framework/runtime/options.h" + +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { +namespace { + +const std::unordered_set kLanguages = { + "en", "es", "fr", "de", "it", "pt", "pl", "tr", "ru", "nl", + "cs", "ar", "zh-cn", "ja", "hu", "ko", "hi", +}; + +std::string normalize_language(std::string language) { + language = engine::io::trim_ascii_whitespace(language); + std::transform(language.begin(), language.end(), language.begin(), [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + if (language == "zh") { + language = "zh-cn"; + } + if (kLanguages.count(language) == 0) { + throw std::runtime_error("unsupported XTTS v2 language: " + language); + } + return language; +} + +} // namespace + +XttsV2Request parse_xtts_v2_request(const runtime::TaskRequest & request) { + XttsV2Request out; + if (request.text_input.has_value()) { + out.text = engine::io::trim_ascii_whitespace(request.text_input->text); + } else if (const auto text = runtime::find_option(request.options, {"text", "prompt"})) { + out.text = engine::io::trim_ascii_whitespace(*text); + } + if (out.text.empty()) { + throw std::runtime_error("XTTS v2 requires text input"); + } + if (!request.voice.has_value() || !request.voice->speaker.has_value() || + !request.voice->speaker->audio.has_value()) { + throw std::runtime_error("XTTS v2 requires --voice-ref or voice.speaker.audio"); + } + out.speaker_audio = *request.voice->speaker->audio; + if (out.speaker_audio.sample_rate <= 0 || out.speaker_audio.channels <= 0 || + out.speaker_audio.samples.empty()) { + throw std::runtime_error("XTTS v2 speaker reference is empty or invalid"); + } + if (const auto language = runtime::find_option(request.options, {"language"})) { + out.language = normalize_language(*language); + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"temperature"})) out.generation.temperature = *value; + if (const auto value = runtime::parse_finite_float_option(request.options, {"top_p"})) out.generation.top_p = *value; + if (const auto value = runtime::parse_int_option(request.options, {"top_k"})) out.generation.top_k = *value; + if (const auto value = runtime::parse_finite_float_option(request.options, {"repetition_penalty"})) out.generation.repetition_penalty = *value; + if (const auto value = runtime::parse_finite_float_option(request.options, {"speed"})) out.generation.speed = *value; + if (const auto value = runtime::parse_int_option(request.options, {"max_tokens"})) out.generation.max_tokens = *value; + if (const auto value = runtime::parse_u32_option(request.options, {"seed"})) { + out.generation.seed = *value; + } else { + out.generation.seed = runtime::random_u32_seed(); + } + if (!(out.generation.temperature > 0.0F) || !(out.generation.top_p > 0.0F && out.generation.top_p <= 1.0F) || + out.generation.top_k <= 0 || out.generation.repetition_penalty < 1.0F || + !(out.generation.speed > 0.0F) || out.generation.max_tokens <= 0 || out.generation.max_tokens > 603) { + throw std::runtime_error("invalid XTTS v2 generation options"); + } + return out; +} + +} // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/tokenizer.cpp b/src/models/xtts_v2/tokenizer.cpp new file mode 100644 index 00000000..09532990 --- /dev/null +++ b/src/models/xtts_v2/tokenizer.cpp @@ -0,0 +1,169 @@ +#include "engine/models/xtts_v2/tokenizer.h" + +#include "engine/framework/io/json.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { +namespace { + +struct PairHash { + size_t operator()(const std::pair & pair) const noexcept { + return std::hash{}(pair.first) ^ (std::hash{}(pair.second) << 1U); + } +}; + +std::vector utf8_symbols(std::string_view text) { + std::vector out; + for (size_t pos = 0; pos < text.size();) { + const auto lead = static_cast(text[pos]); + size_t width = 1; + if ((lead & 0x80U) == 0) width = 1; + else if ((lead & 0xE0U) == 0xC0U) width = 2; + else if ((lead & 0xF0U) == 0xE0U) width = 3; + else if ((lead & 0xF8U) == 0xF0U) width = 4; + else throw std::runtime_error("XTTS v2 tokenizer received invalid UTF-8"); + if (pos + width > text.size()) throw std::runtime_error("XTTS v2 tokenizer received truncated UTF-8"); + out.emplace_back(text.substr(pos, width)); + pos += width; + } + return out; +} + +size_t utf8_width_at(std::string_view text, size_t pos) { + if (pos >= text.size()) throw std::runtime_error("XTTS v2 tokenizer UTF-8 offset is out of range"); + const auto lead = static_cast(text[pos]); + size_t width = 0; + if ((lead & 0x80U) == 0) width = 1; + else if ((lead & 0xE0U) == 0xC0U) width = 2; + else if ((lead & 0xF0U) == 0xE0U) width = 3; + else if ((lead & 0xF8U) == 0xF0U) width = 4; + else throw std::runtime_error("XTTS v2 tokenizer received invalid UTF-8"); + if (pos + width > text.size()) throw std::runtime_error("XTTS v2 tokenizer received truncated UTF-8"); + return width; +} + +std::string clean_text(const std::string & input) { + std::string out; + out.reserve(input.size()); + bool space = false; + for (unsigned char ch : input) { + if (ch == '"') continue; + if (std::isspace(ch) != 0) { + space = !out.empty(); + continue; + } + if (space) out.push_back(' '); + space = false; + out.push_back(ch < 0x80U ? static_cast(std::tolower(ch)) : static_cast(ch)); + } + return out; +} + +} // namespace + +struct XttsV2Tokenizer::Impl { + std::unordered_map vocab; + std::unordered_map, int32_t, PairHash> merges; + std::vector specials; + int32_t unk = 1; + int32_t start = 261; + int32_t stop = 0; + + std::vector encode_piece(const std::string & text) const { + auto symbols = utf8_symbols(text); + while (symbols.size() > 1) { + int32_t rank = std::numeric_limits::max(); + size_t selected = symbols.size(); + for (size_t i = 0; i + 1 < symbols.size(); ++i) { + const auto found = merges.find({symbols[i], symbols[i + 1]}); + if (found != merges.end() && found->second < rank) { + rank = found->second; + selected = i; + } + } + if (selected == symbols.size()) break; + symbols[selected] += symbols[selected + 1]; + symbols.erase(symbols.begin() + static_cast(selected + 1)); + } + std::vector ids; + ids.reserve(symbols.size()); + for (const auto & symbol : symbols) { + const auto found = vocab.find(symbol); + ids.push_back(found == vocab.end() ? unk : found->second); + } + return ids; + } + + std::vector encode_marked(const std::string & text) const { + std::vector ids; + std::string ordinary; + auto flush = [&] { + if (ordinary.empty()) return; + auto encoded = encode_piece(ordinary); + ids.insert(ids.end(), encoded.begin(), encoded.end()); + ordinary.clear(); + }; + for (size_t pos = 0; pos < text.size();) { + const auto special = std::find_if(specials.begin(), specials.end(), [&](const std::string & candidate) { + return pos + candidate.size() <= text.size() && text.compare(pos, candidate.size(), candidate) == 0; + }); + if (special != specials.end()) { + flush(); + ids.push_back(vocab.at(*special)); + pos += special->size(); + continue; + } + const size_t width = utf8_width_at(text, pos); + ordinary.append(text, pos, width); + pos += width; + } + flush(); + return ids; + } +}; + +XttsV2Tokenizer::XttsV2Tokenizer(const std::filesystem::path & path) : impl_(std::make_unique()) { + const auto root = engine::io::json::parse_file(path); + const auto & model = root.require("model"); + if (model.require("type").as_string() != "BPE") throw std::runtime_error("XTTS v2 expects a BPE tokenizer"); + for (const auto & [piece, value] : model.require("vocab").as_object()) impl_->vocab.emplace(piece, static_cast(value.as_i64())); + int32_t rank = 0; + for (const auto & item : model.require("merges").as_array()) { + const auto merge = item.as_string(); + const auto split = merge.find(' '); + if (split == std::string::npos) throw std::runtime_error("XTTS v2 tokenizer has an invalid merge"); + impl_->merges.emplace(std::make_pair(merge.substr(0, split), merge.substr(split + 1)), rank++); + } + for (const auto & item : root.require("added_tokens").as_array()) { + if (const auto * content = item.find("content"); content != nullptr && content->is_string()) impl_->specials.push_back(content->as_string()); + } + std::sort(impl_->specials.begin(), impl_->specials.end(), [](const auto & a, const auto & b) { return a.size() > b.size(); }); + impl_->unk = impl_->vocab.at("[UNK]"); + impl_->start = impl_->vocab.at("[START]"); + impl_->stop = impl_->vocab.at("[STOP]"); +} + +XttsV2Tokenizer::~XttsV2Tokenizer() = default; +XttsV2Tokenizer::XttsV2Tokenizer(XttsV2Tokenizer &&) noexcept = default; +XttsV2Tokenizer & XttsV2Tokenizer::operator=(XttsV2Tokenizer &&) noexcept = default; + +std::vector XttsV2Tokenizer::encode(const std::string & text, const std::string & language) const { + std::string language_tag = language == "zh" ? "zh-cn" : language; + std::string prepared = "[" + language_tag + "]" + clean_text(text); + std::string marked; + marked.reserve(prepared.size() + 16); + for (const char ch : prepared) marked += ch == ' ' ? "[SPACE]" : std::string(1, ch); + return impl_->encode_marked(marked); +} + +int32_t XttsV2Tokenizer::start_token() const noexcept { return impl_->start; } +int32_t XttsV2Tokenizer::stop_token() const noexcept { return impl_->stop; } + +} // namespace engine::models::xtts_v2 From 980ca3d4a9c9afc014966c7ae1631323d8e1167d Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 01:09:33 -0400 Subject: [PATCH 03/11] xtts_v2: implement reference audio frontends --- .../engine/models/xtts_v2/audio_features.h | 35 +++++ src/models/xtts_v2/audio_features.cpp | 147 ++++++++++++++++++ tools/community_models/convert_xtts_v2.py | 7 + 3 files changed, 189 insertions(+) create mode 100644 include/engine/models/xtts_v2/audio_features.h create mode 100644 src/models/xtts_v2/audio_features.cpp diff --git a/include/engine/models/xtts_v2/audio_features.h b/include/engine/models/xtts_v2/audio_features.h new file mode 100644 index 00000000..d589547f --- /dev/null +++ b/include/engine/models/xtts_v2/audio_features.h @@ -0,0 +1,35 @@ +#pragma once + +#include "engine/framework/runtime/model.h" + +#include +#include +#include + +namespace engine::models::xtts_v2 { + +struct XttsV2MelFeatures { + std::vector values; // channel-major [channels, frames] + int64_t channels = 0; + int64_t frames = 0; +}; + +struct XttsV2PreparedReference { + std::vector waveform_22050; + std::vector waveform_16000; +}; + +XttsV2PreparedReference prepare_xtts_v2_reference(const runtime::AudioBuffer & audio); + +XttsV2MelFeatures compute_xtts_v2_conditioning_mel( + const std::vector & waveform_22050, + const std::vector & mel_stats, + size_t threads = 0); + +XttsV2MelFeatures compute_xtts_v2_speaker_mel( + const std::vector & waveform_16000, + const std::vector & window, + const std::vector & mel_filterbank, + size_t threads = 0); + +} // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/audio_features.cpp b/src/models/xtts_v2/audio_features.cpp new file mode 100644 index 00000000..755f2519 --- /dev/null +++ b/src/models/xtts_v2/audio_features.cpp @@ -0,0 +1,147 @@ +#include "engine/models/xtts_v2/audio_features.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/dsp.h" +#include "engine/framework/audio/resampling.h" + +#include +#include +#include + +namespace engine::models::xtts_v2 { +namespace { + +std::vector mono(const runtime::AudioBuffer & audio) { + if (audio.channels <= 0 || audio.samples.empty() || + audio.samples.size() % static_cast(audio.channels) != 0) { + throw std::runtime_error("XTTS v2 reference audio has an invalid shape"); + } + return audio.channels == 1 + ? audio.samples + : engine::audio::mixdown_interleaved_to_mono_average(audio.samples, audio.channels); +} + +std::vector resample(const std::vector & input, int source_rate, int target_rate) { + if (source_rate == target_rate) return input; + auto options = engine::audio::torchaudio_sinc_hann_float32_options(); + return engine::audio::resample_mono_torchaudio_sinc_hann(input, source_rate, target_rate, options); +} + +XttsV2MelFeatures power_mel( + const std::vector & waveform, + int64_t sample_rate, + int64_t n_fft, + int64_t hop, + int64_t win, + int64_t n_mels, + float fmax, + const std::vector & window, + const std::vector & filterbank, + size_t threads) { + const engine::audio::STFTConfig stft{ + n_fft, hop, win, true, engine::audio::STFTPadMode::Reflect, + engine::audio::STFTFamily::Default, + }; + const auto magnitude = engine::audio::STFT{}.compute_magnitude( + waveform, window, 1, static_cast(waveform.size()), stft, threads); + if (magnitude.shape.size() != 3) throw std::runtime_error("XTTS v2 STFT returned an invalid shape"); + const int64_t bins = n_fft / 2 + 1; + const int64_t frames = magnitude.shape[2]; + if (static_cast(filterbank.size()) != bins * n_mels) { + throw std::runtime_error("XTTS v2 mel filterbank shape mismatch"); + } + (void) sample_rate; + (void) fmax; + XttsV2MelFeatures out; + out.channels = n_mels; + out.frames = frames; + out.values.assign(static_cast(n_mels * frames), 0.0F); + for (int64_t m = 0; m < n_mels; ++m) { + for (int64_t t = 0; t < frames; ++t) { + double energy = 0.0; + for (int64_t f = 0; f < bins; ++f) { + const float value = magnitude.values[static_cast(f * frames + t)]; + energy += static_cast(filterbank[static_cast(m * bins + f)]) * value * value; + } + out.values[static_cast(m * frames + t)] = static_cast(energy); + } + } + return out; +} + +} // namespace + +XttsV2PreparedReference prepare_xtts_v2_reference(const runtime::AudioBuffer & audio) { + auto waveform = mono(audio); + for (float & value : waveform) value = std::max(-1.0F, std::min(1.0F, value)); + // Coqui caps each input reference at 30 seconds before joining references. + const size_t max_source = static_cast(audio.sample_rate) * 30U; + if (waveform.size() > max_source) waveform.resize(max_source); + XttsV2PreparedReference out; + out.waveform_22050 = resample(waveform, audio.sample_rate, 22050); + out.waveform_16000 = resample(waveform, audio.sample_rate, 16000); + return out; +} + +XttsV2MelFeatures compute_xtts_v2_conditioning_mel( + const std::vector & waveform_22050, + const std::vector & mel_stats, + size_t threads) { + if (mel_stats.size() != 80) throw std::runtime_error("XTTS v2 conditioning requires 80 mel norms"); + const engine::audio::STFTConfig stft{2048, 256, 1024, true, engine::audio::STFTPadMode::Reflect, engine::audio::STFTFamily::Kokoro}; + const auto & window = engine::audio::get_cached_stft_window(stft); + const auto filterbank = engine::audio::MelFilterbank{}.build({22050, 2048, 80, 0.0F, 8000.0F, true}); + + // The caller supplies one chunk. XTTS v2 runs this frontend and the Perceiver + // independently for each (normally four-second) chunk, then averages latents. + auto out = power_mel(waveform_22050, 22050, 2048, 256, 1024, 80, 8000.0F, window, filterbank.values, threads); + for (int64_t m = 0; m < out.channels; ++m) { + const float norm = mel_stats[static_cast(m)]; + if (norm == 0.0F) throw std::runtime_error("XTTS v2 mel norm contains zero"); + for (int64_t t = 0; t < out.frames; ++t) { + auto & value = out.values[static_cast(m * out.frames + t)]; + value = std::log(std::max(value, 1.0e-5F)) / norm; + } + } + return out; +} + +XttsV2MelFeatures compute_xtts_v2_speaker_mel( + const std::vector & waveform_16000, + const std::vector & window, + const std::vector & mel_filterbank, + size_t threads) { + if (waveform_16000.size() < 2 || window.size() != 400 || mel_filterbank.size() != 257U * 64U) { + throw std::runtime_error("XTTS v2 speaker frontend inputs are invalid"); + } + // ResNetSpeakerEncoder's embedded torchaudio frontend applies reflection-pad + // pre-emphasis [-0.97, 1] before a power mel spectrogram. + std::vector emphasized(waveform_16000.size()); + emphasized[0] = waveform_16000[0] - 0.97F * waveform_16000[1]; + for (size_t i = 1; i < waveform_16000.size(); ++i) { + emphasized[i] = waveform_16000[i] - 0.97F * waveform_16000[i - 1]; + } + auto out = power_mel(emphasized, 16000, 512, 160, 400, 64, 8000.0F, window, mel_filterbank, threads); + for (auto & value : out.values) value = std::log(value + 1.0e-6F); + + // InstanceNorm1d normalizes each mel channel across time without affine terms. + for (int64_t m = 0; m < out.channels; ++m) { + double mean = 0.0; + for (int64_t t = 0; t < out.frames; ++t) mean += out.values[static_cast(m * out.frames + t)]; + mean /= static_cast(out.frames); + double variance = 0.0; + for (int64_t t = 0; t < out.frames; ++t) { + const double delta = out.values[static_cast(m * out.frames + t)] - mean; + variance += delta * delta; + } + variance /= static_cast(out.frames); + const float inv = static_cast(1.0 / std::sqrt(variance + 1.0e-5)); + for (int64_t t = 0; t < out.frames; ++t) { + auto & value = out.values[static_cast(m * out.frames + t)]; + value = (value - static_cast(mean)) * inv; + } + } + return out; +} + +} // namespace engine::models::xtts_v2 diff --git a/tools/community_models/convert_xtts_v2.py b/tools/community_models/convert_xtts_v2.py index 7891d5f1..62b253fd 100644 --- a/tools/community_models/convert_xtts_v2.py +++ b/tools/community_models/convert_xtts_v2.py @@ -146,6 +146,13 @@ def main() -> int: for namespace, prefix in PREFIXES.items(): tensors = extract_group(state, prefix) + if namespace == "gpt": + # Xtts.mel_stats is registered on the model rather than under gpt, + # but it is an inference input to the conditioning encoder. + mel_stats = state.get("mel_stats") + if not isinstance(mel_stats, torch.Tensor) or tuple(mel_stats.shape) != (80,): + raise RuntimeError("checkpoint is missing the 80-bin XTTS mel_stats tensor") + tensors["mel_stats"] = mel_stats.detach().cpu().float().contiguous() destination = output_dir / f"{namespace}.safetensors" save_file(tensors, str(destination)) parameters = sum(t.numel() for t in tensors.values()) From d3734572ece9c8b84c6d57e705c53aabb993ad1b Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 01:19:18 -0400 Subject: [PATCH 04/11] xtts_v2: add conditioning encoder runtime probe --- CMakeLists.txt | 5 + include/engine/models/xtts_v2/conditioning.h | 44 ++++ model_specs/xtts_v2.json | 15 +- src/models/xtts_v2/conditioning.cpp | 252 +++++++++++++++++++ tests/xtts_v2/xtts_v2_conditioning_probe.cpp | 57 +++++ tools/community_models/convert_xtts_v2.py | 23 +- 6 files changed, 388 insertions(+), 8 deletions(-) create mode 100644 include/engine/models/xtts_v2/conditioning.h create mode 100644 src/models/xtts_v2/conditioning.cpp create mode 100644 tests/xtts_v2/xtts_v2_conditioning_probe.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b0cedea3..cec681b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2156,6 +2156,11 @@ if (ENGINE_BUILD_WARMBENCH) add_engine_warmbench(higgs_audio_tts_warm_bench tests/higgs_audio_tts/higgs_audio_tts_warm_bench.cpp) add_engine_warmbench(hviske_asr_warm_bench tests/hviske_asr/hviske_asr_warm_bench.cpp) add_engine_warmbench(index_tts2_warm_bench tests/index_tts2/index_tts2_warm_bench.cpp) + add_engine_warmbench(xtts_v2_conditioning_probe tests/xtts_v2/xtts_v2_conditioning_probe.cpp) + target_sources(xtts_v2_conditioning_probe PRIVATE + src/models/xtts_v2/assets.cpp + src/models/xtts_v2/audio_features.cpp + src/models/xtts_v2/conditioning.cpp) add_engine_warmbench(irodori_tts_warm_bench tests/irodori_tts/irodori_tts_warm_bench.cpp) add_engine_warmbench(marblenet_vad_warm_bench tests/marblenet_vad/marblenet_vad_warm_bench.cpp) add_engine_warmbench(miocodec_warm_bench tests/miocodec/miocodec_warm_bench.cpp) diff --git a/include/engine/models/xtts_v2/conditioning.h b/include/engine/models/xtts_v2/conditioning.h new file mode 100644 index 00000000..f22e2120 --- /dev/null +++ b/include/engine/models/xtts_v2/conditioning.h @@ -0,0 +1,44 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/models/xtts_v2/assets.h" +#include "engine/models/xtts_v2/audio_features.h" + +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { + +struct XttsV2ConditioningLatent { + std::vector values; // frame-major [32, 1024] + int64_t frames = 32; + int64_t dims = 1024; +}; + +class XttsV2ConditioningRuntime { +public: + XttsV2ConditioningRuntime( + const XttsV2Assets & assets, + core::ExecutionContext & execution, + size_t weight_context_bytes, + size_t graph_context_bytes, + assets::TensorStorageType matmul_storage_type, + assets::TensorStorageType conv_storage_type); + ~XttsV2ConditioningRuntime(); + + XttsV2ConditioningLatent encode(const XttsV2MelFeatures & mel); + const std::vector & mel_stats() const noexcept; + +private: + struct Weights; + class Graph; + core::ExecutionContext & execution_; + size_t graph_context_bytes_ = 0; + std::shared_ptr weights_; + std::unique_ptr graph_; +}; + +} // namespace engine::models::xtts_v2 diff --git a/model_specs/xtts_v2.json b/model_specs/xtts_v2.json index a24df8ae..160211be 100644 --- a/model_specs/xtts_v2.json +++ b/model_specs/xtts_v2.json @@ -9,8 +9,9 @@ "modes": ["offline"], "languages": ["en", "es", "fr", "de", "it", "pt", "pl", "tr", "ru", "nl", "cs", "ar", "zh-cn", "ja", "hu", "ko", "hi"], "runtime": {"tags": ["gguf"]}, - "capabilities": {"clone": ["speaker_reference", "cross_lingual"]}, + "capabilities": {"clone": ["speaker_reference"]}, "options": { + "session": [], "request": [ {"name": "language", "type": "string", "description": "Required XTTS language code.", "required": true, "default": "en"}, {"name": "temperature", "type": "float", "description": "GPT sampling temperature.", "required": false, "min": 0.01, "max": 2.0, "default": 0.75}, @@ -21,11 +22,11 @@ {"name": "seed", "type": "int", "description": "Non-negative deterministic sampling seed.", "required": false, "min": 0} ], "load": [ - {"name": "xtts_v2.weight_type", "type": "enum", "preset": "weight_type_full", "description": "GPT matrix storage type.", "required": false, "default": "native"}, - {"name": "xtts_v2.conv_weight_type", "type": "enum", "preset": "weight_type_conv", "description": "Conditioning, decoder, and speaker-encoder convolution storage type.", "required": false, "default": "native"}, - {"name": "xtts_v2.gpt_graph_arena_mb", "type": "int", "description": "GPT graph arena size.", "required": false, "min": 64, "default": 512}, - {"name": "xtts_v2.reference_graph_arena_mb", "type": "int", "description": "Reference-conditioning graph arena size.", "required": false, "min": 32, "default": 256}, - {"name": "xtts_v2.decoder_graph_arena_mb", "type": "int", "description": "HiFiGAN decoder graph arena size.", "required": false, "min": 32, "default": 256} + {"name": "weight_type", "type": "enum", "preset": "weight_type_full", "description": "GPT matrix storage type.", "required": false, "default": "native"}, + {"name": "conv_weight_type", "type": "enum", "preset": "weight_type_conv", "description": "Conditioning, decoder, and speaker-encoder convolution storage type.", "required": false, "default": "native"}, + {"name": "gpt_graph_arena_mb", "type": "int", "description": "GPT graph arena size.", "required": false, "min": 64, "default": 512}, + {"name": "reference_graph_arena_mb", "type": "int", "description": "Reference-conditioning graph arena size.", "required": false, "min": 32, "default": 256}, + {"name": "decoder_graph_arena_mb", "type": "int", "description": "HiFiGAN decoder graph arena size.", "required": false, "min": 32, "default": 256} ] }, "package_defaults": { @@ -73,7 +74,7 @@ "dependencies": [], "ui": { "recommended_package": "xtts_v2_q8_0", - "tags": ["TTS", "Clone", "Multilingual", "Non-commercial", "GGUF"], + "tags": ["TTS", "Clone", "GGUF"], "docs": ["docs/community_models/xtts_v2.md"] } } diff --git a/src/models/xtts_v2/conditioning.cpp b/src/models/xtts_v2/conditioning.cpp new file mode 100644 index 00000000..032b8005 --- /dev/null +++ b/src/models/xtts_v2/conditioning.cpp @@ -0,0 +1,252 @@ +#include "engine/models/xtts_v2/conditioning.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { +namespace { + +namespace binding = engine::modules::binding; +namespace modules = engine::modules; + +constexpr int64_t kDim = 1024; +constexpr int64_t kHeads = 16; +constexpr int64_t kHeadDim = 64; +constexpr int64_t kPerceiverHeads = 8; +constexpr int64_t kPerceiverInner = 512; +constexpr int64_t kPerceiverFf = 2730; +constexpr int64_t kLatents = 32; + +struct AttentionWeights { + modules::NormWeights norm; + modules::Conv1dWeights qkv; + modules::Conv1dWeights out; +}; + +struct PerceiverLayerWeights { + modules::LinearWeights q; + modules::LinearWeights kv; + modules::LinearWeights out; + modules::LinearWeights ff_in; + modules::LinearWeights ff_out; +}; + +struct ContextDeleter { + void operator()(ggml_context * ctx) const noexcept { if (ctx != nullptr) ggml_free(ctx); } +}; + +core::TensorValue reshape_heads(core::ModuleBuildContext & ctx, const core::TensorValue & input, int64_t heads, int64_t dim) { + auto contiguous = core::ensure_backend_addressable_layout(ctx, input); + return core::reshape_tensor(ctx, contiguous, core::TensorShape::from_dims({input.shape.dims[0], input.shape.dims[1], heads, dim})); +} + +core::TensorValue attention( + core::ModuleBuildContext & ctx, + const core::TensorValue & input_bct, + const AttentionWeights & weights) { + auto x = modules::GroupNormModule({kDim, 32, 1.0e-5F, true, true}).build(ctx, input_bct, weights.norm); + auto qkv = modules::Conv1dModule({kDim, 3 * kDim, 1, 1, 0, 1, true}).build(ctx, x, weights.qkv); + auto q = modules::SliceModule({1, 0, kDim}).build(ctx, qkv); + auto k = modules::SliceModule({1, kDim, kDim}).build(ctx, qkv); + auto v = modules::SliceModule({1, 2 * kDim, kDim}).build(ctx, qkv); + q = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, q), kHeads, kHeadDim)); + k = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, k), kHeads, kHeadDim)); + v = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, v), kHeads, kHeadDim)); + auto scores = modules::MatMulModule{}.build(ctx, q, modules::TransposeModule({{0, 1, 3, 2}, 4}).build(ctx, k)); + scores = core::wrap_tensor(ggml_scale(ctx.ggml, scores.tensor, 1.0F / std::sqrt(static_cast(kHeadDim))), scores.shape, GGML_TYPE_F32); + auto probs = core::wrap_tensor(ggml_soft_max(ctx.ggml, core::ensure_backend_addressable_layout(ctx, scores).tensor), scores.shape, GGML_TYPE_F32); + auto mixed = modules::MatMulModule{}.build(ctx, probs, v); + mixed = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, mixed); + mixed = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, mixed), core::TensorShape::from_dims({1, input_bct.shape.dims[2], kDim})); + mixed = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, mixed); + mixed = modules::Conv1dModule({kDim, kDim, 1, 1, 0, 1, true}).build(ctx, mixed, weights.out); + return modules::AddModule{}.build(ctx, x, mixed); +} + +core::TensorValue perceiver_attention( + core::ModuleBuildContext & ctx, + const core::TensorValue & latents, + const core::TensorValue & context, + const PerceiverLayerWeights & weights) { + const auto full = modules::ConcatModule({1}).build(ctx, latents, context); + auto q = modules::LinearModule({kDim, kPerceiverInner, false}).build(ctx, latents, weights.q); + auto kv = modules::LinearModule({kDim, 2 * kPerceiverInner, false}).build(ctx, full, weights.kv); + auto k = modules::SliceModule({2, 0, kPerceiverInner}).build(ctx, kv); + auto v = modules::SliceModule({2, kPerceiverInner, kPerceiverInner}).build(ctx, kv); + q = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, q, kPerceiverHeads, kHeadDim)); + k = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, k, kPerceiverHeads, kHeadDim)); + v = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, v, kPerceiverHeads, kHeadDim)); + auto scores = modules::MatMulModule{}.build(ctx, q, modules::TransposeModule({{0, 1, 3, 2}, 4}).build(ctx, k)); + scores = core::wrap_tensor(ggml_scale(ctx.ggml, scores.tensor, 1.0F / 8.0F), scores.shape, GGML_TYPE_F32); + auto probs = core::wrap_tensor(ggml_soft_max(ctx.ggml, core::ensure_backend_addressable_layout(ctx, scores).tensor), scores.shape, GGML_TYPE_F32); + auto mixed = modules::MatMulModule{}.build(ctx, probs, v); + mixed = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, mixed); + mixed = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, mixed), core::TensorShape::from_dims({1, kLatents, kPerceiverInner})); + return modules::LinearModule({kPerceiverInner, kDim, false}).build(ctx, mixed, weights.out); +} + +core::TensorValue geglu(core::ModuleBuildContext & ctx, const core::TensorValue & input) { + auto value = modules::SliceModule({2, 0, kPerceiverFf}).build(ctx, input); + auto gate = modules::SliceModule({2, kPerceiverFf, kPerceiverFf}).build(ctx, input); + gate = modules::GeluModule({modules::GeluApproximation::ExactErf}).build(ctx, gate); + return modules::MulModule{}.build(ctx, value, gate); +} + +core::TensorValue rms_norm(core::ModuleBuildContext & ctx, const core::TensorValue & input, const core::TensorValue & gamma) { + auto squared = core::wrap_tensor(ggml_sqr(ctx.ggml, input.tensor), input.shape, GGML_TYPE_F32); + auto sum = modules::ReduceSumModule({2}).build(ctx, squared); + sum = core::wrap_tensor(ggml_sqrt(ctx.ggml, sum.tensor), sum.shape, GGML_TYPE_F32); + auto divisor = modules::RepeatModule({input.shape}).build(ctx, sum); + auto normalized = core::wrap_tensor(ggml_div(ctx.ggml, input.tensor, divisor.tensor), input.shape, GGML_TYPE_F32); + normalized = core::wrap_tensor(ggml_scale(ctx.ggml, normalized.tensor, std::sqrt(static_cast(kDim))), normalized.shape, GGML_TYPE_F32); + auto gamma_view = core::reshape_tensor(ctx, gamma, core::TensorShape::from_dims({1, 1, kDim})); + return modules::MulModule{}.build(ctx, normalized, modules::RepeatModule({input.shape}).build(ctx, gamma_view)); +} + +} // namespace + +struct XttsV2ConditioningRuntime::Weights { + std::shared_ptr store; + modules::Conv1dWeights input; + std::vector attention; + core::TensorValue latents; + std::vector perceiver; + core::TensorValue final_gamma; + std::vector mel_stats; +}; + +class XttsV2ConditioningRuntime::Graph { +public: + Graph(core::ExecutionContext & execution, std::shared_ptr weights, int64_t frames, size_t arena) + : execution_(execution), weights_(std::move(weights)), frames_(frames) { + ggml_init_params params{arena, nullptr, true}; + ctx_.reset(ggml_init(params)); + ggml_init_params input_params{4U * 1024U * 1024U, nullptr, true}; + input_ctx_.reset(ggml_init(input_params)); + if (!ctx_ || !input_ctx_) throw std::runtime_error("failed to initialize XTTS v2 conditioning graph contexts"); + core::ModuleBuildContext ctx{ctx_.get(), "xtts_v2.conditioning", execution_.backend_type()}; + core::ModuleBuildContext input_ctx{input_ctx_.get(), "xtts_v2.conditioning.input", execution_.backend_type()}; + input_ = core::make_tensor(input_ctx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 80, frames_})).tensor; + ggml_set_input(input_); + auto x = core::wrap_tensor(input_, core::TensorShape::from_dims({1, 80, frames_}), GGML_TYPE_F32); + x = modules::Conv1dModule({80, kDim, 1, 1, 0, 1, true}).build(ctx, x, weights_->input); + for (const auto & layer : weights_->attention) x = attention(ctx, x, layer); + auto context = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, x); + auto latents = core::reshape_tensor(ctx, weights_->latents, core::TensorShape::from_dims({1, kLatents, kDim})); + latents = modules::RepeatModule({core::TensorShape::from_dims({1, kLatents, kDim})}).build(ctx, latents); + for (const auto & layer : weights_->perceiver) { + latents = modules::AddModule{}.build(ctx, latents, perceiver_attention(ctx, latents, context, layer)); + auto ff = modules::LinearModule({kDim, 2 * kPerceiverFf, true}).build(ctx, latents, layer.ff_in); + ff = geglu(ctx, ff); + ff = modules::LinearModule({kPerceiverFf, kDim, true}).build(ctx, ff, layer.ff_out); + latents = modules::AddModule{}.build(ctx, latents, ff); + } + auto output = rms_norm(ctx, latents, weights_->final_gamma); + output_ = core::ensure_backend_addressable_layout(ctx, output).tensor; + ggml_set_output(output_); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + ggml_build_forward_expand(graph_, output_); + input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); + allocator_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution_.backend())); + if (!input_buffer_ || !allocator_ || !ggml_gallocr_reserve(allocator_, graph_) || !ggml_gallocr_alloc_graph(allocator_, graph_)) { + throw std::runtime_error("failed to allocate XTTS v2 conditioning graph"); + } + } + + ~Graph() { + if (graph_) core::release_backend_graph_resources(execution_.backend(), graph_); + if (allocator_) ggml_gallocr_free(allocator_); + if (input_buffer_) ggml_backend_buffer_free(input_buffer_); + } + + bool matches(int64_t frames) const noexcept { return frames_ == frames; } + + XttsV2ConditioningLatent run(const std::vector & mel) { + if (static_cast(mel.size()) != 80 * frames_) throw std::runtime_error("XTTS v2 conditioning mel shape mismatch"); + ggml_backend_tensor_set(input_, mel.data(), 0, mel.size() * sizeof(float)); + core::set_backend_threads(execution_.backend(), execution_.config().threads); + if (core::compute_backend_graph(execution_.backend(), graph_) != GGML_STATUS_SUCCESS) throw std::runtime_error("XTTS v2 conditioning compute failed"); + ggml_backend_synchronize(execution_.backend()); + XttsV2ConditioningLatent out; + out.values.resize(static_cast(kLatents * kDim)); + ggml_backend_tensor_get(output_, out.values.data(), 0, out.values.size() * sizeof(float)); + return out; + } + +private: + core::ExecutionContext & execution_; + std::shared_ptr weights_; + int64_t frames_ = 0; + std::unique_ptr ctx_; + std::unique_ptr input_ctx_; + ggml_tensor * input_ = nullptr; + ggml_tensor * output_ = nullptr; + ggml_cgraph * graph_ = nullptr; + ggml_gallocr_t allocator_ = nullptr; + ggml_backend_buffer_t input_buffer_ = nullptr; +}; + +XttsV2ConditioningRuntime::XttsV2ConditioningRuntime( + const XttsV2Assets & assets, + core::ExecutionContext & execution, + size_t weight_context_bytes, + size_t graph_context_bytes, + assets::TensorStorageType matmul_storage_type, + assets::TensorStorageType conv_storage_type) + : execution_(execution), graph_context_bytes_(graph_context_bytes) { + auto weights = std::make_shared(); + weights->store = std::make_shared(execution.backend(), execution.backend_type(), "xtts_v2.conditioning.weights", weight_context_bytes); + const auto & source = *assets.gpt; + weights->input = binding::conv1d_from_source(*weights->store, source, "conditioning_encoder.init", conv_storage_type, kDim, 80, 1, true); + weights->attention.reserve(6); + for (int64_t i = 0; i < 6; ++i) { + const std::string prefix = "conditioning_encoder.attn." + std::to_string(i); + AttentionWeights layer; + layer.norm = binding::norm_from_source(*weights->store, source, prefix + ".norm", kDim); + layer.qkv = binding::conv1d_from_source(*weights->store, source, prefix + ".qkv", conv_storage_type, 3 * kDim, kDim, 1, true); + layer.out = binding::conv1d_from_source(*weights->store, source, prefix + ".proj_out", conv_storage_type, kDim, kDim, 1, true); + weights->attention.push_back(std::move(layer)); + } + weights->latents = weights->store->load_f32_tensor(source, "conditioning_perceiver.latents", {kLatents, kDim}); + weights->perceiver.reserve(2); + for (int64_t i = 0; i < 2; ++i) { + const std::string prefix = "conditioning_perceiver.layers." + std::to_string(i); + PerceiverLayerWeights layer; + layer.q = binding::linear_from_source(*weights->store, source, prefix + ".0.to_q", matmul_storage_type, kPerceiverInner, kDim, false); + layer.kv = binding::linear_from_source(*weights->store, source, prefix + ".0.to_kv", matmul_storage_type, 2 * kPerceiverInner, kDim, false); + layer.out = binding::linear_from_source(*weights->store, source, prefix + ".0.to_out", matmul_storage_type, kDim, kPerceiverInner, false); + layer.ff_in = binding::linear_from_source(*weights->store, source, prefix + ".1.0", matmul_storage_type, 2 * kPerceiverFf, kDim, true); + layer.ff_out = binding::linear_from_source(*weights->store, source, prefix + ".1.2", matmul_storage_type, kDim, kPerceiverFf, true); + weights->perceiver.push_back(std::move(layer)); + } + weights->final_gamma = weights->store->load_f32_tensor(source, "conditioning_perceiver.norm.gamma", {kDim}); + weights->mel_stats = source.require_f32("mel_stats", {80}); + weights->store->upload(); + weights_ = std::move(weights); +} + +XttsV2ConditioningRuntime::~XttsV2ConditioningRuntime() = default; + +XttsV2ConditioningLatent XttsV2ConditioningRuntime::encode(const XttsV2MelFeatures & mel) { + if (mel.channels != 80 || mel.frames <= 0) throw std::runtime_error("XTTS v2 conditioning expects [80, frames] mel input"); + if (!graph_ || !graph_->matches(mel.frames)) { + graph_.reset(); + graph_ = std::make_unique(execution_, weights_, mel.frames, graph_context_bytes_); + } + return graph_->run(mel.values); +} + +const std::vector & XttsV2ConditioningRuntime::mel_stats() const noexcept { return weights_->mel_stats; } + +} // namespace engine::models::xtts_v2 diff --git a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp new file mode 100644 index 00000000..de5ed2a7 --- /dev/null +++ b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp @@ -0,0 +1,57 @@ +#include "engine/framework/audio/wav_reader.h" +#include "engine/framework/core/execution_context.h" +#include "engine/models/xtts_v2/assets.h" +#include "engine/models/xtts_v2/audio_features.h" +#include "engine/models/xtts_v2/conditioning.h" + +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char ** argv) try { + if (argc != 3) { + std::cerr << "usage: xtts_v2_conditioning_probe \n"; + return 2; + } + auto assets = engine::models::xtts_v2::load_xtts_v2_assets(argv[1]); + const auto wav = engine::audio::read_wav_f32(std::filesystem::path(argv[2])); + engine::runtime::AudioBuffer audio{wav.sample_rate, wav.channels, wav.samples}; + const auto reference = engine::models::xtts_v2::prepare_xtts_v2_reference(audio); + const size_t chunk_samples = 4U * 22050U; + const std::vector chunk( + reference.waveform_22050.begin(), + reference.waveform_22050.begin() + static_cast( + std::min(chunk_samples, reference.waveform_22050.size()))); + + engine::core::ExecutionContext execution({engine::core::BackendType::Cpu, 0, 4}); + engine::models::xtts_v2::XttsV2ConditioningRuntime runtime( + *assets, execution, 512U * 1024U * 1024U, 512U * 1024U * 1024U, + engine::assets::TensorStorageType::Native, + engine::assets::TensorStorageType::Native); + const auto mel = engine::models::xtts_v2::compute_xtts_v2_conditioning_mel( + chunk, runtime.mel_stats(), 4); + const auto latent = runtime.encode(mel); + double sum = std::accumulate(latent.values.begin(), latent.values.end(), 0.0); + double sq = 0.0; + for (float value : latent.values) sq += static_cast(value) * value; + std::cout << std::setprecision(10) + << "{\"mel_frames\":" << mel.frames + << ",\"frames\":" << latent.frames + << ",\"dims\":" << latent.dims + << ",\"sum\":" << sum + << ",\"rms\":" << std::sqrt(sq / latent.values.size()) + << ",\"first\":["; + for (size_t i = 0; i < std::min(16, latent.values.size()); ++i) { + if (i) std::cout << ','; + std::cout << latent.values[i]; + } + std::cout << "]}\n"; + return 0; +} catch (const std::exception & error) { + std::cerr << "xtts_v2_conditioning_probe failed: " << error.what() << '\n'; + return 1; +} diff --git a/tools/community_models/convert_xtts_v2.py b/tools/community_models/convert_xtts_v2.py index 62b253fd..978a2479 100644 --- a/tools/community_models/convert_xtts_v2.py +++ b/tools/community_models/convert_xtts_v2.py @@ -17,6 +17,7 @@ import argparse import json +import math import shutil import subprocess from pathlib import Path @@ -95,6 +96,25 @@ def validate_config(path: Path) -> None: raise ValueError(f"unsupported XTTS checkpoint configuration: {mismatches}") +def stage_config(source: Path, destination: Path) -> None: + """Write strict JSON; Coqui's config contains non-standard Infinity values.""" + config = json.loads(source.read_text(encoding="utf-8")) + + def sanitize(value): + if isinstance(value, float) and not math.isfinite(value): + return None + if isinstance(value, dict): + return {key: sanitize(item) for key, item in value.items()} + if isinstance(value, list): + return [sanitize(item) for item in value] + return value + + destination.write_text( + json.dumps(sanitize(config), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + def converter_command(output_dir: Path, converter: Path, quant_type: str) -> list[str]: command = [str(converter)] for namespace in PREFIXES: @@ -158,7 +178,8 @@ def main() -> int: parameters = sum(t.numel() for t in tensors.values()) print(f"wrote {destination} ({len(tensors)} tensors, {parameters:,} values)") - for filename in ("config.json", "vocab.json", "LICENSE.txt"): + stage_config(config, root / "config.json") + for filename in ("vocab.json", "LICENSE.txt"): shutil.copyfile(require_file(model_dir / filename, filename), root / filename) print(f"staged config, tokenizer, and CPML license in {root}") From b1105b428d5834bdcfce646209565d96cbcc1718 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 02:47:31 -0400 Subject: [PATCH 05/11] xtts_v2: implement speaker encoder --- CMakeLists.txt | 3 +- .../engine/models/xtts_v2/speaker_encoder.h | 39 ++++ src/models/xtts_v2/audio_features.cpp | 10 +- src/models/xtts_v2/speaker_encoder.cpp | 216 ++++++++++++++++++ tests/xtts_v2/xtts_v2_conditioning_probe.cpp | 30 ++- 5 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 include/engine/models/xtts_v2/speaker_encoder.h create mode 100644 src/models/xtts_v2/speaker_encoder.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cec681b9..3273ffe2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2160,7 +2160,8 @@ if (ENGINE_BUILD_WARMBENCH) target_sources(xtts_v2_conditioning_probe PRIVATE src/models/xtts_v2/assets.cpp src/models/xtts_v2/audio_features.cpp - src/models/xtts_v2/conditioning.cpp) + src/models/xtts_v2/conditioning.cpp + src/models/xtts_v2/speaker_encoder.cpp) add_engine_warmbench(irodori_tts_warm_bench tests/irodori_tts/irodori_tts_warm_bench.cpp) add_engine_warmbench(marblenet_vad_warm_bench tests/marblenet_vad/marblenet_vad_warm_bench.cpp) add_engine_warmbench(miocodec_warm_bench tests/miocodec/miocodec_warm_bench.cpp) diff --git a/include/engine/models/xtts_v2/speaker_encoder.h b/include/engine/models/xtts_v2/speaker_encoder.h new file mode 100644 index 00000000..c22b4f1c --- /dev/null +++ b/include/engine/models/xtts_v2/speaker_encoder.h @@ -0,0 +1,39 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/models/xtts_v2/assets.h" +#include "engine/models/xtts_v2/audio_features.h" + +#include +#include + +namespace engine::models::xtts_v2 { + +struct XttsV2SpeakerEmbedding { + std::vector values; +}; + +class XttsV2SpeakerEncoderRuntime { +public: + XttsV2SpeakerEncoderRuntime( + const XttsV2Assets & assets, + core::ExecutionContext & execution, + size_t weight_context_bytes, + size_t graph_context_bytes, + assets::TensorStorageType matmul_storage_type, + assets::TensorStorageType conv_storage_type); + ~XttsV2SpeakerEncoderRuntime(); + + XttsV2SpeakerEmbedding encode(const XttsV2MelFeatures & mel); + +private: + struct Weights; + class Graph; + core::ExecutionContext & execution_; + size_t graph_context_bytes_; + std::shared_ptr weights_; + std::unique_ptr graph_; +}; + +} // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/audio_features.cpp b/src/models/xtts_v2/audio_features.cpp index 755f2519..d56b7a74 100644 --- a/src/models/xtts_v2/audio_features.cpp +++ b/src/models/xtts_v2/audio_features.cpp @@ -121,7 +121,15 @@ XttsV2MelFeatures compute_xtts_v2_speaker_mel( for (size_t i = 1; i < waveform_16000.size(); ++i) { emphasized[i] = waveform_16000[i] - 0.97F * waveform_16000[i - 1]; } - auto out = power_mel(emphasized, 16000, 512, 160, 400, 64, 8000.0F, window, mel_filterbank, threads); + // Torchaudio stores MelScale.fb as [frequency, mel], whereas power_mel + // consumes a row-major [mel, frequency] matrix. + std::vector transposed_filterbank(64U * 257U); + for (size_t f = 0; f < 257U; ++f) { + for (size_t m = 0; m < 64U; ++m) { + transposed_filterbank[m * 257U + f] = mel_filterbank[f * 64U + m]; + } + } + auto out = power_mel(emphasized, 16000, 512, 160, 400, 64, 8000.0F, window, transposed_filterbank, threads); for (auto & value : out.values) value = std::log(value + 1.0e-6F); // InstanceNorm1d normalizes each mel channel across time without affine terms. diff --git a/src/models/xtts_v2/speaker_encoder.cpp b/src/models/xtts_v2/speaker_encoder.cpp new file mode 100644 index 00000000..2a7c8295 --- /dev/null +++ b/src/models/xtts_v2/speaker_encoder.cpp @@ -0,0 +1,216 @@ +#include "engine/models/xtts_v2/speaker_encoder.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { +namespace { +namespace binding = engine::modules::binding; +namespace modules = engine::modules; + +struct BlockWeights { + modules::Conv2dWeights conv1; + modules::BatchNorm2dEvalWeights bn1; + modules::Conv2dWeights conv2; + modules::BatchNorm2dEvalWeights bn2; + modules::LinearWeights se1; + modules::LinearWeights se2; + std::optional downsample; + std::optional downsample_bn; + int64_t in_channels = 0; + int64_t channels = 0; + int stride = 1; +}; + +struct ContextDeleter { void operator()(ggml_context * p) const noexcept { if (p) ggml_free(p); } }; + +modules::BatchNorm2dEvalWeights load_bn( + core::BackendWeightStore & store, const assets::TensorSource & source, + const std::string & prefix, int64_t channels) { + const auto gamma = source.require_f32(prefix + ".weight", {channels}); + const auto beta = source.require_f32(prefix + ".bias", {channels}); + const auto mean = source.require_f32(prefix + ".running_mean", {channels}); + const auto variance = source.require_f32(prefix + ".running_var", {channels}); + std::vector scale(static_cast(channels)); + std::vector bias(static_cast(channels)); + for (int64_t i = 0; i < channels; ++i) { + const auto j = static_cast(i); + scale[j] = gamma[j] / std::sqrt(variance[j] + 1.0e-5F); + bias[j] = beta[j] - mean[j] * scale[j]; + } + return {store.make_f32(core::TensorShape::from_dims({channels}), scale), + store.make_f32(core::TensorShape::from_dims({channels}), bias)}; +} + +modules::BatchNorm1dEvalWeights load_bn1d( + core::BackendWeightStore & store, const assets::TensorSource & source, + const std::string & prefix, int64_t channels) { + auto fused = load_bn(store, source, prefix, channels); + return {fused.scale, fused.bias}; +} + +core::TensorValue block(core::ModuleBuildContext & ctx, const core::TensorValue & input, const BlockWeights & w) { + auto x = modules::Conv2dModule({w.in_channels, w.channels, 3, 3, w.stride, w.stride, 1, 1, 1, 1, false}).build(ctx, input, w.conv1); + x = modules::ReluModule{}.build(ctx, x); + x = modules::BatchNorm2dEvalModule({w.channels}).build(ctx, x, w.bn1); + x = modules::Conv2dModule({w.channels, w.channels, 3, 3, 1, 1, 1, 1, 1, 1, false}).build(ctx, x, w.conv2); + x = modules::BatchNorm2dEvalModule({w.channels}).build(ctx, x, w.bn2); + + auto gate = modules::ReduceMeanModule({3}).build(ctx, x); + gate = modules::ReduceMeanModule({2}).build(ctx, gate); + gate = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, gate), core::TensorShape::from_dims({1, 1, w.channels})); + gate = modules::LinearModule({w.channels, w.channels / 8, true}).build(ctx, gate, w.se1); + gate = modules::ReluModule{}.build(ctx, gate); + gate = modules::LinearModule({w.channels / 8, w.channels, true}).build(ctx, gate, w.se2); + gate = modules::SigmoidModule{}.build(ctx, gate); + gate = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, gate), core::TensorShape::from_dims({1, w.channels, 1, 1})); + x = modules::MulModule{}.build(ctx, x, modules::RepeatModule({x.shape}).build(ctx, gate)); + + auto residual = input; + if (w.downsample) { + residual = modules::Conv2dModule({w.in_channels, w.channels, 1, 1, w.stride, w.stride, 0, 0, 1, 1, false}).build(ctx, input, *w.downsample); + residual = modules::BatchNorm2dEvalModule({w.channels}).build(ctx, residual, *w.downsample_bn); + } + return modules::ReluModule{}.build(ctx, modules::AddModule{}.build(ctx, x, residual)); +} + +} // namespace + +struct XttsV2SpeakerEncoderRuntime::Weights { + std::shared_ptr store; + modules::Conv2dWeights input; + modules::BatchNorm2dEvalWeights input_bn; + std::vector blocks; + modules::Conv1dWeights attention1; + modules::BatchNorm1dEvalWeights attention_bn; + modules::Conv1dWeights attention2; + modules::LinearWeights output; +}; + +class XttsV2SpeakerEncoderRuntime::Graph { +public: + Graph(core::ExecutionContext & execution, std::shared_ptr weights, int64_t frames, size_t arena) + : execution_(execution), weights_(std::move(weights)), frames_(frames) { + ctx_.reset(ggml_init({arena, nullptr, true})); + input_ctx_.reset(ggml_init({4U * 1024U * 1024U, nullptr, true})); + if (!ctx_ || !input_ctx_) throw std::runtime_error("failed to initialize XTTS v2 speaker graph"); + core::ModuleBuildContext ctx{ctx_.get(), "xtts_v2.speaker", execution_.backend_type()}; + core::ModuleBuildContext ictx{input_ctx_.get(), "xtts_v2.speaker.input", execution_.backend_type()}; + input_ = core::make_tensor(ictx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 1, 64, frames_})).tensor; + ggml_set_input(input_); + auto x = core::wrap_tensor(input_, core::TensorShape::from_dims({1, 1, 64, frames_}), GGML_TYPE_F32); + x = modules::Conv2dModule({1, 32, 3, 3, 1, 1, 1, 1, 1, 1, true}).build(ctx, x, weights_->input); + x = modules::ReluModule{}.build(ctx, x); + x = modules::BatchNorm2dEvalModule({32}).build(ctx, x, weights_->input_bn); + for (const auto & item : weights_->blocks) x = block(ctx, x, item); + const int64_t time = x.shape.dims[3]; + x = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, x), core::TensorShape::from_dims({1, 2048, time})); + auto w = modules::Conv1dModule({2048, 128, 1, 1, 0, 1, true}).build(ctx, x, weights_->attention1); + w = modules::ReluModule{}.build(ctx, w); + w = modules::BatchNorm1dEvalModule({128}).build(ctx, w, weights_->attention_bn); + w = modules::Conv1dModule({128, 2048, 1, 1, 0, 1, true}).build(ctx, w, weights_->attention2); + w = core::wrap_tensor(ggml_soft_max(ctx.ggml, core::ensure_backend_addressable_layout(ctx, w).tensor), w.shape, GGML_TYPE_F32); + auto weighted = modules::MulModule{}.build(ctx, x, w); + auto mu = modules::ReduceSumModule({2}).build(ctx, weighted); + auto squared = core::wrap_tensor(ggml_sqr(ctx.ggml, x.tensor), x.shape, GGML_TYPE_F32); + auto second = modules::ReduceSumModule({2}).build(ctx, modules::MulModule{}.build(ctx, squared, w)); + auto mu2 = core::wrap_tensor(ggml_sqr(ctx.ggml, mu.tensor), mu.shape, GGML_TYPE_F32); + auto variance = core::wrap_tensor(ggml_sub(ctx.ggml, second.tensor, mu2.tensor), second.shape, GGML_TYPE_F32); + variance = core::wrap_tensor(ggml_clamp(ctx.ggml, variance.tensor, 1.0e-5F, INFINITY), variance.shape, GGML_TYPE_F32); + auto sigma = core::wrap_tensor(ggml_sqrt(ctx.ggml, variance.tensor), variance.shape, GGML_TYPE_F32); + auto stats = modules::ConcatModule({1}).build(ctx, mu, sigma); + stats = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, stats), core::TensorShape::from_dims({1, 1, 4096})); + auto output = modules::LinearModule({4096, 512, true}).build(ctx, stats, weights_->output); + auto norm2 = modules::ReduceSumModule({2}).build(ctx, core::wrap_tensor(ggml_sqr(ctx.ggml, output.tensor), output.shape, GGML_TYPE_F32)); + auto norm = core::wrap_tensor(ggml_sqrt(ctx.ggml, norm2.tensor), norm2.shape, GGML_TYPE_F32); + output = core::wrap_tensor(ggml_div(ctx.ggml, output.tensor, modules::RepeatModule({output.shape}).build(ctx, norm).tensor), output.shape, GGML_TYPE_F32); + output_ = core::ensure_backend_addressable_layout(ctx, output).tensor; + ggml_set_output(output_); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + ggml_build_forward_expand(graph_, output_); + input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); + allocator_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution_.backend())); + if (!input_buffer_ || !allocator_ || !ggml_gallocr_reserve(allocator_, graph_) || !ggml_gallocr_alloc_graph(allocator_, graph_)) + throw std::runtime_error("failed to allocate XTTS v2 speaker graph"); + } + ~Graph() { + if (graph_) core::release_backend_graph_resources(execution_.backend(), graph_); + if (allocator_) ggml_gallocr_free(allocator_); + if (input_buffer_) ggml_backend_buffer_free(input_buffer_); + } + bool matches(int64_t frames) const noexcept { return frames == frames_; } + XttsV2SpeakerEmbedding run(const std::vector & mel) { + if (static_cast(mel.size()) != 64 * frames_) throw std::runtime_error("XTTS v2 speaker mel shape mismatch"); + ggml_backend_tensor_set(input_, mel.data(), 0, mel.size() * sizeof(float)); + core::set_backend_threads(execution_.backend(), execution_.config().threads); + if (core::compute_backend_graph(execution_.backend(), graph_) != GGML_STATUS_SUCCESS) throw std::runtime_error("XTTS v2 speaker compute failed"); + ggml_backend_synchronize(execution_.backend()); + XttsV2SpeakerEmbedding result; result.values.resize(512); + ggml_backend_tensor_get(output_, result.values.data(), 0, result.values.size() * sizeof(float)); + return result; + } +private: + core::ExecutionContext & execution_; + std::shared_ptr weights_; + int64_t frames_; + std::unique_ptr ctx_, input_ctx_; + ggml_tensor * input_ = nullptr; ggml_tensor * output_ = nullptr; ggml_cgraph * graph_ = nullptr; + ggml_gallocr_t allocator_ = nullptr; ggml_backend_buffer_t input_buffer_ = nullptr; +}; + +XttsV2SpeakerEncoderRuntime::XttsV2SpeakerEncoderRuntime( + const XttsV2Assets & assets, core::ExecutionContext & execution, + size_t weight_context_bytes, size_t graph_context_bytes, + assets::TensorStorageType matmul_type, assets::TensorStorageType conv_type) + : execution_(execution), graph_context_bytes_(graph_context_bytes) { + auto out = std::make_shared(); + out->store = std::make_shared(execution.backend(), execution.backend_type(), "xtts_v2.speaker.weights", weight_context_bytes); + const auto & source = *assets.speaker_encoder; + out->input = binding::conv2d_from_source(*out->store, source, "conv1", conv_type, 32, 1, 3, 3, true); + out->input_bn = load_bn(*out->store, source, "bn1", 32); + constexpr std::array counts{3, 4, 6, 3}; + constexpr std::array channels{32, 64, 128, 256}; + int64_t in_channels = 32; + for (size_t stage = 0; stage < counts.size(); ++stage) for (int index = 0; index < counts[stage]; ++index) { + const int stride = stage > 0 && index == 0 ? 2 : 1; + const std::string p = "layer" + std::to_string(stage + 1) + "." + std::to_string(index); + BlockWeights item; item.in_channels = in_channels; item.channels = channels[stage]; item.stride = stride; + item.conv1 = binding::conv2d_from_source(*out->store, source, p + ".conv1", conv_type, item.channels, in_channels, 3, 3, false); + item.bn1 = load_bn(*out->store, source, p + ".bn1", item.channels); + item.conv2 = binding::conv2d_from_source(*out->store, source, p + ".conv2", conv_type, item.channels, item.channels, 3, 3, false); + item.bn2 = load_bn(*out->store, source, p + ".bn2", item.channels); + item.se1 = binding::linear_from_source(*out->store, source, p + ".se.fc.0", matmul_type, item.channels / 8, item.channels, true); + item.se2 = binding::linear_from_source(*out->store, source, p + ".se.fc.2", matmul_type, item.channels, item.channels / 8, true); + if (stride != 1 || in_channels != item.channels) { + item.downsample = binding::conv2d_from_source(*out->store, source, p + ".downsample.0", conv_type, item.channels, in_channels, 1, 1, false); + item.downsample_bn = load_bn(*out->store, source, p + ".downsample.1", item.channels); + } + out->blocks.push_back(std::move(item)); in_channels = channels[stage]; + } + out->attention1 = binding::conv1d_from_source(*out->store, source, "attention.0", conv_type, 128, 2048, 1, true); + out->attention_bn = load_bn1d(*out->store, source, "attention.2", 128); + out->attention2 = binding::conv1d_from_source(*out->store, source, "attention.3", conv_type, 2048, 128, 1, true); + out->output = binding::linear_from_source(*out->store, source, "fc", matmul_type, 512, 4096, true); + out->store->upload(); weights_ = std::move(out); +} + +XttsV2SpeakerEncoderRuntime::~XttsV2SpeakerEncoderRuntime() = default; +XttsV2SpeakerEmbedding XttsV2SpeakerEncoderRuntime::encode(const XttsV2MelFeatures & mel) { + if (mel.channels != 64 || mel.frames <= 0) throw std::runtime_error("XTTS v2 speaker encoder expects [64, frames]"); + if (!graph_ || !graph_->matches(mel.frames)) graph_ = std::make_unique(execution_, weights_, mel.frames, graph_context_bytes_); + return graph_->run(mel.values); +} + +} // namespace engine::models::xtts_v2 diff --git a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp index de5ed2a7..b9a7af18 100644 --- a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp +++ b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp @@ -3,18 +3,20 @@ #include "engine/models/xtts_v2/assets.h" #include "engine/models/xtts_v2/audio_features.h" #include "engine/models/xtts_v2/conditioning.h" +#include "engine/models/xtts_v2/speaker_encoder.h" #include #include #include #include +#include #include #include #include int main(int argc, char ** argv) try { - if (argc != 3) { - std::cerr << "usage: xtts_v2_conditioning_probe \n"; + if (argc != 3 && argc != 4) { + std::cerr << "usage: xtts_v2_conditioning_probe [speaker-f32]\n"; return 2; } auto assets = engine::models::xtts_v2::load_xtts_v2_assets(argv[1]); @@ -35,6 +37,28 @@ int main(int argc, char ** argv) try { const auto mel = engine::models::xtts_v2::compute_xtts_v2_conditioning_mel( chunk, runtime.mel_stats(), 4); const auto latent = runtime.encode(mel); + engine::models::xtts_v2::XttsV2SpeakerEncoderRuntime speaker_runtime( + *assets, execution, 128U * 1024U * 1024U, 512U * 1024U * 1024U, + engine::assets::TensorStorageType::Native, + engine::assets::TensorStorageType::Native); + const auto speaker_mel = engine::models::xtts_v2::compute_xtts_v2_speaker_mel( + reference.waveform_16000, + assets->speaker_encoder->require_f32("torch_spec.1.spectrogram.window", {400}), + assets->speaker_encoder->require_f32("torch_spec.1.mel_scale.fb", {257, 64}), + 4); + const auto speaker = speaker_runtime.encode(speaker_mel); + if (argc == 4) { + std::ofstream output(argv[3], std::ios::binary); + output.write(reinterpret_cast(speaker.values.data()), + static_cast(speaker.values.size() * sizeof(float))); + if (!output) throw std::runtime_error("failed to write speaker embedding dump"); + std::ofstream mel_output(std::string(argv[3]) + ".mel", std::ios::binary); + mel_output.write(reinterpret_cast(speaker_mel.values.data()), + static_cast(speaker_mel.values.size() * sizeof(float))); + if (!mel_output) throw std::runtime_error("failed to write speaker mel dump"); + } + double speaker_sq = 0.0; + for (float value : speaker.values) speaker_sq += static_cast(value) * value; double sum = std::accumulate(latent.values.begin(), latent.values.end(), 0.0); double sq = 0.0; for (float value : latent.values) sq += static_cast(value) * value; @@ -42,6 +66,8 @@ int main(int argc, char ** argv) try { << "{\"mel_frames\":" << mel.frames << ",\"frames\":" << latent.frames << ",\"dims\":" << latent.dims + << ",\"speaker_frames\":" << speaker_mel.frames + << ",\"speaker_norm\":" << std::sqrt(speaker_sq) << ",\"sum\":" << sum << ",\"rms\":" << std::sqrt(sq / latent.values.size()) << ",\"first\":["; From 46b5e3f897ed4b97118b3fff6560edea2e769534 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 02:49:20 -0400 Subject: [PATCH 06/11] xtts_v2: bind GPT acoustic model weights --- CMakeLists.txt | 1 + include/engine/models/xtts_v2/gpt.h | 42 ++++++++++++++++ src/models/xtts_v2/gpt.cpp | 50 ++++++++++++++++++++ tests/xtts_v2/xtts_v2_conditioning_probe.cpp | 5 ++ 4 files changed, 98 insertions(+) create mode 100644 include/engine/models/xtts_v2/gpt.h create mode 100644 src/models/xtts_v2/gpt.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3273ffe2..fdd6ff65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2161,6 +2161,7 @@ if (ENGINE_BUILD_WARMBENCH) src/models/xtts_v2/assets.cpp src/models/xtts_v2/audio_features.cpp src/models/xtts_v2/conditioning.cpp + src/models/xtts_v2/gpt.cpp src/models/xtts_v2/speaker_encoder.cpp) add_engine_warmbench(irodori_tts_warm_bench tests/irodori_tts/irodori_tts_warm_bench.cpp) add_engine_warmbench(marblenet_vad_warm_bench tests/marblenet_vad/marblenet_vad_warm_bench.cpp) diff --git a/include/engine/models/xtts_v2/gpt.h b/include/engine/models/xtts_v2/gpt.h new file mode 100644 index 00000000..00f2e60d --- /dev/null +++ b/include/engine/models/xtts_v2/gpt.h @@ -0,0 +1,42 @@ +#pragma once + +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/models/xtts_v2/assets.h" + +#include +#include + +namespace engine::models::xtts_v2 { + +struct XttsV2GptLayerWeights { + modules::NormWeights attn_norm; + modules::LinearWeights qkv; + modules::LinearWeights attn_out; + modules::NormWeights mlp_norm; + modules::LinearWeights mlp_in; + modules::LinearWeights mlp_out; +}; + +struct XttsV2GptWeights { + std::shared_ptr store; + core::TensorValue text_embedding; + core::TensorValue audio_embedding; + core::TensorValue text_positions; + core::TensorValue audio_positions; + std::vector layers; + modules::NormWeights transformer_norm; + modules::NormWeights output_norm; + modules::LinearWeights audio_head; +}; + +std::shared_ptr load_xtts_v2_gpt_weights( + const XttsV2Assets & assets, + core::ExecutionContext & execution, + size_t weight_context_bytes, + assets::TensorStorageType storage_type); + +} // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/gpt.cpp b/src/models/xtts_v2/gpt.cpp new file mode 100644 index 00000000..84f8d476 --- /dev/null +++ b/src/models/xtts_v2/gpt.cpp @@ -0,0 +1,50 @@ +#include "engine/models/xtts_v2/gpt.h" + +#include "engine/framework/modules/weight_binding.h" + +#include +#include + +namespace engine::models::xtts_v2 { + +std::shared_ptr load_xtts_v2_gpt_weights( + const XttsV2Assets & assets, + core::ExecutionContext & execution, + size_t weight_context_bytes, + assets::TensorStorageType storage_type) { + namespace binding = engine::modules::binding; + constexpr int64_t dim = 1024; + constexpr int64_t mlp = 4096; + auto weights = std::make_shared(); + weights->store = std::make_shared( + execution.backend(), execution.backend_type(), "xtts_v2.gpt.weights", weight_context_bytes); + const auto & source = *assets.gpt; + weights->text_embedding = weights->store->load_tensor(source, "text_embedding.weight", storage_type, {6681, dim}); + weights->audio_embedding = weights->store->load_tensor(source, "mel_embedding.weight", storage_type, {1026, dim}); + weights->text_positions = weights->store->load_f32_tensor(source, "text_pos_embedding.emb.weight", {404, dim}); + weights->audio_positions = weights->store->load_f32_tensor(source, "mel_pos_embedding.emb.weight", {608, dim}); + weights->layers.reserve(30); + for (int64_t index = 0; index < 30; ++index) { + const std::string prefix = "gpt.h." + std::to_string(index); + XttsV2GptLayerWeights layer; + layer.attn_norm = binding::norm_from_source(*weights->store, source, prefix + ".ln_1", dim); + layer.qkv = binding::hf_conv1d_linear_from_source( + *weights->store, source, prefix + ".attn.c_attn", storage_type, dim, 3 * dim, true); + layer.attn_out = binding::hf_conv1d_linear_from_source( + *weights->store, source, prefix + ".attn.c_proj", storage_type, dim, dim, true); + layer.mlp_norm = binding::norm_from_source(*weights->store, source, prefix + ".ln_2", dim); + layer.mlp_in = binding::hf_conv1d_linear_from_source( + *weights->store, source, prefix + ".mlp.c_fc", storage_type, dim, mlp, true); + layer.mlp_out = binding::hf_conv1d_linear_from_source( + *weights->store, source, prefix + ".mlp.c_proj", storage_type, mlp, dim, true); + weights->layers.push_back(std::move(layer)); + } + weights->transformer_norm = binding::norm_from_source(*weights->store, source, "gpt.ln_f", dim); + weights->output_norm = binding::norm_from_source(*weights->store, source, "final_norm", dim); + weights->audio_head = binding::linear_from_source( + *weights->store, source, "mel_head", storage_type, 1026, dim, true); + weights->store->upload(); + return weights; +} + +} // namespace engine::models::xtts_v2 diff --git a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp index b9a7af18..f28b8715 100644 --- a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp +++ b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp @@ -3,6 +3,7 @@ #include "engine/models/xtts_v2/assets.h" #include "engine/models/xtts_v2/audio_features.h" #include "engine/models/xtts_v2/conditioning.h" +#include "engine/models/xtts_v2/gpt.h" #include "engine/models/xtts_v2/speaker_encoder.h" #include @@ -47,6 +48,9 @@ int main(int argc, char ** argv) try { assets->speaker_encoder->require_f32("torch_spec.1.mel_scale.fb", {257, 64}), 4); const auto speaker = speaker_runtime.encode(speaker_mel); + const auto gpt_weights = engine::models::xtts_v2::load_xtts_v2_gpt_weights( + *assets, execution, 1536U * 1024U * 1024U, + engine::assets::TensorStorageType::Native); if (argc == 4) { std::ofstream output(argv[3], std::ios::binary); output.write(reinterpret_cast(speaker.values.data()), @@ -68,6 +72,7 @@ int main(int argc, char ** argv) try { << ",\"dims\":" << latent.dims << ",\"speaker_frames\":" << speaker_mel.frames << ",\"speaker_norm\":" << std::sqrt(speaker_sq) + << ",\"gpt_layers\":" << gpt_weights->layers.size() << ",\"sum\":" << sum << ",\"rms\":" << std::sqrt(sq / latent.values.size()) << ",\"first\":["; From d9ed7109c51c135a2d3acc278be1b7012ce78046 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 02:52:32 -0400 Subject: [PATCH 07/11] xtts_v2: implement GPT prefill runtime --- CMakeLists.txt | 3 +- include/engine/models/xtts_v2/gpt.h | 27 ++++ src/models/xtts_v2/gpt.cpp | 132 +++++++++++++++++++ tests/xtts_v2/xtts_v2_conditioning_probe.cpp | 20 ++- 4 files changed, 178 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fdd6ff65..f3eca2cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2162,7 +2162,8 @@ if (ENGINE_BUILD_WARMBENCH) src/models/xtts_v2/audio_features.cpp src/models/xtts_v2/conditioning.cpp src/models/xtts_v2/gpt.cpp - src/models/xtts_v2/speaker_encoder.cpp) + src/models/xtts_v2/speaker_encoder.cpp + src/models/xtts_v2/tokenizer.cpp) add_engine_warmbench(irodori_tts_warm_bench tests/irodori_tts/irodori_tts_warm_bench.cpp) add_engine_warmbench(marblenet_vad_warm_bench tests/marblenet_vad/marblenet_vad_warm_bench.cpp) add_engine_warmbench(miocodec_warm_bench tests/miocodec/miocodec_warm_bench.cpp) diff --git a/include/engine/models/xtts_v2/gpt.h b/include/engine/models/xtts_v2/gpt.h index 00f2e60d..8afce19a 100644 --- a/include/engine/models/xtts_v2/gpt.h +++ b/include/engine/models/xtts_v2/gpt.h @@ -33,10 +33,37 @@ struct XttsV2GptWeights { modules::LinearWeights audio_head; }; +struct XttsV2GptPrefillResult { + std::vector logits; + std::vector latent; +}; + std::shared_ptr load_xtts_v2_gpt_weights( const XttsV2Assets & assets, core::ExecutionContext & execution, size_t weight_context_bytes, assets::TensorStorageType storage_type); +class XttsV2GptRuntime { +public: + XttsV2GptRuntime( + const XttsV2Assets & assets, + core::ExecutionContext & execution, + size_t weight_context_bytes, + size_t graph_context_bytes, + assets::TensorStorageType storage_type); + ~XttsV2GptRuntime(); + + XttsV2GptPrefillResult prefill( + const std::vector & conditioning_latent, + const std::vector & text_tokens); + +private: + class PrefillGraph; + core::ExecutionContext & execution_; + size_t graph_context_bytes_; + std::shared_ptr weights_; + std::unique_ptr prefill_; +}; + } // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/gpt.cpp b/src/models/xtts_v2/gpt.cpp index 84f8d476..43851848 100644 --- a/src/models/xtts_v2/gpt.cpp +++ b/src/models/xtts_v2/gpt.cpp @@ -1,11 +1,64 @@ #include "engine/models/xtts_v2/gpt.h" #include "engine/framework/modules/weight_binding.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include #include #include namespace engine::models::xtts_v2 { +namespace { +namespace modules = engine::modules; + +constexpr int64_t kDim = 1024; +constexpr int64_t kHeads = 16; +constexpr int64_t kHeadDim = 64; +constexpr int64_t kAudioCodes = 1026; + +struct ContextDeleter { void operator()(ggml_context * p) const noexcept { if (p) ggml_free(p); } }; + +core::TensorValue reshape_heads(core::ModuleBuildContext & ctx, const core::TensorValue & x) { + return core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, x), + core::TensorShape::from_dims({x.shape.dims[0], x.shape.dims[1], kHeads, kHeadDim})); +} + +core::TensorValue project(core::ModuleBuildContext & ctx, const core::TensorValue & x, + int64_t input, int64_t output, const modules::LinearWeights & weights) { + return modules::LinearModule({input, output, true, GGML_PREC_F32}).build(ctx, x, weights); +} + +core::TensorValue gpt_layer(core::ModuleBuildContext & ctx, const core::TensorValue & input, + const XttsV2GptLayerWeights & weights) { + auto normed = modules::LayerNormModule({kDim, 1.0e-5F, true, true}).build(ctx, input, weights.attn_norm); + auto qkv = project(ctx, normed, kDim, 3 * kDim, weights.qkv); + auto q = modules::SliceModule({2, 0, kDim}).build(ctx, qkv); + auto k = modules::SliceModule({2, kDim, kDim}).build(ctx, qkv); + auto v = modules::SliceModule({2, 2 * kDim, kDim}).build(ctx, qkv); + q = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, q)); + k = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, k)); + v = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, reshape_heads(ctx, v)); + auto scores = modules::MatMulModule{}.build(ctx, q, modules::TransposeModule({{0, 1, 3, 2}, 4}).build(ctx, k)); + scores = core::wrap_tensor(ggml_scale(ctx.ggml, scores.tensor, 1.0F / std::sqrt(64.0F)), scores.shape, GGML_TYPE_F32); + scores = core::wrap_tensor(ggml_diag_mask_inf(ctx.ggml, scores.tensor, 0), scores.shape, GGML_TYPE_F32); + auto probability = core::wrap_tensor( + ggml_soft_max(ctx.ggml, core::ensure_backend_addressable_layout(ctx, scores).tensor), scores.shape, GGML_TYPE_F32); + auto mixed = modules::MatMulModule{}.build(ctx, probability, v); + mixed = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, mixed); + mixed = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, mixed), input.shape); + auto x = modules::AddModule{}.build(ctx, input, project(ctx, mixed, kDim, kDim, weights.attn_out)); + auto hidden = modules::LayerNormModule({kDim, 1.0e-5F, true, true}).build(ctx, x, weights.mlp_norm); + hidden = project(ctx, hidden, kDim, 4 * kDim, weights.mlp_in); + hidden = modules::GeluModule({modules::GeluApproximation::Tanh}).build(ctx, hidden); + hidden = project(ctx, hidden, 4 * kDim, kDim, weights.mlp_out); + return modules::AddModule{}.build(ctx, x, hidden); +} + +} // namespace std::shared_ptr load_xtts_v2_gpt_weights( const XttsV2Assets & assets, @@ -47,4 +100,83 @@ std::shared_ptr load_xtts_v2_gpt_weights( return weights; } +class XttsV2GptRuntime::PrefillGraph { +public: + PrefillGraph(core::ExecutionContext & execution, std::shared_ptr weights, + int64_t text_count, size_t arena) + : execution_(execution), weights_(std::move(weights)), text_count_(text_count), steps_(32 + text_count + 3) { + ctx_.reset(ggml_init({arena, nullptr, true})); + input_ctx_.reset(ggml_init({8U * 1024U * 1024U, nullptr, true})); + if (!ctx_ || !input_ctx_) throw std::runtime_error("failed to initialize XTTS v2 GPT prefill graph"); + core::ModuleBuildContext ctx{ctx_.get(), "xtts_v2.gpt.prefill", execution_.backend_type()}; + core::ModuleBuildContext ictx{input_ctx_.get(), "xtts_v2.gpt.prefill.inputs", execution_.backend_type()}; + conditioning_ = core::make_tensor(ictx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 32, kDim})).tensor; + text_ids_ = ggml_new_tensor_1d(input_ctx_.get(), GGML_TYPE_I32, text_count_ + 2); + audio_id_ = ggml_new_tensor_1d(input_ctx_.get(), GGML_TYPE_I32, 1); + ggml_set_input(conditioning_); ggml_set_input(text_ids_); ggml_set_input(audio_id_); + auto condition = core::wrap_tensor(conditioning_, core::TensorShape::from_dims({1, 32, kDim}), GGML_TYPE_F32); + auto ids = core::wrap_tensor(text_ids_, core::TensorShape::from_dims({text_count_ + 2}), GGML_TYPE_I32); + auto text = modules::EmbeddingModule({6681, kDim}).build(ctx, ids, weights_->text_embedding); + text = modules::AddModule{}.build(ctx, text, modules::SliceModule({0, 0, text_count_ + 2}).build(ctx, weights_->text_positions)); + text = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, text), core::TensorShape::from_dims({1, text_count_ + 2, kDim})); + auto aid = core::wrap_tensor(audio_id_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto audio = modules::EmbeddingModule({kAudioCodes, kDim}).build(ctx, aid, weights_->audio_embedding); + audio = modules::AddModule{}.build(ctx, audio, modules::SliceModule({0, 0, 1}).build(ctx, weights_->audio_positions)); + audio = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, audio), core::TensorShape::from_dims({1, 1, kDim})); + auto x = modules::ConcatModule({1}).build(ctx, condition, text); + x = modules::ConcatModule({1}).build(ctx, x, audio); + for (const auto & layer : weights_->layers) x = gpt_layer(ctx, x, layer); + x = modules::LayerNormModule({kDim, 1.0e-5F, true, true}).build(ctx, x, weights_->transformer_norm); + x = modules::SliceModule({1, steps_ - 1, 1}).build(ctx, x); + x = modules::LayerNormModule({kDim, 1.0e-5F, true, true}).build(ctx, x, weights_->output_norm); + latent_ = core::ensure_backend_addressable_layout(ctx, x).tensor; + logits_ = core::ensure_backend_addressable_layout(ctx, project(ctx, x, kDim, kAudioCodes, weights_->audio_head)).tensor; + ggml_set_output(latent_); ggml_set_output(logits_); + graph_ = ggml_new_graph_custom(ctx_.get(), 262144, false); + ggml_build_forward_expand(graph_, logits_); ggml_build_forward_expand(graph_, latent_); + input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); + allocator_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution_.backend())); + if (!input_buffer_ || !allocator_ || !ggml_gallocr_reserve(allocator_, graph_) || !ggml_gallocr_alloc_graph(allocator_, graph_)) + throw std::runtime_error("failed to allocate XTTS v2 GPT prefill graph"); + } + ~PrefillGraph() { + if (graph_) core::release_backend_graph_resources(execution_.backend(), graph_); + if (allocator_) ggml_gallocr_free(allocator_); + if (input_buffer_) ggml_backend_buffer_free(input_buffer_); + } + bool matches(int64_t count) const noexcept { return count == text_count_; } + XttsV2GptPrefillResult run(const std::vector & condition, const std::vector & tokens) { + if (condition.size() != 32U * 1024U || static_cast(tokens.size()) != text_count_) + throw std::runtime_error("XTTS v2 GPT prefill input shape mismatch"); + std::vector ids; ids.reserve(tokens.size() + 2); ids.push_back(261); ids.insert(ids.end(), tokens.begin(), tokens.end()); ids.push_back(0); + const int32_t start_audio = 1024; + ggml_backend_tensor_set(conditioning_, condition.data(), 0, condition.size() * sizeof(float)); + ggml_backend_tensor_set(text_ids_, ids.data(), 0, ids.size() * sizeof(int32_t)); + ggml_backend_tensor_set(audio_id_, &start_audio, 0, sizeof(start_audio)); + core::set_backend_threads(execution_.backend(), execution_.config().threads); + if (core::compute_backend_graph(execution_.backend(), graph_) != GGML_STATUS_SUCCESS) throw std::runtime_error("XTTS v2 GPT prefill compute failed"); + ggml_backend_synchronize(execution_.backend()); + XttsV2GptPrefillResult result; result.logits.resize(kAudioCodes); result.latent.resize(kDim); + ggml_backend_tensor_get(logits_, result.logits.data(), 0, result.logits.size() * sizeof(float)); + ggml_backend_tensor_get(latent_, result.latent.data(), 0, result.latent.size() * sizeof(float)); + return result; + } +private: + core::ExecutionContext & execution_; std::shared_ptr weights_; + int64_t text_count_, steps_; std::unique_ptr ctx_, input_ctx_; + ggml_tensor * conditioning_ = nullptr, * text_ids_ = nullptr, * audio_id_ = nullptr, * logits_ = nullptr, * latent_ = nullptr; + ggml_cgraph * graph_ = nullptr; ggml_gallocr_t allocator_ = nullptr; ggml_backend_buffer_t input_buffer_ = nullptr; +}; + +XttsV2GptRuntime::XttsV2GptRuntime(const XttsV2Assets & assets, core::ExecutionContext & execution, + size_t weight_context_bytes, size_t graph_context_bytes, assets::TensorStorageType storage_type) + : execution_(execution), graph_context_bytes_(graph_context_bytes), + weights_(load_xtts_v2_gpt_weights(assets, execution, weight_context_bytes, storage_type)) {} +XttsV2GptRuntime::~XttsV2GptRuntime() = default; +XttsV2GptPrefillResult XttsV2GptRuntime::prefill(const std::vector & condition, const std::vector & tokens) { + if (!prefill_ || !prefill_->matches(static_cast(tokens.size()))) + prefill_ = std::make_unique(execution_, weights_, static_cast(tokens.size()), graph_context_bytes_); + return prefill_->run(condition, tokens); +} + } // namespace engine::models::xtts_v2 diff --git a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp index f28b8715..81f0be10 100644 --- a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp +++ b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp @@ -5,6 +5,7 @@ #include "engine/models/xtts_v2/conditioning.h" #include "engine/models/xtts_v2/gpt.h" #include "engine/models/xtts_v2/speaker_encoder.h" +#include "engine/models/xtts_v2/tokenizer.h" #include #include @@ -48,9 +49,13 @@ int main(int argc, char ** argv) try { assets->speaker_encoder->require_f32("torch_spec.1.mel_scale.fb", {257, 64}), 4); const auto speaker = speaker_runtime.encode(speaker_mel); - const auto gpt_weights = engine::models::xtts_v2::load_xtts_v2_gpt_weights( - *assets, execution, 1536U * 1024U * 1024U, + engine::models::xtts_v2::XttsV2Tokenizer tokenizer(assets->tokenizer_path); + const auto text_tokens = tokenizer.encode("The quick brown fox.", "en"); + engine::models::xtts_v2::XttsV2GptRuntime gpt_runtime( + *assets, execution, 1536U * 1024U * 1024U, 1536U * 1024U * 1024U, engine::assets::TensorStorageType::Native); + const auto prefill = gpt_runtime.prefill(latent.values, text_tokens); + const auto top = std::max_element(prefill.logits.begin(), prefill.logits.end()); if (argc == 4) { std::ofstream output(argv[3], std::ios::binary); output.write(reinterpret_cast(speaker.values.data()), @@ -60,6 +65,15 @@ int main(int argc, char ** argv) try { mel_output.write(reinterpret_cast(speaker_mel.values.data()), static_cast(speaker_mel.values.size() * sizeof(float))); if (!mel_output) throw std::runtime_error("failed to write speaker mel dump"); + const auto write_values = [&](const std::string & suffix, const std::vector & values) { + std::ofstream stream(std::string(argv[3]) + suffix, std::ios::binary); + stream.write(reinterpret_cast(values.data()), + static_cast(values.size() * sizeof(float))); + if (!stream) throw std::runtime_error("failed to write XTTS probe tensor dump"); + }; + write_values(".cond", latent.values); + write_values(".logits", prefill.logits); + write_values(".gptlatent", prefill.latent); } double speaker_sq = 0.0; for (float value : speaker.values) speaker_sq += static_cast(value) * value; @@ -72,7 +86,7 @@ int main(int argc, char ** argv) try { << ",\"dims\":" << latent.dims << ",\"speaker_frames\":" << speaker_mel.frames << ",\"speaker_norm\":" << std::sqrt(speaker_sq) - << ",\"gpt_layers\":" << gpt_weights->layers.size() + << ",\"gpt_prefill_argmax\":" << std::distance(prefill.logits.begin(), top) << ",\"sum\":" << sum << ",\"rms\":" << std::sqrt(sq / latent.values.size()) << ",\"first\":["; From 785d86472a70b5fd226dc218434dedeb51278a17 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 02:54:08 -0400 Subject: [PATCH 08/11] xtts_v2: add autoregressive acoustic sampling --- include/engine/models/xtts_v2/gpt.h | 9 +++ src/models/xtts_v2/gpt.cpp | 78 ++++++++++++++++---- tests/xtts_v2/xtts_v2_conditioning_probe.cpp | 5 ++ 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/include/engine/models/xtts_v2/gpt.h b/include/engine/models/xtts_v2/gpt.h index 8afce19a..27223321 100644 --- a/include/engine/models/xtts_v2/gpt.h +++ b/include/engine/models/xtts_v2/gpt.h @@ -38,6 +38,11 @@ struct XttsV2GptPrefillResult { std::vector latent; }; +struct XttsV2GptGeneration { + std::vector codes; + std::vector latents; // frame-major [codes, 1024] +}; + std::shared_ptr load_xtts_v2_gpt_weights( const XttsV2Assets & assets, core::ExecutionContext & execution, @@ -57,6 +62,10 @@ class XttsV2GptRuntime { XttsV2GptPrefillResult prefill( const std::vector & conditioning_latent, const std::vector & text_tokens); + XttsV2GptGeneration generate( + const std::vector & conditioning_latent, + const std::vector & text_tokens, + const XttsV2GenerationOptions & options); private: class PrefillGraph; diff --git a/src/models/xtts_v2/gpt.cpp b/src/models/xtts_v2/gpt.cpp index 43851848..b0012731 100644 --- a/src/models/xtts_v2/gpt.cpp +++ b/src/models/xtts_v2/gpt.cpp @@ -8,6 +8,9 @@ #include "engine/framework/modules/structural_modules.h" #include +#include +#include +#include #include #include @@ -103,8 +106,9 @@ std::shared_ptr load_xtts_v2_gpt_weights( class XttsV2GptRuntime::PrefillGraph { public: PrefillGraph(core::ExecutionContext & execution, std::shared_ptr weights, - int64_t text_count, size_t arena) - : execution_(execution), weights_(std::move(weights)), text_count_(text_count), steps_(32 + text_count + 3) { + int64_t text_count, int64_t audio_count, size_t arena) + : execution_(execution), weights_(std::move(weights)), text_count_(text_count), audio_count_(audio_count), + steps_(32 + text_count + 2 + audio_count) { ctx_.reset(ggml_init({arena, nullptr, true})); input_ctx_.reset(ggml_init({8U * 1024U * 1024U, nullptr, true})); if (!ctx_ || !input_ctx_) throw std::runtime_error("failed to initialize XTTS v2 GPT prefill graph"); @@ -112,17 +116,17 @@ class XttsV2GptRuntime::PrefillGraph { core::ModuleBuildContext ictx{input_ctx_.get(), "xtts_v2.gpt.prefill.inputs", execution_.backend_type()}; conditioning_ = core::make_tensor(ictx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 32, kDim})).tensor; text_ids_ = ggml_new_tensor_1d(input_ctx_.get(), GGML_TYPE_I32, text_count_ + 2); - audio_id_ = ggml_new_tensor_1d(input_ctx_.get(), GGML_TYPE_I32, 1); + audio_id_ = ggml_new_tensor_1d(input_ctx_.get(), GGML_TYPE_I32, audio_count_); ggml_set_input(conditioning_); ggml_set_input(text_ids_); ggml_set_input(audio_id_); auto condition = core::wrap_tensor(conditioning_, core::TensorShape::from_dims({1, 32, kDim}), GGML_TYPE_F32); auto ids = core::wrap_tensor(text_ids_, core::TensorShape::from_dims({text_count_ + 2}), GGML_TYPE_I32); auto text = modules::EmbeddingModule({6681, kDim}).build(ctx, ids, weights_->text_embedding); text = modules::AddModule{}.build(ctx, text, modules::SliceModule({0, 0, text_count_ + 2}).build(ctx, weights_->text_positions)); text = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, text), core::TensorShape::from_dims({1, text_count_ + 2, kDim})); - auto aid = core::wrap_tensor(audio_id_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto aid = core::wrap_tensor(audio_id_, core::TensorShape::from_dims({audio_count_}), GGML_TYPE_I32); auto audio = modules::EmbeddingModule({kAudioCodes, kDim}).build(ctx, aid, weights_->audio_embedding); - audio = modules::AddModule{}.build(ctx, audio, modules::SliceModule({0, 0, 1}).build(ctx, weights_->audio_positions)); - audio = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, audio), core::TensorShape::from_dims({1, 1, kDim})); + audio = modules::AddModule{}.build(ctx, audio, modules::SliceModule({0, 0, audio_count_}).build(ctx, weights_->audio_positions)); + audio = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, audio), core::TensorShape::from_dims({1, audio_count_, kDim})); auto x = modules::ConcatModule({1}).build(ctx, condition, text); x = modules::ConcatModule({1}).build(ctx, x, audio); for (const auto & layer : weights_->layers) x = gpt_layer(ctx, x, layer); @@ -144,15 +148,18 @@ class XttsV2GptRuntime::PrefillGraph { if (allocator_) ggml_gallocr_free(allocator_); if (input_buffer_) ggml_backend_buffer_free(input_buffer_); } - bool matches(int64_t count) const noexcept { return count == text_count_; } - XttsV2GptPrefillResult run(const std::vector & condition, const std::vector & tokens) { - if (condition.size() != 32U * 1024U || static_cast(tokens.size()) != text_count_) + bool matches(int64_t text_count, int64_t audio_count) const noexcept { + return text_count == text_count_ && audio_count == audio_count_; + } + XttsV2GptPrefillResult run(const std::vector & condition, const std::vector & tokens, + const std::vector & audio_tokens) { + if (condition.size() != 32U * 1024U || static_cast(tokens.size()) != text_count_ || + static_cast(audio_tokens.size()) != audio_count_) throw std::runtime_error("XTTS v2 GPT prefill input shape mismatch"); std::vector ids; ids.reserve(tokens.size() + 2); ids.push_back(261); ids.insert(ids.end(), tokens.begin(), tokens.end()); ids.push_back(0); - const int32_t start_audio = 1024; ggml_backend_tensor_set(conditioning_, condition.data(), 0, condition.size() * sizeof(float)); ggml_backend_tensor_set(text_ids_, ids.data(), 0, ids.size() * sizeof(int32_t)); - ggml_backend_tensor_set(audio_id_, &start_audio, 0, sizeof(start_audio)); + ggml_backend_tensor_set(audio_id_, audio_tokens.data(), 0, audio_tokens.size() * sizeof(int32_t)); core::set_backend_threads(execution_.backend(), execution_.config().threads); if (core::compute_backend_graph(execution_.backend(), graph_) != GGML_STATUS_SUCCESS) throw std::runtime_error("XTTS v2 GPT prefill compute failed"); ggml_backend_synchronize(execution_.backend()); @@ -163,7 +170,7 @@ class XttsV2GptRuntime::PrefillGraph { } private: core::ExecutionContext & execution_; std::shared_ptr weights_; - int64_t text_count_, steps_; std::unique_ptr ctx_, input_ctx_; + int64_t text_count_, audio_count_, steps_; std::unique_ptr ctx_, input_ctx_; ggml_tensor * conditioning_ = nullptr, * text_ids_ = nullptr, * audio_id_ = nullptr, * logits_ = nullptr, * latent_ = nullptr; ggml_cgraph * graph_ = nullptr; ggml_gallocr_t allocator_ = nullptr; ggml_backend_buffer_t input_buffer_ = nullptr; }; @@ -174,9 +181,50 @@ XttsV2GptRuntime::XttsV2GptRuntime(const XttsV2Assets & assets, core::ExecutionC weights_(load_xtts_v2_gpt_weights(assets, execution, weight_context_bytes, storage_type)) {} XttsV2GptRuntime::~XttsV2GptRuntime() = default; XttsV2GptPrefillResult XttsV2GptRuntime::prefill(const std::vector & condition, const std::vector & tokens) { - if (!prefill_ || !prefill_->matches(static_cast(tokens.size()))) - prefill_ = std::make_unique(execution_, weights_, static_cast(tokens.size()), graph_context_bytes_); - return prefill_->run(condition, tokens); + if (!prefill_ || !prefill_->matches(static_cast(tokens.size()), 1)) + prefill_ = std::make_unique(execution_, weights_, static_cast(tokens.size()), 1, graph_context_bytes_); + return prefill_->run(condition, tokens, {1024}); +} + +XttsV2GptGeneration XttsV2GptRuntime::generate( + const std::vector & condition, const std::vector & text, + const XttsV2GenerationOptions & options) { + if (text.empty() || text.size() >= 400) throw std::runtime_error("XTTS v2 GPT text token count is out of range"); + std::vector input{1024}; + XttsV2GptGeneration result; + std::mt19937 random(options.seed); + const int64_t limit = std::min(options.max_tokens, 603); + for (int64_t step = 0; step < limit; ++step) { + if (!prefill_ || !prefill_->matches(static_cast(text.size()), static_cast(input.size()))) + prefill_ = std::make_unique(execution_, weights_, static_cast(text.size()), static_cast(input.size()), graph_context_bytes_); + auto output = prefill_->run(condition, text, input); + std::vector logits = output.logits; + for (int32_t prior : result.codes) { + if (prior >= 0 && prior < 1025) { + float & value = logits[static_cast(prior)]; + value = value < 0.0F ? value * options.repetition_penalty : value / options.repetition_penalty; + } + } + logits[1025] = step < 2 ? -std::numeric_limits::infinity() : logits[1025]; + std::vector order(1026); for (int32_t i = 0; i < 1026; ++i) order[static_cast(i)] = i; + std::partial_sort(order.begin(), order.begin() + std::min(options.top_k, 1026), order.end(), + [&](int32_t a, int32_t b) { return logits[static_cast(a)] > logits[static_cast(b)]; }); + const size_t keep_k = static_cast(std::min(options.top_k, 1026)); + order.resize(keep_k); + const float max_logit = logits[static_cast(order.front())]; + std::vector probability(order.size()); double total = 0.0; + for (size_t i = 0; i < order.size(); ++i) { probability[i] = std::exp((logits[static_cast(order[i])] - max_logit) / options.temperature); total += probability[i]; } + double cumulative = 0.0; size_t nucleus = probability.size(); + for (size_t i = 0; i < probability.size(); ++i) { cumulative += probability[i] / total; if (cumulative >= options.top_p) { nucleus = i + 1; break; } } + probability.resize(nucleus); order.resize(nucleus); + std::discrete_distribution distribution(probability.begin(), probability.end()); + const int32_t next = order[distribution(random)]; + if (next == 1025) break; + result.codes.push_back(next); + result.latents.insert(result.latents.end(), output.latent.begin(), output.latent.end()); + input.push_back(next); + } + return result; } } // namespace engine::models::xtts_v2 diff --git a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp index 81f0be10..509f96e4 100644 --- a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp +++ b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp @@ -55,6 +55,10 @@ int main(int argc, char ** argv) try { *assets, execution, 1536U * 1024U * 1024U, 1536U * 1024U * 1024U, engine::assets::TensorStorageType::Native); const auto prefill = gpt_runtime.prefill(latent.values, text_tokens); + engine::models::xtts_v2::XttsV2GenerationOptions generation_options; + generation_options.max_tokens = 3; + generation_options.seed = 1234; + const auto generation = gpt_runtime.generate(latent.values, text_tokens, generation_options); const auto top = std::max_element(prefill.logits.begin(), prefill.logits.end()); if (argc == 4) { std::ofstream output(argv[3], std::ios::binary); @@ -87,6 +91,7 @@ int main(int argc, char ** argv) try { << ",\"speaker_frames\":" << speaker_mel.frames << ",\"speaker_norm\":" << std::sqrt(speaker_sq) << ",\"gpt_prefill_argmax\":" << std::distance(prefill.logits.begin(), top) + << ",\"generated_codes\":" << generation.codes.size() << ",\"sum\":" << sum << ",\"rms\":" << std::sqrt(sq / latent.values.size()) << ",\"first\":["; From 2ff2a3c84d8c52627636280f03909034dd2bedb2 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 03:01:51 -0400 Subject: [PATCH 09/11] xtts_v2: implement HiFiGAN waveform decoder --- CMakeLists.txt | 1 + include/engine/models/xtts_v2/decoder.h | 32 ++++ src/models/xtts_v2/decoder.cpp | 191 +++++++++++++++++++ tests/xtts_v2/xtts_v2_conditioning_probe.cpp | 9 + 4 files changed, 233 insertions(+) create mode 100644 include/engine/models/xtts_v2/decoder.h create mode 100644 src/models/xtts_v2/decoder.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f3eca2cb..887bd054 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2161,6 +2161,7 @@ if (ENGINE_BUILD_WARMBENCH) src/models/xtts_v2/assets.cpp src/models/xtts_v2/audio_features.cpp src/models/xtts_v2/conditioning.cpp + src/models/xtts_v2/decoder.cpp src/models/xtts_v2/gpt.cpp src/models/xtts_v2/speaker_encoder.cpp src/models/xtts_v2/tokenizer.cpp) diff --git a/include/engine/models/xtts_v2/decoder.h b/include/engine/models/xtts_v2/decoder.h new file mode 100644 index 00000000..b58d4dd9 --- /dev/null +++ b/include/engine/models/xtts_v2/decoder.h @@ -0,0 +1,32 @@ +#pragma once + +#include "engine/framework/core/execution_context.h" +#include "engine/models/xtts_v2/assets.h" +#include "engine/models/xtts_v2/gpt.h" +#include "engine/models/xtts_v2/speaker_encoder.h" + +#include +#include + +namespace engine::models::xtts_v2 { + +class XttsV2DecoderRuntime { +public: + XttsV2DecoderRuntime(const XttsV2Assets & assets, core::ExecutionContext & execution, + size_t weight_context_bytes, size_t graph_context_bytes, + assets::TensorStorageType conv_storage_type); + ~XttsV2DecoderRuntime(); + + std::vector decode(const std::vector & latents, int64_t frames, + const XttsV2SpeakerEmbedding & speaker); + +private: + struct Weights; + class Graph; + core::ExecutionContext & execution_; + size_t graph_context_bytes_; + std::shared_ptr weights_; + std::unique_ptr graph_; +}; + +} // namespace engine::models::xtts_v2 diff --git a/src/models/xtts_v2/decoder.cpp b/src/models/xtts_v2/decoder.cpp new file mode 100644 index 00000000..6f28b15f --- /dev/null +++ b/src/models/xtts_v2/decoder.cpp @@ -0,0 +1,191 @@ +#include "engine/models/xtts_v2/decoder.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { +namespace { +namespace binding = engine::modules::binding; +namespace modules = engine::modules; + +struct ResBlockWeights { + std::array first; + std::array second; + int64_t channels = 0; + int64_t kernel = 0; +}; +struct ContextDeleter { void operator()(ggml_context * p) const noexcept { if (p) ggml_free(p); } }; + +core::TensorValue resblock(core::ModuleBuildContext & ctx, core::TensorValue x, const ResBlockWeights & weights) { + constexpr std::array dilation{1, 3, 5}; + for (size_t i = 0; i < 3; ++i) { + auto y = modules::LeakyReluModule({0.1F}).build(ctx, x); + y = modules::Conv1dModule({weights.channels, weights.channels, weights.kernel, 1, + static_cast((weights.kernel * dilation[i] - dilation[i]) / 2), dilation[i], true}).build(ctx, y, weights.first[i]); + y = modules::LeakyReluModule({0.1F}).build(ctx, y); + y = modules::Conv1dModule({weights.channels, weights.channels, weights.kernel, 1, + static_cast((weights.kernel - 1) / 2), 1, true}).build(ctx, y, weights.second[i]); + x = modules::AddModule{}.build(ctx, x, y); + } + return x; +} + +core::TensorValue upsample(core::ModuleBuildContext & ctx, const core::TensorValue & input, + int64_t in_channels, int64_t out_channels, int64_t kernel, int stride, + const modules::ConvTranspose1dWeights & weights) { + const int padding = static_cast((kernel - stride) / 2); + auto full = modules::ConvTranspose1dModule({in_channels, out_channels, kernel, stride, 0, 1, true}) + .build(ctx, input, weights); + const int64_t cropped = (input.shape.dims[2] - 1) * stride - 2 * padding + kernel; + return modules::SliceModule({2, padding, cropped}).build(ctx, full); +} + +std::vector interpolate_frame_major(const std::vector & input, int64_t input_frames, + int64_t output_frames, double scale) { + std::vector output(static_cast(output_frames * 1024)); + for (int64_t t = 0; t < output_frames; ++t) { + const double source = (static_cast(t) + 0.5) / scale - 0.5; + const int64_t left = std::max(0, std::min(input_frames - 1, static_cast(std::floor(source)))); + const int64_t right = std::max(0, std::min(input_frames - 1, left + 1)); + const float fraction = static_cast(std::max(0.0, std::min(1.0, source - static_cast(left)))); + for (int64_t c = 0; c < 1024; ++c) { + const float a = input[static_cast(left * 1024 + c)]; + const float b = input[static_cast(right * 1024 + c)]; + output[static_cast(t * 1024 + c)] = a + (b - a) * fraction; + } + } + return output; +} +} // namespace + +struct XttsV2DecoderRuntime::Weights { + std::shared_ptr store; + modules::Conv1dWeights pre; + modules::Conv1dWeights condition; + std::array up; + std::array stage_condition; + std::array residual; + modules::Conv1dWeights post; +}; + +class XttsV2DecoderRuntime::Graph { +public: + Graph(core::ExecutionContext & execution, std::shared_ptr weights, int64_t frames, size_t arena) + : execution_(execution), weights_(std::move(weights)), frames_(frames) { + const int64_t ar_interpolated = frames_ * 4; + const int64_t interpolated = ar_interpolated * 24000 / 22050; + ctx_.reset(ggml_init({arena, nullptr, true})); + input_ctx_.reset(ggml_init({8U * 1024U * 1024U, nullptr, true})); + if (!ctx_ || !input_ctx_) throw std::runtime_error("failed to initialize XTTS v2 decoder graph"); + core::ModuleBuildContext ctx{ctx_.get(), "xtts_v2.decoder", execution_.backend_type()}; + core::ModuleBuildContext ictx{input_ctx_.get(), "xtts_v2.decoder.inputs", execution_.backend_type()}; + latent_ = core::make_tensor(ictx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 1024, interpolated})).tensor; + speaker_ = core::make_tensor(ictx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 512, 1})).tensor; + ggml_set_input(latent_); ggml_set_input(speaker_); + auto x = core::wrap_tensor(latent_, core::TensorShape::from_dims({1, 1024, interpolated}), GGML_TYPE_F32); + x = modules::Conv1dModule({1024, 512, 7, 1, 3, 1, true}).build(ctx, x, weights_->pre); + auto speaker = core::wrap_tensor(speaker_, core::TensorShape::from_dims({1, 512, 1}), GGML_TYPE_F32); + auto cond = modules::Conv1dModule({512, 512, 1, 1, 0, 1, true}).build(ctx, speaker, weights_->condition); + x = modules::AddModule{}.build(ctx, x, modules::RepeatModule({x.shape}).build(ctx, cond)); + constexpr std::array strides{8, 8, 2, 2}; + constexpr std::array kernels{16, 16, 4, 4}; + constexpr std::array channels{256, 128, 64, 32}; + int64_t in_channels = 512; + for (size_t stage = 0; stage < 4; ++stage) { + x = modules::LeakyReluModule({0.1F}).build(ctx, x); + x = upsample(ctx, x, in_channels, channels[stage], kernels[stage], strides[stage], weights_->up[stage]); + auto stage_cond = modules::Conv1dModule({512, channels[stage], 1, 1, 0, 1, true}).build(ctx, speaker, weights_->stage_condition[stage]); + x = modules::AddModule{}.build(ctx, x, modules::RepeatModule({x.shape}).build(ctx, stage_cond)); + auto sum = resblock(ctx, x, weights_->residual[stage * 3]); + sum = modules::AddModule{}.build(ctx, sum, resblock(ctx, x, weights_->residual[stage * 3 + 1])); + sum = modules::AddModule{}.build(ctx, sum, resblock(ctx, x, weights_->residual[stage * 3 + 2])); + x = core::wrap_tensor(ggml_scale(ctx.ggml, sum.tensor, 1.0F / 3.0F), sum.shape, GGML_TYPE_F32); + in_channels = channels[stage]; + } + x = modules::LeakyReluModule({0.01F}).build(ctx, x); + x = modules::Conv1dModule({32, 1, 7, 1, 3, 1, false}).build(ctx, x, weights_->post); + x = modules::TanhModule{}.build(ctx, x); + output_frames_ = x.shape.dims[2]; + output_ = core::ensure_backend_addressable_layout(ctx, x).tensor; + ggml_set_output(output_); + graph_ = ggml_new_graph_custom(ctx_.get(), 131072, false); ggml_build_forward_expand(graph_, output_); + input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); + allocator_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution_.backend())); + if (!input_buffer_ || !allocator_ || !ggml_gallocr_reserve(allocator_, graph_) || !ggml_gallocr_alloc_graph(allocator_, graph_)) + throw std::runtime_error("failed to allocate XTTS v2 decoder graph"); + } + ~Graph() { + if (graph_) core::release_backend_graph_resources(execution_.backend(), graph_); + if (allocator_) ggml_gallocr_free(allocator_); if (input_buffer_) ggml_backend_buffer_free(input_buffer_); + } + bool matches(int64_t frames) const noexcept { return frames == frames_; } + std::vector run(const std::vector & latent, const std::vector & speaker) { + const int64_t interpolated = (frames_ * 4) * 24000 / 22050; + if (static_cast(latent.size()) != interpolated * 1024 || speaker.size() != 512) throw std::runtime_error("XTTS v2 decoder input shape mismatch"); + ggml_backend_tensor_set(latent_, latent.data(), 0, latent.size() * sizeof(float)); + ggml_backend_tensor_set(speaker_, speaker.data(), 0, speaker.size() * sizeof(float)); + core::set_backend_threads(execution_.backend(), execution_.config().threads); + if (core::compute_backend_graph(execution_.backend(), graph_) != GGML_STATUS_SUCCESS) throw std::runtime_error("XTTS v2 decoder compute failed"); + ggml_backend_synchronize(execution_.backend()); std::vector output(static_cast(output_frames_)); + ggml_backend_tensor_get(output_, output.data(), 0, output.size() * sizeof(float)); return output; + } +private: + core::ExecutionContext & execution_; std::shared_ptr weights_; int64_t frames_, output_frames_; + std::unique_ptr ctx_, input_ctx_; ggml_tensor * latent_ = nullptr, * speaker_ = nullptr, * output_ = nullptr; + ggml_cgraph * graph_ = nullptr; ggml_gallocr_t allocator_ = nullptr; ggml_backend_buffer_t input_buffer_ = nullptr; +}; + +XttsV2DecoderRuntime::XttsV2DecoderRuntime(const XttsV2Assets & assets, core::ExecutionContext & execution, + size_t weight_context_bytes, size_t graph_context_bytes, assets::TensorStorageType type) + : execution_(execution), graph_context_bytes_(graph_context_bytes) { + auto out = std::make_shared(); out->store = std::make_shared( + execution.backend(), execution.backend_type(), "xtts_v2.decoder.weights", weight_context_bytes); + const auto & source = *assets.decoder; + out->pre = binding::conv1d_from_source(*out->store, source, "conv_pre", type, 512, 1024, 7, true); + out->condition = binding::conv1d_from_source(*out->store, source, "cond_layer", type, 512, 512, 1, true); + constexpr std::array in_channels{512, 256, 128, 64}; + constexpr std::array channels{256, 128, 64, 32}; + constexpr std::array kernels{16, 16, 4, 4}; + for (size_t i = 0; i < 4; ++i) { + out->up[i] = binding::conv_transpose1d_from_source(*out->store, source, "ups." + std::to_string(i), type, in_channels[i], channels[i], kernels[i], true); + out->stage_condition[i] = binding::conv1d_from_source(*out->store, source, "conds." + std::to_string(i), type, channels[i], 512, 1, true); + } + constexpr std::array rb_kernel{3, 7, 11}; constexpr std::array dilation{1, 3, 5}; + for (size_t stage = 0; stage < 4; ++stage) for (size_t branch = 0; branch < 3; ++branch) { + const size_t index = stage * 3 + branch; auto & rb = out->residual[index]; rb.channels = channels[stage]; rb.kernel = rb_kernel[branch]; + for (size_t layer = 0; layer < 3; ++layer) { + const std::string base = "resblocks." + std::to_string(index); + rb.first[layer] = binding::conv1d_from_source(*out->store, source, base + ".convs1." + std::to_string(layer), type, rb.channels, rb.channels, rb.kernel, true); + rb.second[layer] = binding::conv1d_from_source(*out->store, source, base + ".convs2." + std::to_string(layer), type, rb.channels, rb.channels, rb.kernel, true); + } + } + out->post = binding::conv1d_from_source(*out->store, source, "conv_post", type, 1, 32, 7, false); + out->store->upload(); weights_ = std::move(out); +} +XttsV2DecoderRuntime::~XttsV2DecoderRuntime() = default; +std::vector XttsV2DecoderRuntime::decode(const std::vector & latents, int64_t frames, const XttsV2SpeakerEmbedding & speaker) { + if (frames <= 0) throw std::runtime_error("XTTS v2 decoder requires latent frames"); + if (!graph_ || !graph_->matches(frames)) graph_ = std::make_unique(execution_, weights_, frames, graph_context_bytes_); + auto stage1 = interpolate_frame_major(latents, frames, frames * 4, 4.0); + const int64_t stage2_frames = (frames * 4) * 24000 / 22050; + auto stage2 = interpolate_frame_major(stage1, frames * 4, stage2_frames, 24000.0 / 22050.0); + // Convert [time, channels] to the channel-major [B,C,T] input expected by ggml. + std::vector channel_major(stage2.size()); + for (int64_t t = 0; t < stage2_frames; ++t) for (int64_t c = 0; c < 1024; ++c) + channel_major[static_cast(c * stage2_frames + t)] = stage2[static_cast(t * 1024 + c)]; + return graph_->run(channel_major, speaker.values); +} + +} // namespace engine::models::xtts_v2 diff --git a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp index 509f96e4..55e8afae 100644 --- a/tests/xtts_v2/xtts_v2_conditioning_probe.cpp +++ b/tests/xtts_v2/xtts_v2_conditioning_probe.cpp @@ -3,6 +3,7 @@ #include "engine/models/xtts_v2/assets.h" #include "engine/models/xtts_v2/audio_features.h" #include "engine/models/xtts_v2/conditioning.h" +#include "engine/models/xtts_v2/decoder.h" #include "engine/models/xtts_v2/gpt.h" #include "engine/models/xtts_v2/speaker_encoder.h" #include "engine/models/xtts_v2/tokenizer.h" @@ -59,6 +60,11 @@ int main(int argc, char ** argv) try { generation_options.max_tokens = 3; generation_options.seed = 1234; const auto generation = gpt_runtime.generate(latent.values, text_tokens, generation_options); + engine::models::xtts_v2::XttsV2DecoderRuntime decoder_runtime( + *assets, execution, 128U * 1024U * 1024U, 1024U * 1024U * 1024U, + engine::assets::TensorStorageType::Native); + const auto waveform = decoder_runtime.decode( + generation.latents, static_cast(generation.codes.size()), speaker); const auto top = std::max_element(prefill.logits.begin(), prefill.logits.end()); if (argc == 4) { std::ofstream output(argv[3], std::ios::binary); @@ -78,6 +84,8 @@ int main(int argc, char ** argv) try { write_values(".cond", latent.values); write_values(".logits", prefill.logits); write_values(".gptlatent", prefill.latent); + write_values(".generated_latents", generation.latents); + write_values(".wav", waveform); } double speaker_sq = 0.0; for (float value : speaker.values) speaker_sq += static_cast(value) * value; @@ -92,6 +100,7 @@ int main(int argc, char ** argv) try { << ",\"speaker_norm\":" << std::sqrt(speaker_sq) << ",\"gpt_prefill_argmax\":" << std::distance(prefill.logits.begin(), top) << ",\"generated_codes\":" << generation.codes.size() + << ",\"waveform_samples\":" << waveform.size() << ",\"sum\":" << sum << ",\"rms\":" << std::sqrt(sq / latent.values.size()) << ",\"first\":["; From 2c05f45fe7a1ccbd28d88bd09e3c97a452d71358 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 03:11:16 -0400 Subject: [PATCH 10/11] xtts_v2: integrate native voice cloning session --- CMakeLists.txt | 17 +++ include/engine/models/xtts_v2/session.h | 43 ++++++++ model_specs/xtts_v2.json | 3 +- src/models/xtts_v2/session.cpp | 125 ++++++++++++++++++++++ tools/community_models/convert_xtts_v2.py | 9 ++ 5 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 include/engine/models/xtts_v2/session.h create mode 100644 src/models/xtts_v2/session.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 887bd054..ee168200 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1236,6 +1236,23 @@ audiocpp_add_model(index_tts2 engine::models::index_tts2::make_index_tts2_loader ) +audiocpp_add_model(xtts_v2 + SOURCES + src/models/xtts_v2/assets.cpp + src/models/xtts_v2/audio_features.cpp + src/models/xtts_v2/conditioning.cpp + src/models/xtts_v2/decoder.cpp + src/models/xtts_v2/gpt.cpp + src/models/xtts_v2/request.cpp + src/models/xtts_v2/session.cpp + src/models/xtts_v2/speaker_encoder.cpp + src/models/xtts_v2/tokenizer.cpp + INCLUDES + engine/models/xtts_v2/session.h + LOADERS + engine::models::xtts_v2::make_xtts_v2_loader +) + audiocpp_add_model(nemotron_asr SOURCES src/models/nemotron_asr/assets.cpp diff --git a/include/engine/models/xtts_v2/session.h b/include/engine/models/xtts_v2/session.h new file mode 100644 index 00000000..7f014c51 --- /dev/null +++ b/include/engine/models/xtts_v2/session.h @@ -0,0 +1,43 @@ +#pragma once + +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/session_base.h" +#include "engine/models/xtts_v2/assets.h" +#include "engine/models/xtts_v2/conditioning.h" +#include "engine/models/xtts_v2/decoder.h" +#include "engine/models/xtts_v2/gpt.h" +#include "engine/models/xtts_v2/speaker_encoder.h" +#include "engine/models/xtts_v2/tokenizer.h" + +#include + +namespace engine::models::xtts_v2 { + +std::shared_ptr make_xtts_v2_loader(); + +class XttsV2Session final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession { +public: + XttsV2Session(runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + ~XttsV2Session() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + runtime::TaskResult run(const runtime::TaskRequest & request) override; + +private: + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + XttsV2Tokenizer tokenizer_; + std::unique_ptr conditioning_; + std::unique_ptr speaker_; + std::unique_ptr gpt_; + std::unique_ptr decoder_; +}; + +} // namespace engine::models::xtts_v2 diff --git a/model_specs/xtts_v2.json b/model_specs/xtts_v2.json index 160211be..ecccdeca 100644 --- a/model_specs/xtts_v2.json +++ b/model_specs/xtts_v2.json @@ -18,6 +18,7 @@ {"name": "top_k", "type": "int", "description": "GPT top-k sampling cutoff.", "required": false, "min": 1, "default": 50}, {"name": "top_p", "type": "float", "description": "GPT nucleus-sampling cutoff.", "required": false, "min": 0.01, "max": 1.0, "default": 0.85}, {"name": "repetition_penalty", "type": "float", "description": "GPT repetition penalty.", "required": false, "min": 1.0, "default": 5.0}, + {"name": "max_tokens", "type": "int", "description": "Maximum generated acoustic-token count.", "required": false, "min": 1, "max": 603, "default": 603}, {"name": "speed", "type": "float", "description": "Output speaking-rate multiplier.", "required": false, "min": 0.05, "max": 4.0, "default": 1.0}, {"name": "seed", "type": "int", "description": "Non-negative deterministic sampling seed.", "required": false, "min": 0} ], @@ -30,7 +31,7 @@ ] }, "package_defaults": { - "download": {"kind": "huggingface_snapshot", "repo": "audio-cpp/audio.cpp-gguf", "revision": "refs/pr/REPLACE_WITH_HF_PR"} + "download": {"kind": "huggingface_snapshot", "repo": "audio-cpp/audio.cpp-gguf", "revision": "refs/pr/9"} }, "packages": [ { diff --git a/src/models/xtts_v2/session.cpp b/src/models/xtts_v2/session.cpp new file mode 100644 index 00000000..db0d191c --- /dev/null +++ b/src/models/xtts_v2/session.cpp @@ -0,0 +1,125 @@ +#include "engine/models/xtts_v2/session.h" + +#include "engine/framework/runtime/spec_backed_model.h" +#include "engine/models/xtts_v2/audio_features.h" +#include "engine/models/xtts_v2/request.h" + +#include +#include +#include +#include + +namespace engine::models::xtts_v2 { +namespace { +constexpr const char * kFamily = "xtts_v2"; + +std::shared_ptr require_assets(std::shared_ptr assets) { + if (!assets) throw std::runtime_error("XTTS v2 session requires assets"); + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (!contract) throw std::runtime_error("XTTS v2 session requires a model contract"); + return contract; +} + +std::unique_ptr create_session( + const runtime::TaskSpec & task, const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + if (task.mode != runtime::RunMode::Offline || + (task.task != runtime::VoiceTaskKind::Tts && task.task != runtime::VoiceTaskKind::VoiceCloning)) + throw std::runtime_error("XTTS v2 supports offline TTS and voice cloning"); + return std::make_unique(task, options, std::move(assets), std::move(contract)); +} + +XttsV2ConditioningLatent conditioning_for_reference( + XttsV2ConditioningRuntime & runtime, const std::vector & waveform) { + constexpr size_t chunk = 4U * 22050U; + constexpr size_t minimum = 22050U / 3U; + XttsV2ConditioningLatent average; size_t used = 0; + for (size_t offset = 0; offset < waveform.size(); offset += chunk) { + const size_t end = std::min(waveform.size(), offset + chunk); + if (end - offset < minimum) continue; + const std::vector samples(waveform.begin() + static_cast(offset), + waveform.begin() + static_cast(end)); + auto latent = runtime.encode(compute_xtts_v2_conditioning_mel(samples, runtime.mel_stats())); + if (average.values.empty()) { average = latent; std::fill(average.values.begin(), average.values.end(), 0.0F); } + for (size_t i = 0; i < latent.values.size(); ++i) average.values[i] += latent.values[i]; + ++used; + } + if (used == 0) throw std::runtime_error("XTTS v2 reference audio must contain at least 0.33 seconds"); + for (float & value : average.values) value /= static_cast(used); + return average; +} + +std::vector scale_speed(const std::vector & input, int64_t frames, float speed, int64_t & output_frames) { + if (speed == 1.0F) { output_frames = frames; return input; } + output_frames = std::max(1, static_cast(std::floor(static_cast(frames) / speed))); + std::vector output(static_cast(output_frames * 1024)); + const double scale = 1.0 / static_cast(speed); + for (int64_t t = 0; t < output_frames; ++t) { + const double source = (static_cast(t) + 0.5) / scale - 0.5; + const int64_t left = std::max(0, std::min(frames - 1, static_cast(std::floor(source)))); + const int64_t right = std::min(frames - 1, left + 1); + const float fraction = static_cast(std::max(0.0, std::min(1.0, source - left))); + for (int64_t c = 0; c < 1024; ++c) { + const float a = input[static_cast(left * 1024 + c)]; + const float b = input[static_cast(right * 1024 + c)]; + output[static_cast(t * 1024 + c)] = a + (b - a) * fraction; + } + } + return output; +} +} // namespace + +XttsV2Session::XttsV2Session(runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(options), task_(task), assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))), tokenizer_(assets_->tokenizer_path) { + runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "XTTS v2"); + auto & execution = execution_context(); + conditioning_ = std::make_unique(*assets_, execution, + 512U * 1024U * 1024U, 512U * 1024U * 1024U, + assets::TensorStorageType::Native, assets::TensorStorageType::Native); + speaker_ = std::make_unique(*assets_, execution, + 128U * 1024U * 1024U, 512U * 1024U * 1024U, + assets::TensorStorageType::Native, assets::TensorStorageType::Native); + gpt_ = std::make_unique(*assets_, execution, + 1536U * 1024U * 1024U, 1536U * 1024U * 1024U, assets::TensorStorageType::Native); + decoder_ = std::make_unique(*assets_, execution, + 128U * 1024U * 1024U, 1024U * 1024U * 1024U, assets::TensorStorageType::Native); +} +XttsV2Session::~XttsV2Session() = default; +std::string XttsV2Session::family() const { return kFamily; } +runtime::VoiceTaskKind XttsV2Session::task_kind() const { return task_.task; } +runtime::RunMode XttsV2Session::run_mode() const { return task_.mode; } +void XttsV2Session::prepare(const runtime::SessionPreparationRequest & request) { + runtime::validate_spec_backed_request_options(request.options, *contract_, "XTTS v2"); mark_prepared(); +} +runtime::TaskResult XttsV2Session::run(const runtime::TaskRequest & request) { + require_prepared("XTTS v2 run"); runtime::validate_spec_backed_request_options(request.options, *contract_, "XTTS v2"); + const auto parsed = parse_xtts_v2_request(request); + const auto reference = prepare_xtts_v2_reference(parsed.speaker_audio); + const auto condition = conditioning_for_reference(*conditioning_, reference.waveform_22050); + const auto speaker_mel = compute_xtts_v2_speaker_mel(reference.waveform_16000, + assets_->speaker_encoder->require_f32("torch_spec.1.spectrogram.window", {400}), + assets_->speaker_encoder->require_f32("torch_spec.1.mel_scale.fb", {257, 64})); + const auto speaker_embedding = speaker_->encode(speaker_mel); + const auto text = tokenizer_.encode(parsed.text, parsed.language); + const auto generated = gpt_->generate(condition.values, text, parsed.generation); + if (generated.codes.empty()) throw std::runtime_error("XTTS v2 generated no acoustic tokens"); + int64_t decoder_frames = static_cast(generated.codes.size()); + auto latents = scale_speed(generated.latents, decoder_frames, parsed.generation.speed, decoder_frames); + runtime::TaskResult result; runtime::AudioBuffer audio; audio.sample_rate = 24000; audio.channels = 1; + audio.samples = decoder_->decode(latents, decoder_frames, speaker_embedding); result.audio_output = std::move(audio); return result; +} +std::shared_ptr make_xtts_v2_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; config.load_assets = load_xtts_v2_assets; config.create_session = create_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::models::xtts_v2 diff --git a/tools/community_models/convert_xtts_v2.py b/tools/community_models/convert_xtts_v2.py index 978a2479..851a20a8 100644 --- a/tools/community_models/convert_xtts_v2.py +++ b/tools/community_models/convert_xtts_v2.py @@ -125,6 +125,15 @@ def converter_command(output_dir: Path, converter: Path, quant_type: str) -> lis "--type", quant_type, "--output", str(output_dir / f"xtts-v2-{quant_type}.gguf"), ] + # The Perceiver resampler repeatedly normalizes and attends over the reference + # latent. Half precision here can overflow on otherwise ordinary recordings, + # poisoning the entire conditioning prefix with NaNs. Preserve these small, + # numerically sensitive tensors in F32 for every reduced-precision package. + if quant_type != "f32": + command += [ + "--keep-type", "gpt/conditioning_perceiver.*=f32", + "--keep-type", "gpt/mel_stats=f32", + ] # Convolutions have no quantized execution path. Keeping embeddings and the # small output head in F16 also avoids sampling regressions from Q8 logits. if quant_type.startswith("q"): From 91d726eca3131ef88db7ae7a79a2e9f05741e330 Mon Sep 17 00:00:00 2001 From: DrewThomasson Date: Sat, 12 Sep 2026 03:30:41 -0400 Subject: [PATCH 11/11] xtts_v2: match repetition penalty semantics --- src/models/xtts_v2/gpt.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/models/xtts_v2/gpt.cpp b/src/models/xtts_v2/gpt.cpp index b0012731..774d2bc0 100644 --- a/src/models/xtts_v2/gpt.cpp +++ b/src/models/xtts_v2/gpt.cpp @@ -199,10 +199,26 @@ XttsV2GptGeneration XttsV2GptRuntime::generate( prefill_ = std::make_unique(execution_, weights_, static_cast(text.size()), static_cast(input.size()), graph_context_bytes_); auto output = prefill_->run(condition, text, input); std::vector logits = output.logits; + std::vector penalized(1025, false); + // Transformers applies the processor to the complete synthetic input-id + // sequence. XTTS fills its 34+text prefix positions with token 1 and ends + // that prefix with the audio BOS token, even though those ids are replaced + // by cached embeddings inside the model. + penalized[1] = true; + penalized[1024] = true; + for (int32_t prefix_id : {1, 1024}) { + float & value = logits[static_cast(prefix_id)]; + value = value < 0.0F ? value * options.repetition_penalty : value / options.repetition_penalty; + } for (int32_t prior : result.codes) { - if (prior >= 0 && prior < 1025) { + // Hugging Face's RepetitionPenaltyLogitsProcessor applies the + // penalty once per token id present in the history, not once per + // occurrence. Reapplying it to repeated codes quickly destroys the + // acoustic distribution and produces rough, overlong speech. + if (prior >= 0 && prior < 1025 && !penalized[static_cast(prior)]) { float & value = logits[static_cast(prior)]; value = value < 0.0F ? value * options.repetition_penalty : value / options.repetition_penalty; + penalized[static_cast(prior)] = true; } } logits[1025] = step < 2 ? -std::numeric_limits::infinity() : logits[1025];