From f18f7bf8fc056abb9c1f6273d66bdab7a96478aa Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:12:46 -0400 Subject: [PATCH 1/2] Expose built-in audio utilities as model family --- CMakeLists.txt | 9 + app/server/config.cpp | 7 +- docs/audio_tools.md | 53 +++- include/engine/framework/audio/utility_api.h | 21 ++ .../models/builtin_audio_utils/session.h | 30 ++ src/framework/audio/utility_api.cpp | 65 +++++ src/framework/runtime/registry.cpp | 9 +- src/models/builtin_audio_utils/session.cpp | 262 ++++++++++++++++++ 8 files changed, 452 insertions(+), 4 deletions(-) create mode 100644 include/engine/models/builtin_audio_utils/session.h create mode 100644 src/models/builtin_audio_utils/session.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8251497c2..c0d057f71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1697,6 +1697,15 @@ audiocpp_add_model(audiosr engine::models::audiosr::make_audiosr_loader ) +audiocpp_add_model(builtin_audio_utils + SOURCES + src/models/builtin_audio_utils/session.cpp + INCLUDES + engine/models/builtin_audio_utils/session.h + LOADERS + engine::models::builtin_audio_utils::make_builtin_audio_utils_loader +) + audiocpp_add_model(controlfoley SOURCES src/models/controlfoley/assets.cpp diff --git a/app/server/config.cpp b/app/server/config.cpp index 1199dd15d..565bc5e83 100644 --- a/app/server/config.cpp +++ b/app/server/config.cpp @@ -278,11 +278,14 @@ ServerConfig load_server_config(const std::filesystem::path & path) { for (const auto & item : models->as_array()) { ServerModelConfig model; model.id = engine::io::json::require_string(item, "id"); - model.path = resolve_path(base, engine::io::json::require_string(item, "path")); + model.family = engine::io::json::require_string(item, "family"); + const auto raw_path = engine::io::json::require_string(item, "path"); + model.path = model.family == "builtin_audio_utils" + ? std::filesystem::path(raw_path) + : resolve_path(base, raw_path); if (const auto * value = item.find("model_spec_override")) { model.model_spec_override = resolve_path(base, value->as_string()); } - model.family = engine::io::json::require_string(item, "family"); model.task = engine::io::json::optional_string(item, "task", model.task); model.mode = engine::io::json::optional_string(item, "mode", model.mode); model.lazy = engine::io::json::optional_bool(item, "lazy", config.lazy_load); diff --git a/docs/audio_tools.md b/docs/audio_tools.md index b976fdc37..02bf959dc 100644 --- a/docs/audio_tools.md +++ b/docs/audio_tools.md @@ -2,9 +2,10 @@ | Model | Family | Task(s) | Quick Start | |---|---|---|---| +| Built-in audio utilities | `builtin_audio_utils` | `s2s` denoise/enhance/super-resolution | [Built-in audio utilities](#built-in-audio-utilities) | | AudioSR | `audiosr` | `s2s` audio super-resolution | [AudioSR](#audiosr) | | ControlFoley | `controlfoley` | `gen` Foley/SFX generation | [ControlFoley](#controlfoley) | -| GTCRN | `gtcrn`, `gtcrn_dns3`, `gtcrn_vctk`, `gtcrn_streaming` | framework denoise utility | [GTCRN](#gtcrn) | +| GTCRN | `gtcrn`, `gtcrn_dns3`, `gtcrn_vctk`, `gtcrn_streaming` | framework denoise utility API | [GTCRN](#gtcrn) | | MeanVC2 | `meanvc2` | `vc` | [MeanVC2](#meanvc2) | | MioCodec | `miocodec` | `vc`, `s2s` | [MioCodec](#miocodec) | | PersonaPlex | `personaplex` | `s2s` | [PersonaPlex](#personaplex) | @@ -29,6 +30,56 @@ Common CLI shape: audiocpp_cli --task --family --model --backend cuda ... ``` +## Built-in Audio Utilities + +The `builtin_audio_utils` family exposes the built-in framework audio utility +models through the normal CLI and server model-loading path. These utilities do +not use a GGUF or external model directory as `--model`; pass the utility id +directly. + +| Utility id | Operation | Input rate | Output rate | +|---|---|---:|---:| +| `deepfilternet2` | Denoise/enhance | 48 kHz | 48 kHz | +| `rnnoise` | Denoise/enhance | 48 kHz | 48 kHz | +| `zipenhancer` | Denoise/enhance | 16 kHz | 16 kHz | +| `gtcrn` | Denoise/enhance, alias for `gtcrn_streaming` | 16 kHz | 16 kHz | +| `gtcrn_streaming` | Denoise/enhance | 16 kHz | 16 kHz | +| `gtcrn_dns3` | Denoise/enhance | 16 kHz | 16 kHz | +| `gtcrn_vctk` | Denoise/enhance | 16 kHz | 16 kHz | +| `flashsr` | Audio super-resolution | 16 kHz | 48 kHz | + +CLI example: + +```bash +audiocpp_cli --task s2s --family builtin_audio_utils \ + --model rnnoise \ + --backend cuda \ + --audio input.wav \ + --out enhanced.wav \ + --log \ + --log-file rnnoise.log +``` + +Server config example: + +```json +{ + "id": "builtin-rnnoise", + "family": "builtin_audio_utils", + "path": "rnnoise", + "task": "s2s", + "mode": "offline" +} +``` + +Server request example: + +```bash +curl http://127.0.0.1:8080/v1/tasks/run \ + -H 'Content-Type: application/json' \ + -d '{"model":"builtin-rnnoise","request":{"audio":"input.wav"}}' +``` + ## AudioSR AudioSR performs audio super-resolution from an input waveform. See diff --git a/include/engine/framework/audio/utility_api.h b/include/engine/framework/audio/utility_api.h index dddb9e662..0a4d4b528 100644 --- a/include/engine/framework/audio/utility_api.h +++ b/include/engine/framework/audio/utility_api.h @@ -3,12 +3,25 @@ #include "engine/framework/core/backend.h" #include +#include #include #include #include namespace engine::audio { +enum class BuiltinAudioUtilityKind { + Denoise, + SuperResolve, +}; + +struct BuiltinAudioUtilityInfo { + std::string_view id; + BuiltinAudioUtilityKind kind = BuiltinAudioUtilityKind::Denoise; + int input_sample_rate = 0; + int output_sample_rate = 0; +}; + struct AudioUtilityPaths { AudioUtilityPaths() = default; explicit AudioUtilityPaths(std::filesystem::path assets_root_path) @@ -25,6 +38,14 @@ struct AudioUtilityBatchResult { std::vector outputs; }; +std::filesystem::path default_audio_utility_assets_root(); +std::vector list_builtin_audio_utilities(); +std::optional find_builtin_audio_utility(std::string_view model); +BuiltinAudioUtilityInfo require_builtin_audio_utility(std::string_view model); +std::filesystem::path resolve_builtin_audio_utility_asset( + const AudioUtilityPaths & paths, + std::string_view model); + void denoise_file( const std::filesystem::path & input_wav, const std::filesystem::path & output_wav, diff --git a/include/engine/models/builtin_audio_utils/session.h b/include/engine/models/builtin_audio_utils/session.h new file mode 100644 index 000000000..913787f85 --- /dev/null +++ b/include/engine/models/builtin_audio_utils/session.h @@ -0,0 +1,30 @@ +#pragma once + +#include "engine/framework/runtime/model.h" +#include "engine/framework/runtime/session.h" +#include "engine/framework/runtime/session_base.h" + +#include +#include + +namespace engine::models::builtin_audio_utils { + +class BuiltinAudioUtilsLoadedModel final : public runtime::ILoadedVoiceModel { +public: + explicit BuiltinAudioUtilsLoadedModel(std::string model_id); + + const runtime::ModelMetadata & metadata() const noexcept override; + const runtime::CapabilitySet & capabilities() const noexcept override; + std::unique_ptr create_task_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options) const override; + +private: + std::string model_id_; + runtime::ModelMetadata metadata_; + runtime::CapabilitySet capabilities_; +}; + +std::shared_ptr make_builtin_audio_utils_loader(); + +} // namespace engine::models::builtin_audio_utils diff --git a/src/framework/audio/utility_api.cpp b/src/framework/audio/utility_api.cpp index 70b2905ee..73b5f069b 100644 --- a/src/framework/audio/utility_api.cpp +++ b/src/framework/audio/utility_api.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -94,8 +95,72 @@ bool is_gtcrn_model(std::string_view model) { return model == "gtcrn" || model == "gtcrn_dns3" || model == "gtcrn_vctk" || model == "gtcrn_streaming"; } +constexpr std::array kBuiltinUtilities{{ + {"deepfilternet2", BuiltinAudioUtilityKind::Denoise, 48000, 48000}, + {"rnnoise", BuiltinAudioUtilityKind::Denoise, 48000, 48000}, + {"zipenhancer", BuiltinAudioUtilityKind::Denoise, 16000, 16000}, + {"gtcrn", BuiltinAudioUtilityKind::Denoise, 16000, 16000}, + {"gtcrn_streaming", BuiltinAudioUtilityKind::Denoise, 16000, 16000}, + {"gtcrn_dns3", BuiltinAudioUtilityKind::Denoise, 16000, 16000}, + {"gtcrn_vctk", BuiltinAudioUtilityKind::Denoise, 16000, 16000}, + {"flashsr", BuiltinAudioUtilityKind::SuperResolve, 16000, 48000}, +}}; + } // namespace +std::filesystem::path default_audio_utility_assets_root() { + return std::filesystem::path("assets/framework/audio_utilities"); +} + +std::vector list_builtin_audio_utilities() { + return std::vector(kBuiltinUtilities.begin(), kBuiltinUtilities.end()); +} + +std::optional find_builtin_audio_utility(std::string_view model) { + const auto it = std::find_if( + kBuiltinUtilities.begin(), + kBuiltinUtilities.end(), + [&](const BuiltinAudioUtilityInfo & info) { + return info.id == model; + }); + if (it == kBuiltinUtilities.end()) { + return std::nullopt; + } + return *it; +} + +BuiltinAudioUtilityInfo require_builtin_audio_utility(std::string_view model) { + if (auto info = find_builtin_audio_utility(model)) { + return *info; + } + throw_unsupported_model( + "builtin_audio_utils", + model, + "deepfilternet2, rnnoise, zipenhancer, gtcrn, gtcrn_streaming, gtcrn_dns3, gtcrn_vctk, flashsr"); +} + +std::filesystem::path resolve_builtin_audio_utility_asset( + const AudioUtilityPaths & paths, + std::string_view model) { + (void) require_builtin_audio_utility(model); + if (model == "deepfilternet2") { + return require_model_dir(paths, "deepfilternet2"); + } + if (model == "rnnoise") { + return require_model_dir(paths, "rnnoise") / "rnnoise10Gb_15.safetensors"; + } + if (model == "zipenhancer") { + return require_model_dir(paths, "zipenhancer"); + } + if (is_gtcrn_model(model)) { + return gtcrn_checkpoint_for(paths, model); + } + if (model == "flashsr") { + return require_model_dir(paths, "flashsr"); + } + throw_unsupported_model("builtin_audio_utils", model, ""); +} + void denoise_file( const std::filesystem::path & input_wav, const std::filesystem::path & output_wav, diff --git a/src/framework/runtime/registry.cpp b/src/framework/runtime/registry.cpp index 9b7dabcf5..a22d39cc5 100644 --- a/src/framework/runtime/registry.cpp +++ b/src/framework/runtime/registry.cpp @@ -1,6 +1,7 @@ #include "engine/framework/runtime/registry.h" #include "engine/framework/debug/trace.h" +#include "engine/framework/audio/utility_api.h" #include "engine/framework/model_spec/package.h" #include "engine/framework/io/config.h" #include "engine/framework/io/filesystem.h" @@ -149,7 +150,13 @@ std::unique_ptr ModelRegistry::load(const std::filesystem::pa } void ModelRegistry::validate_request(const ModelLoadRequest & request) const { - if (!engine::io::is_existing_file(request.model_path) && !engine::io::is_existing_directory(request.model_path)) { + const bool builtin_audio_utility_id = + request.family_hint.has_value() && + *request.family_hint == "builtin_audio_utils" && + engine::audio::find_builtin_audio_utility(request.model_path.generic_string()).has_value(); + if (!builtin_audio_utility_id && + !engine::io::is_existing_file(request.model_path) && + !engine::io::is_existing_directory(request.model_path)) { throw std::runtime_error("model path does not exist: " + request.model_path.string()); } if (request.family_hint.has_value() && !supports_family(*request.family_hint)) { diff --git a/src/models/builtin_audio_utils/session.cpp b/src/models/builtin_audio_utils/session.cpp new file mode 100644 index 000000000..24915d078 --- /dev/null +++ b/src/models/builtin_audio_utils/session.cpp @@ -0,0 +1,262 @@ +#include "engine/models/builtin_audio_utils/session.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/deepfilternet2.h" +#include "engine/framework/audio/flashsr.h" +#include "engine/framework/audio/gtcrn.h" +#include "engine/framework/audio/rnnoise.h" +#include "engine/framework/audio/utility_api.h" +#include "engine/framework/audio/zipenhancer.h" +#include "engine/framework/debug/profiler.h" + +#include +#include +#include +#include +#include + +namespace engine::models::builtin_audio_utils { +namespace { + +using UtilityRuntime = std::variant< + engine::audio::DeepFilterNet2Model, + engine::audio::RnnoiseModel, + engine::audio::ZipEnhancerModel, + engine::audio::GTCRNModel, + engine::audio::FlashSrModel>; + +runtime::CapabilitySet builtin_audio_utils_capabilities() { + runtime::CapabilitySet out; + out.supported_tasks = { + {runtime::VoiceTaskKind::SpeechToSpeech, {runtime::RunMode::Offline}}, + }; + return out; +} + +runtime::ModelMetadata builtin_audio_utils_metadata(const std::string & model_id) { + const auto info = engine::audio::require_builtin_audio_utility(model_id); + runtime::ModelMetadata out; + out.family = "builtin_audio_utils"; + out.variant = model_id; + out.description = info.kind == engine::audio::BuiltinAudioUtilityKind::SuperResolve + ? "Built-in audio utility: super resolution." + : "Built-in audio utility: denoise/enhance."; + return out; +} + +std::string operation_name(engine::audio::BuiltinAudioUtilityKind kind) { + switch (kind) { + case engine::audio::BuiltinAudioUtilityKind::Denoise: + return "denoise"; + case engine::audio::BuiltinAudioUtilityKind::SuperResolve: + return "super_resolve"; + } + return "unknown"; +} + +UtilityRuntime load_utility_runtime( + const std::string & model_id, + const engine::audio::AudioUtilityPaths & paths) { + const auto asset = engine::audio::resolve_builtin_audio_utility_asset(paths, model_id); + if (model_id == "deepfilternet2") { + return engine::audio::DeepFilterNet2Model::load_from_directory(asset, paths.backend); + } + if (model_id == "rnnoise") { + return engine::audio::RnnoiseModel::load_from_safetensors(asset, paths.backend); + } + if (model_id == "zipenhancer") { + return engine::audio::ZipEnhancerModel::load_from_directory(asset, paths.backend); + } + if (model_id == "gtcrn" || model_id == "gtcrn_streaming" || + model_id == "gtcrn_dns3" || model_id == "gtcrn_vctk") { + return engine::audio::GTCRNModel::load_from_safetensors(asset, paths.backend); + } + if (model_id == "flashsr") { + return engine::audio::FlashSrModel::load_from_directory(asset, paths.backend); + } + (void) engine::audio::require_builtin_audio_utility(model_id); + throw std::runtime_error("unreachable builtin audio utility model: " + model_id); +} + +runtime::AudioBuffer audio_buffer_from_mono( + int sample_rate, + std::vector samples) { + runtime::AudioBuffer output; + output.sample_rate = sample_rate; + output.channels = 1; + output.samples = std::move(samples); + return output; +} + +class BuiltinAudioUtilsSession final + : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession { +public: + BuiltinAudioUtilsSession( + runtime::TaskSpec task, + const runtime::SessionOptions & options, + std::string model_id) + : runtime::RuntimeSessionBase(options), + task_(task), + model_id_(std::move(model_id)), + info_(engine::audio::require_builtin_audio_utility(model_id_)), + runtime_(load_utility_runtime( + model_id_, + engine::audio::AudioUtilityPaths{ + engine::audio::default_audio_utility_assets_root(), + options.backend})) { + if (task_.task != runtime::VoiceTaskKind::SpeechToSpeech) { + throw std::runtime_error("builtin_audio_utils only supports --task s2s"); + } + if (task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("builtin_audio_utils only supports offline mode"); + } + } + + std::string family() const override { + return "builtin_audio_utils"; + } + + runtime::VoiceTaskKind task_kind() const override { + return task_.task; + } + + runtime::RunMode run_mode() const override { + return task_.mode; + } + + void prepare(const runtime::SessionPreparationRequest &) override { + mark_prepared(); + } + + runtime::TaskResult run(const runtime::TaskRequest & request) override { + require_prepared("builtin_audio_utils run"); + if (!request.audio_input.has_value()) { + throw std::runtime_error("builtin_audio_utils requires --audio"); + } + const auto & input = *request.audio_input; + const auto prep_start = std::chrono::steady_clock::now(); + const auto mono = engine::audio::convert_interleaved_audio_to_mono_linear_resampled( + input.samples, + input.sample_rate, + input.channels, + info_.input_sample_rate); + engine::debug::timing_log_scalar( + "builtin_audio_utils.audio_prepare_ms", + engine::debug::elapsed_ms(prep_start)); + + const auto compute_start = std::chrono::steady_clock::now(); + runtime::TaskResult result; + if (model_id_ == "deepfilternet2") { + const auto output = std::get(runtime_).run_mono_48k(mono); + result.audio_output = audio_buffer_from_mono(output.sample_rate, output.samples); + } else if (model_id_ == "rnnoise") { + const auto output = std::get(runtime_).process_mono_48k(mono); + result.audio_output = audio_buffer_from_mono(output.sample_rate, output.samples); + } else if (model_id_ == "zipenhancer") { + const auto output = std::get(runtime_).denoise_mono_16k(mono); + result.audio_output = audio_buffer_from_mono(output.sample_rate, output.samples); + } else if (model_id_ == "gtcrn" || model_id_ == "gtcrn_streaming" || + model_id_ == "gtcrn_dns3" || model_id_ == "gtcrn_vctk") { + const auto output = std::get(runtime_).denoise_mono_16k(mono); + result.audio_output = audio_buffer_from_mono(output.sample_rate, output.samples); + } else if (model_id_ == "flashsr") { + const auto output = std::get(runtime_).super_resolve_mono_16k(mono); + result.audio_output = audio_buffer_from_mono(output.sample_rate, output.samples); + } else { + (void) engine::audio::require_builtin_audio_utility(model_id_); + } + engine::debug::timing_log_scalar( + "builtin_audio_utils." + operation_name(info_.kind) + "_ms", + engine::debug::elapsed_ms(compute_start)); + return result; + } + +private: + runtime::TaskSpec task_; + std::string model_id_; + engine::audio::BuiltinAudioUtilityInfo info_; + UtilityRuntime runtime_; +}; + +class BuiltinAudioUtilsLoader final : public runtime::IVoiceModelLoader { +public: + std::string family() const override { + return "builtin_audio_utils"; + } + + runtime::CapabilitySet advertised_capabilities() const override { + return builtin_audio_utils_capabilities(); + } + + bool can_load(const runtime::ModelLoadRequest & request) const override { + if (!request.family_hint.has_value() || *request.family_hint != family()) { + return false; + } + return engine::audio::find_builtin_audio_utility(request.model_path.generic_string()).has_value(); + } + + runtime::ModelInspection inspect(const runtime::ModelLoadRequest & request) const override { + const auto model_id = request.model_path.generic_string(); + const auto info = engine::audio::require_builtin_audio_utility(model_id); + runtime::ModelInspection inspection; + inspection.model_root = engine::audio::default_audio_utility_assets_root(); + inspection.metadata = builtin_audio_utils_metadata(model_id); + inspection.capabilities = builtin_audio_utils_capabilities(); + inspection.discovered_weights.push_back(runtime::NamedAsset{ + model_id, + engine::audio::resolve_builtin_audio_utility_asset( + engine::audio::AudioUtilityPaths{engine::audio::default_audio_utility_assets_root()}, + model_id)}); + inspection.cli.request_options.push_back(runtime::CliOptionInfo{ + "audio", + "wav", + "Input WAV audio to process.", + true}); + inspection.cli.request_options.push_back(runtime::CliOptionInfo{ + "out", + "wav", + "Output WAV path.", + true}); + inspection.cli.load_options.push_back(runtime::CliOptionInfo{ + "operation", + operation_name(info.kind), + "Selected built-in audio utility operation.", + false, + operation_name(info.kind)}); + return inspection; + } + + std::unique_ptr load(const runtime::ModelLoadRequest & request) const override { + return std::make_unique(request.model_path.generic_string()); + } +}; + +} // namespace + +BuiltinAudioUtilsLoadedModel::BuiltinAudioUtilsLoadedModel(std::string model_id) + : model_id_(std::move(model_id)), + metadata_(builtin_audio_utils_metadata(model_id_)), + capabilities_(builtin_audio_utils_capabilities()) { + (void) engine::audio::require_builtin_audio_utility(model_id_); +} + +const runtime::ModelMetadata & BuiltinAudioUtilsLoadedModel::metadata() const noexcept { + return metadata_; +} + +const runtime::CapabilitySet & BuiltinAudioUtilsLoadedModel::capabilities() const noexcept { + return capabilities_; +} + +std::unique_ptr BuiltinAudioUtilsLoadedModel::create_task_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options) const { + return std::make_unique(task, options, model_id_); +} + +std::shared_ptr make_builtin_audio_utils_loader() { + return std::make_shared(); +} + +} // namespace engine::models::builtin_audio_utils From c515e9e08bde2567e747aa2dd5ec20fff224da0d Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:18:34 -0400 Subject: [PATCH 2/2] Add builtin audio utility model spec --- model_specs/builtin_audio_utils.json | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 model_specs/builtin_audio_utils.json diff --git a/model_specs/builtin_audio_utils.json b/model_specs/builtin_audio_utils.json new file mode 100644 index 000000000..7b5a574c8 --- /dev/null +++ b/model_specs/builtin_audio_utils.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "family": "builtin_audio_utils", + "display_name": "Built-in Audio Utilities", + "description": "Built-in denoise, enhancement, and audio super-resolution utilities packaged with audio.cpp.", + "category": "audio_tools", + "status": "experimental", + "tasks": [ + "s2s" + ], + "modes": [ + "offline" + ], + "languages": [ + "auto" + ], + "runtime": { + "tags": [ + "cpu", + "cuda" + ] + }, + "capabilities": { + "s2s": [ + "audio_enhancement" + ] + }, + "options": { + "request": [], + "session": [], + "load": [] + }, + "packages": [], + "dependencies": [], + "ui": { + "tags": [], + "docs": [ + "docs/audio_tools.md" + ] + }, + "sources": [] +}