Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ add_library(engine_core OBJECT
src/framework/runtime/model.cpp
src/framework/model_spec/metadata.cpp
src/framework/runtime/session.cpp
src/framework/runtime/task_vocabulary.cpp
src/framework/runtime/artifacts.cpp
src/framework/runtime/cache.cpp
src/framework/runtime/graph_executor.cpp
Expand Down Expand Up @@ -2538,6 +2539,10 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
add_test(NAME transformer_kv_ring_test COMMAND transformer_kv_ring_test)

add_engine_unittest(model_spec_system_test tests/unittests/test_model_spec_system.cpp)
# So the test can read the specs this repository ships, rather than a
# copy of them staged somewhere.
target_compile_definitions(model_spec_system_test
PRIVATE AUDIOCPP_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
target_include_directories(model_spec_system_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests)
add_test(NAME model_spec_system_test COMMAND model_spec_system_test)
endif()
Expand Down
67 changes: 64 additions & 3 deletions include/audiocpp.h
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,33 @@ AUDIOCPP_API audiocpp_status audiocpp_registry_family(const audiocpp_registry *
size_t index,
const char ** out_family);

/* ---- Task vocabulary -------------------------------------------------------
*
* The tasks this build knows, askable without a model. Every other task-aware
* entry point takes an audiocpp_model, so a caller that is deciding what to
* install -- reading model_specs/*.json to build a picker, say -- has had
* nothing to ask and has had to hardcode a copy of the table in
* src/framework/model_spec/metadata.cpp.
*
* Model specs and this ABI use different spellings for the same kinds: a spec
* says "music", "sfx", "edit" or "audio_generation" where this says "gen",
* "clone" for "clon", "design" for "vdes", "speaker" for "spk". Nine of the
* fourteen are identical, which is what makes comparing them directly appear
* to work.
*/

/* How many task tokens this build accepts. */
AUDIOCPP_API size_t audiocpp_task_count(void);

/* The canonical token at `index`, or NULL when out of range. The returned
* pointer is static and outlives any call. */
AUDIOCPP_API const char * audiocpp_task_name(size_t index);

/* The canonical token for a model-spec task name ("music" -> "gen"), or NULL
* when the name names no task kind -- which is also how a caller detects a
* spec declaring a task this build cannot serve. */
AUDIOCPP_API const char * audiocpp_task_from_spec_name(const char * spec_task);

/* ------------------------------------------------------------------ */
/* Model */
/* ------------------------------------------------------------------ */
Expand All @@ -129,6 +156,7 @@ AUDIOCPP_API audiocpp_status audiocpp_registry_family(const audiocpp_registry *
* config_id --config, for packages that ship several configs
* weight_id --weight, for packages that ship several weight sets
* model_spec_override --model-spec-override */

typedef struct audiocpp_model_config {
const char * family_hint;
const char * config_id;
Expand All @@ -147,9 +175,20 @@ AUDIOCPP_API void audiocpp_model_free(audiocpp_model * model);
AUDIOCPP_API const char * audiocpp_model_family(const audiocpp_model * model);
AUDIOCPP_API const char * audiocpp_model_description(const audiocpp_model * model);

/* Capability queries. task/mode are the same spellings the CLI accepts,
* e.g. "tts", "asr", "vad", "diarization", "alignment" / "offline",
* "streaming". Returns 1 when supported, 0 when not or when unrecognised. */
/* Capability queries. `task` is one of the tokens audiocpp_task_name()
* enumerates -- "vad", "asr", "diar", "sep", "gen", "tts", "clon", "vc",
* "s2s", "align", "vdes", "spk", "svc", "midi" -- and `mode` is "offline" or
* "streaming".
*
* Returns 1 when supported and 0 otherwise, which includes a task or mode this
* build does not recognise: a caller cannot tell a misspelled question from a
* negative answer. Validate against audiocpp_task_name() first if that
* distinction matters.
*
* (This comment previously gave "diarization" and "alignment" as examples.
* Neither has ever parsed, so a caller following it was told "no" for every
* model that does diarize, with nothing to indicate the question was
* malformed.) */
AUDIOCPP_API int audiocpp_model_supports(const audiocpp_model * model,
const char * task,
const char * mode);
Expand Down Expand Up @@ -236,10 +275,32 @@ AUDIOCPP_API void audiocpp_request_free(audiocpp_request * request
* "language" option, because that is what audiocpp_cli's --language does and
* some families read only the option. A later audiocpp_request_set_option with
* the same key overrides it. */
/* Sets the request text, and -- when `language` is non-NULL and non-empty --
* both the transcript language and options["language"], mirroring what
* audiocpp_cli's --language does. Some families read only the option, so the
* two travel together by default.
*
* That coupling cannot be undone through this call: pass NULL and use
* audiocpp_request_set_text_language() below to set the transcript language
* alone. Needed because "does this model declare a language option" and "does
* this model need a transcript language" are different questions with
* different answers -- parakeet_tdt validates its request options strictly and
* refuses a language it does not declare, while qwen3_forced_aligner declares
* no language option and requires the transcript language anyway. */
AUDIOCPP_API audiocpp_status audiocpp_request_set_text(audiocpp_request * request,
const char * text,
const char * language);

/* Sets the transcript language without touching options["language"].
*
* The reference server needs exactly this and reaches around the ABI for it
* (drop_unsupported_language_option in app/server/runtime.cpp erases the option
* and keeps the transcript language). A C ABI client could not express that:
* set_text is the only way to reach the transcript language and it writes the
* option as a side effect, so the two arrived together or not at all. */
AUDIOCPP_API audiocpp_status audiocpp_request_set_text_language(audiocpp_request * request,
const char * language);

/* Interleaved float PCM. Copied into the request, so `samples` need not
* outlive the call. `frames` is per-channel. */
AUDIOCPP_API audiocpp_status audiocpp_request_set_audio(audiocpp_request * request,
Expand Down
56 changes: 56 additions & 0 deletions include/engine/framework/runtime/task_vocabulary.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#pragma once

#include "engine/framework/runtime/session.h"

#include <cstddef>
#include <string_view>

namespace engine::runtime {

/// One task kind, with every name any layer accepts for it.
///
/// The canonical token is what the C ABI and the CLI take and what `to_string`
/// returns. The aliases are the spellings a model spec may use: the two
/// vocabularies differ for historical reasons (`gen` vs `music`/`sfx`/`edit`),
/// and this is the only place that difference is written down.
/// Every view here must refer to a string literal.
///
/// The C ABI hands `token.data()` straight to callers as a `const char *`, so a
/// view over anything that is not NUL-terminated and statically allocated would
/// return a pointer into a temporary or an unterminated buffer. That holds for
/// the table below; it is a constraint on anything added to it.
struct TaskVocabularyEntry {
VoiceTaskKind kind;
std::string_view token;
/// Spec-side spellings, `aliases[alias_count]` onwards unused.
std::string_view aliases[4];
std::size_t alias_count;
};

/// Every task kind this build knows, in enum order.
///
/// This is the single definition of the task vocabulary. `to_string`,
/// `parse_voice_task_kind`, the model-spec schema's allowed task set, the
/// model-spec parser and the C ABI's enumeration all read it, so they cannot
/// disagree with one another.
///
/// They used to. The schema accepted `codec`, which no parser mapped to a kind,
/// so `model_specs/miocodec.json` threw at load; the spec parser accepted
/// `audio_generation`, which the schema rejected, so a spec using it failed
/// validation instead; and `include/audiocpp.h` documented `"diarization"` and
/// `"alignment"`, which nothing accepts. Four hand-maintained lists of the same
/// fourteen things drift in four directions, and nothing was comparing them.
const TaskVocabularyEntry * task_vocabulary(std::size_t & count) noexcept;

/// The canonical token for a spec-side task name, or an empty view when the
/// name names no task kind.
///
/// `"music"` -> `"gen"`. Exposed because a caller that reads `model_specs/*.json`
/// -- a package browser, an installer, a binding building a picker before
/// anything is loaded -- needs the mapping and has no model to ask.
std::string_view task_token_for_spec_name(std::string_view spec_task) noexcept;

/// Whether a spec may declare this task name.
bool is_spec_task_name(std::string_view value) noexcept;

} // namespace engine::runtime
1 change: 0 additions & 1 deletion model_specs/miocodec.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
"category": "audio_tools",
"status": "supported",
"tasks": [
"codec",
"vc",
"s2s"
],
Expand Down
2 changes: 1 addition & 1 deletion model_specs/moss_voicegen.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"category": "community",
"status": "community",
"tasks": [
"vdes"
"design"
],
"modes": [
"offline"
Expand Down
42 changes: 42 additions & 0 deletions src/capi/audiocpp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

#include "audiocpp.h"

#include "engine/framework/runtime/task_vocabulary.h"

#include "engine/framework/core/backend.h"
#include "engine/framework/core/module.h"
#include "engine/framework/runtime/model.h"
Expand Down Expand Up @@ -586,6 +588,46 @@ void audiocpp_request_free(audiocpp_request * request) {
delete request;
}

size_t audiocpp_task_count(void) {
size_t count = 0;
(void) rt::task_vocabulary(count);
return count;
}

const char * audiocpp_task_name(size_t index) {
size_t count = 0;
const auto * entries = rt::task_vocabulary(count);
if (index >= count) {
return nullptr;
}
/* Every token is a string literal in the table, so this outlives any call
* and the caller never owns it. */
return entries[index].token.data();
}

const char * audiocpp_task_from_spec_name(const char * spec_task) {
if (spec_task == nullptr) {
return nullptr;
}
const auto token = rt::task_token_for_spec_name(spec_task);
return token.empty() ? nullptr : token.data();
}

audiocpp_status audiocpp_request_set_text_language(audiocpp_request * request, const char * language) {
if (request == nullptr) {
return fail(AUDIOCPP_ERR_INVALID_ARGUMENT, "request must be non-null");
}
return guard([&] {
/* Deliberately not touching request->request.options: that is the whole
* difference between this and set_text's language argument. */
if (!request->request.text_input.has_value()) {
request->request.text_input = rt::Transcript{};
}
request->request.text_input->language = language != nullptr ? language : "";
return AUDIOCPP_OK;
});
}

audiocpp_status audiocpp_request_set_text(audiocpp_request * request, const char * text, const char * language) {
if (request == nullptr || text == nullptr) {
return fail(AUDIOCPP_ERR_INVALID_ARGUMENT, "request and text must be non-null");
Expand Down
61 changes: 19 additions & 42 deletions src/framework/model_spec/metadata.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "engine/framework/model_spec/metadata.h"
#include "engine/framework/runtime/task_vocabulary.h"

#include "engine/framework/model_spec/options.h"
#include "engine/framework/model_spec/package.h"
Expand All @@ -19,49 +20,25 @@ namespace {
namespace json = engine::io::json;

runtime::VoiceTaskKind parse_task_kind(const std::string & value) {
if (value == "vad") {
return runtime::VoiceTaskKind::Vad;
}
if (value == "asr") {
return runtime::VoiceTaskKind::Asr;
}
if (value == "diar") {
return runtime::VoiceTaskKind::Diarization;
}
if (value == "sep") {
return runtime::VoiceTaskKind::SourceSeparation;
}
if (value == "audio_generation" || value == "music" || value == "sfx" || value == "edit") {
return runtime::VoiceTaskKind::AudioGeneration;
}
if (value == "tts") {
return runtime::VoiceTaskKind::Tts;
}
if (value == "clone") {
return runtime::VoiceTaskKind::VoiceCloning;
}
if (value == "vc") {
return runtime::VoiceTaskKind::VoiceConversion;
}
if (value == "s2s") {
return runtime::VoiceTaskKind::SpeechToSpeech;
}
if (value == "align") {
return runtime::VoiceTaskKind::Alignment;
}
if (value == "design") {
return runtime::VoiceTaskKind::VoiceDesign;
}
if (value == "speaker") {
return runtime::VoiceTaskKind::SpeakerRecognition;
}
if (value == "svc") {
return runtime::VoiceTaskKind::Svc;
}
if (value == "midi") {
return runtime::VoiceTaskKind::Midi;
// Spec names, not ABI tokens: "music" here is "gen" there. The mapping is
// in runtime::task_vocabulary, which the schema's allowed set reads too, so
// a name the schema accepts cannot be one this rejects.
const auto token = runtime::task_token_for_spec_name(value);
if (!token.empty()) {
return runtime::parse_voice_task_kind(std::string(token));
}
std::string expected;
std::size_t count = 0;
const auto * entries = runtime::task_vocabulary(count);
for (std::size_t i = 0; i < count; ++i) {
for (std::size_t alias = 0; alias < entries[i].alias_count; ++alias) {
if (!expected.empty()) {
expected += ", ";
}
expected.append(entries[i].aliases[alias]);
}
}
throw std::runtime_error("unknown model spec task: " + value);
throw std::runtime_error("unknown model spec task: " + value + " (expected one of " + expected + ")");
}

runtime::RunMode parse_run_mode(const std::string & value) {
Expand Down
19 changes: 15 additions & 4 deletions src/framework/model_spec/schema.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "engine/framework/model_spec/schema.h"
#include "engine/framework/runtime/task_vocabulary.h"
#include "engine/framework/model_spec/options.h"

#include <algorithm>
Expand Down Expand Up @@ -65,10 +66,20 @@ void require_spec_number(const json::Value & value, std::string_view path) {
}

const std::unordered_set<std::string> & tasks() {
static const std::unordered_set<std::string> values = {
"vad", "asr", "diar", "sep", "music", "sfx", "edit", "tts", "clone", "vc",
"s2s", "align", "design", "speaker", "svc", "codec", "midi",
};
// Built from the one vocabulary rather than typed out again. The hand-kept
// copy had drifted both ways: it allowed "codec", which no parser maps to a
// task kind, and omitted "audio_generation", which the parser accepts.
static const std::unordered_set<std::string> values = [] {
std::unordered_set<std::string> names;
std::size_t count = 0;
const auto * entries = engine::runtime::task_vocabulary(count);
for (std::size_t i = 0; i < count; ++i) {
for (std::size_t alias = 0; alias < entries[i].alias_count; ++alias) {
names.emplace(entries[i].aliases[alias]);
}
}
return names;
}();
return values;
}

Expand Down
Loading
Loading