Skip to content

Add CUDA lowering implementations #533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
0xShug0 wants to merge 3 commits into
base: ggml-wan-lowering-core-apis
Choose a base branch
from
Open

Add CUDA lowering implementations #533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9e37a1b
Release v0.7.4
0xShug0 Sep 12, 2026
78458ef
Add explicit ggml lowering APIs
0xShug0 Sep 12, 2026
01842a6
Add CUDA lowering implementations
0xShug0 Sep 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 47 additions & 26 deletions CONTRIBUTING.md
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 2 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@ ModelScope repo mirror: https://www.modelscope.cn/models/HereIsMark/audio.cpp-gg

> [!IMPORTANT]
>
> **2026-09-10 - Dev testing: Yue2 3B:** Yue2 native song generation is available on the [dev branch](https://github.com/0xShug0/audio.cpp/tree/dev) for community testing and optimization.
>
> **2026-09-09 - VibeVoice ASR Streaming 7B and Irodori-TTS v4.1 Anime:** New GGUF packages are available for streaming VibeVoice ASR 7B and the anime fine-tuned Irodori-TTS v4.1 Small variant.
> **2026-09-10 - Dev testing: Yue2 3B:** Yue2 native song generation is available on the [dev branch](https://github.com/0xShug0/audio.cpp/tree/dev) for community testing and optimization. Dev binaries are available from [Actions -> Release -> latest dev artifacts](https://github.com/0xShug0/audio.cpp/actions/workflows/release.yml?query=branch%3Adev).
>
> **Arena UI:** The new Arena tab makes it easier to compare local models side by side for TTS, voice conversion, and ASR. Use one shared input, queue multiple models or GGUF variants, then review outputs with metrics!
>
Expand Down Expand Up @@ -53,7 +51,7 @@ audio.cpp would not be moving this quickly without generous contributors bringin
## News

> [!IMPORTANT]
> **2026-09-04 - Release 0.7.2:** This release adds BreezeTTS 2, CosyVoice3, Chatterbox Turbo TTS, Audio8 TTS, and Audio8 ASR, plus the new multipart audio alignment endpoint.
> **2026-09-12 - Release 0.7.4:** This release adds VibeVoice ASR Streaming 7B, Irodori-TTS v4.1 Anime, Moonshine Streaming ASR, and Kokoro 82M, plus GGUF/package and UI updates for the latest community models. Thanks [@DrewThomasson](https://github.com/DrewThomasson) for the Colab UI!
>
> **2026-08-26 - Release 0.7:** This release adds MiniMax Music 3, MagpieTTS, PersonaPlex, MeanVC2, AudioSR, ControlFoley, FireRedTTS3, FireRedAudio, MiDashengLM-Gen, F5-TTS/Habibi, Granite Speech 5.0 TurboCTC, MMS Forced Aligner, and MOSS-VoiceGenerator, plus DotTTS Edit and ACE-Step 1.5 XL variants, bringing audio.cpp to **62** total model families and **85+** model variants! It also introduces the new Arena UI for side-by-side TTS, voice-conversion, and ASR comparison with shared inputs, queued runs, metrics, and result sorting.
>
Expand Down
117 changes: 115 additions & 2 deletions external/ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,8 +448,14 @@ extern "C" {

// op hint
enum ggml_op_hint {
GGML_HINT_NONE = 0,
GGML_HINT_SRC0_IS_HADAMARD = 1,
GGML_HINT_NONE = 0,
GGML_HINT_SRC0_IS_HADAMARD = 1,
};

enum ggml_mul_mat_lowering {
GGML_MUL_MAT_LOWERING_DEFAULT = 0,
GGML_MUL_MAT_LOWERING_CUDA_NVFP4_F16_ACTIVATION = 2,
GGML_MUL_MAT_LOWERING_CUDA_TILE_F16_ACCUM_OUTPUT = 3,
};

// model file types
Expand Down Expand Up @@ -604,6 +610,11 @@ extern "C" {
GGML_OP_MUL_MAT_ADD,
GGML_OP_MUL_MAT_ADD_RELU,
GGML_OP_IM2COL_ASYM,
GGML_OP_CONV_3D_CONCAT_PAD_SPATIAL_GEMM,
GGML_OP_RMS_NORM_CHANNELS,
GGML_OP_RMS_NORM_CHANNELS_SILU,
GGML_OP_RMS_NORM_CHANNELS_ADD_BIAS_SILU,
GGML_OP_ROPE_INTERLEAVED_PAIRS,

GGML_OP_COUNT,
};
Expand Down Expand Up @@ -678,6 +689,33 @@ extern "C" {
GGML_TRI_TYPE_LOWER = 3
};

enum ggml_concat_lowering {
GGML_CONCAT_LOWERING_DEFAULT = 0,
GGML_CONCAT_LOWERING_CUDA_CONTIGUOUS_4D = 1,
};

enum ggml_im2col_2d_lowering {
GGML_IM2COL_2D_LOWERING_DEFAULT = 0,
GGML_IM2COL_2D_LOWERING_CUDA_N_K3_PAD1_X8 = 1,
GGML_IM2COL_2D_LOWERING_CUDA_N_K3_NOPAD_X8 = 2,
};

enum ggml_im2col_3d_lowering {
GGML_IM2COL_3D_LOWERING_DEFAULT = 0,
GGML_IM2COL_3D_LOWERING_CUDA_N1_K3_NOPAD_X8 = 1,
};

enum ggml_rms_norm_channels_lowering {
GGML_RMS_NORM_CHANNELS_LOWERING_DEFAULT = 0,
GGML_RMS_NORM_CHANNELS_LOWERING_CUDA_COALESCED = 1,
};

enum ggml_conv_3d_concat_pad_spatial_gemm_lowering {
GGML_CONV_3D_CONCAT_PAD_SPATIAL_GEMM_LOWERING_DEFAULT = 0,
GGML_CONV_3D_CONCAT_PAD_SPATIAL_GEMM_LOWERING_CUDA_C48 = 1,
GGML_CONV_3D_CONCAT_PAD_SPATIAL_GEMM_LOWERING_CUDA_TILED_C48 = 2,
};

struct ggml_init_params {
// memory pool
size_t mem_size; // bytes
Expand Down Expand Up @@ -1110,6 +1148,46 @@ extern "C" {
struct ggml_tensor * b,
int dim);

GGML_API struct ggml_tensor * ggml_rope_interleaved_pairs(
struct ggml_context * ctx,
struct ggml_tensor * even,
struct ggml_tensor * odd,
struct ggml_tensor * cos,
struct ggml_tensor * sin);

GGML_API void ggml_concat_set_lowering(
struct ggml_tensor * tensor,
enum ggml_concat_lowering lowering);

GGML_API struct ggml_tensor * ggml_conv_3d_concat_pad_spatial_gemm(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * w,
int lp0,
int rp0,
int lp1,
int rp1,
int lp2,
int rp2);

GGML_API struct ggml_tensor * ggml_conv_3d_concat_pad_spatial_gemm_ex(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * w,
int lp0,
int rp0,
int lp1,
int rp1,
int lp2,
int rp2,
enum ggml_type dst_type);

GGML_API void ggml_conv_3d_concat_pad_spatial_gemm_set_lowering(
struct ggml_tensor * tensor,
enum ggml_conv_3d_concat_pad_spatial_gemm_lowering lowering);

GGML_API struct ggml_tensor * ggml_abs(
struct ggml_context * ctx,
struct ggml_tensor * a);
Expand Down Expand Up @@ -1406,6 +1484,29 @@ extern "C" {
struct ggml_tensor * a,
float eps);

GGML_API struct ggml_tensor * ggml_rms_norm_channels(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * gamma,
float eps);

GGML_API struct ggml_tensor * ggml_rms_norm_channels_silu(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * gamma,
float eps);

GGML_API struct ggml_tensor * ggml_rms_norm_channels_add_bias_silu(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * bias,
struct ggml_tensor * gamma,
float eps);

GGML_API void ggml_rms_norm_channels_set_lowering(
struct ggml_tensor * tensor,
enum ggml_rms_norm_channels_lowering lowering);

// group normalize along ne0*ne1*n_groups
// used in stable-diffusion
GGML_API struct ggml_tensor * ggml_group_norm(
Expand Down Expand Up @@ -1464,6 +1565,10 @@ extern "C" {
struct ggml_tensor * a,
enum ggml_op_hint hint);

GGML_API void ggml_mul_mat_set_lowering(
struct ggml_tensor * a,
enum ggml_mul_mat_lowering lowering);

// indirect matrix multiplication
GGML_API struct ggml_tensor * ggml_mul_mat_id(
struct ggml_context * ctx,
Expand Down Expand Up @@ -2028,6 +2133,10 @@ extern "C" {
bool is_2D,
enum ggml_type dst_type);

GGML_API void ggml_im2col_2d_set_lowering(
struct ggml_tensor * tensor,
enum ggml_im2col_2d_lowering lowering);

GGML_API struct ggml_tensor * ggml_im2col_back(
struct ggml_context * ctx,
struct ggml_tensor * a, // convolution kernel
Expand Down Expand Up @@ -2125,6 +2234,10 @@ extern "C" {
int d2, // dilation depth
enum ggml_type dst_type);

GGML_API void ggml_im2col_3d_set_lowering(
struct ggml_tensor * tensor,
enum ggml_im2col_3d_lowering lowering);

// a: [OC*IC, KD, KH, KW]
// b: [N*IC, ID, IH, IW]
// result: [N*OC, OD, OH, OW]
Expand Down
Loading