diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index b9d9c898..6c8217dc 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -1,10 +1,10 @@ --- title: "Diffusion 共享规则" created: 2026-07-20 -updated: 2026-10-06 +updated: 2026-10-09 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #7648", "PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", "PR #6989", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_multiproc_engine_concurrency.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150", "PR #6410", tests/diffusion/layers/test_activation.py, tests/diffusion/layers/test_fused_qk_norm_rope_npu.py] +sources: ["PR #7648", "PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", "PR #6989", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_multiproc_engine_concurrency.py, "PR #4755", "PR #5990", vllm_omni/diffusion/layers/fused_qk_norm_rope.py, vllm_omni/diffusion/cache/teacache/extractors.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, tests/diffusion/layers/test_fused_qk_norm_rope.py, "PR #6165", "PR #5677", vllm_omni/diffusion/quantization/hsdp_fp8.py, tests/diffusion/quantization/test_hsdp_fp8.py, "PR #4845", "PR #6173", "PR #6070", vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/model_loader/hub_prefetch.py, "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", vllm_omni/diffusion/vllm_config.py, "PR #6283", vllm_omni/diffusion/layers/activation.py, "PR #6281", vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py, "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150", "PR #6410", tests/diffusion/layers/test_activation.py, tests/diffusion/layers/test_fused_qk_norm_rope_npu.py, "PR #8510"] confidence: high --- @@ -231,3 +231,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 验收：schema 回归断言 mutable 参数无默认值；staging 关闭/开启均执行 eager 与 `fullgraph=True`。验证 tensor 内容变化不重编译时固定 host scalar 和 history phase， 将空历史→满历史引发的合法 scalar specialization 与 tensor recompile 分开。 ^[PR #7648] + +## VLLM-OMNI-PR8510-MHC-FP32-MUSA — MAGI-2 mHC 收缩的 MUSA 形态必须保留 einsum 回退、误差界与 stream-major 布局 + +- 触发：修改 `vllm_omni/diffusion/models/magi2/layers.py` 中 `MHCHandler` 的 `apply_pre`、stream mix（`hyper_connect`）、`compute_logits`、`_stream_major_normed` 或 `__init__` 里的平台分派；或改动 mHC stream contraction 的精度、dtype 与内存布局。 +- 强制：MUSA 分派只在 `MHCHandler.__init__` 判定一次（`fp32_stream_contractions`）。仅 MUSA 上 `apply_pre` 与编译态 stream mix 使用 FP32 逐元素乘加，`apply_pre` 输出保持流的输入 dtype；`compute_logits` 按流拆 K 为分批 matmul（`bmm`）加求和，其 dtype 跟随操作数：`_stream_major_normed` 把归一输出 cast 到 `self.dtype`，`bmm` 使用的 `phi_fused` 不做额外 FP32 cast，因此只有这些操作数本身是 FP32 时该 matmul 才是 FP32，不得写成无条件 FP32 matmul。FP32 逐元素形态只接受 exact-shaped `[T, N]` 系数、流 dtype（fp32/bf16/fp16）且不在 autocast 中；该准入不含流数条件，exact-shaped `[T, 1]` 单流系数同样满足 FP32 分支。broadcast 系数、float64、mixed-dtype、autocast、额外 rank 输入必须继续走 einsum 路径并保留其原有错误（mixed-dtype/额外 rank 抛 RuntimeError）。非 MUSA 平台必须与 einsum/matmul 形式逐位一致（rtol=0、atol=0）；共享层 `vllm_omni/diffusion/layers/mhc.py` 不得为此改动，eager MUSA 的 mix 仍走 #7261 的 Triton kernel。 +- 强制：编译态下 `_stream_major_normed` 在 `[tokens, streams, hidden]` 视图上做 norm，并以 `as_strided` 钉住 stream-major buffer 的 strides，让 fused norm kernel 自己写出稠密 `[streams, tokens, hidden]` 布局、`bmm` 原地读取；数值型 compile 测试必须钉住 Inductor 确定性（`torch._inductor.config.deterministic` 与 `test_configs.force_filter_reduction_configs`），否则每次编译 benchmark reduction config 会改变位数。 +- 禁止：把 MUSA 形态做成唯一路径或改变非 MUSA 数值；把 `compute_logits` 声称或审成无条件 FP32 matmul——其 dtype 由 `self.dtype` 的归一输出与未 cast 的 `phi_fused` 决定；eager 中先把乘积 round 到流 dtype 再求和（乘积必须在 FP32 中形成）；吞掉 einsum 路径对非法输入的报错；给 FP32 分支加入流数排除，或声称/要求单流输入走 einsum 路径——单流只受数值约束，测试不断言其走了哪条分支；用裸 `.contiguous()` 声称已消除 Inductor 下的拷贝（clone 会融合进 reduction 并保持 token-major 布局）；声称端到端 bitwise 一致——生成结果相对 einsum 形态确有差异（视频 PSNR 24.9 dB、音频 37.4 dB），幅度与无关的 SP4xCFG2→SP8xCFG1 切换（24.0 dB、38.0 dB）相当，只能按此格式论证差异可接受。 +- 验收：CPU 测试 monkeypatch `current_omni_platform.is_musa` 构造两种形态：非 MUSA 三种流 dtype 下 `apply_pre` 与 einsum、`compute_logits` 与 `flattened @ phi` 参考逐位一致；`is_compiling()` 下 MUSA 的 mix 走 `_mhc_mix_fp32`、其他平台走 `forward_native`；MUSA `apply_pre` 对 FP64 参考满足“一次输出舍入 + 2×项数×2^-24×幅值 + tiny”的界，且与先 round 再求和的结果不逐位相等；broadcast/float64 保持 einsum 数值，单流结果与 einsum 完全一致（仅数值断言，不断言分支），mixed-dtype/额外 rank 仍抛 RuntimeError；`_mhc_mix_fp32` 保留 float64；`compute_logits` 在 FP32 操作数下按流拆 K 并满足同类 FP64 误差界。CPU+monkeypatch 只证明形态与误差界，不构成真实 MUSA 硬件上的性能或 parity 证据。^[PR #8510] + +