From 290117b6b9fd737e596ce6ba8a4c24ac0e587ac6 Mon Sep 17 00:00:00 2001 From: vllm-omni-review-bot Date: Mon, 21 Sep 2026 16:07:24 +0800 Subject: [PATCH] knowledge: daily intake batch 2026-09-21 --- knowledge/repos/vllm-omni/benchmark/rules.md | 7 +++++++ .../components/diffusion/rules-output-lifecycle.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/rules.md | 9 ++++++++- .../model-executor/rules-runtime-hot-paths.md | 9 ++++++++- knowledge/repos/vllm-omni/models/bagel/rules.md | 10 +++++++++- .../repos/vllm-omni/models/hunyuan-image3/rules.md | 9 ++++++++- .../vllm-omni/models/minimax-h3/rules-cache-task.md | 9 ++++++++- knowledge/repos/vllm-omni/models/moss-tts/rules.md | 9 ++++++++- 8 files changed, 64 insertions(+), 7 deletions(-) diff --git a/knowledge/repos/vllm-omni/benchmark/rules.md b/knowledge/repos/vllm-omni/benchmark/rules.md index d8146763f..744b9f7ec 100644 --- a/knowledge/repos/vllm-omni/benchmark/rules.md +++ b/knowledge/repos/vllm-omni/benchmark/rules.md @@ -72,3 +72,10 @@ confidence: high determinism 或 eager 数值等价证明。 - 验收：产物能追溯各 arm 的真实 source/config/environment，报告候选差值和控制漂移； 稳态统计可从相同筛选重算，正确性另由对应 parity 测试支持。 ^[PR #7648] + +## BENCH-1g — duplex/eval 媒体解码必须用捆绑 PyAV，不得依赖 host ffmpeg + +- 触发：修改 `omni-duplex-eval` 或同类 benchmark 的视频时长、帧抽取、音频 PCM 解码，或重新引入 `ffmpeg`/`ffprobe` subprocess。 +- 强制：媒体路径经 PyAV（`av`）打开/seek/decode/resample；结果不得依赖主机安装的 ffmpeg 版本。帧抽取语义对齐“首个 PTS ≥ 目标时刻”；非 WAV 音频经 `AudioResampler` 并在循环后 `resample(None)` flush 尾部样本。 +- 禁止：`subprocess` 调用 host `ffmpeg`/`ffprobe` 作为默认路径；把“本机 ffmpeg 能跑”当作可复现证据；在已知会挂起的旧 HEVC decoder 上无超时地阻塞 generate 阶段。 +- 验收：覆盖 duration/JPEG/PCM 合同与失败 raise；至少用曾触发 host ffmpeg 4.4.2 挂起的样本证明不再无限忙等。^[PR #7504] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 12223046b..745324915 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -1,7 +1,7 @@ --- title: "Diffusion output 与 multiprocess runtime 规则" created: 2026-09-02 -updated: 2026-09-08 +updated: 2026-09-21 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", "PR #6847", "PR #6953", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py, "PR #6294"] @@ -176,3 +176,10 @@ confidence: high - 强制：runner 与 pipeline 共有同一 `InteractionCoordinator`；仅对已注册 modality 构造 handler；`enqueue` 按 modality 路由；`apply_at_chunk_boundary` 以稳定顺序推进（prompt 优先，再及其他 modality）；per-request session 落在 `StepRequestState.interaction_sessions`，chunk ACK 元数据写入 `interaction_chunk_metadata`。未注册 modality 必须 fail closed。当前公开行为保持 prompt-only，直到另有 PR 注册并验收非 prompt handler。 - 禁止：在 denoise 中途旁路 coordinator 直接改 `prompt_embeds`；在非 chunk 边界静默应用 interaction；未注册就接受 camera/`multi_modal_data`；或把内部 handler 重构当成用户 API 已扩展。 - 验收：覆盖 prompt enqueue→chunk-boundary apply→ACK event id、未注册 modality 拒绝，以及无 session 的 modality 跳过；不得用本框架落地证明 camera/MM 路径已可用。^[PR #6294] + +## DIFF-1ai — 复合 mid-stream interaction 必须原子入队，非懒会话在 chunk0 前物化 + +- 触发：修改 `InteractionCoordinator.enqueue_parts`、`lazy_initialize_session`、camera/`multi_modal_data` 与 prompt 同事件入队、`synchronized_monotonic_time`，或 chunk-boundary apply 的 media/latent 计数。 +- 强制：含多 modality 的同一事件先全部 `validate_payload`，再入队；任一路不支持或非法不得部分改队列。`received_at`/boundary 时钟经 rank 同步，使 USP/SP 分片共享同一到达时刻。`lazy_initialize_session=False` 的 handler（如 camera）必须在首个 denoise chunk 前 `maybe_prepare_initial_session`，即使尚无 client enqueue。chunk apply 区分 `num_media_frames` 与 `num_latent_frames`，需要者才 `peek_chunk_media`。 +- 禁止：先入队再校验导致半更新；把未注册 camera 当已可用；懒会话与必须预创建会话混用同一 skip 逻辑；用未同步的 per-rank `time.monotonic()` 驱动跨分片时间线。 +- 验收：复合 prompt+camera 原子成功/失败、未注册拒绝、非懒会话在 chunk0 前存在、介质计数字段传播。^[PR #7198] diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index d693a2d07..e96a82b8f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -1,7 +1,7 @@ --- title: "Distributed 传输规则" created: 2026-08-05 -updated: 2026-09-08 +updated: 2026-09-21 type: rule tags: [vllm-omni, components, distributed] sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", "PR #6834", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529", "PR #7136"] @@ -142,3 +142,10 @@ confidence: high - 强制：CUDA Mooncake 依赖必须钉在包含 acknowledged TCP writes 的版本（`0.3.12` 起）；`batch_transfer_sync_write` 返回成功后，destination 内存必须已更新到可读状态，不能只表示本地 send 入队。回归必须在 receiver 被挂起时证明 completion 不会提前返回，恢复后再断言目的端字节一致。 - 禁止：降级到无 ACK 的 TCP 实现却不更新测试合同；把 RDMA 路径的 completion 语义外推到 TCP；或在 completion 前启动依赖 destination 内容的零拷贝消费。 - 验收：强制 TCP 的挂起-恢复测试覆盖「停止期间不完成、恢复后成功且字节匹配」；依赖声明与该测试同步变更。^[PR #7136] + +## DIST-1l — 原生 Mooncake AR→DiT KV 必须关 async_chunk 并按物理页序释放 + +- 触发：部署或修改 `kv_transfer_config`/`MooncakeConnector`、AR→DiT 原生 KV handoff、`OmniARScheduler` 的 block free，或与 legacy `OmniKVTransferManager` 并存的路径。 +- 强制：存在 stage `kv_transfer_config` 时必须 `async_chunk=False`。原生路径由 scheduler 拥有 diffusion KV 页；connector 失败 fail-close diffusion engine 后再拆页。非 caching P/D 池释放时按递增 `block_id` 归还，便于相邻页合并；deferred free 存逆序以匹配 vLLM drain。原生 transfer 边界用已确认 computed tokens，排除 async output placeholders；需要时短暂改写 request status/token 计数并在 `finally` 恢复。 +- 禁止：`async_chunk=True` 搭配原生 KV transfer；把 eviction-priority free 顺序当成无缓存池的物理布局；用含 placeholder 的 token 计数作为 transfer 边界；connector 失败后仍继续拆页当成功。 +- 验收：配置拒绝 async_chunk+kv_transfer；物理序 free；confirmed-token transfer 边界；失败 fail-close。^[PR #7166] diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md index 5b07098bb..37ce6d199 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -1,7 +1,7 @@ --- title: "运行时热路径合同" created: 2026-09-04 -updated: 2026-09-08 +updated: 2026-09-21 type: rule tags: [vllm-omni, components, model-executor] sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6424", "PR #6454", vllm_omni/data_entry_keys.py, "vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py", "vllm_omni/model_executor/stage_input_processors/cosyvoice3.py", "PR #6458", "PR #6317", "PR #7136"] @@ -101,3 +101,10 @@ confidence: high - 强制：上述字段为 Python `int`/`bool`，按真实 prompt 进度逐行生产。长度为 1 的 **prefill** 行（含 chunked prefill 的最后一 token）必须保留 preprocess embedding，且不得进入 runner-managed MTP；只有单 token **decode** 行才可走 MTP。out-of-tree 模型虽见下划线前缀，仍按公开 contract 消费。 - 禁止：用当前 span 长度把 one-token prefill 当成 decode；只在模型 helper 手喂 metadata 却不测 runner producer；或把该合同当成已为所有模型开启 chunked prefill。 - 验收：直接调用生产 `_preprocess`，在 mixed batch 中对照应/不应进入 MTP 的行，覆盖单 token prompt/tail、cached progress、normal/batched hooks；真实权重 E2E 至少锁一条 one-token prefill tail。^[PR #7136] + +## EXEC-11i — 请求 seed 必须驱动模型内 CFM/噪声流，且与 batch 位置无关 + +- 触发：TTS/talker 在 `forward()` 内绘制 CFM、flow-matching 或同类噪声，且公开协议声称 `seed` 可复现；修改 `_omni_seed`、`SamplingParams.seed`、per-request `torch.Generator` 或 `deterministic_cfm_noise`。 +- 强制：`SamplingParams.seed` 到达 vLLM sampler 不等于到达模型内噪声点。有 seed 的请求必须在首次 prefill 从 runner 的 `_omni_seed` 构造请求私有 `Generator`，并在每个噪声 fill 站点（eager、cuda-graph、batched prefill/decode、unified replay）对 shape `(1, …)` 的行切片使用该 generator。同 text+seed 的噪声流必须与 batch 组成、request id 无关；无 seed 保持全局 RNG；`deterministic_cfm_noise` 的 request-id 哈希只服务 replay，且请求 seed 优先。 +- 禁止：仅依赖 `SamplingParams.seed` 或全局 `normal_()` 宣称 speech seed 合同；用 request-id 哈希冒充用户 seed；让 seeded 行的噪声依赖同 batch 其他请求。 +- 验收：跨 request 同 seed 字节一致、异 seed 敏感、batch 位次无关、seed 优先于 replay 哈希；覆盖所有噪声站点。^[PR #7866] diff --git a/knowledge/repos/vllm-omni/models/bagel/rules.md b/knowledge/repos/vllm-omni/models/bagel/rules.md index 478792595..118b3188b 100644 --- a/knowledge/repos/vllm-omni/models/bagel/rules.md +++ b/knowledge/repos/vllm-omni/models/bagel/rules.md @@ -1,7 +1,7 @@ --- title: "BAGEL 实现规则" created: 2026-09-02 -updated: 2026-09-05 +updated: 2026-09-21 type: rule tags: [vllm-omni, models, diffusion] sources: ["PR #5775", "PR #5884", "PR #6359", "PR #7049", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/diffusion/models/lance/lance_transformer.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/test_diffusion_engine_dummy_run.py, tests/diffusion/models/bagel/test_step_execution.py] @@ -93,3 +93,11 @@ step request abort/finish 后，runner 必须先清理 request state、paged dif step request finish/abort 后的 explicit text full-forward fallback。schedule regression 必须断言 `N < 2` 在 complete 和 step image path 均被拒绝，且不将 denoise update 数写成 `N`；dummy-run regression 必须对 request 和 step 两种 mode 都断言两个 inference steps。^[PR #6359] ^[PR #7049] + +## BAGEL-4 — 显式输出画布优先于 KV image_shape 与源图尺寸 + +**适用范围**：`/v1/images/edits` 与 BAGEL img2img/`get_bagel_pre_process_func`/`_forward_single` 的 height/width 决议，含 two-stage 的 `kv_metadata["image_shape"]`。 + +**合同**：画布优先级为 **显式请求 > `kv_metadata["image_shape"]` > 输入图像推导**。`size=auto` 在 API 层解析为源尺寸后，必须用 `height_not_provided`/`width_not_provided` 标记非意图画布；preprocessor 对派生画布同样置位。显式边长按 latent stride 向下对齐并告警；超过 checkpoint 上限以 `OmniClientError`（HTTP 400）拒绝。源图仍可为 VAE/ViT prefill 做 stride 对齐，但不得覆盖已请求的输出画布。非流式 generation/edit 响应的 `size` 必须来自实际生成图像。 + +**验收**：显式 `size` 不被源图或 AR 发布的 KV shape 覆盖；`size=auto` 与纯离线 img2img 保持派生对齐；越界拒绝；响应 `size` 与解码图像一致。^[PR #7287] diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md index 9e1f4a20b..c23e9437c 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md @@ -1,7 +1,7 @@ --- title: "HunyuanImage3 开发规则" created: 2026-07-13 -updated: 2026-09-08 +updated: 2026-09-21 type: rule tags: [vllm-omni, models, hunyuan-image3] sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306", "PR #6102", "PR #6563", "PR #4048", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py, "PR #7021"] @@ -170,3 +170,10 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif - 强制：公开 `get_expert_mapping()` 必须返回 flat `list[tuple[str, str, int, str]]`（非 MoE 返回 `[]`），供上游 loader 消费；Hunyuan checkpoint 的 `gate_proj`/`up_proj` → `gate_and_up_proj` 分片 remapping 只能放在独立的 `_get_expert_weights_remapping()`，由 `load_weights` 显式取用。AR 与 diffusion transformer 两侧同一合同。 - 禁止：让 `get_expert_mapping()` 再返回 `(mapping, remapping)` 二元组，或把 Omni-local remapping 塞进上游期望的 public API。 - 验收：对 `HunyuanModel` 与 `HunyuanImage3Model` 断言 public mapping 为 flat list、local remapping 字典独立，并覆盖非 MoE 空返回。^[PR #7021] + +## HY3-10a — 文本-only chat 必须按请求标为 comprehension，不得走图像 stage 强制 + +- 触发：修改 HunyuanImage3 AR `sample()`、`engine_output_type="latent"` 的 generation 部署上的 chat completions、`SamplingParams.extra_args["ar_task_mode"]`，或 AR runner 向 custom sampler 转发 `extra_args`。 +- 强制：任务模式是 per-request 属性。serving 对 modalities 仅为 text 的 chat 在普通 `SamplingParams` 上 `setdefault("ar_task_mode", "comprehension")`，不覆盖调用方显式值，也不标记 image/audio/video 输出请求。声明 `model_sampler_wants_extra_args` 的模型必须经共享 dispatcher 按 `req_ids` 顺序收到每行 `extra_args`。comprehension 行只做无状态图像脚手架 token 屏蔽（含 ``/``/``/`` 等），禁止 `→` 等 generation stage 强制。 +- 禁止：仅凭引擎级 `engine_output_type` 让文本答案走图像 stage 转移；把 DiT scaffold token 漏进文本答案；破坏既有两参数 custom sampler 合同。 +- 验收：混合 comprehension/generation batch、生产 chat 入口的 text-only 路由、GPU/NPU 转发与未标记行的 generation 行为保持不变。^[PR #6111] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md index 969a0c607..4335377c6 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -1,7 +1,7 @@ --- title: "MiniMax H3 缓存与任务生命周期规则" created: 2026-09-04 -updated: 2026-09-08 +updated: 2026-09-21 type: rule tags: [vllm-omni, models, diffusion] sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", "PR #6666", "PR #6714", "PR #6909", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/fasth3.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/npu/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/entrypoints/openai/video_api_utils.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_fasth3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_native_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py, tests/entrypoints/openai_api/test_video_api_utils.py, "PR #7162", "PR #7062"] @@ -231,3 +231,10 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 - 强制：`--lora-path` 指向单文件，或只含一个可解析 Turbo artifact 的目录；多候选必须 fail closed。`ref2v` 只能挂在 Ref2VA-only server；combined/`transformers_ref` 拓扑不得绑定只注入 `transformer` 的 Turbo adapter。ComfyUI 融合 QKV/SwiGLU 导出按名拒绝。 - 禁止：继续要求单一 `4step_v1.0_768p` 文件名或 `alpha==128`；在目录里静默挑选；把未声明 alpha 当 rank；或让 Ref2VA 请求跑到未蒸馏 DiT 的 few-step schedule。 - 验收：覆盖多 artifact 文件名矩阵、缺/错 alpha、多文件目录拒绝、task/steps/shift 负向门禁、Ref2VA-only vs combined 拒绝，以及 Diffusers-layout packing；不得把单一 artifact 通过外推为全矩阵质量证据。^[PR #7062] + +## MMH3-2r — 长视频 continuation 必须按全局时间线切窗并同步 AV 边界 + +- 触发：修改 MiniMax-H3 `long_video`/`continuation_*`、`audio_mode=lock_source`、窗口 RoPE `temporal_offset`、`continuation_prompts`，或 fused QK-norm 的 token 索引宽度。 +- 强制：默认 latent-tail continuation 只在 Ref2VA request execution（非 step）下启用；window/overlap 必须落在 `17n+5` 网格且 window∈[107,345]、overlap