From 094dbbe13810e2cc7bc5ca6a22beb892362082d5 Mon Sep 17 00:00:00 2001 From: Maulana Date: Tue, 22 Sep 2026 22:24:18 +0700 Subject: [PATCH 1/3] feat: add generic workflow model cache and serverless lifecycle --- .dockerignore | 4 + .github/workflows/test.yml | 45 +- Dockerfile | 20 +- README.md | 227 +++- docker-bake.hcl | 22 +- docs/configuration.md | 7 +- docs/customization.md | 92 ++ docs/deployment.md | 82 ++ handler.py | 33 +- requirements.txt | 2 + scripts/install-workflow-nodes.py | 33 + src/media_output.py | 42 + src/start.sh | 94 +- src/workflow_models.py | 223 ++++ tests/Dockerfile | 10 + tests/test_serverless_lifecycle.py | 123 ++ tests/test_workflow_models.py | 157 +++ workflow/README.md | 27 + workflow/ltx25-flf-v1.json | 605 ++++++++++ workflow/ltx25-i2v-v1.json | 707 +++++++++++ workflow/ltx25-t2v-v1.json | 607 ++++++++++ workflow/ltx25.yaml | 23 + workflow/minimax-h3.yaml | 5 + workflow/video_minimax_h3_r2v.json | 1745 ++++++++++++++++++++++++++++ 24 files changed, 4878 insertions(+), 57 deletions(-) create mode 100644 scripts/install-workflow-nodes.py create mode 100644 src/media_output.py create mode 100644 src/workflow_models.py create mode 100644 tests/Dockerfile create mode 100644 tests/test_serverless_lifecycle.py create mode 100644 tests/test_workflow_models.py create mode 100644 workflow/README.md create mode 100644 workflow/ltx25-flf-v1.json create mode 100644 workflow/ltx25-i2v-v1.json create mode 100644 workflow/ltx25-t2v-v1.json create mode 100644 workflow/ltx25.yaml create mode 100644 workflow/minimax-h3.yaml create mode 100644 workflow/video_minimax_h3_r2v.json diff --git a/.dockerignore b/.dockerignore index 4aab96452..b17fd900d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,7 @@ node_modules .changeset .git .github + +__pycache__ +.pytest_cache +.venv diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da1cfab74..6312ff46d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,19 +1,26 @@ -name: Tests - -on: - workflow_dispatch: - push: - branches-ignore: [main] - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Placeholder test - run: echo "Tests temporarily disabled - placeholder to satisfy required checks" +name: Tests + +on: + workflow_dispatch: + push: + branches-ignore: [main] + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install test dependencies + run: python -m pip install -r requirements.txt pytest + - name: Worker contracts and lifecycle + run: python -m pytest tests -q + - name: Startup syntax + run: bash -n src/start.sh diff --git a/Dockerfile b/Dockerfile index 1dd71a953..978c691d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -110,10 +110,12 @@ ADD src/extra_model_paths.yaml ./ WORKDIR / # Install Python runtime dependencies for the handler -RUN uv pip install runpod requests websocket-client +COPY requirements.txt /requirements.txt +RUN uv pip install -r /requirements.txt # Add application code and scripts -ADD src/start.sh src/network_volume.py handler.py test_input.json ./ +ADD src/start.sh src/network_volume.py src/workflow_models.py src/media_output.py handler.py test_input.json ./ +COPY workflow/ /workflow/ RUN chmod +x /start.sh # Add script to install custom nodes @@ -130,6 +132,13 @@ RUN chmod +x /usr/local/bin/comfy-manager-set-mode # Set the default command to run when starting the container CMD ["/start.sh"] +# Optional custom-node dependencies, chosen by manifest at build time. +# No workflow, model family or weights are selected by default. +ARG CUSTOM_NODE_MANIFESTS="" +COPY scripts/install-workflow-nodes.py /install-workflow-nodes.py +RUN python /install-workflow-nodes.py --manifests "$CUSTOM_NODE_MANIFESTS" + +# Legacy opt-in targets used by existing release jobs; not built by default. # Stage 2: Download models FROM base AS downloader @@ -180,7 +189,10 @@ RUN if [ "$MODEL_TYPE" = "z-image-turbo" ]; then \ fi # Stage 3: Final image -FROM base AS final +FROM base AS baked-models # Copy models from stage 2 to the final image -COPY --from=downloader /comfyui/models /comfyui/models \ No newline at end of file +COPY --from=downloader /comfyui/models /comfyui/models + +# Default: generic worker with no model weights. +FROM base AS final diff --git a/README.md b/README.md index b4da1825c..b353fb61b 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ This project allows you to run ComfyUI workflows as a serverless API endpoint on ## Table of Contents -- [Quickstart](#quickstart) +- [Alur worker generik: dari workflow sampai endpoint](#alur-worker-generik-dari-workflow-sampai-endpoint) +- [Quickstart upstream](#quickstart-upstream) - [Available Docker Images](#available-docker-images) - [API Specification](#api-specification) - [Usage](#usage) @@ -23,7 +24,229 @@ This project allows you to run ComfyUI workflows as a serverless API endpoint on --- -## Quickstart +## Alur worker generik: dari workflow sampai endpoint + +Alur yang digunakan repo ini: + +**Pilih workflow → tes lokal → build image → checklist model → isi cache sekali → jalankan endpoint `cache-only` → hasil dikirim → worker dihentikan.** + +Image default berisi runtime ComfyUI dan handler, tanpa bobot model dan tanpa +workflow aktif. Menambahkan workflow tidak memerlukan target Docker baru. +Semua perintah berikut dijalankan dari direktori `worker-comfyui/`. + +### 1. Pilih workflow yang akan dilayani + +| Workflow | Nilai `WORKFLOWS` | Sumber daftar model | Dependency tambahan saat build | +| --- | --- | --- | --- | +| MiniMax H3 R2V | `minimax-h3.yaml` | Metadata di `video_minimax_h3_r2v.json`; 5 aset | Tidak ada custom node dalam graph yang disediakan | +| LTX 2.5 Senai | `ltx25.yaml` | Enam aset untuk graph T2V, I2V, dan FLF | `CUSTOM_NODE_MANIFESTS=ltx25.yaml` | + +Contoh berikut memakai MiniMax: + +```bash +export WORKFLOWS=minimax-h3.yaml +export WORKER_IMAGE=worker-comfyui:generic +``` + +`WORKFLOWS=video_minimax_h3_r2v.json` juga valid untuk membaca metadata model +langsung. File lain dalam folder `workflow/` tidak otomatis diunduh. +`WORKFLOW_MANIFESTS` tetap didukung sebagai alias lama; `WORKFLOWS` diprioritaskan. + +**Bedakan dua format:** JSON editor MiniMax dipakai untuk menemukan aset model. +Handler tetap menerima **JSON API** dalam `input.workflow`; export melalui +ComfyUI sebelum mengirim job. Pemilihan env menyiapkan aset, bukan otomatis +menjalankan graph atau mengubah format editor menjadi API. + +### 2. Jalankan tes lokal tanpa GPU dan tanpa download model + +```bash +docker build --platform linux/amd64 \ + -f tests/Dockerfile -t worker-comfyui-tests . +docker run --rm worker-comfyui-tests +``` + +Tes mencakup pemilihan aset, cache, download dengan data tiruan, respons handler, +SDK Runpod `stopPod`, serta cleanup proses ketika selesai, crash, dan SIGTERM. +Hasil terakhir: **49 tes lulus**. Layanan ComfyUI/GPU, download dan S3 dimock; +ini belum membuktikan inference atau penghentian worker di cloud. + +### 3. Build image runtime generik + +Untuk MiniMax: + +```bash +docker build --platform linux/amd64 -t "$WORKER_IMAGE" . +``` + +Untuk LTX, gunakan Dockerfile yang sama dengan dependency terpilih: + +```bash +export WORKFLOWS=ltx25.yaml +export WORKER_IMAGE=worker-comfyui:with-ltx-nodes +docker build --platform linux/amd64 \ + --build-arg CUSTOM_NODE_MANIFESTS=ltx25.yaml \ + -t "$WORKER_IMAGE" . +``` + +Build arg memasang custom node yang dipin dalam YAML, bukan bobot model atau +pilihan workflow runtime. Tidak ada target `--target ltx25`. Perubahan dependency +Python/custom node membutuhkan rebuild; perubahan model tidak. + +### 4. Periksa checklist model terlebih dahulu + +```bash +docker run --rm \ + -e WORKFLOWS="$WORKFLOWS" \ + -e MODEL_DOWNLOAD_CHECK_ONLY=true \ + "$WORKER_IMAGE" +``` + +Container mencetak aset terpilih lalu berhenti, tanpa GPU check, download, +ComfyUI, atau handler. MiniMax harus menampilkan lima aset MiniMax saja. + +**Untuk tahap validasi tanpa bobot nyata, berhenti di sini.** Langkah pengisian +cache berikut memang melakukan download; lanjutkan ketika siap mengunduh model. + +### 5. Isi cache sekali, terpisah dari worker yang melayani job + +Contoh cache lokal: + +```bash +mkdir -p model-cache +docker run --rm \ + -v "$PWD/model-cache:/runpod-volume" \ + -e WORKFLOWS="$WORKFLOWS" \ + -e PREPARE_MODELS_ONLY=true \ + -e MODEL_DOWNLOAD_POLICY=missing \ + -e MODEL_DOWNLOAD_CONCURRENCY=4 \ + "$WORKER_IMAGE" +``` + +Untuk model yang memerlukan autentikasi, isi `HF_TOKEN` di lingkungan shell dan +tambahkan `-e HF_TOKEN` pada perintah tersebut. Proses preparation tidak memerlukan +GPU dan berhenti setelah selesai. Bobot tersimpan di `model-cache/models/`. + +Download menggunakan file sementara, penggantian atomik, dan lock per model. +Receipt `.worker-cache.json` disimpan di samping bobot. Pertahankan receipt ketika +menggunakan kembali cache; `cache-only` memeriksa receipt, sumber, ukuran, dan +mtime tanpa membaca seluruh bobot untuk menghitung ulang hash. + +Untuk Runpod, isi **network volume yang sama** dengan yang akan dipasang pada +endpoint. Folder Docker lokal di atas bukan network volume Runpod. Jalankan image +preparation pada compute sementara yang memasang volume tersebut dan atur +`COMFY_MODEL_ROOT` ke lokasi aktualnya. Contoh: bila volume pada Pod terpasang di +`/workspace`, gunakan `/workspace/models`; isi volume itu nantinya terlihat sebagai +`/runpod-volume/models` pada serverless. Hentikan compute preparation setelah selesai. + +Jika cache dipindahkan dan mtime berubah, jalankan preparation lagi terhadap +lokasi tujuan sebelum serving. Untuk unduhan yang dapat direproduksi, pin revision +URL dan isi SHA256 dalam manifest; contoh awal masih memakai URL upstream `main`. + +### 6. Siapkan image dan konfigurasi endpoint + +Push image hasil build ke registry milik Anda: + +```bash +export REGISTRY_IMAGE=ghcr.io/your-account/worker-comfyui:generic-v1 +docker login ghcr.io +docker tag "$WORKER_IMAGE" "$REGISTRY_IMAGE" +docker push "$REGISTRY_IMAGE" +``` + +Ganti `your-account` dan tag sesuai registry tujuan. Di Runpod, buat template +serverless menggunakan image tersebut, lalu endpoint queue-based dengan: + +- Network volume yang sudah diisi pada langkah 5. +- GPU dan kapasitas disk yang sesuai dengan workflow. +- Active Workers **0**, Max Workers **1** untuk pengujian awal. +- Idle timeout **5 detik** dan satu job per worker pada satu waktu. + +Isi env endpoint (contoh MiniMax): + +```dotenv +WORKFLOWS=minimax-h3.yaml +MODEL_DOWNLOAD_POLICY=cache-only +COMFY_MODEL_ROOT=/runpod-volume/models +REFRESH_WORKER=true +``` + +Jangan aktifkan `PREPARE_MODELS_ONLY`, `MODEL_DOWNLOAD_CHECK_ONLY`, atau +`SERVE_API_LOCALLY` pada endpoint serving. Cache yang belum siap akan menghasilkan +error yang meminta preparation; worker tidak diam-diam mengunduh saat cold start. + +Untuk konsumen dengan kontrak respons Senai, tambahkan: + +```dotenv +OUTPUT_FORMAT=senai +AWS_BUCKET_NAME=your-output-bucket +AWS_ENDPOINT_URL=https://your-account.r2.cloudflarestorage.com +AWS_DEFAULT_REGION=auto +``` + +Simpan `AWS_ACCESS_KEY_ID` dan `AWS_SECRET_ACCESS_KEY` sebagai secret endpoint. +Sesuaikan endpoint/region untuk storage yang digunakan. Senai harus mengizinkan +hostname hasil di `allowed_output_hosts`; output base64 ditolak oleh Senai. +`OUTPUT_FORMAT=senai` hanya mengatur format respons, bukan menambahkan dukungan +model MiniMax ke katalog atau codec aplikasi Senai. + +### 7. Kirim job API dan verifikasi sampai worker berhenti + +Siapkan `request.json` dengan envelope `{"input":{"workflow": ...}}` berisi +graph API lengkap. Untuk R2V/I2V, tambahkan `input.images` berisi nama dan base64 +gambar referensi yang cocok dengan node `LoadImage`. Jangan mengirim JSON editor +MiniMax langsung sebagai payload. + +Dengan `RUNPOD_API_KEY` dan `RUNPOD_ENDPOINT_ID` sudah diisi pada shell: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + -H "Content-Type: application/json" \ + --data-binary @request.json \ + "https://api.runpod.ai/v2/$RUNPOD_ENDPOINT_ID/run" +``` + +Salin `id` dari respons, kemudian periksa status sampai terminal: + +```bash +export RUNPOD_JOB_ID=replace-with-returned-job-id +curl --fail-with-body \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + "https://api.runpod.ai/v2/$RUNPOD_ENDPOINT_ID/status/$RUNPOD_JOB_ID" +``` + +Pastikan hasil video tersedia, URL storage tetap dapat dibaca setelah worker +berhenti, dan jumlah worker aktif/idle kembali nol saat antrean kosong. +`REFRESH_WORKER=true` meminta retirement melalui SDK setelah hasil dikembalikan; +Runpod yang melakukan penghentian cloud. `start.sh` membersihkan proses anak pada +exit/SIGTERM. Verifikasi ini tetap diperlukan di staging. + +Cache menghilangkan download dan hashing seluruh bobot saat cold start, tetapi +image pull, startup ComfyUI, dan pemuatan bobot ke GPU tetap membutuhkan waktu. +Build produksi GPU, download bobot nyata, dan latensi cloud belum diverifikasi +oleh tes CPU di atas. + +### 8. Tambahkan atau ganti workflow berikutnya + +1. Tambahkan JSON ke `workflow/`. Jika editor JSON memiliki `properties.models`, + buat YAML yang merujuk file itu, seperti [`minimax-h3.yaml`](workflow/minimax-h3.yaml). + Untuk graph tanpa URL model, tulis `models` secara eksplisit seperti + [`ltx25.yaml`](workflow/ltx25.yaml). +2. Pasang definisi workflow melalui volume dengan `WORKFLOW_DIR` yang sesuai, + atau rebuild image untuk menyertakan file baru di `/workflow`. +3. Jika ada custom node baru, deklarasikan repo dan commit dalam YAML, lalu build + dengan `CUSTOM_NODE_MANIFESTS` yang sesuai. Dependency tidak dipasang saat boot. +4. Jalankan checklist dan preparation untuk workflow baru pada cache yang sama. +5. Ubah `WORKFLOWS` pada endpoint dan tetap gunakan `cache-only`. Beberapa workflow + bisa dipilih dengan koma; hanya pilih yang memang akan dilayani. + +Detail schema dan cache ada di [Customization Guide](docs/customization.md#generic-workflow-selection-and-model-cache), +dan checklist staging ada di [Deployment Guide](docs/deployment.md#generic-worker-prepare-assets-separately-from-serving). + +## Quickstart upstream + +Bagian berikut menjelaskan image upstream yang sudah dipublikasikan. Untuk build +repo ini dan alur cache baru, gunakan panduan worker generik di atas. 1. 🐳 Choose one of the [available Docker images](#available-docker-images) for your serverless endpoint (e.g., `runpod/worker-comfyui:-sd3`). 2. 📄 Follow the [Deployment Guide](docs/deployment.md) to set up your RunPod template and endpoint. diff --git a/docker-bake.hcl b/docker-bake.hcl index b463dbac3..9958dd793 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -31,12 +31,16 @@ variable "PYTORCH_INDEX_URL" { default = "" } +variable "CUSTOM_NODE_MANIFESTS" { + default = "" +} + variable "HUGGINGFACE_ACCESS_TOKEN" { default = "" } group "default" { - targets = ["base", "sdxl", "sd3", "flux1-schnell", "flux1-dev", "flux1-dev-fp8", "z-image-turbo", "base-cuda12-8-1"] + targets = ["base"] } target "base" { @@ -50,7 +54,7 @@ target "base" { CUDA_VERSION_FOR_COMFY = "${CUDA_VERSION_FOR_COMFY}" ENABLE_PYTORCH_UPGRADE = "${ENABLE_PYTORCH_UPGRADE}" PYTORCH_INDEX_URL = "${PYTORCH_INDEX_URL}" - MODEL_TYPE = "base" + CUSTOM_NODE_MANIFESTS = "${CUSTOM_NODE_MANIFESTS}" } tags = ["${DOCKERHUB_REPO}/${DOCKERHUB_IMG}:${RELEASE_VERSION}-base"] } @@ -58,7 +62,7 @@ target "base" { target "sdxl" { context = "." dockerfile = "Dockerfile" - target = "final" + target = "baked-models" args = { BASE_IMAGE = "${BASE_IMAGE}" COMFYUI_VERSION = "${COMFYUI_VERSION}" @@ -74,7 +78,7 @@ target "sdxl" { target "sd3" { context = "." dockerfile = "Dockerfile" - target = "final" + target = "baked-models" args = { BASE_IMAGE = "${BASE_IMAGE}" COMFYUI_VERSION = "${COMFYUI_VERSION}" @@ -91,7 +95,7 @@ target "sd3" { target "flux1-schnell" { context = "." dockerfile = "Dockerfile" - target = "final" + target = "baked-models" args = { BASE_IMAGE = "${BASE_IMAGE}" COMFYUI_VERSION = "${COMFYUI_VERSION}" @@ -108,7 +112,7 @@ target "flux1-schnell" { target "flux1-dev" { context = "." dockerfile = "Dockerfile" - target = "final" + target = "baked-models" args = { BASE_IMAGE = "${BASE_IMAGE}" COMFYUI_VERSION = "${COMFYUI_VERSION}" @@ -125,7 +129,7 @@ target "flux1-dev" { target "flux1-dev-fp8" { context = "." dockerfile = "Dockerfile" - target = "final" + target = "baked-models" args = { BASE_IMAGE = "${BASE_IMAGE}" COMFYUI_VERSION = "${COMFYUI_VERSION}" @@ -141,7 +145,7 @@ target "flux1-dev-fp8" { target "z-image-turbo" { context = "." dockerfile = "Dockerfile" - target = "final" + target = "baked-models" args = { BASE_IMAGE = "${BASE_IMAGE}" COMFYUI_VERSION = "${COMFYUI_VERSION}" @@ -166,7 +170,7 @@ target "base-cuda12-8-1" { CUDA_VERSION_FOR_COMFY = "" ENABLE_PYTORCH_UPGRADE = "true" PYTORCH_INDEX_URL = "https://download.pytorch.org/whl/cu128" - MODEL_TYPE = "base" + CUSTOM_NODE_MANIFESTS = "${CUSTOM_NODE_MANIFESTS}" } tags = ["${DOCKERHUB_REPO}/${DOCKERHUB_IMG}:${RELEASE_VERSION}-base-cuda12.8.1"] } diff --git a/docs/configuration.md b/docs/configuration.md index 25bed9e70..f7c36890d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,7 +6,7 @@ This document outlines the environment variables available for configuring the ` | Environment Variable | Description | Default | | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| `REFRESH_WORKER` | When `true`, the worker pod will stop after each completed job to ensure a clean state for the next job. See the [RunPod documentation](https://docs.runpod.io/docs/handler-additional-controls#refresh-worker) for details. | `false` | +| `REFRESH_WORKER` | When `true`, the worker pod will stop after each completed job to ensure a clean state for the next job. See the [RunPod documentation](https://docs.runpod.io/docs/handler-additional-controls#refresh-worker) for details. | `true` | | `SERVE_API_LOCALLY` | When `true`, enables a local HTTP server simulating the RunPod environment for development and testing. See the [Development Guide](development.md#local-api) for more details. | `false` | | `COMFY_ORG_API_KEY` | Comfy.org API key to enable ComfyUI API Nodes. If set, it is sent with each workflow; clients can override per request via `input.api_key_comfy_org`. | – | @@ -67,3 +67,8 @@ If the S3 environment variables (`BUCKET_ENDPOINT_URL`, `BUCKET_ACCESS_KEY_ID`, ``` The `data` field contains the presigned URL to the uploaded image file in your S3 bucket. The path usually includes the job ID. + +## Workflow model bootstrap + +See [runtime manifests](customization.md#generic-workflow-selection-and-model-cache) for `WORKFLOWS` (legacy alias `WORKFLOW_MANIFESTS`), `WORKFLOW_DIR`, `COMFY_MODEL_ROOT`, `HF_TOKEN`, `PREPARE_MODELS_ONLY`, `MODEL_DOWNLOAD_POLICY`, `MODEL_DOWNLOAD_CONCURRENCY`, and `MODEL_DOWNLOAD_CHECK_ONLY`. +`OUTPUT_FORMAT=senai` enables the Senai media envelope and AWS S3/R2 output configuration. diff --git a/docs/customization.md b/docs/customization.md index 715791e49..a8b464920 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -110,3 +110,95 @@ Using a Network Volume is primarily useful if you want to manage **models** sepa > > - When a Network Volume is correctly attached, ComfyUI running inside the worker container will automatically detect and load models from the standard directories (`/runpod-volume/models/...`) within that volume (for serverless workers). For directory mapping details and troubleshooting, see [Network Volumes & Model Paths](network-volumes.md). > - This method is **not suitable for installing custom nodes**; use the Custom Dockerfile method for that. + +## Generic workflow selection and model cache + +The default Docker image contains no model weights and selects no workflow. +Adding a workflow does not add a Docker stage or download unrelated models. +`WORKFLOWS` selects comma-separated YAML manifests or editor JSON files from +`WORKFLOW_DIR` (default `/workflow`). `WORKFLOW_MANIFESTS` remains a legacy alias; +`WORKFLOWS` takes precedence. Files elsewhere in the directory are not scanned. + +```dotenv +WORKFLOWS=minimax-h3.yaml +# Alternatively: WORKFLOWS=video_minimax_h3_r2v.json +``` + +MiniMax's editor JSON already contains `properties.models` with names, URLs and +model directories. The generic loader reads these (also inside subgraphs). +`minimax-h3.yaml` references that JSON without duplicating its model catalog. +This selects five MiniMax assets and no LTX weights. For API JSONs without model +metadata, use explicit model entries as in `ltx25.yaml`: + +```yaml +version: 1 +workflows: + - my-api-workflow.json +models: + - path: diffusion_models/my-model.safetensors + url: https://huggingface.co/org/repo/resolve/COMMIT/my-model.safetensors + # sha256: <64 hexadecimal characters> +``` + +Paths are relative to `COMFY_MODEL_ROOT`; any category or nested filename is +supported. Model references ending in `.safetensors`, `.gguf`, `.ckpt`, `.pt`, +`.pth`, or `.bin` must have download metadata. Other auxiliary files must be +listed explicitly. Missing metadata fails before downloading anything. + +Editor JSON is used for **asset discovery only**. `handler.py` still requires +an API-format graph in `input.workflow`; export it from ComfyUI for execution. +The MiniMax reference images must also be supplied for actual inference. + +### Prepare once, reuse on every cold start + +When `/runpod-volume` exists, startup defaults to `/runpod-volume/models`; +otherwise it uses `/comfyui/models`. An explicit `COMFY_MODEL_ROOT` overrides it. +The generated extra-model-path configuration registers selected categories. + +- `PREPARE_MODELS_ONLY=true`: populate the cache and exit, without a GPU check, + ComfyUI or the Runpod handler. Use this in a separate preparation job/container. +- `MODEL_DOWNLOAD_POLICY=missing` (default): download missing/stale assets. +- `MODEL_DOWNLOAD_POLICY=cache-only`: require previously prepared cache entries; + fail immediately if any are absent/stale, with no model network calls. +- `MODEL_DOWNLOAD_CONCURRENCY=4`: bounded parallel downloads (1–16). +- `MODEL_DOWNLOAD_CHECK_ONLY=true`: print the selected checklist and exit, + without downloads or startup. + +Downloads use HTTPS, bounded retries, per-model file locks, temporary files and +atomic replacement. Workers sharing a volume serialize downloads of the same +asset; different assets can download concurrently. `HF_TOKEN` or +`HUGGINGFACE_ACCESS_TOKEN` is sent only to Hugging Face. + +Successful preparation records source URL, optional SHA256, size and modification +time in adjacent `.worker-cache.json` receipts. Subsequent starts compare metadata +without rereading tens of GB to hash them. Changed URLs/checksums invalidate the +receipt. Changed file size/mtime also invalidate it. This is a trusted-volume +optimization, not tamper-proof verification. Files without receipts are adopted +only during preparation; optional SHA256 is checked once. Use pinned revisions +and SHA256 for integrity; `main` URLs cannot reveal upstream changes automatically. + +### Custom nodes remain build-time dependencies + +Core-node workflows such as the supplied MiniMax JSON need no custom-node install. +For workflows that require extra packages, declare pinned dependencies in YAML: + +```yaml +custom_nodes: + - name: ComfyUI-LTXVideo + repo: https://github.com/Lightricks/ComfyUI-LTXVideo.git + revision: ac4d99839020b983e956a8ab67ec38aec1b6e65a +``` + +Build with `--build-arg CUSTOM_NODE_MANIFESTS=ltx25.yaml` to include those selected +dependencies. This does not select runtime workflows or download weights. +A model/graph change needs no rebuild when using a mounted `WORKFLOW_DIR`; new +Python/custom-node dependencies do require a compatible image. Manager stays offline. + +### Handler and Senai output + +Keep `OUTPUT_FORMAT=senai` explicit on Senai endpoints; it is not tied to a model. +It returns `status: success` with media entries under `output`. For signed S3/R2 +URLs configure `AWS_BUCKET_NAME`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, +`AWS_ENDPOINT_URL` and `AWS_DEFAULT_REGION` (R2: `auto`). Senai must allow the +storage hostname. Small outputs without a bucket are base64; Senai rejects them. +Other callers retain the existing image response format by default. diff --git a/docs/deployment.md b/docs/deployment.md index 7e020eb81..d3d1d17ce 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -105,3 +105,85 @@ RunPod offers a seamless way to deploy directly from your GitHub repository cont 5. **Deploy:** RunPod will clone the repository, build the image from your specified branch and Dockerfile, push it to a temporary registry, and deploy the endpoint. Every `git push` to the configured branch will automatically trigger a new build and update your RunPod endpoint. For more details, refer to the [RunPod GitHub Integration Documentation](https://docs.runpod.io/serverless/github-integration). + +## Generic worker: prepare assets separately from serving + +Build the model-free image once (MiniMax uses only core nodes): + +```bash +docker build --platform linux/amd64 -t worker-comfyui:generic . +``` + +For extra node dependencies, use the same Dockerfile with +`--build-arg CUSTOM_NODE_MANIFESTS=ltx25.yaml`. There is no `ltx25` target and no +hardcoded LTX runtime environment. Default `docker buildx bake` builds only base. +Historical baked-model release targets remain explicit opt-ins; adding another +workflow does not require adding one of these targets. + +Inspect MiniMax's model checklist without a GPU or download: + +```bash +docker run --rm -e WORKFLOWS=minimax-h3.yaml \ + -e MODEL_DOWNLOAD_CHECK_ONLY=true worker-comfyui:generic +``` + +Populate a shared volume once, outside serving workers: + +```bash +docker run --rm -v "$PWD/model-cache:/runpod-volume" \ + -e WORKFLOWS=minimax-h3.yaml -e PREPARE_MODELS_ONLY=true \ + -e MODEL_DOWNLOAD_POLICY=missing worker-comfyui:generic +``` + +On Runpod, perform equivalent preparation against the **same network volume** +attached to the endpoint; a local Docker directory is not that remote volume. +Then configure the serving endpoint: + +```dotenv +WORKFLOWS=minimax-h3.yaml +MODEL_DOWNLOAD_POLICY=cache-only +COMFY_MODEL_ROOT=/runpod-volume/models +REFRESH_WORKER=true +# For Senai consumers, plus AWS storage credentials: +OUTPUT_FORMAT=senai +``` + +Switch to `WORKFLOWS=ltx25.yaml` for LTX after preparing its cache and ensuring +its node dependencies are installed. Mounted workflow definitions can change +without a rebuild. Select only workflows the endpoint actually serves. + +`cache-only` removes download and whole-file hashing from cold start, but does +not remove image pull, Python/ComfyUI initialization or model-to-GPU loading. +No GPU latency measurements have been made locally. + +`REFRESH_WORKER=true` requests retirement after each success or error response. +The SDK sends the result with `stopPod`; `start.sh` cleans up child processes on +exit/SIGTERM. Set endpoint Active Workers = 0, Max Workers = 1 initially and idle +timeout = 5 seconds. Cloud retirement still needs a live acceptance check. + +### Local verification and staging checklist + +```bash +docker build --platform linux/amd64 -f tests/Dockerfile -t worker-comfyui-tests . +docker run --rm worker-comfyui-tests +``` + +Tests use the real startup script, handler and Runpod SDK, with external services +mocked. They cover selected-only downloads, MiniMax editor metadata, all three +Senai LTX graphs, cache-only/no-network startup, cache invalidation, shared-volume +locking, handler responses, SDK `stopPod`, crash/SIGTERM and process cleanup. + +- [ ] Download real weights and pin source revisions/checksums. +- [ ] Build the production GPU image and validate required node registrations. +- [ ] Prepare the remote network volume, then boot with `cache-only`. +- [ ] Submit API-format workflows and reference assets for actual GPU inference. +- [ ] Verify storage URLs remain readable after retirement. +- [ ] Observe workers scale to zero and measure cold start on the staging endpoint. + +Real model downloads remain skipped. No endpoint is created by the local tests. + +Local verification (2026-09-22): **49 tests passed** in the Python 3.12 CPU +container. MiniMax checklist selects five assets only. `docker buildx bake +--print` selects only `base`; startup syntax and whitespace checks passed. +Production GPU build, actual weight downloads and cloud cold-start timing have +not been measured. diff --git a/handler.py b/handler.py index ec80043dc..a0fd9c9ad 100644 --- a/handler.py +++ b/handler.py @@ -37,7 +37,7 @@ # Fallback retry limit when PID file is unavailable and retries=0 COMFY_API_FALLBACK_MAX_RETRIES = 500 # PID file written by start.sh so we can detect if ComfyUI has crashed -COMFY_PID_FILE = "/tmp/comfyui.pid" +COMFY_PID_FILE = os.environ.get("COMFY_PID_FILE", "/tmp/comfyui.pid") # Websocket reconnection behaviour (can be overridden through environment variables) # NOTE: more attempts and diagnostics improve debuggability whenever ComfyUI crashes mid-job. # • WEBSOCKET_RECONNECT_ATTEMPTS sets how many times we will try to reconnect. @@ -57,7 +57,7 @@ COMFY_HOST = "127.0.0.1:8188" # Enforce a clean state after each job is done # see https://docs.runpod.io/docs/handler-additional-controls#refresh-worker -REFRESH_WORKER = os.environ.get("REFRESH_WORKER", "false").lower() == "true" +REFRESH_WORKER = os.environ.get("REFRESH_WORKER", "true").lower() == "true" # --------------------------------------------------------------------------- # Model loader nodes — used for pre-flight validation of workflow model refs @@ -68,6 +68,8 @@ MODEL_LOADER_NODES = { "CheckpointLoaderSimple": ("checkpoints", ("ckpt_name",)), "LoraLoader": ("loras", ("lora_name",)), + "CLIPLoader": ("text_encoders", ("clip_name",)), + "LatentUpscaleModelLoader": ("latent_upscale_models", ("model_name",)), "VAELoader": ("vae", ("vae_name",)), "DualCLIPLoader": ("text_encoders", ("clip_name1", "clip_name2")), "TripleCLIPLoader": ("text_encoders", ("clip_name1", "clip_name2", "clip_name3")), @@ -806,7 +808,7 @@ def get_image_data(filename, subfolder, image_type): return None -def handler(job): +def _handle_job(job): """ Handles a job using ComfyUI via websockets for status and image retrieval. @@ -995,7 +997,16 @@ def handler(job): errors.append(warning_msg) print(f"worker-comfyui - Processing {len(outputs)} output nodes...") + if os.getenv("OUTPUT_FORMAT") == "senai": + from media_output import collect_senai_outputs + return collect_senai_outputs(outputs, job_id, get_image_data, errors) + for node_id, node_output in outputs.items(): + node_output = dict(node_output) + node_output["images"] = [ + entry for key in ("images", "videos", "gifs", "audio") + for entry in node_output.get(key, []) + ] if "images" in node_output: print( f"worker-comfyui - Node {node_id} contains {len(node_output['images'])} image(s)" @@ -1085,7 +1096,7 @@ def handler(job): errors.append(error_msg) # Check for other output types - other_keys = [k for k in node_output.keys() if k != "images"] + other_keys = [k for k in node_output.keys() if k not in ("images", "videos", "gifs", "audio")] if other_keys: warn_msg = ( f"Node {node_id} produced unhandled output keys: {other_keys}." @@ -1142,6 +1153,20 @@ def handler(job): return final_result +def handler(job): + """Return results before asking Runpod to retire this worker, on every path.""" + try: + result = _handle_job(job) + except Exception: + logger.exception("Unhandled job failure") + result = {"error": "Unexpected job failure"} + if os.getenv("OUTPUT_FORMAT") == "senai" and "error" in result: + result["status"] = "error" + if REFRESH_WORKER: + result["refresh_worker"] = True + return result + + if __name__ == "__main__": print("worker-comfyui - Starting handler...") runpod.serverless.start({"handler": handler}) diff --git a/requirements.txt b/requirements.txt index cb5677360..3c7eca8b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ runpod~=1.7.12 websocket-client requests +PyYAML>=6.0,<7 +boto3>=1.40,<2 diff --git a/scripts/install-workflow-nodes.py b/scripts/install-workflow-nodes.py new file mode 100644 index 000000000..08d1419fe --- /dev/null +++ b/scripts/install-workflow-nodes.py @@ -0,0 +1,33 @@ +"""Install only explicitly selected, revision-pinned custom node dependencies.""" +import argparse +from pathlib import Path +import re +import subprocess +import sys +sys.path.insert(0, '/') +from workflow_models import contained +import yaml + +parser = argparse.ArgumentParser() +parser.add_argument('--manifests', default='') +args = parser.parse_args() +installed = {} +for selection in filter(None, args.manifests.split(',')): + manifest = yaml.safe_load(contained(Path('/workflow'), selection.strip()).read_text()) + for node in manifest.get('custom_nodes', []): + name, repo, revision = node['name'], node['repo'], node['revision'] + if not re.fullmatch(r'[A-Za-z0-9_-]+', name) or not re.fullmatch(r'[a-f0-9]{40}', revision) or not repo.startswith('https://'): + raise ValueError('Custom nodes require a safe name, HTTPS repo and full commit SHA') + if name in installed: + if installed[name] != (repo, revision): + raise ValueError('Conflicting custom node revisions') + continue + target = '/comfyui/custom_nodes/' + name + subprocess.run(['git', 'clone', '--filter=blob:none', '--no-checkout', repo, target], check=True) + subprocess.run(['git', '-C', target, 'checkout', revision], check=True) + requirements = Path(target) / 'requirements.txt' + if requirements.exists(): + subprocess.run(['uv', 'pip', 'install', '-r', str(requirements)], check=True) + installed[name] = (repo, revision) +if installed: + subprocess.run(['timeout', '300', sys.executable, 'main.py', '--quick-test-for-ci', '--cpu'], cwd='/comfyui', check=True) diff --git a/src/media_output.py b/src/media_output.py new file mode 100644 index 000000000..864725504 --- /dev/null +++ b/src/media_output.py @@ -0,0 +1,42 @@ +"""Senai-compatible media envelope; legacy image responses remain the default.""" +import base64 +import mimetypes +import os +from pathlib import PurePosixPath + +import boto3 + + +def collect_senai_outputs(outputs, job_id, fetch, errors): + if errors: + return {'status': 'error', 'error': '; '.join(errors)} + result = {} + bucket = os.getenv('AWS_BUCKET_NAME') + client = boto3.client('s3') if bucket else None + index = 0 + for node in outputs.values(): + for category in ('images', 'videos', 'gifs', 'audio'): + for item in node.get(category, []): + if item.get('type') == 'temp': + continue + filename = item.get('filename') + if not filename: + raise ValueError('Output has no filename') + data = fetch(filename, item.get('subfolder', ''), item.get('type', 'output')) + if not data: + raise ValueError('Failed to fetch output media') + media_type = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + entry = {'filename': filename, 'subfolder': item.get('subfolder', ''), 'media_type': media_type} + if client: + key = f'renders/{job_id}/{index:02d}-{PurePosixPath(filename).name}' + client.put_object(Bucket=bucket, Key=key, Body=data, ContentType=media_type) + entry.update(type='url', data=client.generate_presigned_url('get_object', Params={'Bucket': bucket, 'Key': key}, ExpiresIn=86400)) + else: + if len(data) > 5 * 1024 * 1024: + raise ValueError('Configure AWS_BUCKET_NAME for outputs larger than 5 MiB') + entry.update(type='base64', data=base64.b64encode(data).decode('ascii')) + result.setdefault(category, []).append(entry) + index += 1 + if not result: + return {'status': 'error', 'error': 'Workflow produced no media'} + return {'status': 'success', 'output': result} diff --git a/src/start.sh b/src/start.sh index 4d10703d9..f5085fa42 100644 --- a/src/start.sh +++ b/src/start.sh @@ -1,5 +1,53 @@ #!/usr/bin/env bash +set -eo pipefail + +WORKER_ROOT="${WORKER_ROOT:-/}" +COMFY_ROOT="${COMFY_ROOT:-/comfyui}" +if [ -d /runpod-volume ]; then + default_model_root=/runpod-volume/models +else + default_model_root="$COMFY_ROOT/models" +fi +export COMFY_MODEL_ROOT="${COMFY_MODEL_ROOT:-$default_model_root}" +export COMFY_PID_FILE="${COMFY_PID_FILE:-/tmp/comfyui.pid}" +export REFRESH_WORKER="${REFRESH_WORKER:-true}" + +# A checklist is an explicit standalone mode: never start a worker without models. +if [ "${MODEL_DOWNLOAD_CHECK_ONLY:-false}" = "true" ]; then + exec python3 "$WORKER_ROOT/workflow_models.py" --check +fi + +if [ "${PREPARE_MODELS_ONLY:-false}" = "true" ]; then + exec python3 "$WORKER_ROOT/workflow_models.py" +fi + +comfy_pid="" +handler_pid="" +cleanup() { + trap - EXIT TERM INT + for pid in "$handler_pid" "$comfy_pid"; do + [ -z "$pid" ] || kill -TERM "$pid" 2>/dev/null || true + done + # Bound shutdown even if a child ignores SIGTERM. + for attempt in {1..50}; do + alive=false + for pid in "$handler_pid" "$comfy_pid"; do + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then alive=true; fi + done + [ "$alive" = true ] || break + sleep 0.1 + done + for pid in "$handler_pid" "$comfy_pid"; do + [ -z "$pid" ] || kill -KILL "$pid" 2>/dev/null || true + [ -z "$pid" ] || wait "$pid" 2>/dev/null || true + done + rm -f "$COMFY_PID_FILE" +} +trap cleanup EXIT +trap 'exit 143' TERM +trap 'exit 130' INT + # Start SSH server if PUBLIC_KEY is set (enables remote access and dev-sync.sh) if [ -n "$PUBLIC_KEY" ]; then mkdir -p ~/.ssh @@ -19,8 +67,8 @@ if [ -n "$PUBLIC_KEY" ]; then fi # Use libtcmalloc for better memory management -TCMALLOC="$(ldconfig -p | grep -Po "libtcmalloc.so.\d" | head -n 1)" -export LD_PRELOAD="${TCMALLOC}" +TCMALLOC="$(ldconfig -p | grep -Po "libtcmalloc.so.\d" | head -n 1 || true)" +if [ -n "$TCMALLOC" ]; then export LD_PRELOAD="${TCMALLOC}"; fi # --------------------------------------------------------------------------- # GPU pre-flight check @@ -56,6 +104,11 @@ except Exception as e: fi echo "worker-comfyui: GPU available — $GPU_CHECK" +python3 "$WORKER_ROOT/workflow_models.py" & +handler_pid=$! +wait "$handler_pid" +handler_pid="" + # Ensure ComfyUI-Manager runs in offline network mode inside the container comfy-manager-set-mode offline || echo "worker-comfyui - Could not set ComfyUI-Manager network_mode" >&2 @@ -64,20 +117,25 @@ echo "worker-comfyui: Starting ComfyUI" # Allow operators to tweak verbosity; default is DEBUG. : "${COMFY_LOG_LEVEL:=DEBUG}" -# PID file used by the handler to detect if ComfyUI is still running -COMFY_PID_FILE="/tmp/comfyui.pid" - -# Serve the API and don't shutdown the container -if [ "$SERVE_API_LOCALLY" == "true" ]; then - python -u /comfyui/main.py --disable-auto-launch --disable-metadata --listen --verbose "${COMFY_LOG_LEVEL}" --log-stdout & - echo $! > "$COMFY_PID_FILE" - - echo "worker-comfyui: Starting RunPod Handler" - python -u /handler.py --rp_serve_api --rp_api_host=0.0.0.0 -else - python -u /comfyui/main.py --disable-auto-launch --disable-metadata --verbose "${COMFY_LOG_LEVEL}" --log-stdout & - echo $! > "$COMFY_PID_FILE" +comfy_args=(--disable-auto-launch --disable-metadata --verbose "$COMFY_LOG_LEVEL" --log-stdout) +if [ -n "${WORKFLOWS:-${WORKFLOW_MANIFESTS:-}}" ]; then + comfy_args+=(--extra-model-paths-config "${WORKFLOW_MODEL_PATHS:-/tmp/workflow_model_paths.yaml}") +fi +handler_args=() +if [ "${SERVE_API_LOCALLY:-false}" = "true" ]; then + comfy_args+=(--listen) + handler_args+=(--rp_serve_api --rp_api_host=0.0.0.0) +fi +python -u "$COMFY_ROOT/main.py" "${comfy_args[@]}" & +comfy_pid=$! +echo "$comfy_pid" > "$COMFY_PID_FILE" +echo "worker-comfyui: Starting RunPod Handler" +python -u "$WORKER_ROOT/handler.py" "${handler_args[@]}" "$@" & +handler_pid=$! - echo "worker-comfyui: Starting RunPod Handler" - python -u /handler.py -fi \ No newline at end of file +# Exit when either service exits; the EXIT trap stops the other child. +status=0 +finished="" +wait -n -p finished "$comfy_pid" "$handler_pid" || status=$? +if [ "$finished" = "$comfy_pid" ] && [ "$status" = 0 ]; then status=1; fi +exit "$status" diff --git a/src/workflow_models.py b/src/workflow_models.py new file mode 100644 index 000000000..ffcbfd8ba --- /dev/null +++ b/src/workflow_models.py @@ -0,0 +1,223 @@ +"""Validated, declarative model downloads selected by WORKFLOWS.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import tempfile +import time +from urllib.parse import urlparse + +import requests +import yaml + + +def contained(root, relative): + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise ValueError("Expected a nonempty relative path") + result = (root / relative).resolve() + if not result.is_relative_to(root.resolve()) or result == root.resolve(): + raise ValueError("Path escapes configured root") + return result + + +MODEL_SUFFIXES = ('.safetensors', '.gguf', '.ckpt', '.pt', '.pth', '.bin') + + +def graph_assets(graph): + """Read API graphs or editor metadata, including nested subgraphs.""" + assets, references = [], set() + if isinstance(graph.get('nodes'), list): + nodes = list(graph['nodes']) + for subgraph in graph.get('definitions', {}).get('subgraphs', []): + extra_assets, extra_refs = graph_assets(subgraph) + assets.extend(extra_assets) + references.update(extra_refs) + for node in nodes: + for model in node.get('properties', {}).get('models', []): + assets.append({'path': model['directory'] + '/' + model['name'], + 'url': model['url'], 'sha256': model.get('sha256')}) + if node.get('type') in ('MarkdownNote', 'Note'): + continue + for value in node.get('widgets_values') or []: + if isinstance(value, str) and value.endswith(MODEL_SUFFIXES): + references.add(value) + else: + if not graph or not all(isinstance(n, dict) and 'class_type' in n for n in graph.values()): + raise ValueError('Expected a ComfyUI API or editor workflow') + for node in graph.values(): + for value in node.get('inputs', {}).values(): + if isinstance(value, str) and value.endswith(MODEL_SUFFIXES): + references.add(value) + return assets, references + + +def load_plan(selection, workflow_root, model_root): + """Only selected YAML/JSON files contribute assets; never scan the folder.""" + plan = {} + for name in selection.split(','): + name = name.strip() + document = yaml.safe_load(contained(workflow_root, name).read_text()) + if name.endswith('.json'): + models, references = graph_assets(document) + else: + if not isinstance(document, dict) or document.get('version') != 1: + raise ValueError(f'Unsupported manifest version: {name}') + models = list(document.get('models', [])) + references = set() + graphs = document.get('workflows') + if not isinstance(graphs, list) or not graphs: + raise ValueError('Manifest must reference at least one workflow') + for graph_path in graphs: + graph = json.loads(contained(workflow_root, graph_path).read_text()) + embedded, refs = graph_assets(graph) + models.extend(embedded) + references.update(refs) + declared = set() + for model in models: + target = contained(model_root, model['path']) + if len(Path(model['path']).parts) < 2: + raise ValueError('Model path must include category and filename') + parsed = urlparse(model['url']) + if parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password: + raise ValueError('Model URL must use HTTPS without embedded credentials') + digest = model.get('sha256') + if digest is not None and not re.fullmatch(r'[a-fA-F0-9]{64}', str(digest)): + raise ValueError('Invalid sha256') + item = {'path': model['path'], 'url': model['url'], 'sha256': digest} + if target in plan and plan[target] != item: + raise ValueError(f"Conflicting model definitions: {model['path']}") + plan[target] = item + declared.add('/'.join(Path(model['path']).parts[1:])) + missing = references - declared + if missing: + raise ValueError('Workflow model missing from manifest: ' + ', '.join(sorted(missing))) + return plan + + +def cache_record(path, model): + stat = path.stat() + return {**model, 'size': stat.st_size, 'mtime_ns': stat.st_mtime_ns} + + +def receipt_path(path): + return path.with_name(path.name + '.worker-cache.json') + + +def cached(path, model): + if not path.is_file() or path.stat().st_size == 0: + return False + try: + return json.loads(receipt_path(path).read_text()) == cache_record(path, model) + except (OSError, ValueError): + return False + + +def record_cache(path, model): + # Called under the per-model lock, after successful verification/download. + receipt = receipt_path(path) + temporary = receipt.with_suffix('.tmp') + temporary.write_text(json.dumps(cache_record(path, model))) + temporary.replace(receipt) + + + +def valid_file(path, digest): + if not path.is_file() or path.stat().st_size == 0: + return False + if digest: + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() == digest.lower() + return True + + +def _download(path, model, attempts=3): + if cached(path, model): + print(f"[cached] {model['path']}") + return + if not receipt_path(path).exists() and valid_file(path, model['sha256']): + record_cache(path, model) + print(f"[adopted] {model['path']}") + return + token = os.getenv('HF_TOKEN') or os.getenv('HUGGINGFACE_ACCESS_TOKEN') + headers = {} + if token and urlparse(model['url']).hostname == 'huggingface.co': + headers['Authorization'] = f'Bearer {token}' + for attempt in range(attempts): + temporary = None + try: + with requests.get(model['url'], headers=headers, stream=True, timeout=(30, 120)) as response: + response.raise_for_status() + if 'text/html' in response.headers.get('Content-Type', ''): + raise ValueError('Server returned HTML instead of a model') + with tempfile.NamedTemporaryFile(dir=path.parent, prefix=path.name + '.', suffix='.part', delete=False) as out: + temporary = Path(out.name) + for chunk in response.iter_content(8 * 1024 * 1024): + out.write(chunk) + if not valid_file(temporary, model['sha256']): + raise ValueError('Empty download or checksum mismatch') + length = response.headers.get('Content-Length') + if length and not response.headers.get('Content-Encoding') and temporary.stat().st_size != int(length): + raise ValueError('Incomplete download') + temporary.replace(path) + record_cache(path, model) + print(f"[downloaded] {model['path']}") + return + except (requests.RequestException, OSError, ValueError): + if attempt + 1 == attempts: + # Never print signed URLs or authorization headers from exceptions. + raise RuntimeError(f"Download failed: {model['path']} ({attempts} attempts)") from None + time.sleep(2 ** attempt) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def download(path, model, attempts=3): + path.parent.mkdir(parents=True, exist_ok=True) + # Shared-volume workers must not download the same asset concurrently. + with path.with_name(path.name + '.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + _download(path, model, attempts) + + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--check', action='store_true', help='Validate and print checklist; no downloads or worker startup') + args = parser.parse_args() + selection = os.getenv('WORKFLOWS') or os.getenv('WORKFLOW_MANIFESTS', '') + if not selection: + print('No workflows selected; using existing models') + return + root = Path(os.getenv('WORKFLOW_DIR', '/workflow')) + models = Path(os.getenv('COMFY_MODEL_ROOT', '/comfyui/models')) + plan = load_plan(selection, root, models) + policy = os.getenv('MODEL_DOWNLOAD_POLICY', 'missing') + if policy not in ('missing', 'cache-only'): + raise ValueError('MODEL_DOWNLOAD_POLICY must be missing or cache-only') + if args.check: + for path, model in plan.items(): + state = 'cached' if cached(path, model) else 'prepare required' + print(f"[ ] {model['path']} ({state})") + elif policy == 'cache-only': + missing = [item['path'] for path, item in plan.items() if not cached(path, item)] + if missing: + raise RuntimeError('Prepare model cache before starting worker: ' + ', '.join(missing)) + else: + workers = int(os.getenv('MODEL_DOWNLOAD_CONCURRENCY', '4')) + if not 1 <= workers <= 16: + raise ValueError('MODEL_DOWNLOAD_CONCURRENCY must be between 1 and 16') + with ThreadPoolExecutor(max_workers=workers) as pool: + list(pool.map(lambda pair: download(*pair), plan.items())) + if not args.check: + categories = {Path(item['path']).parts[0] for item in plan.values()} + config = {'workflow_models': {'base_path': str(models.resolve()), **{key: key + '/' for key in sorted(categories)}}} + Path(os.getenv('WORKFLOW_MODEL_PATHS', '/tmp/workflow_model_paths.yaml')).write_text(yaml.safe_dump(config)) + + +if __name__ == '__main__': + main() diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 000000000..96205d79c --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,10 @@ +# CPU-only contract/lifecycle tests. No ComfyUI or model weights required. +FROM python:3.12-slim +WORKDIR /worker +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt pytest +COPY handler.py ./ +COPY src/ ./src/ +COPY workflow/ ./workflow/ +COPY tests/ ./tests/ +CMD ["python", "-m", "pytest", "tests", "-q"] diff --git a/tests/test_serverless_lifecycle.py b/tests/test_serverless_lifecycle.py new file mode 100644 index 000000000..90e518f0e --- /dev/null +++ b/tests/test_serverless_lifecycle.py @@ -0,0 +1,123 @@ +import json +import os +from pathlib import Path +import subprocess +import sys +import time +from unittest.mock import MagicMock, patch + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT), str(ROOT / 'src')] +import handler +import media_output + + +@pytest.mark.parametrize('result', [{'images': []}, {'error': 'bad input'}]) +def test_refresh_all_results(result): + with patch.object(handler, '_handle_job', return_value=result.copy()), patch.object(handler, 'REFRESH_WORKER', True): + assert handler.handler({})['refresh_worker'] is True + + +def test_refresh_exception_and_opt_out(): + with patch.object(handler, '_handle_job', side_effect=RuntimeError('oops')), patch.object(handler, 'REFRESH_WORKER', True): + assert handler.handler({})['refresh_worker'] is True + with patch.object(handler, '_handle_job', return_value={'images': []}), patch.object(handler, 'REFRESH_WORKER', False): + assert 'refresh_worker' not in handler.handler({}) + + +def test_ltx_graph_to_senai_video_response(monkeypatch): + monkeypatch.setenv('OUTPUT_FORMAT', 'senai') + monkeypatch.setenv('AWS_BUCKET_NAME', 'staging') + graph = json.loads((ROOT / 'workflow/ltx25-t2v-v1.json').read_text()) + ws = MagicMock() + ws.recv.return_value = json.dumps({'type': 'executing', 'data': {'node': None, 'prompt_id': 'p'}}) + s3 = MagicMock() + s3.generate_presigned_url.return_value = 'https://media.example/video.mp4' + with patch.object(handler, 'check_server', return_value=True), patch.object(handler, 'validate_workflow_models', return_value=None), patch.object(handler, 'queue_workflow', return_value={'prompt_id': 'p'}) as queue, patch.object(handler.websocket, 'WebSocket', return_value=ws), patch.object(handler, 'get_history', return_value={'p': {'outputs': {'75': {'images': [{'filename': 'video.mp4', 'type': 'output'}]}}}}), patch.object(handler, 'get_image_data', return_value=b'fake mp4'), patch.object(media_output.boto3, 'client', return_value=s3): + result = handler.handler({'id': 'test', 'input': {'workflow': graph}}) + assert result['status'] == 'success' + assert result['refresh_worker'] is True + assert result['output']['images'][0]['media_type'] == 'video/mp4' + assert result['output']['images'][0]['type'] == 'url' + assert queue.call_args.args[0] == graph + s3.put_object.assert_called_once() + + +@pytest.mark.parametrize('scenario', ['completed', 'comfy_crash', 'signal']) +def test_start_supervises_and_reaps_children(tmp_path, scenario): + # Run the real shell entrypoint against lightweight process doubles. + bin_dir = tmp_path / 'bin' + bin_dir.mkdir() + fake = bin_dir / 'python' + fake.write_text('''#!/usr/bin/env python3 +import os,sys,time +from pathlib import Path +if '-c' in sys.argv: + print('OK: mock GPU'); sys.exit(0) +if any('workflow_models.py' in arg for arg in sys.argv): sys.exit(0) +role = 'comfy' if any('main.py' in arg for arg in sys.argv) else 'handler' +Path(os.environ['TEST_ROOT'], role+'.pid').write_text(str(os.getpid())) +if os.environ['SCENARIO'] == 'completed' and role == 'handler': + time.sleep(.3); sys.exit(0) +if os.environ['SCENARIO'] == 'comfy_crash' and role == 'comfy': + time.sleep(.3); sys.exit(7) +time.sleep(60) +''') + fake.chmod(0o755) + # start.sh uses both python and python3; shebang must avoid recursion. + fake.write_text(fake.read_text().replace('#!/usr/bin/env python3', '#!' + sys.executable)) + (bin_dir / 'python3').symlink_to(fake) + env = {**os.environ, 'PATH': str(bin_dir) + ':' + os.environ['PATH'], 'TEST_ROOT': str(tmp_path), 'SCENARIO': scenario, 'PUBLIC_KEY': '', 'WORKFLOW_MANIFESTS': '', 'COMFY_PID_FILE': str(tmp_path / 'comfyui.pid')} + proc = subprocess.Popen(['bash', str(ROOT / 'src/start.sh')], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + try: + if scenario == 'signal': + deadline = time.monotonic() + 5 + while not (tmp_path / 'handler.pid').exists() and time.monotonic() < deadline: + time.sleep(.02) + assert (tmp_path / 'handler.pid').exists() + proc.terminate() + output, _ = proc.communicate(timeout=10) + assert proc.returncode == {'completed': 0, 'comfy_crash': 7, 'signal': 143}[scenario], output.decode() + for role in ('comfy', 'handler'): + pid = int((tmp_path / (role + '.pid')).read_text()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + assert not (tmp_path / 'comfyui.pid').exists() + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + + +@pytest.mark.parametrize('result', [{'images': [{'filename': 'x.png'}]}, {'error': 'invalid workflow'}]) +def test_real_runpod_sdk_reports_result_with_stop_pod(result): + import asyncio + from runpod.serverless.modules.rp_job import run_job + with patch.object(handler, '_handle_job', return_value=result.copy()), patch.object(handler, 'REFRESH_WORKER', True): + reply = asyncio.run(run_job(handler.handler, {'id': 'local-sdk-test', 'input': {}})) + assert reply['stopPod'] is True + if 'error' in result: + assert reply['error'] == result['error'] + else: + assert reply['output'] == result + + +def test_preparation_entrypoint_needs_no_gpu_or_handler(tmp_path): + import workflow_models + plan = workflow_models.load_plan('minimax-h3.yaml', ROOT / 'workflow', tmp_path) + for path, item in plan.items(): + path.parent.mkdir(exist_ok=True) + path.write_bytes(b'prepared-model') + workflow_models.record_cache(path, item) + env = {**os.environ, 'PATH': str(Path(sys.executable).parent) + ':' + os.environ['PATH'], + 'WORKER_ROOT': str(ROOT / 'src'), 'WORKFLOW_DIR': str(ROOT / 'workflow'), + 'WORKFLOWS': 'minimax-h3.yaml', 'COMFY_MODEL_ROOT': str(tmp_path), + 'WORKFLOW_MODEL_PATHS': str(tmp_path / 'paths.yaml'), + 'PREPARE_MODELS_ONLY': 'true', 'MODEL_DOWNLOAD_POLICY': 'cache-only'} + result = subprocess.run(['bash', str(ROOT / 'src/start.sh')], env=env, capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'paths.yaml').exists() + assert 'GPU' not in result.stdout + assert 'Starting RunPod Handler' not in result.stdout diff --git a/tests/test_workflow_models.py b/tests/test_workflow_models.py new file mode 100644 index 000000000..1f2842718 --- /dev/null +++ b/tests/test_workflow_models.py @@ -0,0 +1,157 @@ +import hashlib +import json +from pathlib import Path +import sys +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +import workflow_models as models + +ROOT = Path(__file__).resolve().parents[1] + + +def test_senai_all_modes_six_models(tmp_path): + plan = models.load_plan('ltx25.yaml', ROOT / 'workflow', tmp_path) + assert len(plan) == 6 + assert {p.parent.name for p in plan} == {'diffusion_models', 'text_encoders', 'vae', 'latent_upscale_models'} + + +def test_manifest_missing_model_and_escape(tmp_path): + (tmp_path / 'graph.json').write_text(json.dumps({'1': {'class_type': 'NewLoader', 'inputs': {'model': 'missing.gguf'}}})) + manifest = {'version': 1, 'workflows': ['graph.json'], 'models': [{'path': 'vae/a.pt', 'url': 'https://example.com/a.pt'}]} + (tmp_path / 'm.yaml').write_text(yaml.safe_dump(manifest)) + with pytest.raises(ValueError, match='missing from manifest'): + models.load_plan('m.yaml', tmp_path, tmp_path / 'models') + manifest['models'][0]['path'] = '../escape.pt' + (tmp_path / 'm.yaml').write_text(yaml.safe_dump(manifest)) + with pytest.raises(ValueError, match='escapes'): + models.load_plan('m.yaml', tmp_path, tmp_path / 'models') + + +def test_download_atomic_cached_and_auth(tmp_path, monkeypatch): + monkeypatch.setenv('HF_TOKEN', 'secret') + content = b'mock model weights' + item = {'path': 'vae/a.pt', 'url': 'https://huggingface.co/a/b/resolve/main/a.pt', 'sha256': hashlib.sha256(content).hexdigest()} + response = MagicMock() + response.__enter__.return_value = response + response.headers = {'Content-Length': str(len(content))} + response.iter_content.return_value = [content] + target = tmp_path / 'vae/a.pt' + with patch.object(models.requests, 'get', return_value=response) as get: + models.download(target, item) + models.download(target, item) + assert get.call_count == 1 + assert get.call_args.kwargs['headers']['Authorization'] == 'Bearer secret' + assert target.read_bytes() == content + assert not list(target.parent.glob('*.part')) + + +def test_failed_download_preserves_existing_and_cleans_partial(tmp_path): + target = tmp_path / 'a.pt' + target.write_bytes(b'previous') + item = {'path': 'a.pt', 'url': 'https://example.com/a.pt', 'sha256': '0' * 64} + response = MagicMock() + response.__enter__.return_value = response + response.headers = {} + response.iter_content.return_value = [b'corrupt'] + with patch.object(models.requests, 'get', return_value=response), pytest.raises(RuntimeError): + models.download(target, item, attempts=1) + assert target.read_bytes() == b'previous' + assert not list(tmp_path.glob('*.part')) + + +def test_check_does_not_download_or_write(tmp_path, monkeypatch, capsys): + monkeypatch.setenv('WORKFLOW_MANIFESTS', 'ltx25.yaml') + monkeypatch.setenv('WORKFLOW_DIR', str(ROOT / 'workflow')) + monkeypatch.setenv('COMFY_MODEL_ROOT', str(tmp_path / 'models')) + monkeypatch.setattr(sys, 'argv', ['workflow_models.py', '--check']) + with patch.object(models.requests, 'get') as get: + models.main() + get.assert_not_called() + assert capsys.readouterr().out.count('[ ]') == 6 + assert not (tmp_path / 'models').exists() + + +def test_multiple_manifests_deduplicate_and_reject_conflicts(tmp_path): + (tmp_path / 'graph.json').write_text(json.dumps({'1': {'class_type': 'NewLoader', 'inputs': {'model': 'a.pt'}}})) + manifest = {'version': 1, 'workflows': ['graph.json'], 'models': [{'path': 'custom/a.pt', 'url': 'https://example.com/a.pt'}]} + for name in ('a.yaml', 'b.yaml'): + (tmp_path / name).write_text(yaml.safe_dump(manifest)) + assert len(models.load_plan('a.yaml,b.yaml', tmp_path, tmp_path / 'models')) == 1 + manifest['models'][0]['url'] = 'https://example.com/different.pt' + (tmp_path / 'b.yaml').write_text(yaml.safe_dump(manifest)) + with pytest.raises(ValueError, match='Conflicting'): + models.load_plan('a.yaml,b.yaml', tmp_path, tmp_path / 'models') + + +def test_custom_model_root_is_registered_with_comfy(tmp_path, monkeypatch): + monkeypatch.setenv('WORKFLOW_MANIFESTS', 'ltx25.yaml') + monkeypatch.setenv('WORKFLOW_DIR', str(ROOT / 'workflow')) + monkeypatch.setenv('COMFY_MODEL_ROOT', str(tmp_path / 'persistent-models')) + monkeypatch.setenv('WORKFLOW_MODEL_PATHS', str(tmp_path / 'paths.yaml')) + monkeypatch.setattr(sys, 'argv', ['workflow_models.py']) + with patch.object(models, 'download') as download: + models.main() + assert download.call_count == 6 + config = yaml.safe_load((tmp_path / 'paths.yaml').read_text())['workflow_models'] + assert config['base_path'] == str(tmp_path / 'persistent-models') + assert config['latent_upscale_models'] == 'latent_upscale_models/' + assert config['text_encoders'] == 'text_encoders/' + + +def test_minimax_editor_metadata_does_not_include_ltx(tmp_path): + plan = models.load_plan('minimax-h3.yaml', ROOT / 'workflow', tmp_path) + direct = models.load_plan('video_minimax_h3_r2v.json', ROOT / 'workflow', tmp_path) + assert plan == direct + assert len(plan) == 5 + assert all('MiniMax-H3' in item['url'] for item in plan.values()) + assert not any('ltx' in item['path'].lower() for item in plan.values()) + + +def test_cache_only_no_network_no_hash_and_detects_changed_file(tmp_path, monkeypatch): + monkeypatch.setenv('WORKFLOWS', 'minimax-h3.yaml') + monkeypatch.setenv('WORKFLOW_DIR', str(ROOT / 'workflow')) + monkeypatch.setenv('COMFY_MODEL_ROOT', str(tmp_path)) + monkeypatch.setenv('WORKFLOW_MODEL_PATHS', str(tmp_path / 'paths.yaml')) + monkeypatch.setenv('MODEL_DOWNLOAD_POLICY', 'cache-only') + monkeypatch.setattr(sys, 'argv', ['workflow_models.py']) + plan = models.load_plan('minimax-h3.yaml', ROOT / 'workflow', tmp_path) + for path, item in plan.items(): + path.parent.mkdir(exist_ok=True) + path.write_bytes(b'prepared') + models.record_cache(path, item) + with patch.object(models.requests, 'get') as get, patch.object(models.hashlib, 'file_digest') as digest: + models.main() + get.assert_not_called() + digest.assert_not_called() + next(iter(plan)).write_bytes(b'modified') + with patch.object(models.requests, 'get') as get, pytest.raises(RuntimeError, match='Prepare model cache'): + models.main() + get.assert_not_called() + + +def test_cache_identity_changes_when_source_changes(tmp_path): + path = tmp_path / 'model.pt' + path.write_bytes(b'weights') + item = {'path': 'vae/model.pt', 'url': 'https://example.com/v1', 'sha256': None} + models.record_cache(path, item) + assert models.cached(path, item) + assert not models.cached(path, {**item, 'url': 'https://example.com/v2'}) + + +def test_concurrent_workers_download_once(tmp_path): + from concurrent.futures import ThreadPoolExecutor + item = {'path': 'vae/model.pt', 'url': 'https://example.com/model.pt', 'sha256': None} + response = MagicMock() + response.__enter__.return_value = response + response.headers = {} + response.iter_content.return_value = [b'weights'] + target = tmp_path / 'vae/model.pt' + with patch.object(models.requests, 'get', return_value=response) as get: + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(lambda _: models.download(target, item), range(2))) + assert get.call_count == 1 + assert models.cached(target, item) diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 000000000..560e98bfe --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,27 @@ +# Workflow manifests + +`WORKFLOW_MANIFESTS=ltx25.yaml` selects the initial Senai LTX 2.5 stack. + +The three JSON API graphs were copied unchanged from +`senai/backend/src/senai/modules/providers/runpod/workflows/` on 2026-09-22. +Their upstream contract pins `vavo/LTX2.5-serverless` revision +`5e6a88f116c6611420a7073a74cea684870ef01c`. Download locations follow that +checkout's `src/bootstrap_ltx25.sh`. + +The YAML prepares assets; the request's `input.workflow` remains the execution +graph. Adding a workflow requires its API JSON and a manifest listing all model +assets, plus installing any required custom nodes in the image. Multiple +manifests can share a model only when its URL and checksum agree. + +The initial URLs use upstream `main`; real downloads and hashes have not been +verified. Pin revisions and `sha256` values after commissioning. See +[customization](../docs/customization.md#generic-workflow-selection-and-model-cache) and +[deployment](../docs/deployment.md#generic-worker-prepare-assets-separately-from-serving). + +## Additional workflows + +`WORKFLOWS=minimax-h3.yaml` (or `video_minimax_h3_r2v.json`) reads the five model +assets directly from editor metadata. It never selects LTX assets. Editor JSON +is not a handler payload: export an API graph before inference. The YAML is +only a reference and has no duplicated model URLs. See the generic cache +preparation instructions in the deployment guide. diff --git a/workflow/ltx25-flf-v1.json b/workflow/ltx25-flf-v1.json new file mode 100644 index 000000000..b708f17e6 --- /dev/null +++ b/workflow/ltx25-flf-v1.json @@ -0,0 +1,605 @@ +{ + "75": { + "class_type": "SaveVideo", + "_meta": { + "title": "Save LTX 2.5 video" + }, + "inputs": { + "filename_prefix": "video/LTX-2.5_flf", + "format": "auto", + "codec": "auto", + "video": [ + "398:370", + 0 + ] + } + }, + "395": { + "class_type": "LoadImage", + "_meta": { + "title": "Load first frame" + }, + "inputs": { + "image": "source-image.png" + } + }, + "398:393": { + "class_type": "CLIPLoader", + "_meta": { + "title": "CLIPLoader" + }, + "inputs": { + "clip_name": "gemma4_e2b_it_bf16.safetensors", + "type": "ltxv", + "device": "default" + } + }, + "398:380": { + "class_type": "TextGenerateLTX2Prompt", + "_meta": { + "title": "TextGenerateLTX2Prompt" + }, + "inputs": { + "clip": [ + "398:393", + 0 + ], + "image": [ + "398:350", + 0 + ], + "prompt": [ + "398:376", + 0 + ], + "max_length": 600, + "sampling_mode": "on", + "sampling_mode.temperature": 0.7, + "sampling_mode.top_k": 64, + "sampling_mode.top_p": 0.95, + "sampling_mode.min_p": 0.05, + "sampling_mode.repetition_penalty": 1.15, + "sampling_mode.seed": 0 + } + }, + "398:383": { + "class_type": "PrimitiveBoolean", + "_meta": { + "title": "Boolean (Enable Prompt Enhance)" + }, + "inputs": { + "value": true + } + }, + "398:364": { + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIPTextEncode" + }, + "inputs": { + "clip": [ + "398:387", + 0 + ], + "text": [ + "398:382", + 0 + ] + } + }, + "398:376": { + "class_type": "PrimitiveStringMultiline", + "_meta": { + "title": "Prompt" + }, + "inputs": { + "value": "A cinematic subject comes alive from the first frame with natural motion, synchronized sound, and no text." + } + }, + "398:382": { + "class_type": "ComfySwitchNode", + "_meta": { + "title": "ComfySwitchNode" + }, + "inputs": { + "on_false": [ + "398:376", + 0 + ], + "on_true": [ + "398:380", + 0 + ], + "switch": [ + "398:383", + 0 + ] + } + }, + "398:384": { + "class_type": "UNETLoader", + "_meta": { + "title": "UNETLoader" + }, + "inputs": { + "unet_name": "ltx-2.5-22b-distilled-transformer-comfy-int8-convrot.safetensors", + "weight_dtype": "default" + } + }, + "398:362": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Duration" + }, + "inputs": { + "value": 5 + } + }, + "398:365": { + "class_type": "LTXVConditioning", + "_meta": { + "title": "LTXVConditioning" + }, + "inputs": { + "positive": [ + "398:364", + 0 + ], + "negative": [ + "398:373", + 0 + ], + "frame_rate": [ + "398:359", + 0 + ] + } + }, + "398:385": { + "class_type": "VAELoader", + "_meta": { + "title": "VAELoader" + }, + "inputs": { + "vae_name": "ltx-2.5-video-vae-bf16.safetensors" + } + }, + "398:386": { + "class_type": "VAELoader", + "_meta": { + "title": "VAELoader" + }, + "inputs": { + "vae_name": "ltx-2.5-audio-vae-bf16.safetensors" + } + }, + "398:372": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Width" + }, + "inputs": { + "value": 1280 + } + }, + "398:373": { + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIPTextEncode" + }, + "inputs": { + "clip": [ + "398:387", + 0 + ], + "text": "pc game, console game, video game, cartoon, childish, ugly" + } + }, + "398:387": { + "class_type": "CLIPLoader", + "_meta": { + "title": "CLIPLoader" + }, + "inputs": { + "clip_name": "gemma4-12b-with-proj-ltx-2.5-comfy-int8-convrot.safetensors", + "type": "ltxv", + "device": "default" + } + }, + "398:360": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Height" + }, + "inputs": { + "value": 720 + } + }, + "398:352": { + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + }, + "inputs": { + "sampler_name": "euler_ancestral" + } + }, + "398:339": { + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + }, + "inputs": { + "noise_seed": 875362541677469 + } + }, + "398:397": { + "class_type": "ManualSigmas", + "_meta": { + "title": "ManualSigmas" + }, + "inputs": { + "sigmas": "1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0" + } + }, + "398:361": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Frame Rate" + }, + "inputs": { + "value": 24 + } + }, + "398:388": { + "class_type": "LTXVDualCFGGuider", + "_meta": { + "title": "LTXVDualCFGGuider" + }, + "inputs": { + "positive": [ + "398:901", + 0 + ], + "negative": [ + "398:901", + 1 + ], + "model": [ + "398:384", + 0 + ], + "video_cfg": 1, + "audio_cfg": 1 + } + }, + "398:344": { + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + }, + "inputs": { + "noise": [ + "398:339", + 0 + ], + "guider": [ + "398:388", + 0 + ], + "sampler": [ + "398:352", + 0 + ], + "sigmas": [ + "398:397", + 0 + ], + "latent_image": [ + "398:377", + 0 + ] + } + }, + "398:366": { + "class_type": "LTXVEmptyLatentAudio", + "_meta": { + "title": "LTXVEmptyLatentAudio" + }, + "inputs": { + "audio_vae": [ + "398:386", + 0 + ], + "frames_number": [ + "398:378", + 1 + ], + "frame_rate": [ + "398:359", + 1 + ], + "batch_size": 1 + } + }, + "398:367": { + "class_type": "LTXVSeparateAVLatent", + "_meta": { + "title": "LTXVSeparateAVLatent" + }, + "inputs": { + "av_latent": [ + "398:344", + 0 + ] + } + }, + "398:377": { + "class_type": "LTXVConcatAVLatent", + "_meta": { + "title": "LTXVConcatAVLatent" + }, + "inputs": { + "video_latent": [ + "398:901", + 2 + ], + "audio_latent": [ + "398:366", + 0 + ] + } + }, + "398:351": { + "class_type": "ResizeImageMaskNode", + "_meta": { + "title": "ResizeImageMaskNode" + }, + "inputs": { + "input": [ + "395", + 0 + ], + "resize_type": "scale longer dimension", + "resize_type.longer_size": 1536, + "scale_method": "lanczos" + } + }, + "398:350": { + "class_type": "LTXVPreprocess", + "_meta": { + "title": "LTXVPreprocess" + }, + "inputs": { + "image": [ + "398:351", + 0 + ], + "img_compression": 18 + } + }, + "398:356": { + "class_type": "EmptyLTXVLatentVideo", + "_meta": { + "title": "EmptyLTXVLatentVideo" + }, + "inputs": { + "width": [ + "398:372", + 0 + ], + "height": [ + "398:360", + 0 + ], + "length": [ + "398:378", + 1 + ], + "batch_size": 1 + } + }, + "398:359": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "Math Expression (fps)" + }, + "inputs": { + "values.a": [ + "398:361", + 0 + ], + "expression": "a" + } + }, + "398:378": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "Math Expression (length)" + }, + "inputs": { + "values.a": [ + "398:362", + 0 + ], + "values.b": [ + "398:361", + 0 + ], + "expression": "a * b + 1" + } + }, + "398:374": { + "class_type": "VAEDecodeTiled", + "_meta": { + "title": "VAEDecodeTiled" + }, + "inputs": { + "samples": [ + "398:902", + 2 + ], + "vae": [ + "398:385", + 0 + ], + "tile_size": 512, + "overlap": 64, + "temporal_size": 64, + "temporal_overlap": 16 + } + }, + "398:358": { + "class_type": "LTXVAudioVAEDecode", + "_meta": { + "title": "LTXVAudioVAEDecode" + }, + "inputs": { + "samples": [ + "398:367", + 1 + ], + "audio_vae": [ + "398:386", + 0 + ] + } + }, + "398:370": { + "class_type": "CreateVideo", + "_meta": { + "title": "CreateVideo" + }, + "inputs": { + "images": [ + "398:374", + 0 + ], + "audio": [ + "398:358", + 0 + ], + "fps": [ + "398:359", + 0 + ] + } + }, + "396": { + "class_type": "LoadImage", + "_meta": { + "title": "Load last frame" + }, + "inputs": { + "image": "end-image.png" + } + }, + "398:451": { + "class_type": "ResizeImageMaskNode", + "_meta": { + "title": "ResizeImageMaskNode" + }, + "inputs": { + "input": [ + "396", + 0 + ], + "resize_type": "scale longer dimension", + "resize_type.longer_size": 1536, + "scale_method": "lanczos" + } + }, + "398:450": { + "class_type": "LTXVPreprocess", + "_meta": { + "title": "LTXVPreprocess" + }, + "inputs": { + "image": [ + "398:451", + 0 + ], + "img_compression": 18 + } + }, + "398:900": { + "class_type": "LTXVAddGuide", + "_meta": { + "title": "Guide: first frame" + }, + "inputs": { + "positive": [ + "398:365", + 0 + ], + "negative": [ + "398:365", + 1 + ], + "vae": [ + "398:385", + 0 + ], + "latent": [ + "398:356", + 0 + ], + "image": [ + "398:350", + 0 + ], + "frame_idx": 0, + "strength": 0.7 + } + }, + "398:901": { + "class_type": "LTXVAddGuide", + "_meta": { + "title": "Guide: last frame" + }, + "inputs": { + "positive": [ + "398:900", + 0 + ], + "negative": [ + "398:900", + 1 + ], + "vae": [ + "398:385", + 0 + ], + "latent": [ + "398:900", + 2 + ], + "image": [ + "398:450", + 0 + ], + "frame_idx": -1, + "strength": 0.7 + } + }, + "398:902": { + "class_type": "LTXVCropGuides", + "_meta": { + "title": "Crop guides" + }, + "inputs": { + "positive": [ + "398:901", + 0 + ], + "negative": [ + "398:901", + 1 + ], + "latent": [ + "398:367", + 0 + ] + } + } +} \ No newline at end of file diff --git a/workflow/ltx25-i2v-v1.json b/workflow/ltx25-i2v-v1.json new file mode 100644 index 000000000..17b395d60 --- /dev/null +++ b/workflow/ltx25-i2v-v1.json @@ -0,0 +1,707 @@ +{ + "75": { + "class_type": "SaveVideo", + "_meta": { + "title": "Save LTX 2.5 video" + }, + "inputs": { + "filename_prefix": "video/LTX-2.5_i2v", + "format": "auto", + "codec": "auto", + "video": [ + "398:370", + 0 + ] + } + }, + "395": { + "class_type": "LoadImage", + "_meta": { + "title": "Load first frame" + }, + "inputs": { + "image": "source-image.png" + } + }, + "398:393": { + "class_type": "CLIPLoader", + "_meta": { + "title": "CLIPLoader" + }, + "inputs": { + "clip_name": "gemma4_e2b_it_bf16.safetensors", + "type": "ltxv", + "device": "default" + } + }, + "398:380": { + "class_type": "TextGenerateLTX2Prompt", + "_meta": { + "title": "TextGenerateLTX2Prompt" + }, + "inputs": { + "clip": [ + "398:393", + 0 + ], + "image": [ + "398:350", + 0 + ], + "prompt": [ + "398:376", + 0 + ], + "max_length": 600, + "sampling_mode": "on", + "sampling_mode.temperature": 0.7, + "sampling_mode.top_k": 64, + "sampling_mode.top_p": 0.95, + "sampling_mode.min_p": 0.05, + "sampling_mode.repetition_penalty": 1.15, + "sampling_mode.seed": 0 + } + }, + "398:383": { + "class_type": "PrimitiveBoolean", + "_meta": { + "title": "Boolean (Enable Prompt Enhance)" + }, + "inputs": { + "value": true + } + }, + "398:364": { + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIPTextEncode" + }, + "inputs": { + "clip": [ + "398:387", + 0 + ], + "text": [ + "398:382", + 0 + ] + } + }, + "398:376": { + "class_type": "PrimitiveStringMultiline", + "_meta": { + "title": "Prompt" + }, + "inputs": { + "value": "A cinematic subject comes alive from the first frame with natural motion, synchronized sound, and no text." + } + }, + "398:382": { + "class_type": "ComfySwitchNode", + "_meta": { + "title": "ComfySwitchNode" + }, + "inputs": { + "on_false": [ + "398:376", + 0 + ], + "on_true": [ + "398:380", + 0 + ], + "switch": [ + "398:383", + 0 + ] + } + }, + "398:384": { + "class_type": "UNETLoader", + "_meta": { + "title": "UNETLoader" + }, + "inputs": { + "unet_name": "ltx-2.5-22b-distilled-transformer-comfy-int8-convrot.safetensors", + "weight_dtype": "default" + } + }, + "398:362": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Duration" + }, + "inputs": { + "value": 5 + } + }, + "398:363": { + "class_type": "PrimitiveBoolean", + "_meta": { + "title": "Switch to Text to Video?" + }, + "inputs": { + "value": false + } + }, + "398:365": { + "class_type": "LTXVConditioning", + "_meta": { + "title": "LTXVConditioning" + }, + "inputs": { + "positive": [ + "398:364", + 0 + ], + "negative": [ + "398:373", + 0 + ], + "frame_rate": [ + "398:359", + 0 + ] + } + }, + "398:385": { + "class_type": "VAELoader", + "_meta": { + "title": "VAELoader" + }, + "inputs": { + "vae_name": "ltx-2.5-video-vae-bf16.safetensors" + } + }, + "398:386": { + "class_type": "VAELoader", + "_meta": { + "title": "VAELoader" + }, + "inputs": { + "vae_name": "ltx-2.5-audio-vae-bf16.safetensors" + } + }, + "398:372": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Width" + }, + "inputs": { + "value": 1280 + } + }, + "398:373": { + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIPTextEncode" + }, + "inputs": { + "clip": [ + "398:387", + 0 + ], + "text": "pc game, console game, video game, cartoon, childish, ugly" + } + }, + "398:387": { + "class_type": "CLIPLoader", + "_meta": { + "title": "CLIPLoader" + }, + "inputs": { + "clip_name": "gemma4-12b-with-proj-ltx-2.5-comfy-int8-convrot.safetensors", + "type": "ltxv", + "device": "default" + } + }, + "398:360": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Height" + }, + "inputs": { + "value": 720 + } + }, + "398:352": { + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + }, + "inputs": { + "sampler_name": "euler_ancestral" + } + }, + "398:339": { + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + }, + "inputs": { + "noise_seed": 875362541677469 + } + }, + "398:397": { + "class_type": "ManualSigmas", + "_meta": { + "title": "ManualSigmas" + }, + "inputs": { + "sigmas": "1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0" + } + }, + "398:361": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Frame Rate" + }, + "inputs": { + "value": 24 + } + }, + "398:371": { + "class_type": "LatentUpscaleModelLoader", + "_meta": { + "title": "LatentUpscaleModelLoader" + }, + "inputs": { + "model_name": "ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors" + } + }, + "398:388": { + "class_type": "LTXVDualCFGGuider", + "_meta": { + "title": "LTXVDualCFGGuider" + }, + "inputs": { + "positive": [ + "398:365", + 0 + ], + "negative": [ + "398:365", + 1 + ], + "model": [ + "398:384", + 0 + ], + "video_cfg": 1, + "audio_cfg": 1 + } + }, + "398:344": { + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + }, + "inputs": { + "noise": [ + "398:339", + 0 + ], + "guider": [ + "398:388", + 0 + ], + "sampler": [ + "398:352", + 0 + ], + "sigmas": [ + "398:397", + 0 + ], + "latent_image": [ + "398:377", + 0 + ] + } + }, + "398:357": { + "class_type": "LTXVImgToVideoInplace", + "_meta": { + "title": "LTXVImgToVideoInplace" + }, + "inputs": { + "vae": [ + "398:385", + 0 + ], + "image": [ + "398:350", + 0 + ], + "latent": [ + "398:356", + 0 + ], + "bypass": [ + "398:363", + 0 + ], + "strength": 0.7 + } + }, + "398:366": { + "class_type": "LTXVEmptyLatentAudio", + "_meta": { + "title": "LTXVEmptyLatentAudio" + }, + "inputs": { + "audio_vae": [ + "398:386", + 0 + ], + "frames_number": [ + "398:378", + 1 + ], + "frame_rate": [ + "398:359", + 1 + ], + "batch_size": 1 + } + }, + "398:367": { + "class_type": "LTXVSeparateAVLatent", + "_meta": { + "title": "LTXVSeparateAVLatent" + }, + "inputs": { + "av_latent": [ + "398:344", + 0 + ] + } + }, + "398:377": { + "class_type": "LTXVConcatAVLatent", + "_meta": { + "title": "LTXVConcatAVLatent" + }, + "inputs": { + "video_latent": [ + "398:357", + 0 + ], + "audio_latent": [ + "398:366", + 0 + ] + } + }, + "398:351": { + "class_type": "ResizeImageMaskNode", + "_meta": { + "title": "ResizeImageMaskNode" + }, + "inputs": { + "input": [ + "395", + 0 + ], + "resize_type": "scale longer dimension", + "resize_type.longer_size": 1536, + "scale_method": "lanczos" + } + }, + "398:353": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "ComfyMathExpression" + }, + "inputs": { + "values.a": [ + "398:372", + 0 + ], + "expression": "a/2" + } + }, + "398:355": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "ComfyMathExpression" + }, + "inputs": { + "values.a": [ + "398:360", + 0 + ], + "expression": "a/2" + } + }, + "398:350": { + "class_type": "LTXVPreprocess", + "_meta": { + "title": "LTXVPreprocess" + }, + "inputs": { + "image": [ + "398:351", + 0 + ], + "img_compression": 18 + } + }, + "398:356": { + "class_type": "EmptyLTXVLatentVideo", + "_meta": { + "title": "EmptyLTXVLatentVideo" + }, + "inputs": { + "width": [ + "398:353", + 1 + ], + "height": [ + "398:355", + 1 + ], + "length": [ + "398:378", + 1 + ], + "batch_size": 1 + } + }, + "398:348": { + "class_type": "LTXVLatentUpsampler", + "_meta": { + "title": "LTXVLatentUpsampler" + }, + "inputs": { + "samples": [ + "398:367", + 0 + ], + "upscale_model": [ + "398:371", + 0 + ], + "vae": [ + "398:385", + 0 + ] + } + }, + "398:349": { + "class_type": "LTXVImgToVideoInplace", + "_meta": { + "title": "LTXVImgToVideoInplace" + }, + "inputs": { + "image": [ + "398:350", + 0 + ], + "latent": [ + "398:348", + 0 + ], + "bypass": [ + "398:363", + 0 + ], + "vae": [ + "398:385", + 0 + ], + "strength": 1 + } + }, + "398:359": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "Math Expression (fps)" + }, + "inputs": { + "values.a": [ + "398:361", + 0 + ], + "expression": "a" + } + }, + "398:378": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "Math Expression (length)" + }, + "inputs": { + "values.a": [ + "398:362", + 0 + ], + "values.b": [ + "398:361", + 0 + ], + "expression": "a * b + 1" + } + }, + "398:338": { + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + }, + "inputs": { + "noise_seed": 42 + } + }, + "398:340": { + "class_type": "LTXVConcatAVLatent", + "_meta": { + "title": "LTXVConcatAVLatent" + }, + "inputs": { + "video_latent": [ + "398:349", + 0 + ], + "audio_latent": [ + "398:367", + 1 + ] + } + }, + "398:396": { + "class_type": "ManualSigmas", + "_meta": { + "title": "ManualSigmas" + }, + "inputs": { + "sigmas": "0.85, 0.7250, 0.4219, 0.0" + } + }, + "398:391": { + "class_type": "LTXVDualCFGGuider", + "_meta": { + "title": "LTXVDualCFGGuider" + }, + "inputs": { + "positive": [ + "398:365", + 0 + ], + "negative": [ + "398:365", + 1 + ], + "model": [ + "398:384", + 0 + ], + "video_cfg": 1, + "audio_cfg": 1 + } + }, + "398:341": { + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + }, + "inputs": { + "sampler_name": "euler_ancestral" + } + }, + "398:368": { + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + }, + "inputs": { + "noise": [ + "398:338", + 0 + ], + "guider": [ + "398:391", + 0 + ], + "sampler": [ + "398:341", + 0 + ], + "sigmas": [ + "398:396", + 0 + ], + "latent_image": [ + "398:340", + 0 + ] + } + }, + "398:369": { + "class_type": "LTXVSeparateAVLatent", + "_meta": { + "title": "LTXVSeparateAVLatent" + }, + "inputs": { + "av_latent": [ + "398:368", + 0 + ] + } + }, + "398:374": { + "class_type": "VAEDecodeTiled", + "_meta": { + "title": "VAEDecodeTiled" + }, + "inputs": { + "samples": [ + "398:369", + 0 + ], + "vae": [ + "398:385", + 0 + ], + "tile_size": 512, + "overlap": 64, + "temporal_size": 64, + "temporal_overlap": 16 + } + }, + "398:358": { + "class_type": "LTXVAudioVAEDecode", + "_meta": { + "title": "LTXVAudioVAEDecode" + }, + "inputs": { + "samples": [ + "398:369", + 1 + ], + "audio_vae": [ + "398:386", + 0 + ] + } + }, + "398:370": { + "class_type": "CreateVideo", + "_meta": { + "title": "CreateVideo" + }, + "inputs": { + "images": [ + "398:374", + 0 + ], + "audio": [ + "398:358", + 0 + ], + "fps": [ + "398:359", + 0 + ] + } + } +} diff --git a/workflow/ltx25-t2v-v1.json b/workflow/ltx25-t2v-v1.json new file mode 100644 index 000000000..d47e98002 --- /dev/null +++ b/workflow/ltx25-t2v-v1.json @@ -0,0 +1,607 @@ +{ + "75": { + "class_type": "SaveVideo", + "_meta": { + "title": "Save LTX 2.5 video" + }, + "inputs": { + "filename_prefix": "video/LTX-2.5_t2v", + "format": "auto", + "codec": "auto", + "video": [ + "398:370", + 0 + ] + } + }, + "398:393": { + "class_type": "CLIPLoader", + "_meta": { + "title": "CLIPLoader" + }, + "inputs": { + "clip_name": "gemma4_e2b_it_bf16.safetensors", + "type": "ltxv", + "device": "default" + } + }, + "398:380": { + "class_type": "TextGenerateLTX2Prompt", + "_meta": { + "title": "TextGenerateLTX2Prompt" + }, + "inputs": { + "clip": [ + "398:393", + 0 + ], + "prompt": [ + "398:376", + 0 + ], + "max_length": 600, + "sampling_mode": "on", + "sampling_mode.temperature": 0.7, + "sampling_mode.top_k": 64, + "sampling_mode.top_p": 0.95, + "sampling_mode.min_p": 0.05, + "sampling_mode.repetition_penalty": 1.15, + "sampling_mode.seed": 0 + } + }, + "398:383": { + "class_type": "PrimitiveBoolean", + "_meta": { + "title": "Boolean (Enable Prompt Enhance)" + }, + "inputs": { + "value": true + } + }, + "398:364": { + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIPTextEncode" + }, + "inputs": { + "clip": [ + "398:387", + 0 + ], + "text": [ + "398:382", + 0 + ] + } + }, + "398:376": { + "class_type": "PrimitiveStringMultiline", + "_meta": { + "title": "Prompt" + }, + "inputs": { + "value": "A cinematic subject comes alive from the first frame with natural motion, synchronized sound, and no text." + } + }, + "398:382": { + "class_type": "ComfySwitchNode", + "_meta": { + "title": "ComfySwitchNode" + }, + "inputs": { + "on_false": [ + "398:376", + 0 + ], + "on_true": [ + "398:380", + 0 + ], + "switch": [ + "398:383", + 0 + ] + } + }, + "398:384": { + "class_type": "UNETLoader", + "_meta": { + "title": "UNETLoader" + }, + "inputs": { + "unet_name": "ltx-2.5-22b-distilled-transformer-comfy-int8-convrot.safetensors", + "weight_dtype": "default" + } + }, + "398:362": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Duration" + }, + "inputs": { + "value": 5 + } + }, + "398:365": { + "class_type": "LTXVConditioning", + "_meta": { + "title": "LTXVConditioning" + }, + "inputs": { + "positive": [ + "398:364", + 0 + ], + "negative": [ + "398:373", + 0 + ], + "frame_rate": [ + "398:359", + 0 + ] + } + }, + "398:385": { + "class_type": "VAELoader", + "_meta": { + "title": "VAELoader" + }, + "inputs": { + "vae_name": "ltx-2.5-video-vae-bf16.safetensors" + } + }, + "398:386": { + "class_type": "VAELoader", + "_meta": { + "title": "VAELoader" + }, + "inputs": { + "vae_name": "ltx-2.5-audio-vae-bf16.safetensors" + } + }, + "398:372": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Width" + }, + "inputs": { + "value": 1280 + } + }, + "398:373": { + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIPTextEncode" + }, + "inputs": { + "clip": [ + "398:387", + 0 + ], + "text": "pc game, console game, video game, cartoon, childish, ugly" + } + }, + "398:387": { + "class_type": "CLIPLoader", + "_meta": { + "title": "CLIPLoader" + }, + "inputs": { + "clip_name": "gemma4-12b-with-proj-ltx-2.5-comfy-int8-convrot.safetensors", + "type": "ltxv", + "device": "default" + } + }, + "398:360": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Height" + }, + "inputs": { + "value": 720 + } + }, + "398:352": { + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + }, + "inputs": { + "sampler_name": "euler_ancestral" + } + }, + "398:339": { + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + }, + "inputs": { + "noise_seed": 875362541677469 + } + }, + "398:397": { + "class_type": "ManualSigmas", + "_meta": { + "title": "ManualSigmas" + }, + "inputs": { + "sigmas": "1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0" + } + }, + "398:361": { + "class_type": "PrimitiveInt", + "_meta": { + "title": "Frame Rate" + }, + "inputs": { + "value": 24 + } + }, + "398:371": { + "class_type": "LatentUpscaleModelLoader", + "_meta": { + "title": "LatentUpscaleModelLoader" + }, + "inputs": { + "model_name": "ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors" + } + }, + "398:388": { + "class_type": "LTXVDualCFGGuider", + "_meta": { + "title": "LTXVDualCFGGuider" + }, + "inputs": { + "positive": [ + "398:365", + 0 + ], + "negative": [ + "398:365", + 1 + ], + "model": [ + "398:384", + 0 + ], + "video_cfg": 1, + "audio_cfg": 1 + } + }, + "398:344": { + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + }, + "inputs": { + "noise": [ + "398:339", + 0 + ], + "guider": [ + "398:388", + 0 + ], + "sampler": [ + "398:352", + 0 + ], + "sigmas": [ + "398:397", + 0 + ], + "latent_image": [ + "398:377", + 0 + ] + } + }, + "398:366": { + "class_type": "LTXVEmptyLatentAudio", + "_meta": { + "title": "LTXVEmptyLatentAudio" + }, + "inputs": { + "audio_vae": [ + "398:386", + 0 + ], + "frames_number": [ + "398:378", + 1 + ], + "frame_rate": [ + "398:359", + 1 + ], + "batch_size": 1 + } + }, + "398:367": { + "class_type": "LTXVSeparateAVLatent", + "_meta": { + "title": "LTXVSeparateAVLatent" + }, + "inputs": { + "av_latent": [ + "398:344", + 0 + ] + } + }, + "398:377": { + "class_type": "LTXVConcatAVLatent", + "_meta": { + "title": "LTXVConcatAVLatent" + }, + "inputs": { + "video_latent": [ + "398:356", + 0 + ], + "audio_latent": [ + "398:366", + 0 + ] + } + }, + "398:353": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "ComfyMathExpression" + }, + "inputs": { + "values.a": [ + "398:372", + 0 + ], + "expression": "a/2" + } + }, + "398:355": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "ComfyMathExpression" + }, + "inputs": { + "values.a": [ + "398:360", + 0 + ], + "expression": "a/2" + } + }, + "398:356": { + "class_type": "EmptyLTXVLatentVideo", + "_meta": { + "title": "EmptyLTXVLatentVideo" + }, + "inputs": { + "width": [ + "398:353", + 1 + ], + "height": [ + "398:355", + 1 + ], + "length": [ + "398:378", + 1 + ], + "batch_size": 1 + } + }, + "398:348": { + "class_type": "LTXVLatentUpsampler", + "_meta": { + "title": "LTXVLatentUpsampler" + }, + "inputs": { + "samples": [ + "398:367", + 0 + ], + "upscale_model": [ + "398:371", + 0 + ], + "vae": [ + "398:385", + 0 + ] + } + }, + "398:359": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "Math Expression (fps)" + }, + "inputs": { + "values.a": [ + "398:361", + 0 + ], + "expression": "a" + } + }, + "398:378": { + "class_type": "ComfyMathExpression", + "_meta": { + "title": "Math Expression (length)" + }, + "inputs": { + "values.a": [ + "398:362", + 0 + ], + "values.b": [ + "398:361", + 0 + ], + "expression": "a * b + 1" + } + }, + "398:338": { + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + }, + "inputs": { + "noise_seed": 42 + } + }, + "398:340": { + "class_type": "LTXVConcatAVLatent", + "_meta": { + "title": "LTXVConcatAVLatent" + }, + "inputs": { + "video_latent": [ + "398:348", + 0 + ], + "audio_latent": [ + "398:367", + 1 + ] + } + }, + "398:396": { + "class_type": "ManualSigmas", + "_meta": { + "title": "ManualSigmas" + }, + "inputs": { + "sigmas": "0.85, 0.7250, 0.4219, 0.0" + } + }, + "398:391": { + "class_type": "LTXVDualCFGGuider", + "_meta": { + "title": "LTXVDualCFGGuider" + }, + "inputs": { + "positive": [ + "398:365", + 0 + ], + "negative": [ + "398:365", + 1 + ], + "model": [ + "398:384", + 0 + ], + "video_cfg": 1, + "audio_cfg": 1 + } + }, + "398:341": { + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + }, + "inputs": { + "sampler_name": "euler_ancestral" + } + }, + "398:368": { + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + }, + "inputs": { + "noise": [ + "398:338", + 0 + ], + "guider": [ + "398:391", + 0 + ], + "sampler": [ + "398:341", + 0 + ], + "sigmas": [ + "398:396", + 0 + ], + "latent_image": [ + "398:340", + 0 + ] + } + }, + "398:369": { + "class_type": "LTXVSeparateAVLatent", + "_meta": { + "title": "LTXVSeparateAVLatent" + }, + "inputs": { + "av_latent": [ + "398:368", + 0 + ] + } + }, + "398:374": { + "class_type": "VAEDecodeTiled", + "_meta": { + "title": "VAEDecodeTiled" + }, + "inputs": { + "samples": [ + "398:369", + 0 + ], + "vae": [ + "398:385", + 0 + ], + "tile_size": 512, + "overlap": 64, + "temporal_size": 64, + "temporal_overlap": 16 + } + }, + "398:358": { + "class_type": "LTXVAudioVAEDecode", + "_meta": { + "title": "LTXVAudioVAEDecode" + }, + "inputs": { + "samples": [ + "398:369", + 1 + ], + "audio_vae": [ + "398:386", + 0 + ] + } + }, + "398:370": { + "class_type": "CreateVideo", + "_meta": { + "title": "CreateVideo" + }, + "inputs": { + "images": [ + "398:374", + 0 + ], + "audio": [ + "398:358", + 0 + ], + "fps": [ + "398:359", + 0 + ] + } + } +} diff --git a/workflow/ltx25.yaml b/workflow/ltx25.yaml new file mode 100644 index 000000000..c8421e102 --- /dev/null +++ b/workflow/ltx25.yaml @@ -0,0 +1,23 @@ +# Source: Senai workflows; upstream vavo/LTX2.5-serverless@5e6a88f. +version: 1 +workflows: + - ltx25-t2v-v1.json + - ltx25-i2v-v1.json + - ltx25-flf-v1.json +models: + - path: text_encoders/gemma4_e2b_it_bf16.safetensors + url: https://huggingface.co/Comfy-Org/gemma-4/resolve/main/text_encoders/gemma4_e2b_it_bf16.safetensors + - path: diffusion_models/ltx-2.5-22b-distilled-transformer-comfy-int8-convrot.safetensors + url: https://huggingface.co/Lightricks/LTX-2.5/resolve/main/diffusion_models/ltx-2.5-22b-distilled-transformer-comfy-int8-convrot.safetensors + - path: vae/ltx-2.5-video-vae-bf16.safetensors + url: https://huggingface.co/Lightricks/LTX-2.5/resolve/main/vae/ltx-2.5-video-vae-bf16.safetensors + - path: vae/ltx-2.5-audio-vae-bf16.safetensors + url: https://huggingface.co/Lightricks/LTX-2.5/resolve/main/vae/ltx-2.5-audio-vae-bf16.safetensors + - path: text_encoders/gemma4-12b-with-proj-ltx-2.5-comfy-int8-convrot.safetensors + url: https://huggingface.co/Lightricks/LTX-2.5/resolve/main/text_encoders/gemma4-12b-with-proj-ltx-2.5-comfy-int8-convrot.safetensors + - path: latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors + url: https://huggingface.co/Lightricks/LTX-2.5/resolve/main/latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors +custom_nodes: + - name: ComfyUI-LTXVideo + repo: https://github.com/Lightricks/ComfyUI-LTXVideo.git + revision: ac4d99839020b983e956a8ab67ec38aec1b6e65a diff --git a/workflow/minimax-h3.yaml b/workflow/minimax-h3.yaml new file mode 100644 index 000000000..1f5979922 --- /dev/null +++ b/workflow/minimax-h3.yaml @@ -0,0 +1,5 @@ +version: 1 +workflows: + - video_minimax_h3_r2v.json +# Model URLs and directories are read from the workflow's properties.models. +# This workflow declares only comfy-core nodes; no extra node repository. diff --git a/workflow/video_minimax_h3_r2v.json b/workflow/video_minimax_h3_r2v.json new file mode 100644 index 000000000..2febb50ce --- /dev/null +++ b/workflow/video_minimax_h3_r2v.json @@ -0,0 +1,1745 @@ +{ + "id": "e3f2b845-8f2c-4b5a-9caf-eac1029d3e7e", + "revision": 0, + "last_node_id": 146, + "last_link_id": 291, + "nodes": [ + { + "id": 92, + "type": "SaveVideo", + "pos": [ + 750, + 4870 + ], + "size": [ + 1380, + 106 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 260 + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.33.0" + }, + "widgets_values": [ + "video/MiniMax_H3", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "video/MiniMax_H3", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 115, + "type": "ResolutionSelector", + "pos": [ + -1490, + 6060 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 0, + "mode": 0, + "showAdvanced": false, + "inputs": [], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 276 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 277 + ] + } + ], + "title": "Resolution Selector (Size)", + "properties": { + "Node name for S&R": "ResolutionSelector", + "cnr_id": "comfy-core", + "ver": "0.33.0" + }, + "widgets_values": [ + "16:9 (Widescreen)", + 0.4, + 32 + ], + "widgets_values_named": { + "aspect_ratio": "16:9 (Widescreen)", + "megapixels": 0.4, + "multiple": 32 + }, + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 116, + "type": "MarkdownNote", + "pos": [ + -2030, + 4850 + ], + "size": [ + 450, + 890 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note: MiniMax H3", + "properties": {}, + "widgets_values": [ + "## MiniMax H3\n\n[MiniMax H3](https://www.minimax.io/blog/minimax-h3) is MiniMax's general-purpose, omni-modal generation model. It jointly understands text, image, video, and audio, and generates video with **native stereo audio**: voice, sound effects, and music are modeled jointly in a single forward pass, not layered on afterward. Output is up to 2K resolution, 24fps, and up to about 15 seconds.\n\n## ComfyUI links\n- [ComfyUI#15224](https://github.com/Comfy-Org/ComfyUI/pull/15224)\n- [🤗 Comfy-Org/MiniMax-H3](https://huggingface.co/Comfy-Org/MiniMax-H3)\n\n## About this workflow\n\nThis template runs the **reference-to-video (ref2va)** task using the `MiniMaxH3ReferenceToVideo` node. It takes any mix of reference images, videos, and standalone audio, and weaves them into the generation to lock in a character's identity, a style, a motion, a camera move, or a voice.\n\n**Key inputs**\n\n- **ref_images / ref_videos / ref_video_audios / ref_audios**: up to 9 reference images, 3 reference videos (each may carry its own paired soundtrack), and 3 standalone reference audio clips\n- **prompt**: reference the inputs by tag, in the exact order they were connected, for example ``, `