feat: auto-detect VLM checkpoints on the CLI, and fix a false positiv… #427
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Pipeline | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build_and_unit_test: | |
| runs-on: macos-15 | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Metal Toolchain | |
| run: xcodebuild -downloadComponent MetalToolchain || true | |
| - name: Cache Swift packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: .build | |
| key: ${{ runner.os }}-spm-SwiftLM-v3-${{ hashFiles('Package.resolved') }} | |
| restore-keys: | | |
| ${{ runner.os }}-spm-SwiftLM-v3- | |
| - name: Clear stale module cache | |
| run: find .build -type d -name ModuleCache -exec rm -rf {} + 2>/dev/null || true | |
| - name: Resolve dependencies | |
| run: swift package resolve | |
| - name: Build (Release) | |
| run: swift build -c release | |
| - name: Verify binary | |
| run: | | |
| ls -lh .build/release/SwiftLM | |
| file .build/release/SwiftLM | |
| - name: TurboQuant unit tests | |
| run: | | |
| clang++ -std=c++17 -O2 -o /tmp/tq_test tests/test_turbo_quant.cpp | |
| /tmp/tq_test | |
| - name: Build Test Harness | |
| run: swift build --build-tests | |
| - name: Install MLX Metal library | |
| run: | | |
| python3 -m venv /tmp/mlx_venv | |
| /tmp/mlx_venv/bin/pip install --quiet mlx | |
| cp /tmp/mlx_venv/lib/python*/site-packages/mlx/lib/mlx.metallib .build/release/ | |
| find .build -type d -name "MacOS" -exec cp /tmp/mlx_venv/lib/python*/site-packages/mlx/lib/mlx.metallib {}/ \; | |
| - name: SwiftBuddy Tests (MemPalace & Lifecycle) | |
| run: swift test --skip-build --filter SwiftBuddyTests --disable-swift-testing | |
| - name: SwiftLM Server Tests (Streaming & SSE) | |
| run: swift test --skip-build --filter SwiftLMTests --disable-swift-testing | |
| - name: Upload Binary Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: swiftlm-architecture | |
| path: .build/release/ | |
| retention-days: 1 | |
| integration_matrix: | |
| needs: build_and_unit_test | |
| runs-on: macos-15 | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # `models` is prefetched before the test runs. Without it the SwiftLM server | |
| # downloads the model itself inside the test's 180 s startup window, which the | |
| # 4.9 GB gemma-4 builds cannot make (audio/omni/opencode). Keep in sync with the | |
| # model each tests/test-<modality>.sh actually loads. | |
| include: | |
| - modality: server | |
| models: mlx-community/Qwen2.5-0.5B-Instruct-4bit | |
| # Same smoke test, KV-shared Gemma 4. e2b ships k_proj/v_proj for its shared | |
| # layers while e4b omits them, so it exercises a weight layout no other job | |
| # covers — a regression that broke it entirely still passed CI and shipped. | |
| - modality: server | |
| name_suffix: -gemma-e2b | |
| models: mlx-community/gemma-4-e2b-it-4bit | |
| test_model: mlx-community/gemma-4-e2b-it-4bit | |
| # LFM2.5 used to be absent here and was fetched by the server mid-test, which | |
| # is how it silently picked up a republished revision whose chat template was | |
| # one brace short of valid and took main red. Prefetching it at a pinned | |
| # revision is what makes this job depend on our code rather than on what | |
| # upstream published that afternoon. | |
| - modality: vision | |
| models: mlx-community/Qwen2-VL-2B-Instruct-4bit LiquidAI/LFM2.5-VL-450M-MLX-4bit@10ce3604e42cd595497c47aaf67b7890e1e2a3b4 | |
| - modality: audio | |
| models: mlx-community/gemma-4-e4b-it-4bit | |
| - modality: graph | |
| models: "" | |
| # Synthetic checkpoints committed to the repository, a few hundred KB each, | |
| # so this entry downloads nothing and needs no model cache. It covers the | |
| # weight-and-config shapes behind #118, #120 and the b674 regression — the | |
| # class of defect #128 found the unit suite has never caught, and which real | |
| # checkpoints are too large to cover here (gemma-4-e2b alone is 3.6 GB | |
| # against a 10 GB per-repository cache budget). | |
| - modality: fixtures | |
| models: "" | |
| # OpenAI-compatibility contract: what the client receives over the wire, once | |
| # real tokenisation decides chunk boundaries. Runs on the smallest model — | |
| # these assert server behaviour, not model quality (issue #128). | |
| - modality: contract | |
| models: mlx-community/Qwen2.5-0.5B-Instruct-4bit | |
| - modality: omni | |
| models: mlx-community/gemma-4-e4b-it-4bit | |
| # No longer continue-on-error: Test 2 used to shell out to the opencode npm | |
| # CLI, which the runner OOM-killed after the server had already answered | |
| # correctly. It now issues the request shape opencode sends, so a failure here | |
| # means SwiftLM misbehaved and should block. | |
| - modality: opencode | |
| models: mlx-community/gemma-4-e4b-it-4bit | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Download Binary Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: swiftlm-architecture | |
| path: .build/release/ | |
| - name: Restore Architecture Privileges | |
| run: chmod +x .build/release/SwiftLM | |
| # The key is per-modality and derived from the test script, so changing the model a | |
| # test loads rotates the key automatically. The previous static | |
| # `mlx-model-qwen2.5-0.5b-4bit` key could never be re-saved once created | |
| # (actions/cache does not overwrite an existing key), so the gemma-4 modalities kept | |
| # restoring a cache that had never contained their model. | |
| - name: Cache MLX models (${{ matrix.modality }}) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: hf-models-v2-${{ matrix.modality }}${{ matrix.name_suffix }}-${{ hashFiles(format('tests/test-{0}.sh', matrix.modality)) }} | |
| restore-keys: | | |
| hf-models-v2-${{ matrix.modality }}${{ matrix.name_suffix }}- | |
| - name: Set up HuggingFace CLI | |
| if: matrix.models != '' | |
| run: | | |
| python3 -m venv /tmp/hf_venv | |
| /tmp/hf_venv/bin/pip install --quiet huggingface_hub hf | |
| - name: Pre-download models (${{ matrix.modality }}) | |
| if: matrix.models != '' | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "60" | |
| run: | | |
| source /tmp/hf_venv/bin/activate | |
| chmod +x scripts/ci-download-models.sh | |
| scripts/ci-download-models.sh ${{ matrix.models }} | |
| - name: Run E2E tests (${{ matrix.modality }}${{ matrix.name_suffix }}) | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "600" | |
| SWIFTLM_TEST_MODEL: ${{ matrix.test_model }} | |
| run: | | |
| chmod +x tests/test-${{ matrix.modality }}.sh | |
| for attempt in 1 2 3; do | |
| echo "Attempt $attempt of 3..." | |
| if tests/test-${{ matrix.modality }}.sh .build/release/SwiftLM 15413; then exit 0; fi | |
| if [ "$attempt" -eq 3 ]; then echo "All attempts failed"; exit 1; fi | |
| sleep 10 | |
| done | |
| - name: Upload test logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ci-test-logs-${{ matrix.modality }} | |
| path: /tmp/SwiftLM-test-*.log | |
| retention-days: 7 | |
| # ── Speculative Decoding E2E (dual-model: 0.8B draft + 4B main) ── | |
| # Uses the standard macos-15 runner (7 GB RAM). | |
| # We test the 4B main model which safely fits within memory. | |
| speculative-decoding: | |
| runs-on: macos-15 | |
| timeout-minutes: 45 | |
| needs: build_and_unit_test # Run in parallel with integration_matrix | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Metal Toolchain | |
| run: xcodebuild -downloadComponent MetalToolchain || true | |
| - name: Cache Swift packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: .build | |
| key: ${{ runner.os }}-spm-SwiftLM-v3-${{ hashFiles('Package.resolved') }} | |
| restore-keys: | | |
| ${{ runner.os }}-spm-SwiftLM-v3- | |
| - name: Clear stale module cache | |
| run: find .build -type d -name ModuleCache -exec rm -rf {} + 2>/dev/null || true | |
| - name: Resolve dependencies | |
| run: swift package resolve | |
| - name: Build (Release) | |
| run: swift build -c release | |
| - name: Compile and install custom MLX Metal library | |
| run: | | |
| # cmake-based build from SharpAI fork — mirrors build.sh (PR #58) | |
| if [ -d "mlx-swift/Source/Cmlx/mlx" ]; then | |
| MLX_SRC="mlx-swift/Source/Cmlx/mlx" | |
| else | |
| MLX_SRC=".build/checkouts/mlx-swift/Source/Cmlx/mlx" | |
| fi | |
| mkdir -p .build/metallib_build | |
| pushd .build/metallib_build | |
| cmake "../../$MLX_SRC" \ | |
| -DMLX_BUILD_TESTS=OFF \ | |
| -DMLX_BUILD_EXAMPLES=OFF \ | |
| -DMLX_BUILD_BENCHMARKS=OFF \ | |
| -DMLX_BUILD_PYTHON_BINDINGS=OFF \ | |
| -DMLX_METAL_JIT=OFF \ | |
| -DMLX_ENABLE_NAX=1 \ | |
| -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -20 | |
| make mlx-metallib -j$(sysctl -n hw.ncpu) 2>&1 | tail -20 | |
| popd | |
| BUILT=$(find .build/metallib_build -name "mlx.metallib" | head -1) | |
| cp "$BUILT" .build/release/mlx.metallib | |
| # Install hf for model pre-download | |
| python3 -m venv /tmp/mlx_venv | |
| /tmp/mlx_venv/bin/pip install --quiet huggingface_hub hf | |
| - name: Cache MLX models (draft + main) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: mlx-speculative-qwen35-2b-0.8b-v2 | |
| - name: Pre-download HuggingFace models | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "60" | |
| run: | | |
| source /tmp/mlx_venv/bin/activate | |
| chmod +x scripts/ci-download-models.sh | |
| scripts/ci-download-models.sh \ | |
| mlx-community/Qwen3.5-2B-4bit \ | |
| mlx-community/Qwen3.5-0.8B-MLX-4bit | |
| - name: Run speculative decoding E2E | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "900" | |
| SWIFTLM_TOP_K: "4" | |
| run: | | |
| chmod +x tests/test-speculative.sh | |
| for attempt in 1 2 3; do | |
| echo "Attempt $attempt of 3..." | |
| if tests/test-speculative.sh .build/release/SwiftLM 15414; then | |
| exit 0 | |
| fi | |
| if [ "$attempt" -lt 3 ]; then | |
| echo "Test failed, retrying in 10s..." | |
| sleep 10 | |
| fi | |
| done | |
| echo "All attempts failed" | |
| exit 1 | |
| - name: Upload speculative test logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: speculative-test-logs | |
| path: /tmp/SwiftLM-test-speculative.log | |
| retention-days: 7 | |
| # ── DFlash Speculative Decoding E2E ── | |
| # Uses the standard macos-15 runner (7 GB RAM). | |
| dflash-speculative-decoding: | |
| runs-on: macos-15 | |
| timeout-minutes: 45 | |
| needs: build_and_unit_test | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Metal Toolchain | |
| run: xcodebuild -downloadComponent MetalToolchain || true | |
| - name: Cache Swift packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: .build | |
| key: ${{ runner.os }}-spm-SwiftLM-v3-${{ hashFiles('Package.resolved') }} | |
| restore-keys: | | |
| ${{ runner.os }}-spm-SwiftLM-v3- | |
| - name: Clear stale module cache | |
| run: find .build -type d -name ModuleCache -exec rm -rf {} + 2>/dev/null || true | |
| - name: Resolve dependencies | |
| run: swift package resolve | |
| - name: Build (Release) | |
| run: swift build -c release | |
| - name: Compile and install custom MLX Metal library | |
| run: | | |
| if [ -d "mlx-swift/Source/Cmlx/mlx" ]; then | |
| MLX_SRC="mlx-swift/Source/Cmlx/mlx" | |
| else | |
| MLX_SRC=".build/checkouts/mlx-swift/Source/Cmlx/mlx" | |
| fi | |
| mkdir -p .build/metallib_build | |
| pushd .build/metallib_build | |
| cmake "../../$MLX_SRC" \ | |
| -DMLX_BUILD_TESTS=OFF \ | |
| -DMLX_BUILD_EXAMPLES=OFF \ | |
| -DMLX_BUILD_BENCHMARKS=OFF \ | |
| -DMLX_BUILD_PYTHON_BINDINGS=OFF \ | |
| -DMLX_METAL_JIT=OFF \ | |
| -DMLX_ENABLE_NAX=1 \ | |
| -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -20 | |
| make mlx-metallib -j$(sysctl -n hw.ncpu) 2>&1 | tail -20 | |
| popd | |
| BUILT=$(find .build/metallib_build -name "mlx.metallib" | head -1) | |
| cp "$BUILT" .build/release/mlx.metallib | |
| python3 -m venv /tmp/mlx_venv | |
| /tmp/mlx_venv/bin/pip install --quiet huggingface_hub hf | |
| - name: Cache MLX models (dflash + main) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: mlx-dflash-qwen35-4b-v2 | |
| - name: Pre-download HuggingFace models | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "60" | |
| run: | | |
| source /tmp/mlx_venv/bin/activate | |
| chmod +x scripts/ci-download-models.sh | |
| scripts/ci-download-models.sh \ | |
| mlx-community/Qwen3.5-4B-4bit \ | |
| z-lab/Qwen3.5-4B-DFlash | |
| - name: Run DFlash E2E | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "900" | |
| run: | | |
| chmod +x tests/test-dflash.sh | |
| for attempt in 1 2 3; do | |
| echo "Attempt $attempt of 3..." | |
| if tests/test-dflash.sh .build/release/SwiftLM 15415; then | |
| exit 0 | |
| fi | |
| if [ "$attempt" -lt 3 ]; then | |
| echo "Test failed, retrying in 10s..." | |
| sleep 10 | |
| fi | |
| done | |
| echo "All attempts failed" | |
| exit 1 | |
| - name: Upload dflash test logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dflash-test-logs | |
| path: /tmp/SwiftLM-test-dflash.log | |
| retention-days: 7 | |
| # ── Speculative Decoding Memory Evaluation ── | |
| # Runs the 2B model with NUM_DRAFT_TOKENS=2 to check peak | |
| # memory compression/efficiency. Emits vm_stat readings as step summary. | |
| speculative-decoding-eval: | |
| runs-on: macos-15 | |
| timeout-minutes: 45 | |
| needs: build_and_unit_test | |
| continue-on-error: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Metal Toolchain | |
| run: xcodebuild -downloadComponent MetalToolchain || true | |
| - name: Cache Swift packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: .build | |
| key: ${{ runner.os }}-spm-SwiftLM-v3-${{ hashFiles('Package.resolved') }} | |
| restore-keys: | | |
| ${{ runner.os }}-spm-SwiftLM-v3- | |
| - name: Clear stale module cache | |
| run: find .build -type d -name ModuleCache -exec rm -rf {} + 2>/dev/null || true | |
| - name: Resolve dependencies | |
| run: swift package resolve | |
| - name: Build (Release) | |
| run: swift build -c release | |
| - name: Compile and install custom MLX Metal library | |
| run: | | |
| # cmake-based build from SharpAI fork — mirrors build.sh (PR #58) | |
| if [ -d "mlx-swift/Source/Cmlx/mlx" ]; then | |
| MLX_SRC="mlx-swift/Source/Cmlx/mlx" | |
| else | |
| MLX_SRC=".build/checkouts/mlx-swift/Source/Cmlx/mlx" | |
| fi | |
| mkdir -p .build/metallib_build | |
| pushd .build/metallib_build | |
| cmake "../../$MLX_SRC" \ | |
| -DMLX_BUILD_TESTS=OFF \ | |
| -DMLX_BUILD_EXAMPLES=OFF \ | |
| -DMLX_BUILD_BENCHMARKS=OFF \ | |
| -DMLX_BUILD_PYTHON_BINDINGS=OFF \ | |
| -DMLX_METAL_JIT=OFF \ | |
| -DMLX_ENABLE_NAX=1 \ | |
| -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -20 | |
| make mlx-metallib -j$(sysctl -n hw.ncpu) 2>&1 | tail -20 | |
| popd | |
| BUILT=$(find .build/metallib_build -name "mlx.metallib" | head -1) | |
| cp "$BUILT" .build/release/mlx.metallib | |
| # Install hf for model pre-download | |
| python3 -m venv /tmp/mlx_venv | |
| /tmp/mlx_venv/bin/pip install --quiet huggingface_hub hf | |
| - name: Cache MLX models (draft + 2B) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: mlx-speculative-eval-qwen35-2b-0.8b-v2 | |
| - name: Pre-download HuggingFace models | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "60" | |
| run: | | |
| source /tmp/mlx_venv/bin/activate | |
| chmod +x scripts/ci-download-models.sh | |
| scripts/ci-download-models.sh \ | |
| mlx-community/Qwen3.5-2B-4bit \ | |
| mlx-community/Qwen3.5-0.8B-MLX-4bit | |
| - name: Snapshot RAM before test | |
| id: ram_before | |
| run: | | |
| PAGE_SIZE=$(sysctl -n hw.pagesize) | |
| RAM=$(vm_stat | awk -v page_size="$PAGE_SIZE" ' | |
| /Pages active:/ { v=$3; gsub(/\./, "", v); act=v+0 } | |
| /Pages wired down:/ { v=$4; gsub(/\./, "", v); wire=v+0 } | |
| /Pages occupied by compressor:/ { v=$5; gsub(/\./, "", v); comp=v+0 } | |
| END { printf "%.2f", (act+wire+comp)*page_size/1073741824 } | |
| ') | |
| echo "ram_before=$RAM" >> $GITHUB_OUTPUT | |
| echo "RAM before eval: ${RAM} GB" | |
| - name: Run speculative evaluation E2E | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "900" | |
| SWIFTLM_TOP_K: "2" | |
| MAIN_MODEL: "mlx-community/Qwen3.5-2B-4bit" | |
| NUM_DRAFT_TOKENS: "2" | |
| run: | | |
| chmod +x tests/test-speculative-eval.sh | |
| for attempt in 1 2 3; do | |
| echo "Attempt $attempt of 3..." | |
| if tests/test-speculative-eval.sh .build/release/SwiftLM 15414; then | |
| exit 0 | |
| fi | |
| if [ "$attempt" -lt 3 ]; then | |
| echo "Test failed, retrying in 10s..." | |
| sleep 10 | |
| fi | |
| done | |
| echo "All attempts failed" | |
| exit 1 | |
| - name: Snapshot RAM after test | |
| if: always() | |
| id: ram_after | |
| run: | | |
| PAGE_SIZE=$(sysctl -n hw.pagesize) | |
| RAM=$(vm_stat | awk -v page_size="$PAGE_SIZE" ' | |
| /Pages active:/ { v=$3; gsub(/\./, "", v); act=v+0 } | |
| /Pages wired down:/ { v=$4; gsub(/\./, "", v); wire=v+0 } | |
| /Pages occupied by compressor:/ { v=$5; gsub(/\./, "", v); comp=v+0 } | |
| END { printf "%.2f", (act+wire+comp)*page_size/1073741824 } | |
| ') | |
| echo "ram_after=$RAM" >> $GITHUB_OUTPUT | |
| echo "RAM after eval: ${RAM} GB" | |
| - name: Emit memory summary | |
| if: always() | |
| run: | | |
| BEFORE="${{ steps.ram_before.outputs.ram_before }}" | |
| AFTER="${{ steps.ram_after.outputs.ram_after }}" | |
| TOTAL=$(sysctl -n hw.memsize | awk '{printf "%.1f", $1/1073741824}') | |
| { | |
| echo "## 📊 Speculative Eval — Memory Readings" | |
| echo "| Metric | Value |" | |
| echo "|--------|-------|" | |
| echo "| Runner physical RAM | ${TOTAL} GB |" | |
| echo "| RAM before test | ${BEFORE} GB |" | |
| echo "| RAM after test | ${AFTER} GB |" | |
| echo "| Delta | $(echo "$AFTER $BEFORE" | awk '{printf "%.2f", $1-$2}') GB |" | |
| } >> $GITHUB_STEP_SUMMARY | |
| - name: Upload speculative eval logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: speculative-eval-logs | |
| path: /tmp/SwiftLM-test-speculative-eval.log | |
| # ── Issue #72 Regression: SSD streaming + draft model RAM guard ────────────── | |
| # Mandatory (not continue-on-error). Enforces the auto-cap-to-1 fix and the | |
| # memoryLimit sentinel on every PR. Uses tiny models (2B main + 0.8B draft) | |
| # sized for the 7 GB macos-15 runner. | |
| # | |
| # Three checks mirror the local Test 10 in run_benchmark.sh: | |
| # [1] Auto-cap warning present in server log | |
| # [2] Peak RAM ≤ 85% of runner physical RAM during inference | |
| # [3] /v1/chat/completions returns valid content | |
| ssd-draft-memory-guard: | |
| runs-on: macos-15 | |
| timeout-minutes: 45 | |
| needs: build_and_unit_test | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Download Binary Artifact | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true # fall back to building if artifact expired | |
| with: | |
| name: swiftlm-architecture | |
| path: .build/release/ | |
| - name: Build (Release) if artifact missing | |
| run: | | |
| if [ ! -f ".build/release/SwiftLM" ]; then | |
| swift build -c release | |
| fi | |
| chmod +x .build/release/SwiftLM | |
| - name: Install MLX Metal library | |
| run: | | |
| python3 -m venv /tmp/mlx_venv | |
| /tmp/mlx_venv/bin/pip install --quiet mlx huggingface_hub hf | |
| cp /tmp/mlx_venv/lib/python*/site-packages/mlx/lib/mlx.metallib .build/release/ | |
| - name: Cache MLX models (2B main + 0.8B draft) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: mlx-ssd-draft-guard-qwen35-2b-0.8b-v2 | |
| - name: Pre-download models | |
| env: | |
| HF_HUB_DOWNLOAD_TIMEOUT: "60" | |
| run: | | |
| source /tmp/mlx_venv/bin/activate | |
| chmod +x scripts/ci-download-models.sh | |
| scripts/ci-download-models.sh \ | |
| mlx-community/Qwen3.5-2B-4bit \ | |
| mlx-community/Qwen3.5-0.8B-MLX-4bit | |
| - name: Snapshot RAM baseline | |
| id: ram_base | |
| run: | | |
| PAGE_SIZE=$(sysctl -n hw.pagesize) | |
| RAM=$(vm_stat | awk -v page_size="$PAGE_SIZE" ' | |
| /Pages active:/ { v=$3; gsub(/\./, "", v); act=v+0 } | |
| /Pages wired down:/ { v=$4; gsub(/\./, "", v); wire=v+0 } | |
| /Pages occupied by compressor:/ { v=$5; gsub(/\./, "", v); comp=v+0 } | |
| END { printf "%.2f", (act+wire+comp)*page_size/1073741824 } | |
| ') | |
| TOTAL=$(sysctl -n hw.memsize | awk '{printf "%.0f", $1/1073741824}') | |
| LIMIT=$(echo "$TOTAL * 0.85" | bc | cut -d. -f1) | |
| echo "ram_base=$RAM" >> $GITHUB_OUTPUT | |
| echo "runner_ram=$TOTAL" >> $GITHUB_OUTPUT | |
| echo "ram_limit=$LIMIT" >> $GITHUB_OUTPUT | |
| echo "Baseline RAM: ${RAM} GB | Runner: ${TOTAL} GB | Limit: ${LIMIT} GB" | |
| - name: Start SSD + draft server (Issue #72 scenario) | |
| id: server | |
| run: | | |
| # Launch with --num-draft-tokens 4 intentionally — the auto-cap should | |
| # silently reduce it to 1 and log the advisory message. | |
| .build/release/SwiftLM \ | |
| --model mlx-community/Qwen3.5-2B-4bit \ | |
| --draft-model mlx-community/Qwen3.5-0.8B-MLX-4bit \ | |
| --stream-experts \ | |
| --num-draft-tokens 4 \ | |
| --port 15473 \ | |
| --max-tokens 64 \ | |
| > /tmp/ssd_draft_guard.log 2>&1 & | |
| PID=$! | |
| echo "server_pid=$PID" >> $GITHUB_OUTPUT | |
| echo "Waiting for server (up to 300s)..." | |
| for i in $(seq 1 300); do | |
| if ! kill -0 $PID 2>/dev/null; then | |
| echo "Server died early:" | |
| cat /tmp/ssd_draft_guard.log | |
| exit 1 | |
| fi | |
| if curl -sf http://127.0.0.1:15473/health >/dev/null 2>&1; then | |
| echo "Server ready after ${i}s" | |
| break | |
| fi | |
| sleep 1 | |
| if [ "$i" -eq 300 ]; then echo "Timeout"; exit 1; fi | |
| done | |
| - name: Snapshot RAM after model load | |
| id: ram_loaded | |
| run: | | |
| PAGE_SIZE=$(sysctl -n hw.pagesize) | |
| RAM=$(vm_stat | awk -v page_size="$PAGE_SIZE" ' | |
| /Pages active:/ { v=$3; gsub(/\./, "", v); act=v+0 } | |
| /Pages wired down:/ { v=$4; gsub(/\./, "", v); wire=v+0 } | |
| /Pages occupied by compressor:/ { v=$5; gsub(/\./, "", v); comp=v+0 } | |
| END { printf "%.2f", (act+wire+comp)*page_size/1073741824 } | |
| ') | |
| echo "ram_loaded=$RAM" >> $GITHUB_OUTPUT | |
| echo "RAM after load: ${RAM} GB" | |
| - name: "[1/3] Verify draft model loaded in server log" | |
| run: | | |
| # The guard's intent is that a draft model can be loaded alongside the main | |
| # model without exceeding RAM. On non-MoE models, --stream-experts is | |
| # silently disabled (and auto-capping is not needed); what matters is that | |
| # the draft model was actually picked up by the server. | |
| if grep -q "draft" /tmp/ssd_draft_guard.log; then | |
| echo "✅ Draft model reference found in server log" | |
| else | |
| echo "❌ Draft model not mentioned in server log — server may have rejected it" | |
| echo "--- Last 20 lines of server log ---" | |
| tail -20 /tmp/ssd_draft_guard.log | |
| exit 1 | |
| fi | |
| - name: "[2/3] Run inference and snapshot peak RAM" | |
| id: ram_peak | |
| run: | | |
| RESULT=$(curl -sf --max-time 90 http://127.0.0.1:15473/v1/chat/completions \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"model":"test","messages":[{"role":"user","content":"What is 2+2? One word."}],"max_tokens":32,"stream":false}' \ | |
| 2>/dev/null || echo "{}") | |
| echo "$RESULT" > /tmp/inf_result.json | |
| PAGE_SIZE=$(sysctl -n hw.pagesize) | |
| RAM=$(vm_stat | awk -v page_size="$PAGE_SIZE" ' | |
| /Pages active:/ { v=$3; gsub(/\./, "", v); act=v+0 } | |
| /Pages wired down:/ { v=$4; gsub(/\./, "", v); wire=v+0 } | |
| /Pages occupied by compressor:/ { v=$5; gsub(/\./, "", v); comp=v+0 } | |
| END { printf "%.2f", (act+wire+comp)*page_size/1073741824 } | |
| ') | |
| echo "ram_peak=$RAM" >> $GITHUB_OUTPUT | |
| echo "RAM after inference: ${RAM} GB" | |
| LIMIT="${{ steps.ram_base.outputs.ram_limit }}" | |
| OK=$(echo "$RAM <= $LIMIT" | bc -l) | |
| if [ "$OK" = "1" ]; then | |
| echo "✅ RAM=${RAM}GB ≤ ${LIMIT}GB (85% of ${{ steps.ram_base.outputs.runner_ram }}GB runner RAM)" | |
| else | |
| echo "❌ RAM=${RAM}GB EXCEEDS limit ${LIMIT}GB — Issue #72 regression detected" | |
| echo " (memoryLimit sentinel or auto-cap may have regressed)" | |
| exit 1 | |
| fi | |
| - name: "[3/3] Validate inference response" | |
| run: | | |
| RESULT=$(cat /tmp/inf_result.json) | |
| if echo "$RESULT" | grep -q '"content"'; then | |
| TEXT=$(echo "$RESULT" | python3 -c \ | |
| "import sys,json;d=json.load(sys.stdin);print(d['choices'][0]['message']['content'])" \ | |
| 2>/dev/null || echo "(parse error)") | |
| echo "✅ Response: $TEXT" | |
| else | |
| echo "❌ No content in response — server may have crashed or returned empty" | |
| echo "Raw: ${RESULT:0:300}" | |
| exit 1 | |
| fi | |
| - name: Stop server | |
| if: always() | |
| run: kill ${{ steps.server.outputs.server_pid }} 2>/dev/null || true | |
| - name: Emit memory summary to step summary | |
| if: always() | |
| run: | | |
| BASE="${{ steps.ram_base.outputs.ram_base }}" | |
| LOADED="${{ steps.ram_loaded.outputs.ram_loaded }}" | |
| PEAK="${{ steps.ram_peak.outputs.ram_peak }}" | |
| TOTAL="${{ steps.ram_base.outputs.runner_ram }}" | |
| LIMIT="${{ steps.ram_base.outputs.ram_limit }}" | |
| { | |
| echo "## 🛡️ Issue #72 — SSD + Draft Model RAM Guard" | |
| echo "| Metric | Value | Threshold |" | |
| echo "|--------|-------|-----------|" | |
| echo "| Runner physical RAM | ${TOTAL} GB | — |" | |
| echo "| RAM baseline (before server) | ${BASE} GB | — |" | |
| echo "| RAM after model load | ${LOADED} GB | — |" | |
| echo "| RAM after inference (peak) | ${PEAK} GB | ≤ ${LIMIT} GB (85%) |" | |
| echo "| Load delta | $(echo "$LOADED $BASE" | awk '{printf "%.2f", $1-$2}') GB | — |" | |
| echo "| Inference delta | $(echo "$PEAK $LOADED" | awk '{printf "%.2f", $1-$2}') GB | — |" | |
| } >> $GITHUB_STEP_SUMMARY | |
| - name: Upload server log on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ssd-draft-guard-log | |
| path: /tmp/ssd_draft_guard.log | |
| retention-days: 7 | |