From 2811e96892bf7dc1ca8c91c8c71cd40bfeebcac7 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:31:14 -0700 Subject: [PATCH 01/14] feat: stage K2-Horizon llama.cpp integration --- .github/workflows/docs_and_style.yml | 4 +- .github/workflows/validate_llamacpp.yml | 15 ++ docs/embeddable/backends.md | 4 +- docs/guide/configuration/README.md | 4 +- docs/guide/configuration/llamacpp.md | 100 ++++++++ src/cpp/resources/server_models.json | 34 +++ test/test_validate_llamacpp_selection.py | 292 +++++++++++++++++++++++ test/utils/validation_model_selection.py | 110 +++++++++ test/validate_llamacpp.py | 101 +++++--- 9 files changed, 626 insertions(+), 38 deletions(-) create mode 100644 test/test_validate_llamacpp_selection.py create mode 100644 test/utils/validation_model_selection.py diff --git a/.github/workflows/docs_and_style.yml b/.github/workflows/docs_and_style.yml index b9690d679b..cad0fcd73d 100644 --- a/.github/workflows/docs_and_style.yml +++ b/.github/workflows/docs_and_style.yml @@ -76,8 +76,8 @@ jobs: with: python-version: '3.x' - run: pip install pre-commit - - name: Run the comment-slop unit tests - run: python -m unittest test.test_comment_slop -v + - name: Run focused Python unit tests + run: python -m unittest test.test_comment_slop test.test_validate_llamacpp_selection -v - name: Run pre-commit on the changed files # Scoped to what the change touches; running repo-wide would fail on drift that diff --git a/.github/workflows/validate_llamacpp.yml b/.github/workflows/validate_llamacpp.yml index 0862c4fdb6..e951210f85 100644 --- a/.github/workflows/validate_llamacpp.yml +++ b/.github/workflows/validate_llamacpp.yml @@ -7,6 +7,11 @@ on: description: 'Run in lite mode (smallest model only)' type: boolean default: false + models: + description: 'Optional comma-separated model IDs (empty uses hot models)' + type: string + required: false + default: '' pull_request: types: [opened, synchronize, reopened, labeled] merge_group: @@ -717,6 +722,8 @@ jobs: - name: Run validation with lemond shell: PowerShell + env: + LLAMACPP_TEST_MODELS: ${{ inputs.models || '' }} run: | $ErrorActionPreference = "Stop" $port = 13305 @@ -800,6 +807,14 @@ jobs: $validationArgs += $channel } + foreach ($modelValue in ($env:LLAMACPP_TEST_MODELS -split ",")) { + $model = $modelValue.Trim() + if ($model) { + $validationArgs += "--model" + $validationArgs += $model + } + } + if ("${{ env.LITE_MODE }}" -eq "true") { $validationArgs += "--lite" } diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index 688cc64b44..f99595d5cc 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -159,7 +159,7 @@ For example, to use your own Vulkan `llama-server` in place of Lemonade's: lemond.exe ./ REM Set the llama-server vulkan binary path - lemonade.exe config set llamacpp.vulkan_bin C:\path\to\bins + lemonade.exe config set llamacpp.vulkan_bin C:\path\to\llama-server.exe ``` === "Linux (bash)" @@ -169,7 +169,7 @@ For example, to use your own Vulkan `llama-server` in place of Lemonade's: ./lemond ./ # Set the llama-server vulkan binary path - ./lemonade config set llamacpp.vulkan_bin /path/to/bins + ./lemonade config set llamacpp.vulkan_bin /path/to/llama-server ``` See the `*_bin` settings in the [Configuration Guide](../guide/configuration/README.md) for the full set of customization options. diff --git a/docs/guide/configuration/README.md b/docs/guide/configuration/README.md index 4d42ad5e38..47b0a15b28 100644 --- a/docs/guide/configuration/README.md +++ b/docs/guide/configuration/README.md @@ -345,7 +345,7 @@ Every `*_bin` key (e.g. `llamacpp.vulkan_bin`, `whispercpp.cpu_bin`, `sdcpp.rocm | `""` | Same as `"builtin"`. | | `"latest"` | Resolve to the most-recent upstream GitHub release on first install or first status query for that backend, then install on demand. The resolved tag is recorded in `/bin///version.txt`. | | `"b8664"` / `"v1.8.2"` / etc. | A specific upstream release tag. Lemonade downloads that exact version from GitHub. | -| `"/path/to/bin"` | A directory you populated yourself (e.g. a local build). Lemonade uses the executable inside this directory and never downloads. The path must exist when set. | +| `"/path/to/llama-server"` | The full path to an executable you built or installed. Lemonade uses this file and never downloads a replacement. The path must exist when set. | > Note: the `latest` setting is experimental. @@ -361,7 +361,7 @@ lemonade config set llamacpp.vulkan_bin=latest lemonade config set llamacpp.vulkan_bin=b8664 # Use your own llama.cpp build -lemonade config set llamacpp.vulkan_bin=/home/me/llama.cpp/build/bin +lemonade config set llamacpp.vulkan_bin=/home/me/llama.cpp/build/bin/llama-server # Revert to the version lemonade ships lemonade config set llamacpp.vulkan_bin=builtin diff --git a/docs/guide/configuration/llamacpp.md b/docs/guide/configuration/llamacpp.md index 416f1047c2..002a2a99e1 100644 --- a/docs/guide/configuration/llamacpp.md +++ b/docs/guide/configuration/llamacpp.md @@ -2,6 +2,106 @@ Lemonade uses [llama.cpp](https://github.com/ggerganov/llama.cpp) as its primary LLM inference backend, supporting multiple hardware acceleration options. This document explains the available backends and how to choose between them. +## K2-Horizon (IFM branch build) + +Lemonade includes these official IFM BF16 GGUF models: + +| Model ID | Weight size | Maximum context | Suggested starting values | +| --- | ---: | ---: | --- | +| `K2-Horizon-0.9B-GGUF` | 2.16 GB | 131,072 tokens | high reasoning effort, temperature 0.6, top-p 0.95 | +| `K2-Horizon-3.7B-GGUF` | 10.13 GB | 524,288 tokens | high reasoning effort, temperature 1.0, top-p 0.95 | +| `K2-Horizon-7B-GGUF` | 18.01 GB | 524,288 tokens | high reasoning effort, temperature 1.0, top-p 0.95 | + +These files need the K2-Horizon implementation from the +[`MBZUAI-IFM/llama.cpp` `model/K2Horizon` branch](https://github.com/MBZUAI-IFM/llama.cpp/tree/model/K2Horizon). +Use commit `35999d101cf2233fc54f09c3c8d599da7303ce02` for a reproducible build. +The standard llama.cpp binaries that Lemonade currently downloads do not contain this +implementation. Build the required server and configure Lemonade to use its executable: + +> Do not run these model IDs with Lemonade's current managed llama.cpp binaries. The +> catalog changes are intended to ship only after branch-derived binaries are published +> and pinned for every supported backend. + +```bash +git clone --branch model/K2Horizon --single-branch \ + https://github.com/MBZUAI-IFM/llama.cpp.git k2-horizon-llama.cpp +git -C k2-horizon-llama.cpp checkout --detach \ + 35999d101cf2233fc54f09c3c8d599da7303ce02 + +cmake -S k2-horizon-llama.cpp -B k2-horizon-llama.cpp/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_UI=OFF \ + -DLLAMA_USE_PREBUILT_UI=OFF \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_SERVER=ON +cmake --build k2-horizon-llama.cpp/build --config Release \ + --target llama-server -j 8 +``` + +The command above enables Metal automatically on macOS. Add the applicable llama.cpp CMake +option for another accelerator: `-DGGML_VULKAN=ON`, `-DGGML_CUDA=ON`, or +`-DGGML_HIP=ON`. + +Set the `*_bin` value to the full absolute path of `llama-server`, not its directory. For +example, use this command for macOS: + +```bash +lemonade config set \ + llamacpp.metal_bin=/absolute/path/to/k2-horizon-llama.cpp/build/bin/llama-server +``` + +Use `cpu_bin`, `vulkan_bin`, `cuda_bin`, or `rocm_bin` for the applicable backend. On +Windows, include `llama-server.exe` in the path. You can then start a model with a bounded +context: + +```bash +lemonade run K2-Horizon-0.9B-GGUF --ctx-size 8192 +``` + +You can also call the OpenAI-compatible API: + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:13305/api/v1", api_key="not-needed") +response = client.chat.completions.create( + model="K2-Horizon-0.9B-GGUF", + messages=[ + {"role": "user", "content": "What is 17 plus 25? Return the number only."} + ], + temperature=0.6, + top_p=0.95, + max_tokens=256, + extra_body={ + "chat_template_kwargs": { + "reasoning_effort": "high", + "tool_call_format": "xml", + } + }, +) +message = response.choices[0].message +reasoning = getattr(message, "reasoning_content", None) +if reasoning: + print(reasoning) +print(message.content) +``` + +Use high reasoning effort and the default `xml` tool-call format with the pinned branch +commit. Its llama.cpp response parser does not correctly handle the template's `medium`, +`low`, `json`, or `xml_typed` modes. Medium and low reasoning can expose IFM control tags, +and the non-default tool formats can cause parser errors. To disable thinking, set +`chat_template_kwargs.enable_thinking` to `false`. Use a large output allowance for long +reasoning tasks. Do not use a 32K output allowance as a server-wide default. + +The table gives only the weight size. The context cache and backend buffers need more memory. +The maximum 128K and 512K contexts need substantially more memory than the 8K example. + +An older custom llama.cpp executable cannot load these models. This integration does not +support K2-Horizon Uno. + ## Available Backends ### CPU diff --git a/src/cpp/resources/server_models.json b/src/cpp/resources/server_models.json index 9f12321427..cd49fafc14 100644 --- a/src/cpp/resources/server_models.json +++ b/src/cpp/resources/server_models.json @@ -731,6 +731,40 @@ ], "size": 13.4 }, + "K2-Horizon-0.9B-GGUF": { + "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": [ + "chat", + "reasoning", + "tool-calling", + "hot" + ], + "size": 2.16 + }, + "K2-Horizon-3.7B-GGUF": { + "checkpoint": "IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": [ + "chat", + "reasoning", + "tool-calling" + ], + "size": 10.13 + }, + "K2-Horizon-7B-GGUF": { + "checkpoint": "IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": [ + "chat", + "reasoning", + "tool-calling" + ], + "size": 18.01 + }, "MiniCPM5-1B-GGUF": { "source": "modelscope", "checkpoint": "OpenBMB/MiniCPM5-1B-GGUF:MiniCPM5-1B-Q4_K_M.gguf", diff --git a/test/test_validate_llamacpp_selection.py b/test/test_validate_llamacpp_selection.py new file mode 100644 index 0000000000..8f14250efd --- /dev/null +++ b/test/test_validate_llamacpp_selection.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Unit tests for llama.cpp validation model selection.""" + +from __future__ import annotations + +import argparse +import contextlib +import io +import json +import unittest +from pathlib import Path + +from test.utils import validation_model_selection as selection + +MODEL_REGISTRY = ( + Path(__file__).resolve().parents[1] + / "src" + / "cpp" + / "resources" + / "server_models.json" +) + + +class LlamaCppValidationSelectionTests(unittest.TestCase): + def setUp(self) -> None: + self.catalog = [ + { + "id": "Zulu-Hot", + "recipe": "llamacpp", + "labels": ["chat", "hot"], + "checkpoint": "example/zulu:Zulu.gguf", + "size": 4.0, + }, + { + "id": "Alpha-Hot", + "recipe": "llamacpp", + "labels": ["hot", "chat"], + "checkpoint": "example/alpha:Alpha.gguf", + "size": 1.0, + }, + { + "id": "Cold-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/cold:Cold.gguf", + "size": 2.0, + }, + { + "id": "Hot-Image", + "recipe": "sdcpp", + "labels": ["hot", "image"], + "checkpoint": "example/image:Image.gguf", + "size": 0.5, + }, + ] + + def select(self, requested_model_ids=None, lite=False): + def resolve_builtin_model(canonical_model_id): + model_id = canonical_model_id.removeprefix("builtin.") + return next( + (model for model in self.catalog if model["id"] == model_id), None + ) + + return selection.select_llamacpp_models( + self.catalog, + requested_model_ids=requested_model_ids, + lite=lite, + builtin_model_resolver=resolve_builtin_model, + ) + + def test_default_selection_filters_and_sorts_hot_llamacpp_models(self) -> None: + selected = self.select() + + self.assertEqual([model["id"] for model in selected], ["Alpha-Hot", "Zulu-Hot"]) + + def test_lite_selection_uses_smallest_hot_llamacpp_model(self) -> None: + selected = self.select(lite=True) + + self.assertEqual([model["id"] for model in selected], ["Alpha-Hot"]) + + def test_explicit_selection_preserves_order_and_accepts_non_hot_model(self) -> None: + selected = self.select(["Cold-Llama", "Alpha-Hot"]) + + self.assertEqual( + [model["id"] for model in selected], ["Cold-Llama", "Alpha-Hot"] + ) + + def test_explicit_selection_preserves_duplicate_ids(self) -> None: + selected = self.select(["Alpha-Hot", "Alpha-Hot"]) + + self.assertEqual( + [model["id"] for model in selected], ["Alpha-Hot", "Alpha-Hot"] + ) + + def test_explicit_selection_resolves_and_loads_canonical_builtin(self) -> None: + resolved_ids = [] + shadow = { + "id": "K2-Horizon-0.9B-GGUF", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "user/model:shadow.gguf", + } + built_in = { + "id": "K2-Horizon-0.9B-GGUF", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:model.gguf", + } + + def resolve_builtin_model(canonical_model_id): + resolved_ids.append(canonical_model_id) + return built_in + + selected = selection.select_llamacpp_models( + [shadow], + requested_model_ids=["K2-Horizon-0.9B-GGUF"], + builtin_model_resolver=resolve_builtin_model, + ) + + self.assertEqual(resolved_ids, ["builtin.K2-Horizon-0.9B-GGUF"]) + self.assertEqual(selected[0]["id"], "K2-Horizon-0.9B-GGUF") + self.assertEqual(selected[0]["load_id"], "builtin.K2-Horizon-0.9B-GGUF") + self.assertEqual( + selected[0]["checkpoint"], + "IFM/K2-Horizon-0.9B-GGUF:model.gguf", + ) + + def test_explicit_selection_rejects_unverified_bare_catalog_entry(self) -> None: + registered_model = [ + { + "id": "K2-Horizon-0.9B-GGUF", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "user/model:shadow.gguf", + } + ] + + with self.assertRaisesRegex(ValueError, "built-in"): + selection.select_llamacpp_models( + registered_model, + requested_model_ids=["K2-Horizon-0.9B-GGUF"], + ) + + def test_unknown_explicit_model_has_clear_error(self) -> None: + with self.assertRaisesRegex(ValueError, "Missing-Model"): + self.select(["Missing-Model"]) + + def test_non_llamacpp_explicit_model_has_clear_error(self) -> None: + with self.assertRaisesRegex(ValueError, "Hot-Image.*sdcpp"): + self.select(["Hot-Image"]) + + def test_invalid_checkpoint_marks_explicit_model_unpullable(self) -> None: + records = [ + {"id": "Broken-Llama", "recipe": "llamacpp", "labels": ["chat"]}, + { + "id": "Broken-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "", + }, + { + "id": "Broken-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": 123, + }, + { + "id": "Broken-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoints": {"main": ""}, + }, + ] + for record in records: + with self.subTest(record=record): + with self.assertRaisesRegex(ValueError, "Broken-Llama.*checkpoint"): + selection.select_llamacpp_models( + [], + requested_model_ids=["Broken-Llama"], + builtin_model_resolver=lambda _model_id, value=record: value, + ) + + def test_explicit_selection_accepts_checkpoints_main(self) -> None: + model = { + "id": "Multi-Checkpoint-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoints": { + "main": "example/model:main.gguf", + "draft": "example/model:draft.gguf", + }, + } + + selected = selection.select_llamacpp_models( + [], + requested_model_ids=["Multi-Checkpoint-Llama"], + builtin_model_resolver=lambda _model_id: model, + ) + + self.assertEqual(selected[0]["id"], "Multi-Checkpoint-Llama") + + def test_lite_selection_does_not_validate_unselected_models(self) -> None: + self.catalog.append( + { + "id": "Broken-Large-Hot", + "recipe": "llamacpp", + "labels": ["chat", "hot"], + "size": 100.0, + } + ) + + selected = self.select(lite=True) + + self.assertEqual([model["id"] for model in selected], ["Alpha-Hot"]) + + def test_empty_explicit_list_uses_default_selection(self) -> None: + selected = self.select([]) + + self.assertEqual([model["id"] for model in selected], ["Alpha-Hot", "Zulu-Hot"]) + + def test_model_argument_is_repeatable(self) -> None: + parser = argparse.ArgumentParser() + selection.add_model_selection_arguments(parser) + + args = parser.parse_args(["--model", "Cold-Llama", "--model", "Alpha-Hot"]) + + self.assertEqual(args.model, ["Cold-Llama", "Alpha-Hot"]) + self.assertFalse(args.lite) + + def test_model_and_lite_arguments_are_mutually_exclusive(self) -> None: + parser = argparse.ArgumentParser() + selection.add_model_selection_arguments(parser) + + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + parser.parse_args(["--model", "Alpha-Hot", "--lite"]) + + self.assertEqual(raised.exception.code, 2) + + +class K2HorizonCatalogTests(unittest.TestCase): + def test_k2_horizon_bf16_models_have_exact_catalog_contract(self) -> None: + models = json.loads(MODEL_REGISTRY.read_text(encoding="utf-8")) + expected = { + "K2-Horizon-0.9B-GGUF": { + "checkpoint": ("IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf"), + "recipe": "llamacpp", + "suggested": True, + "labels": ["chat", "reasoning", "tool-calling", "hot"], + "size": 2.16, + }, + "K2-Horizon-3.7B-GGUF": { + "checkpoint": ("IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf"), + "recipe": "llamacpp", + "suggested": True, + "labels": ["chat", "reasoning", "tool-calling"], + "size": 10.13, + }, + "K2-Horizon-7B-GGUF": { + "checkpoint": ("IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf"), + "recipe": "llamacpp", + "suggested": True, + "labels": ["chat", "reasoning", "tool-calling"], + "size": 18.01, + }, + } + + for model_id, expected_record in expected.items(): + with self.subTest(model_id=model_id): + self.assertEqual(models.get(model_id), expected_record) + + def test_model_registry_has_no_duplicate_ids(self) -> None: + duplicate_ids = [] + + def unique_object(pairs): + parsed = {} + for key, value in pairs: + if key in parsed: + duplicate_ids.append(key) + parsed[key] = value + return parsed + + json.loads( + MODEL_REGISTRY.read_text(encoding="utf-8"), + object_pairs_hook=unique_object, + ) + + self.assertEqual(duplicate_ids, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/utils/validation_model_selection.py b/test/utils/validation_model_selection.py new file mode 100644 index 0000000000..78a7fc8cf3 --- /dev/null +++ b/test/utils/validation_model_selection.py @@ -0,0 +1,110 @@ +"""Model selection shared by llama.cpp validation entry points.""" + +from __future__ import annotations + +import argparse +import math +from typing import Any, Callable, Iterable + +BuiltinModelResolver = Callable[[str], dict[str, Any] | None] + + +class ModelSelectionError(ValueError): + """Raised when requested validation models cannot be selected safely.""" + + +def add_model_selection_arguments(parser: argparse.ArgumentParser) -> None: + selection = parser.add_mutually_exclusive_group() + selection.add_argument( + "--model", + action="append", + default=None, + help="Built-in llama.cpp model to validate. May be repeated.", + ) + selection.add_argument( + "--lite", + action="store_true", + help="Lite mode: only test the smallest hot model", + ) + + +def _model_size(model: dict[str, Any]) -> float: + size = model.get("size") + if isinstance(size, (int, float)) and not isinstance(size, bool): + return float(size) + return math.inf + + +def _require_pullable_checkpoint(model: dict[str, Any]) -> None: + checkpoint = model.get("checkpoint") + if isinstance(checkpoint, str) and checkpoint.strip(): + return + + checkpoints = model.get("checkpoints") + if isinstance(checkpoints, dict): + main_checkpoint = checkpoints.get("main") + if isinstance(main_checkpoint, str) and main_checkpoint.strip(): + return + + model_id = model.get("id", "") + raise ModelSelectionError( + f"Model '{model_id}' cannot be pulled because its catalog checkpoint " + "is missing or invalid" + ) + + +def select_llamacpp_models( + catalog: Iterable[dict[str, Any]], + requested_model_ids: list[str] | None = None, + lite: bool = False, + builtin_model_resolver: BuiltinModelResolver | None = None, +) -> list[dict[str, Any]]: + models = list(catalog) + + if requested_model_ids: + if builtin_model_resolver is None: + raise ModelSelectionError( + "Explicit model selection requires a built-in model resolver" + ) + selected = [] + for requested_model_id in requested_model_ids: + canonical_model_id = ( + requested_model_id + if requested_model_id.startswith("builtin.") + else f"builtin.{requested_model_id}" + ) + model = builtin_model_resolver(canonical_model_id) + if model is None: + raise ModelSelectionError( + f"Model '{requested_model_id}' was not found in the built-in " + "model catalog" + ) + model = dict(model) + model["id"] = requested_model_id + model["load_id"] = canonical_model_id + recipe = model.get("recipe") + if recipe != "llamacpp": + raise ModelSelectionError( + f"Model '{requested_model_id}' uses recipe '{recipe}', " + "expected 'llamacpp'" + ) + _require_pullable_checkpoint(model) + selected.append(model) + return selected + + selected = [ + model + for model in models + if model.get("recipe") == "llamacpp" and "hot" in model.get("labels", []) + ] + selected.sort(key=lambda model: model["id"]) + if not selected: + raise ModelSelectionError("No hot llama.cpp models found in the model catalog") + + if lite: + selected = [min(selected, key=_model_size)] + + for model in selected: + _require_pullable_checkpoint(model) + + return selected diff --git a/test/validate_llamacpp.py b/test/validate_llamacpp.py index a321730f52..cbf1b902eb 100644 --- a/test/validate_llamacpp.py +++ b/test/validate_llamacpp.py @@ -5,14 +5,15 @@ Usage: python test/validate_llamacpp.py --backend vulkan python test/validate_llamacpp.py --backend rocm + python test/validate_llamacpp.py --backend vulkan --model MODEL_ID This script expects `lemond` to already be running on the target port. This script: -1. Queries `/api/v1/models?show_all=true` and selects models with recipe - `llamacpp` and label `hot` +1. Queries `/api/v1/models?show_all=true` and selects either the requested + llama.cpp models or all models with recipe `llamacpp` and label `hot` 2. Installs the requested llamacpp backend via `POST /api/v1/install` -3. For each hot model, loads it with the requested backend, sends a +3. For each selected model, loads it with the requested backend, sends a `chat/completions` request, queries `/api/v1/stats`, and unloads it 4. Outputs a JSON results file for CI consumption """ @@ -24,14 +25,21 @@ import shutil import sys import tempfile +from urllib.parse import quote import requests from utils.server_base import _auth_headers, unload_all_models, wait_for_server from utils.test_models import PORT, TIMEOUT_DEFAULT +from utils.validation_model_selection import ( + ModelSelectionError, + add_model_selection_arguments, + select_llamacpp_models, +) TIMEOUT_HEALTH = 60 TIMEOUT_INFERENCE = 1800 # 30 minutes; large models may need 60+ GB download +VALIDATION_CTX_SIZE = 4096 CHAT_PROMPT = [ {"role": "user", "content": "What is 2+2? Reply in one sentence."}, ] @@ -92,8 +100,8 @@ def require_running_server(base_url, port): ) -def get_hot_llamacpp_models(base_url): - """Return the catalog entries for hot llamacpp models from the API.""" +def get_model_catalog(base_url): + """Return all model catalog entries from the API.""" response, body = request_json( "GET", f"{base_url}/models?show_all=true", @@ -104,14 +112,29 @@ def get_hot_llamacpp_models(base_url): f"Failed to query model catalog: HTTP {response.status_code} - {body}" ) - hot_models = [] - for model in body.get("data", []): - labels = model.get("labels", []) - if model.get("recipe") == "llamacpp" and "hot" in labels: - hot_models.append(model) + return body.get("data", []) - hot_models.sort(key=lambda model: model["id"]) - return hot_models + +def get_builtin_model(base_url, canonical_model_id): + """Resolve one canonical built-in model from the API.""" + encoded_model_id = quote(canonical_model_id, safe="") + response, body = request_json( + "GET", + f"{base_url}/models/{encoded_model_id}", + timeout=TIMEOUT_DEFAULT, + ) + if response.status_code == 404: + return None + if response.status_code != 200: + raise RuntimeError( + f"Failed to query built-in model '{canonical_model_id}': " + f"HTTP {response.status_code} - {body}" + ) + if not isinstance(body, dict): + raise RuntimeError( + f"Built-in model '{canonical_model_id}' returned an invalid response" + ) + return body def set_rocm_channel(base_url, channel): @@ -172,7 +195,11 @@ def test_model(base_url, model_name, backend, max_tokens=50): "POST", f"{base_url}/load", timeout=TIMEOUT_INFERENCE, - json={"model_name": model_name, "llamacpp_backend": backend}, + json={ + "model_name": model_name, + "llamacpp_backend": backend, + "ctx_size": VALIDATION_CTX_SIZE, + }, ) if load_resp.status_code != 200: return ( @@ -192,6 +219,8 @@ def test_model(base_url, model_name, backend, max_tokens=50): for model in health_body.get("all_models_loaded", []) } loaded_model = loaded.get(model_name, {}) + if not loaded_model and model_name.startswith("builtin."): + loaded_model = loaded.get(model_name.removeprefix("builtin."), {}) recipe_options = loaded_model.get("recipe_options", {}) actual_backend = recipe_options.get("llamacpp_backend") if actual_backend and actual_backend != backend: @@ -252,7 +281,7 @@ def test_model(base_url, model_name, backend, max_tokens=50): def main(): parser = argparse.ArgumentParser( - description="Validate a llama.cpp backend against hot Lemonade models" + description="Validate a llama.cpp backend against selected Lemonade models" ) parser.add_argument( "--backend", @@ -294,11 +323,7 @@ def main(): default=None, help="Directory to collect server log files into (for CI artifact upload)", ) - parser.add_argument( - "--lite", - action="store_true", - help="Lite mode: only test the smallest hot model", - ) + add_model_selection_arguments(parser) args = parser.parse_args() # Label used for output filenames and artifact names @@ -318,23 +343,35 @@ def main(): except requests.RequestException as exc: print(f"Warning: failed to unload pre-existing models: {exc}", flush=True) - hot_models = get_hot_llamacpp_models(base_url) - print(f"Found {len(hot_models)} hot llamacpp models:", flush=True) - for model in hot_models: - print(f" - {model['id']} ({model.get('size', '?')} GB)", flush=True) - - if not hot_models: - print("ERROR: No hot llamacpp models found!", file=sys.stderr, flush=True) + catalog = get_model_catalog(base_url) if not args.model else [] + try: + selected_models = select_llamacpp_models( + catalog, + requested_model_ids=args.model, + lite=args.lite, + builtin_model_resolver=lambda model_id: get_builtin_model( + base_url, model_id + ), + ) + except ModelSelectionError as exc: + print(f"ERROR: {exc}", file=sys.stderr, flush=True) sys.exit(1) - if args.lite and len(hot_models) > 1: - smallest = min(hot_models, key=lambda m: m.get("size", float("inf"))) + selection_mode = "explicitly selected" if args.model else "hot" + print( + f"Found {len(selected_models)} {selection_mode} llamacpp models:", + flush=True, + ) + for model in selected_models: + print(f" - {model['id']} ({model.get('size', '?')} GB)", flush=True) + + if args.lite: + selected = selected_models[0] print( f"Lite mode: testing only smallest model: " - f"{smallest['id']} ({smallest.get('size', '?')} GB)", + f"{selected['id']} ({selected.get('size', '?')} GB)", flush=True, ) - hot_models = [smallest] if not args.skip_install: install_backend(base_url, args.backend) @@ -342,11 +379,11 @@ def main(): results = [] all_passed = True try: - for model in hot_models: + for model in selected_models: model_name = model["id"] print(f"\nTesting: {model_name}", flush=True) success, response_text, stats = test_model( - base_url, model_name, args.backend + base_url, model.get("load_id", model_name), args.backend ) result = { "model": model_name, From 2965c6b3a0eaca24c8ce11d5fc759374625908ae Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:53:48 -0700 Subject: [PATCH 02/14] fix: harden K2-Horizon validation gates --- .github/scripts/llamacpp_validation_args.ps1 | 30 ++ .github/scripts/validate_backend_bench.py | 20 +- .github/workflows/docs_and_style.yml | 11 +- .github/workflows/validate_llamacpp.yml | 205 +++------- docs/embeddable/backends.md | 4 +- docs/guide/configuration/README.md | 2 +- docs/guide/configuration/llamacpp.md | 69 +--- test/test_llamacpp_release_assets.py | 121 ++++++ test/test_llamacpp_validation_args.ps1 | 76 ++++ test/test_validate_llamacpp_selection.py | 384 +++++++++++++++++++ test/test_validate_llamacpp_workflow_plan.py | 100 +++++ test/utils/llamacpp_release_assets.py | 148 +++++++ test/utils/llamacpp_validation_plan.py | 140 +++++++ test/utils/validation_model_selection.py | 13 +- test/validate_llamacpp.py | 111 +++++- 15 files changed, 1205 insertions(+), 229 deletions(-) create mode 100644 .github/scripts/llamacpp_validation_args.ps1 create mode 100644 test/test_llamacpp_release_assets.py create mode 100644 test/test_llamacpp_validation_args.ps1 create mode 100644 test/test_validate_llamacpp_workflow_plan.py create mode 100644 test/utils/llamacpp_release_assets.py create mode 100644 test/utils/llamacpp_validation_plan.py diff --git a/.github/scripts/llamacpp_validation_args.ps1 b/.github/scripts/llamacpp_validation_args.ps1 new file mode 100644 index 0000000000..1a567ca37d --- /dev/null +++ b/.github/scripts/llamacpp_validation_args.ps1 @@ -0,0 +1,30 @@ +function Get-LlamaCppValidationModelArguments { + [CmdletBinding()] + param( + [AllowEmptyString()] + [string] $Models = "", + [bool] $Lite = $false + ) + + $modelIds = @( + $Models -split "," | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } + ) + + if ($modelIds.Count -gt 0 -and $Lite) { + throw "Explicit models and lite mode are mutually exclusive." + } + + $validationArguments = @() + foreach ($modelId in $modelIds) { + $validationArguments += "--model" + $validationArguments += $modelId + } + + if ($Lite) { + $validationArguments += "--lite" + } + + return $validationArguments +} diff --git a/.github/scripts/validate_backend_bench.py b/.github/scripts/validate_backend_bench.py index 19b71543df..ae749840df 100644 --- a/.github/scripts/validate_backend_bench.py +++ b/.github/scripts/validate_backend_bench.py @@ -4,9 +4,9 @@ Uses `lemonade bench` against each tracked fork's prebuilt binary. The fork binary is downloaded, then routed into lemond via -`lemonade config set llamacpp._bin ` (done by the workflow +`lemonade config set llamacpp._bin=` (done by the workflow before lemond starts). lemond then serves --backend from that -binary instead of downloading its own. There is no LEMONADE_*_BIN env var. +binary instead of downloading its own. Usage: python .github/scripts/validate_backend_bench.py --forks src/cpp/resources/benchmark_forks.json --output ci/results @@ -15,7 +15,6 @@ """ import argparse -import hashlib import json import os import platform @@ -161,7 +160,7 @@ def find_lemonade_bin() -> str: def get_models_from_registry(base_url: str) -> list[str]: - """Mirror validate_llamacpp.py:get_hot_llamacpp_models() — reads live registry.""" + """Select hot llama.cpp models from the live registry.""" try: req = urllib.request.Request(f"{base_url}/api/v1/models?show_all=true") with urllib.request.urlopen(req, timeout=30) as r: @@ -210,7 +209,7 @@ def install_fork_binary( fork_id = fork["fork_id"] exe_name = fork["binary_exe_windows"] if IS_WINDOWS else fork["binary_exe_linux"] if not exe_name: - print(f" [skip] No binary for this platform") + print(" [skip] No binary for this platform") return None # One slot per fork — overwrite on version change, no disk accumulation @@ -304,9 +303,8 @@ def run_bench( # llamacpp exposes {system, cuda, vulkan, rocm, cpu} — "rocm-nightly" and # "rocm-gfx11" are NOT backend keys (nightly/stable are rocm *channels*), so # forks declare install_as to map to the real key. The fork binary itself is - # routed via `lemonade config set llamacpp._bin ` (done by the - # workflow before lemond starts); there is no LEMONADE_*_BIN env var — - # lemond never reads one. + # routed via `lemonade config set llamacpp._bin=` by the + # workflow before lemond starts. bench_as = fork.get("install_as", backend) cmd = [ @@ -610,7 +608,7 @@ def main() -> int: print(f"[skip] {fork_id} requires {runner_os}, running on {current_os}") continue - print(f"{'='*60}") + print(f"{'=' * 60}") print(f"Fork: {fork_id} ({fork['label']})") print(f"Repo: {fork['repo']}") @@ -653,7 +651,7 @@ def main() -> int: continue else: print( - f" Skipping POST /install — fork provides its own prebuilt binary" + " Skipping POST /install — fork provides its own prebuilt binary" ) # Fetch model list from registry now that lemond is confirmed running @@ -752,7 +750,7 @@ def main() -> int: ) print(f"Regression : {rr_out}") - print(f"\n{'='*60}") + print(f"\n{'=' * 60}") print("SUMMARY") criticals = [a for a in all_alerts if a["severity"] == "CRITICAL"] warns = [a for a in all_alerts if a["severity"] == "WARN"] diff --git a/.github/workflows/docs_and_style.yml b/.github/workflows/docs_and_style.yml index cad0fcd73d..cdcfda761b 100644 --- a/.github/workflows/docs_and_style.yml +++ b/.github/workflows/docs_and_style.yml @@ -77,7 +77,16 @@ jobs: python-version: '3.x' - run: pip install pre-commit - name: Run focused Python unit tests - run: python -m unittest test.test_comment_slop test.test_validate_llamacpp_selection -v + run: >- + python -m unittest + test.test_comment_slop + test.test_llamacpp_release_assets + test.test_validate_llamacpp_selection + test.test_validate_llamacpp_workflow_plan + -v + - name: Run PowerShell validation argument tests + shell: pwsh + run: ./test/test_llamacpp_validation_args.ps1 - name: Run pre-commit on the changed files # Scoped to what the change touches; running repo-wide would fail on drift that diff --git a/.github/workflows/validate_llamacpp.yml b/.github/workflows/validate_llamacpp.yml index e951210f85..31915b81c5 100644 --- a/.github/workflows/validate_llamacpp.yml +++ b/.github/workflows/validate_llamacpp.yml @@ -31,9 +31,27 @@ env: LEMONADE_CI_MODE: "True" PYTHONIOENCODING: utf-8 HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} - LITE_MODE: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' || (github.event_name == 'workflow_dispatch' && inputs.lite == true) }} jobs: + plan: + name: Plan validation + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@v5 + - name: Build validation matrix + id: plan + env: + EVENT_NAME: ${{ github.event_name }} + MANUAL_MODELS: ${{ inputs.models || '' }} + MANUAL_LITE: ${{ inputs.lite || false }} + run: >- + python -m test.utils.llamacpp_validation_plan + --event "$EVENT_NAME" + --models "$MANUAL_MODELS" + --lite "$MANUAL_LITE" + # ======================================================================== # Discover the latest llama.cpp release for each upstream repo # ======================================================================== @@ -153,34 +171,12 @@ jobs: python3 - <<'PY' import json import os - import re from pathlib import Path - RELEASE_RE = re.compile(r"^b[0-9]+$") - CUDA_SMS = ( - "sm_75", - "sm_80", - "sm_86", - "sm_89", - "sm_90", - "sm_100", - "sm_120", + from test.utils.llamacpp_release_assets import ( + build_asset_requirements, + evaluate_asset_group, ) - ROCM_NIGHTLY_ARCHES = ( - "gfx1151", - "gfx1150", - "gfx120X", - "gfx110X", - "gfx103X", - "gfx90a", - "gfx908", - ) - - def require_release(name): - value = os.environ[name].strip() - if not RELEASE_RE.match(value): - raise SystemExit(f"Invalid {name}: {value!r}") - return value def read_assets(path): return set(Path(path).read_text(encoding="utf-8").splitlines()) @@ -193,24 +189,23 @@ jobs: for key, value in values.items(): out.write(f"{key}={value}\n") - def report_group(name, release, backend_requirements): - update_backends = [] - missing_by_backend = {} + def report_group(name, available_assets, backend_requirements): + update_backends, missing_by_backend = evaluate_asset_group( + available_assets, backend_requirements + ) for backend, required in backend_requirements.items(): - missing = [asset for asset in required if asset not in release] - missing_by_backend[backend] = missing + missing = missing_by_backend[backend] if missing: print(f"{name}: {backend} assets incomplete; keeping existing pin.") for asset in missing: print(f" missing: {asset}") else: print(f"{name}: {backend} assets complete; pin may update.") - update_backends.append(backend) return update_backends, missing_by_backend - ggml_release = require_release("GGML_RELEASE") - rocm_release = require_release("ROCM_RELEASE") - lemonade_release = require_release("LEMONADE_RELEASE") + ggml_release = os.environ["GGML_RELEASE"] + rocm_release = os.environ["ROCM_RELEASE"] + lemonade_release = os.environ["LEMONADE_RELEASE"] ggml_assets = read_assets("ggml_assets.txt") rocm_assets = read_assets("rocm_assets.txt") @@ -219,67 +214,22 @@ jobs: with open("src/cpp/resources/backend_versions.json", encoding="utf-8") as f: versions = json.load(f) - therock = str(versions.get("therock", {}).get("version", "")).strip() - if therock.startswith("v"): - therock = therock[1:] - therock_parts = therock.split(".") - if len(therock_parts) >= 2: - therock = ".".join(therock_parts[:2]) - if not therock: - raise SystemExit("backend_versions.json is missing therock.version") - - asset_families = versions.get("rocm_asset_families", {}) - if not isinstance(asset_families, dict): - raise SystemExit("backend_versions.json rocm_asset_families must be an object") - - rocm_nightly_targets = sorted({asset_families.get(arch, arch) for arch in ROCM_NIGHTLY_ARCHES}) - - ggml_requirements = { - "vulkan": [ - f"llama-{ggml_release}-bin-win-vulkan-x64.zip", - f"llama-{ggml_release}-bin-ubuntu-vulkan-x64.tar.gz", - f"llama-{ggml_release}-bin-ubuntu-vulkan-arm64.tar.gz", - ], - "cpu": [ - f"llama-{ggml_release}-bin-win-cpu-x64.zip", - f"llama-{ggml_release}-bin-ubuntu-x64.tar.gz", - f"llama-{ggml_release}-bin-ubuntu-arm64.tar.gz", - ], - "metal": [ - f"llama-{ggml_release}-bin-macos-arm64.tar.gz", - ], - } - - rocm_nightly_requirements = { - "rocm-nightly": [ - asset - for target in rocm_nightly_targets - for asset in ( - f"llama-{rocm_release}-windows-rocm-{target}-x64.zip", - f"llama-{rocm_release}-ubuntu-rocm-{target}-x64.zip", - ) - ], - } - - lemonade_requirements = { - "rocm-stable": [ - f"llama-{lemonade_release}-bin-win-rocm-{therock}-x64.zip", - f"llama-{lemonade_release}-bin-ubuntu-rocm-{therock}-x64.tar.gz", - ], - "cuda": [ - asset - for sm in CUDA_SMS - for asset in ( - f"llama-{lemonade_release}-windows-cuda-{sm}-x64.7z", - f"llama-{lemonade_release}-ubuntu-cuda-{sm}-x64.tar.xz", - f"llama-{lemonade_release}-ubuntu-cuda-{sm}-arm64.tar.xz", - ) - ], - } + requirements = build_asset_requirements( + ggml_release=ggml_release, + rocm_release=rocm_release, + lemonade_release=lemonade_release, + backend_versions=versions, + ) - ggml_updates, ggml_missing = report_group("ggml-org/llama.cpp", ggml_assets, ggml_requirements) - rocm_updates, rocm_missing = report_group("lemonade-sdk/llamacpp-rocm", rocm_assets, rocm_nightly_requirements) - lemonade_updates, lemonade_missing = report_group("lemonade-sdk/llama.cpp", lemonade_assets, lemonade_requirements) + ggml_updates, ggml_missing = report_group( + "ggml-org/llama.cpp", ggml_assets, requirements["ggml"] + ) + rocm_updates, rocm_missing = report_group( + "lemonade-sdk/llamacpp-rocm", rocm_assets, requirements["rocm"] + ) + lemonade_updates, lemonade_missing = report_group( + "lemonade-sdk/llama.cpp", lemonade_assets, requirements["lemonade"] + ) values = { "ggml_update_backends": ",".join(ggml_updates), @@ -305,23 +255,11 @@ jobs: build: name: Build - needs: [get-latest-releases, verify-release-assets] + needs: [get-latest-releases, verify-release-assets, plan] # Label a PR `ci:upgrades` to opt it back in. if: (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:upgrades')) && always() && !failure() && !cancelled() runs-on: windows-latest - outputs: - lite: ${{ steps.lite.outputs.lite }} steps: - - name: Resolve lite mode - # The env context is unavailable in a job's `strategy` block, so the - # validate matrix reads LITE_MODE through this output to size its - # runners from the same source of truth the validation itself uses. - id: lite - shell: PowerShell - run: | - "lite=$env:LITE_MODE" | - Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - - name: Prepare Windows runner for long paths and stale workspace caches if: runner.os == 'Windows' shell: PowerShell @@ -438,39 +376,12 @@ jobs: # ======================================================================== validate: name: Validate ${{ matrix.backend }}${{ matrix.channel && format(' ({0})', matrix.channel) || '' }} - needs: [get-latest-releases, build] + needs: [get-latest-releases, build, plan] if: always() && needs.build.result == 'success' runs-on: ${{ matrix.runner }} strategy: fail-fast: ${{ github.event_name == 'merge_group' }} - matrix: - # Full runs walk every hot model, topping out at gpt-oss-120b (59 GiB), - # which only fits the 128gb runner. Lite runs test just the smallest hot - # model (0.6 GiB), so they stay off that single contended machine. - include: - - backend: vulkan - runner: >- - ${{ fromJSON(needs.build.outputs.lite == 'true' - && '["self-hosted","Windows","stx-halo","vulkan","lemon-prod"]' - || '["self-hosted","Windows","128gb","vulkan","lemon-prod"]') }} - - backend: rocm - channel: stable - runner: >- - ${{ fromJSON(needs.build.outputs.lite == 'true' - && '["self-hosted","Windows","stx-halo","rocm","lemon-prod"]' - || '["self-hosted","Windows","128gb","rocm","lemon-prod"]') }} - - backend: rocm - channel: nightly - runner: >- - ${{ fromJSON(needs.build.outputs.lite == 'true' - && '["self-hosted","Windows","stx-halo","rocm","lemon-prod"]' - || '["self-hosted","Windows","128gb","rocm","lemon-prod"]') }} - # CUDA validation is prepared but intentionally disabled until an - # NVIDIA self-hosted runner is available. To enable it, uncomment this - # matrix entry. - # - # - backend: cuda - # runner: [self-hosted, Windows, 128gb, cuda] + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} steps: - name: Capture Vulkan runner configuration if: matrix.backend == 'vulkan' @@ -723,7 +634,8 @@ jobs: - name: Run validation with lemond shell: PowerShell env: - LLAMACPP_TEST_MODELS: ${{ inputs.models || '' }} + LLAMACPP_TEST_MODELS: ${{ join(matrix.models, ',') }} + LITE_MODE: ${{ matrix.lite }} run: | $ErrorActionPreference = "Stop" $port = 13305 @@ -807,17 +719,10 @@ jobs: $validationArgs += $channel } - foreach ($modelValue in ($env:LLAMACPP_TEST_MODELS -split ",")) { - $model = $modelValue.Trim() - if ($model) { - $validationArgs += "--model" - $validationArgs += $model - } - } - - if ("${{ env.LITE_MODE }}" -eq "true") { - $validationArgs += "--lite" - } + . ".github/scripts/llamacpp_validation_args.ps1" + $validationArgs += @(Get-LlamaCppValidationModelArguments ` + -Models $env:LLAMACPP_TEST_MODELS ` + -Lite ($env:LITE_MODE -eq "true")) & $venvPython @validationArgs $validationExitCode = $LASTEXITCODE @@ -1144,7 +1049,7 @@ jobs: # to run on ordinary pull request pushes. validation-gate: name: llama.cpp validation - needs: [build, validate] + needs: [build, validate, plan] if: always() runs-on: ubuntu-latest steps: diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index f99595d5cc..13aa714fa3 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -159,7 +159,7 @@ For example, to use your own Vulkan `llama-server` in place of Lemonade's: lemond.exe ./ REM Set the llama-server vulkan binary path - lemonade.exe config set llamacpp.vulkan_bin C:\path\to\llama-server.exe + lemonade.exe config set llamacpp.vulkan_bin=C:\path\to\llama-server.exe ``` === "Linux (bash)" @@ -169,7 +169,7 @@ For example, to use your own Vulkan `llama-server` in place of Lemonade's: ./lemond ./ # Set the llama-server vulkan binary path - ./lemonade config set llamacpp.vulkan_bin /path/to/llama-server + ./lemonade config set llamacpp.vulkan_bin=/path/to/llama-server ``` See the `*_bin` settings in the [Configuration Guide](../guide/configuration/README.md) for the full set of customization options. diff --git a/docs/guide/configuration/README.md b/docs/guide/configuration/README.md index 47b0a15b28..adb8a00eef 100644 --- a/docs/guide/configuration/README.md +++ b/docs/guide/configuration/README.md @@ -345,7 +345,7 @@ Every `*_bin` key (e.g. `llamacpp.vulkan_bin`, `whispercpp.cpu_bin`, `sdcpp.rocm | `""` | Same as `"builtin"`. | | `"latest"` | Resolve to the most-recent upstream GitHub release on first install or first status query for that backend, then install on demand. The resolved tag is recorded in `/bin///version.txt`. | | `"b8664"` / `"v1.8.2"` / etc. | A specific upstream release tag. Lemonade downloads that exact version from GitHub. | -| `"/path/to/llama-server"` | The full path to an executable you built or installed. Lemonade uses this file and never downloads a replacement. The path must exist when set. | +| `"/full/path/to/backend-executable"` | The full path to an executable you built or installed. Lemonade uses this file and never downloads a replacement. The path must exist when set. | > Note: the `latest` setting is experimental. diff --git a/docs/guide/configuration/llamacpp.md b/docs/guide/configuration/llamacpp.md index 002a2a99e1..5874a3df8d 100644 --- a/docs/guide/configuration/llamacpp.md +++ b/docs/guide/configuration/llamacpp.md @@ -2,60 +2,30 @@ Lemonade uses [llama.cpp](https://github.com/ggerganov/llama.cpp) as its primary LLM inference backend, supporting multiple hardware acceleration options. This document explains the available backends and how to choose between them. -## K2-Horizon (IFM branch build) +- [K2-Horizon](#k2-horizon) +- [Available Backends](#available-backends) +- [ROCm Channel Configuration](#rocm-channel-configuration) +- [Choosing the Right Backend](#choosing-the-right-backend) +- [Platform Specifics](#platform-specifics) + +## K2-Horizon Lemonade includes these official IFM BF16 GGUF models: -| Model ID | Weight size | Maximum context | Suggested starting values | +| Model ID | Weight Size | Maximum Context | Suggested Starting Values | | --- | ---: | ---: | --- | | `K2-Horizon-0.9B-GGUF` | 2.16 GB | 131,072 tokens | high reasoning effort, temperature 0.6, top-p 0.95 | | `K2-Horizon-3.7B-GGUF` | 10.13 GB | 524,288 tokens | high reasoning effort, temperature 1.0, top-p 0.95 | | `K2-Horizon-7B-GGUF` | 18.01 GB | 524,288 tokens | high reasoning effort, temperature 1.0, top-p 0.95 | -These files need the K2-Horizon implementation from the -[`MBZUAI-IFM/llama.cpp` `model/K2Horizon` branch](https://github.com/MBZUAI-IFM/llama.cpp/tree/model/K2Horizon). -Use commit `35999d101cf2233fc54f09c3c8d599da7303ce02` for a reproducible build. -The standard llama.cpp binaries that Lemonade currently downloads do not contain this -implementation. Build the required server and configure Lemonade to use its executable: - -> Do not run these model IDs with Lemonade's current managed llama.cpp binaries. The -> catalog changes are intended to ship only after branch-derived binaries are published -> and pinned for every supported backend. - -```bash -git clone --branch model/K2Horizon --single-branch \ - https://github.com/MBZUAI-IFM/llama.cpp.git k2-horizon-llama.cpp -git -C k2-horizon-llama.cpp checkout --detach \ - 35999d101cf2233fc54f09c3c8d599da7303ce02 - -cmake -S k2-horizon-llama.cpp -B k2-horizon-llama.cpp/build \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLAMA_BUILD_TESTS=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF \ - -DLLAMA_BUILD_APP=OFF \ - -DLLAMA_BUILD_UI=OFF \ - -DLLAMA_USE_PREBUILT_UI=OFF \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_SERVER=ON -cmake --build k2-horizon-llama.cpp/build --config Release \ - --target llama-server -j 8 -``` +These are caller-supplied starting points. Lemonade does not install per-model sampling defaults. -The command above enables Metal automatically on macOS. Add the applicable llama.cpp CMake -option for another accelerator: `-DGGML_VULKAN=ON`, `-DGGML_CUDA=ON`, or -`-DGGML_HIP=ON`. - -Set the `*_bin` value to the full absolute path of `llama-server`, not its directory. For -example, use this command for macOS: - -```bash -lemonade config set \ - llamacpp.metal_bin=/absolute/path/to/k2-horizon-llama.cpp/build/bin/llama-server -``` +These model IDs must ship only with managed llama.cpp binaries built from accepted K2-Horizon +support on current upstream llama.cpp. The catalog entries are staged while that upstream +port, the coordinated backend pins, and packaged runtime validation are completed. Do not +publish them with the current managed pins or substitute the obsolete IFM development branch. -Use `cpu_bin`, `vulkan_bin`, `cuda_bin`, or `rocm_bin` for the applicable backend. On -Windows, include `llama-server.exe` in the path. You can then start a model with a bounded -context: +After compatible managed binaries are pinned, start a model with a bounded context: ```bash lemonade run K2-Horizon-0.9B-GGUF --ctx-size 8192 @@ -89,10 +59,11 @@ if reasoning: print(message.content) ``` -Use high reasoning effort and the default `xml` tool-call format with the pinned branch -commit. Its llama.cpp response parser does not correctly handle the template's `medium`, -`low`, `json`, or `xml_typed` modes. Medium and low reasoning can expose IFM control tags, -and the non-default tool formats can cause parser errors. To disable thinking, set +The upstream parser and packaged-runtime gate must prove high, medium, low, and disabled +reasoning plus the `json`, `xml`, and `xml_typed` tool-call formats before publication. The +obsolete IFM development branch does not pass that complete matrix: medium and low reasoning +can expose IFM control tags, and the non-default tool formats can cause parser errors. To +disable thinking after the compatible managed release ships, set `chat_template_kwargs.enable_thinking` to `false`. Use a large output allowance for long reasoning tasks. Do not use a 32K output allowance as a server-wide default. @@ -139,7 +110,7 @@ support K2-Horizon Uno. - **Use Case**: NVIDIA GPU-optimized inference - **Performance**: Optimized for NVIDIA hardware, typically outperforms Vulkan on supported GPUs - **Source**: Per-architecture builds from [lemonade-sdk/llama.cpp](https://github.com/lemonade-sdk/llama.cpp) -- **Binaries**: Compute-capability-specific builds (sm_75, sm_80, sm_86, sm_89, sm_90, sm_100, sm_120) +- **Binaries**: Compute-capability-specific builds (sm_75, sm_80, sm_86, sm_89, sm_90, sm_100, sm_120, sm_121) - **Runtime**: Bundled CUDA runtime libraries (no system-wide CUDA toolkit installation required) - **Notes**: On Windows, .7z extraction requires the bsdtar bundled with Windows 11 22H2+. On Linux, the build is shipped as .tar.xz and extracts with the system `tar`. diff --git a/test/test_llamacpp_release_assets.py b/test/test_llamacpp_release_assets.py new file mode 100644 index 0000000000..51073854d2 --- /dev/null +++ b/test/test_llamacpp_release_assets.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Unit tests for the managed llama.cpp release-asset contract.""" + +from __future__ import annotations + +import unittest + +from test.utils import llamacpp_release_assets as assets + + +class LlamaCppReleaseAssetTests(unittest.TestCase): + def setUp(self) -> None: + self.requirements = assets.build_asset_requirements( + ggml_release="b100", + rocm_release="b200", + lemonade_release="b300", + backend_versions={ + "therock": {"version": "7.14.0"}, + "rocm_asset_families": {}, + }, + ) + + def test_cuda_covers_every_published_compute_capability_and_platform(self) -> None: + required = set(self.requirements["lemonade"]["cuda"]) + + for compute_capability in ( + "sm_75", + "sm_80", + "sm_86", + "sm_89", + "sm_90", + "sm_100", + "sm_120", + "sm_121", + ): + with self.subTest(compute_capability=compute_capability): + self.assertIn( + f"llama-b300-windows-cuda-{compute_capability}-x64.7z", + required, + ) + self.assertIn( + f"llama-b300-ubuntu-cuda-{compute_capability}-x64.tar.xz", + required, + ) + self.assertIn( + f"llama-b300-ubuntu-cuda-{compute_capability}-arm64.tar.xz", + required, + ) + + self.assertEqual(len(required), 24) + + def test_rocm_nightly_covers_advertised_platform_targets(self) -> None: + required = set(self.requirements["rocm"]["rocm-nightly"]) + both_platforms = ( + "gfx1151", + "gfx1150", + "gfx120X", + "gfx110X", + "gfx103X", + "gfx90a", + "gfx908", + "gfx1152", + ) + + for target in both_platforms: + with self.subTest(target=target): + self.assertIn(f"llama-b200-windows-rocm-{target}-x64.zip", required) + self.assertIn(f"llama-b200-ubuntu-rocm-{target}-x64.zip", required) + + self.assertIn("llama-b200-ubuntu-rocm-gfx942-x64.zip", required) + self.assertNotIn("llama-b200-windows-rocm-gfx942-x64.zip", required) + self.assertFalse(any("gfx950" in name for name in required)) + self.assertEqual(len(required), 17) + + def test_rocm_targets_use_resource_family_mapping_without_duplicates(self) -> None: + requirements = assets.build_asset_requirements( + ggml_release="b100", + rocm_release="b200", + lemonade_release="b300", + backend_versions={ + "therock": {"version": "7.14.0"}, + "rocm_asset_families": { + "gfx1151": "gfx115X", + "gfx1152": "gfx115X", + }, + }, + ) + required = requirements["rocm"]["rocm-nightly"] + + self.assertEqual(required.count("llama-b200-windows-rocm-gfx115X-x64.zip"), 1) + self.assertEqual(required.count("llama-b200-ubuntu-rocm-gfx115X-x64.zip"), 1) + + def test_missing_new_target_keeps_rocm_nightly_pin_ineligible(self) -> None: + required = self.requirements["rocm"]["rocm-nightly"] + available = set(required) + missing_asset = "llama-b200-ubuntu-rocm-gfx942-x64.zip" + available.remove(missing_asset) + + eligible, missing = assets.evaluate_asset_group( + available, + self.requirements["rocm"], + ) + + self.assertEqual(eligible, []) + self.assertEqual(missing, {"rocm-nightly": [missing_asset]}) + + def test_invalid_release_tag_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "Invalid GGML_RELEASE"): + assets.build_asset_requirements( + ggml_release="main", + rocm_release="b200", + lemonade_release="b300", + backend_versions={ + "therock": {"version": "7.14.0"}, + "rocm_asset_families": {}, + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_llamacpp_validation_args.ps1 b/test/test_llamacpp_validation_args.ps1 new file mode 100644 index 0000000000..cff27a553b --- /dev/null +++ b/test/test_llamacpp_validation_args.ps1 @@ -0,0 +1,76 @@ +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot/../.github/scripts/llamacpp_validation_args.ps1" + +function Assert-Arguments { + param( + [string[]] $Actual, + [string[]] $Expected, + [string] $Case + ) + + if ($Actual.Count -ne $Expected.Count) { + throw "$Case returned $($Actual.Count) arguments; expected $($Expected.Count)." + } + + for ($index = 0; $index -lt $Expected.Count; $index++) { + if ($Actual[$index] -ne $Expected[$index]) { + throw "$Case argument $index was '$($Actual[$index])'; expected '$($Expected[$index])'." + } + } +} + +Assert-Arguments ` + -Actual @(Get-LlamaCppValidationModelArguments -Models "" -Lite $false) ` + -Expected @() ` + -Case "default mode" + +Assert-Arguments ` + -Actual @( + Get-LlamaCppValidationModelArguments ` + -Models " K2-Horizon-0.9B-GGUF, ,K2-Horizon-3.7B-GGUF " ` + -Lite $false + ) ` + -Expected @( + "--model", + "K2-Horizon-0.9B-GGUF", + "--model", + "K2-Horizon-3.7B-GGUF" + ) ` + -Case "explicit models" + +Assert-Arguments ` + -Actual @( + Get-LlamaCppValidationModelArguments ` + -Models "K2-Horizon-0.9B-GGUF,K2-Horizon-0.9B-GGUF" ` + -Lite $false + ) ` + -Expected @( + "--model", + "K2-Horizon-0.9B-GGUF", + "--model", + "K2-Horizon-0.9B-GGUF" + ) ` + -Case "duplicate models" + +Assert-Arguments ` + -Actual @(Get-LlamaCppValidationModelArguments -Models " , " -Lite $true) ` + -Expected @("--lite") ` + -Case "lite mode" + +$conflict = $null +try { + Get-LlamaCppValidationModelArguments ` + -Models "K2-Horizon-0.9B-GGUF" ` + -Lite $true | Out-Null +} catch { + $conflict = $_ +} + +if ($null -eq $conflict) { + throw "Explicit models and lite mode must be rejected." +} + +if ($conflict.Exception.Message -notmatch "mutually exclusive") { + throw "Conflict error must explain that explicit models and lite mode are mutually exclusive." +} diff --git a/test/test_validate_llamacpp_selection.py b/test/test_validate_llamacpp_selection.py index 8f14250efd..6fca472551 100644 --- a/test/test_validate_llamacpp_selection.py +++ b/test/test_validate_llamacpp_selection.py @@ -5,10 +5,15 @@ import argparse import contextlib +import importlib.util import io import json +import sys +import tempfile +import types import unittest from pathlib import Path +from unittest import mock from test.utils import validation_model_selection as selection @@ -21,6 +26,44 @@ ) +def load_validation_module(): + requests_module = types.ModuleType("requests") + requests_module.RequestException = RuntimeError + + server_base_module = types.ModuleType("utils.server_base") + server_base_module._auth_headers = lambda: {} + server_base_module.unload_all_models = lambda **_kwargs: None + server_base_module.wait_for_server = lambda **_kwargs: None + + test_models_module = types.ModuleType("utils.test_models") + test_models_module.PORT = 13305 + test_models_module.TIMEOUT_DEFAULT = 60 + + utils_package = types.ModuleType("utils") + utils_package.__path__ = [] + module_path = Path(__file__).resolve().parent / "validate_llamacpp.py" + spec = importlib.util.spec_from_file_location( + "validate_llamacpp_under_test", module_path + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load {module_path}") + + module = importlib.util.module_from_spec(spec) + stubs = { + "requests": requests_module, + "utils": utils_package, + "utils.server_base": server_base_module, + "utils.test_models": test_models_module, + "utils.validation_model_selection": selection, + } + with mock.patch.dict(sys.modules, stubs): + spec.loader.exec_module(module) + return module + + +VALIDATION = load_validation_module() + + class LlamaCppValidationSelectionTests(unittest.TestCase): def setUp(self) -> None: self.catalog = [ @@ -92,6 +135,10 @@ def test_explicit_selection_preserves_duplicate_ids(self) -> None: [model["id"] for model in selected], ["Alpha-Hot", "Alpha-Hot"] ) + def test_explicit_selection_rejects_lite_mode(self) -> None: + with self.assertRaisesRegex(ValueError, "mutually exclusive"): + self.select(["Alpha-Hot"], lite=True) + def test_explicit_selection_resolves_and_loads_canonical_builtin(self) -> None: resolved_ids = [] shadow = { @@ -125,6 +172,23 @@ def resolve_builtin_model(canonical_model_id): "IFM/K2-Horizon-0.9B-GGUF:model.gguf", ) + def test_shadowed_builtin_keeps_canonical_id_for_validation_evidence(self) -> None: + built_in = { + "id": "builtin.Shadowed-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/builtin:Shadowed-Llama.gguf", + } + + selected = selection.select_llamacpp_models( + [], + requested_model_ids=["Shadowed-Llama"], + builtin_model_resolver=lambda _model_id: built_in, + ) + + self.assertEqual(selected[0]["id"], "builtin.Shadowed-Llama") + self.assertEqual(selected[0]["load_id"], "builtin.Shadowed-Llama") + def test_explicit_selection_rejects_unverified_bare_catalog_entry(self) -> None: registered_model = [ { @@ -149,6 +213,26 @@ def test_non_llamacpp_explicit_model_has_clear_error(self) -> None: with self.assertRaisesRegex(ValueError, "Hot-Image.*sdcpp"): self.select(["Hot-Image"]) + def test_non_chat_llamacpp_explicit_models_have_clear_errors(self) -> None: + for model_id, label in ( + ("Embed-Llama", "embeddings"), + ("Rerank-Llama", "reranking"), + ): + with self.subTest(label=label): + self.catalog.append( + { + "id": model_id, + "recipe": "llamacpp", + "labels": [label], + "checkpoint": f"example/{label}:model.gguf", + } + ) + + with self.assertRaisesRegex( + ValueError, f"{model_id}.*does not support chat" + ): + self.select([model_id]) + def test_invalid_checkpoint_marks_explicit_model_unpullable(self) -> None: records = [ {"id": "Broken-Llama", "recipe": "llamacpp", "labels": ["chat"]}, @@ -213,6 +297,25 @@ def test_lite_selection_does_not_validate_unselected_models(self) -> None: self.assertEqual([model["id"] for model in selected], ["Alpha-Hot"]) + def test_default_selection_keeps_hot_models_without_catalog_checkpoints( + self, + ) -> None: + self.catalog.append( + { + "id": "Local-Hot", + "recipe": "llamacpp", + "labels": ["chat", "hot"], + "size": 3.0, + } + ) + + selected = self.select() + + self.assertEqual( + [model["id"] for model in selected], + ["Alpha-Hot", "Local-Hot", "Zulu-Hot"], + ) + def test_empty_explicit_list_uses_default_selection(self) -> None: selected = self.select([]) @@ -288,5 +391,286 @@ def unique_object(pairs): self.assertEqual(duplicate_ids, []) +class LlamaCppValidationRuntimeTests(unittest.TestCase): + @staticmethod + def response(status_code=200): + return types.SimpleNamespace(status_code=status_code) + + def request_json_for_chat(self, chat_message, operations=None): + loaded = False + + def request_json(method, url, timeout, **_kwargs): + nonlocal loaded + del method, timeout + operation = url.rsplit("/", maxsplit=1)[-1] + if operations is not None: + operations.append(operation) + if operation == "load": + loaded = True + return self.response(), {} + if operation == "unload": + loaded = False + return self.response(), {} + if operation == "health": + return self.response(), { + "all_models_loaded": ( + [ + { + "model_name": "builtin.Test-Llama", + "recipe_options": {"llamacpp_backend": "vulkan"}, + } + ] + if loaded + else [] + ) + } + if operation == "completions": + return self.response(), {"choices": [{"message": chat_message}]} + if operation == "stats": + return self.response(), {"output_tokens": 4} + return self.response(), {} + + return request_json + + def test_model_rejects_raw_ifm_markers_in_response_fields(self) -> None: + cases = { + "content": ( + {"content": "answer hidden"}, + "content", + ), + "nested_content": ( + { + "content": [ + {"type": "text", "text": "answer "}, + ] + }, + "content", + ), + "reasoning_content": ( + { + "content": "answer", + "reasoning_content": "hidden", + }, + "reasoning_content", + ), + "reasoning": ( + { + "content": "answer", + "reasoning": {"text": "hidden "}, + }, + "reasoning", + ), + "tool_calls": ( + { + "content": "answer", + "tool_calls": [ + { + "function": { + "name": "lookup", + "arguments": '{"query":""}', + } + } + ], + }, + "tool_calls", + ), + "unexpected_field": ( + { + "content": "answer", + "metadata": {"debug": " None: + message = { + "content": "I will use the lookup tool.", + "reasoning_content": "The tool can answer this request.", + "tool_calls": [ + { + "function": { + "name": "lookup", + "arguments": '{"query":"weather"}', + } + } + ], + } + + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat(message), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, response_text, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + + self.assertTrue(success) + self.assertEqual(response_text, message["content"]) + + def run_validation(self, explicit): + operations = [] + model = { + "id": "Test-Llama", + "recipe": "llamacpp", + "labels": ["chat", "hot"], + "checkpoint": "example/Test-Llama:model.gguf", + } + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--skip-install", + ] + if explicit: + argv.extend(["--model", "Test-Llama"]) + + with tempfile.TemporaryDirectory() as temp_dir: + argv.extend(["--output", str(Path(temp_dir) / "results.json")]) + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object( + VALIDATION, "get_model_catalog", return_value=[model] + ), + mock.patch.object(VALIDATION, "get_builtin_model", return_value=model), + mock.patch.object(VALIDATION, "unload_all_models"), + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat( + {"content": "The answer is 4."}, operations + ), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + VALIDATION.main() + + return operations + + def test_explicit_models_reload_after_unload_but_hot_models_do_not(self) -> None: + expected_loads = {"explicit": 2, "hot": 1} + + for selection_mode, expected_load_count in expected_loads.items(): + with self.subTest(selection_mode=selection_mode): + operations = self.run_validation(selection_mode == "explicit") + + self.assertEqual(operations.count("load"), expected_load_count) + self.assertEqual(operations.count("unload"), expected_load_count) + if selection_mode == "explicit": + first_unload = operations.index("unload") + second_load = operations.index("load", first_unload + 1) + self.assertLess(first_unload, second_load) + + def test_reload_fails_when_the_first_model_remains_loaded(self) -> None: + health = { + "all_models_loaded": [{"model_name": "builtin.Test-Llama"}], + } + with ( + mock.patch.object( + VALIDATION, + "test_model", + return_value=(True, "answer", {}), + ) as test_model, + mock.patch.object( + VALIDATION, + "request_json", + return_value=(self.response(), health), + ), + ): + success, error, _stats = VALIDATION.validate_model_lifecycle( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + reload_after_unload=True, + ) + + self.assertFalse(success) + self.assertRegex(error, "still loaded") + test_model.assert_called_once() + + def test_reload_requires_a_valid_loaded_model_inventory(self) -> None: + for health in ({}, {"all_models_loaded": "invalid"}): + with self.subTest(health=health): + with ( + mock.patch.object( + VALIDATION, + "test_model", + return_value=(True, "answer", {}), + ) as test_model, + mock.patch.object( + VALIDATION, + "request_json", + return_value=(self.response(), health), + ), + ): + success, error, _stats = VALIDATION.validate_model_lifecycle( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + reload_after_unload=True, + ) + + self.assertFalse(success) + self.assertRegex(error, "Could not verify unload") + test_model.assert_called_once() + + +class LlamaCppValidationSideEffectTests(unittest.TestCase): + def test_invalid_explicit_model_is_rejected_before_server_mutation(self) -> None: + argv = [ + "validate_llamacpp.py", + "--backend", + "rocm", + "--channel", + "nightly", + "--model", + "Missing-Llama", + ] + + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object(VALIDATION, "get_builtin_model", return_value=None), + mock.patch.object(VALIDATION, "set_rocm_channel") as set_channel, + mock.patch.object(VALIDATION, "unload_all_models") as unload_models, + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit) as raised, + ): + VALIDATION.main() + + self.assertEqual(raised.exception.code, 1) + set_channel.assert_not_called() + unload_models.assert_not_called() + + if __name__ == "__main__": unittest.main() diff --git a/test/test_validate_llamacpp_workflow_plan.py b/test/test_validate_llamacpp_workflow_plan.py new file mode 100644 index 0000000000..57b898ff53 --- /dev/null +++ b/test/test_validate_llamacpp_workflow_plan.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Unit tests for the llama.cpp validation workflow plan.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from test.utils import llamacpp_validation_plan as planning + +K2_SMALL = "K2-Horizon-0.9B-GGUF" +K2_MEDIUM = "K2-Horizon-3.7B-GGUF" +K2_LARGE = "K2-Horizon-7B-GGUF" + + +class LlamaCppValidationPlanTests(unittest.TestCase): + def assert_active_lanes(self, rows): + self.assertEqual( + [(row["backend"], row["channel"]) for row in rows], + [("vulkan", ""), ("rocm", "stable"), ("rocm", "nightly")], + ) + + def test_pull_request_and_merge_group_require_k2(self) -> None: + for event_name in ("pull_request", "merge_group"): + with self.subTest(event_name=event_name): + rows = planning.create_validation_plan(event_name)["include"] + + self.assert_active_lanes(rows) + self.assertEqual(rows[0]["models"], [K2_SMALL, K2_MEDIUM, K2_LARGE]) + self.assertEqual(rows[1]["models"], [K2_SMALL]) + self.assertEqual(rows[2]["models"], [K2_SMALL]) + self.assertTrue(all(not row["lite"] for row in rows)) + self.assertIn("128gb", rows[0]["runner"]) + + def test_schedule_and_default_dispatch_keep_full_hot_selection(self) -> None: + for event_name in ("schedule", "workflow_dispatch"): + with self.subTest(event_name=event_name): + rows = planning.create_validation_plan(event_name)["include"] + + self.assert_active_lanes(rows) + self.assertTrue(all(row["models"] == [] for row in rows)) + self.assertTrue(all(not row["lite"] for row in rows)) + self.assertTrue(all("128gb" in row["runner"] for row in rows)) + + def test_manual_lite_selection_uses_small_runners(self) -> None: + rows = planning.create_validation_plan("workflow_dispatch", lite=True)[ + "include" + ] + + self.assert_active_lanes(rows) + self.assertTrue(all(row["models"] == [] for row in rows)) + self.assertTrue(all(row["lite"] for row in rows)) + self.assertTrue(all("stx-halo" in row["runner"] for row in rows)) + + def test_manual_models_are_trimmed_and_keep_order_and_duplicates(self) -> None: + rows = planning.create_validation_plan( + "workflow_dispatch", + models_csv=" Zulu, ,Alpha,Zulu ", + )["include"] + + self.assert_active_lanes(rows) + self.assertTrue(all(row["models"] == ["Zulu", "Alpha", "Zulu"] for row in rows)) + self.assertTrue(all(not row["lite"] for row in rows)) + self.assertTrue(all("128gb" in row["runner"] for row in rows)) + + def test_manual_models_and_lite_are_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "mutually exclusive"): + planning.create_validation_plan( + "workflow_dispatch", + models_csv=K2_SMALL, + lite=True, + ) + + def test_non_manual_events_reject_manual_inputs(self) -> None: + with self.assertRaisesRegex(ValueError, "workflow_dispatch"): + planning.create_validation_plan("schedule", models_csv=K2_SMALL) + + def test_unknown_event_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "Unsupported"): + planning.create_validation_plan("push") + + def test_github_output_contains_compact_matrix_json(self) -> None: + plan = planning.create_validation_plan("merge_group") + with tempfile.TemporaryDirectory() as directory: + output_path = Path(directory) / "github-output.txt" + + planning.write_github_output(output_path, plan) + + key, payload = ( + output_path.read_text(encoding="utf-8").rstrip().split("=", 1) + ) + self.assertEqual(key, "matrix") + self.assertEqual(json.loads(payload), plan) + self.assertNotIn("\n", payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/utils/llamacpp_release_assets.py b/test/utils/llamacpp_release_assets.py new file mode 100644 index 0000000000..d090b430c4 --- /dev/null +++ b/test/utils/llamacpp_release_assets.py @@ -0,0 +1,148 @@ +"""Managed llama.cpp release-asset requirements used by CI.""" + +from __future__ import annotations + +import re +from collections.abc import Collection, Iterable, Mapping + +_RELEASE_RE = re.compile(r"^b[0-9]+$") +_CUDA_COMPUTE_CAPABILITIES = ( + "sm_75", + "sm_80", + "sm_86", + "sm_89", + "sm_90", + "sm_100", + "sm_120", + "sm_121", +) +_ROCM_NIGHTLY_TARGETS_BY_PLATFORM = { + "windows": ( + "gfx1151", + "gfx1150", + "gfx120X", + "gfx110X", + "gfx103X", + "gfx90a", + "gfx908", + "gfx1152", + ), + "ubuntu": ( + "gfx1151", + "gfx1150", + "gfx120X", + "gfx110X", + "gfx103X", + "gfx90a", + "gfx908", + "gfx1152", + "gfx942", + ), +} + + +def _validated_release(value: str, name: str) -> str: + release = value.strip() + if not _RELEASE_RE.fullmatch(release): + raise ValueError(f"Invalid {name}: {release!r}") + return release + + +def _therock_major_minor(backend_versions: Mapping[str, object]) -> str: + therock = backend_versions.get("therock", {}) + if not isinstance(therock, Mapping): + raise ValueError("backend_versions.json therock must be an object") + version = str(therock.get("version", "")).strip().removeprefix("v") + major_minor = ".".join(version.split(".")[:2]) + if not major_minor: + raise ValueError("backend_versions.json is missing therock.version") + return major_minor + + +def _mapped_rocm_targets( + targets: Iterable[str], asset_families: Mapping[str, object] +) -> list[str]: + mapped = [] + for target in targets: + family = asset_families.get(target, target) + if not isinstance(family, str) or not family: + raise ValueError(f"Invalid ROCm asset family for {target}: {family!r}") + if family not in mapped: + mapped.append(family) + return mapped + + +def build_asset_requirements( + *, + ggml_release: str, + rocm_release: str, + lemonade_release: str, + backend_versions: Mapping[str, object], +) -> dict[str, dict[str, list[str]]]: + """Return required asset names grouped by publisher and Lemonade backend.""" + ggml_release = _validated_release(ggml_release, "GGML_RELEASE") + rocm_release = _validated_release(rocm_release, "ROCM_RELEASE") + lemonade_release = _validated_release(lemonade_release, "LEMONADE_RELEASE") + therock = _therock_major_minor(backend_versions) + + asset_families = backend_versions.get("rocm_asset_families", {}) + if not isinstance(asset_families, Mapping): + raise ValueError("backend_versions.json rocm_asset_families must be an object") + + rocm_nightly = [] + for platform, targets in _ROCM_NIGHTLY_TARGETS_BY_PLATFORM.items(): + for target in _mapped_rocm_targets(targets, asset_families): + rocm_nightly.append( + f"llama-{rocm_release}-{platform}-rocm-{target}-x64.zip" + ) + + cuda = [] + for compute_capability in _CUDA_COMPUTE_CAPABILITIES: + cuda.extend( + ( + f"llama-{lemonade_release}-windows-cuda-{compute_capability}-x64.7z", + f"llama-{lemonade_release}-ubuntu-cuda-{compute_capability}-x64.tar.xz", + f"llama-{lemonade_release}-ubuntu-cuda-{compute_capability}-arm64.tar.xz", + ) + ) + + return { + "ggml": { + "vulkan": [ + f"llama-{ggml_release}-bin-win-vulkan-x64.zip", + f"llama-{ggml_release}-bin-ubuntu-vulkan-x64.tar.gz", + f"llama-{ggml_release}-bin-ubuntu-vulkan-arm64.tar.gz", + ], + "cpu": [ + f"llama-{ggml_release}-bin-win-cpu-x64.zip", + f"llama-{ggml_release}-bin-ubuntu-x64.tar.gz", + f"llama-{ggml_release}-bin-ubuntu-arm64.tar.gz", + ], + "metal": [ + f"llama-{ggml_release}-bin-macos-arm64.tar.gz", + ], + }, + "rocm": {"rocm-nightly": rocm_nightly}, + "lemonade": { + "rocm-stable": [ + f"llama-{lemonade_release}-bin-win-rocm-{therock}-x64.zip", + f"llama-{lemonade_release}-bin-ubuntu-rocm-{therock}-x64.tar.gz", + ], + "cuda": cuda, + }, + } + + +def evaluate_asset_group( + available_assets: Collection[str], + backend_requirements: Mapping[str, Iterable[str]], +) -> tuple[list[str], dict[str, list[str]]]: + """Return eligible backends and their missing required assets.""" + eligible = [] + missing_by_backend = {} + for backend, required_assets in backend_requirements.items(): + missing = [asset for asset in required_assets if asset not in available_assets] + missing_by_backend[backend] = missing + if not missing: + eligible.append(backend) + return eligible, missing_by_backend diff --git a/test/utils/llamacpp_validation_plan.py b/test/utils/llamacpp_validation_plan.py new file mode 100644 index 0000000000..8e44ced78e --- /dev/null +++ b/test/utils/llamacpp_validation_plan.py @@ -0,0 +1,140 @@ +"""Build the llama.cpp validation matrix for GitHub Actions.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +K2_SMALL = "K2-Horizon-0.9B-GGUF" +K2_MEDIUM = "K2-Horizon-3.7B-GGUF" +K2_LARGE = "K2-Horizon-7B-GGUF" +SUPPORTED_EVENTS = {"merge_group", "pull_request", "schedule", "workflow_dispatch"} + +LARGE_VULKAN_RUNNER = [ + "self-hosted", + "Windows", + "128gb", + "vulkan", + "lemon-prod", +] +LARGE_ROCM_RUNNER = [ + "self-hosted", + "Windows", + "128gb", + "rocm", + "lemon-prod", +] +SMALL_VULKAN_RUNNER = [ + "self-hosted", + "Windows", + "stx-halo", + "vulkan", + "lemon-prod", +] +SMALL_ROCM_RUNNER = [ + "self-hosted", + "Windows", + "stx-halo", + "rocm", + "lemon-prod", +] + + +def _parse_models(models_csv: str) -> list[str]: + return [model.strip() for model in models_csv.split(",") if model.strip()] + + +def _row( + backend: str, + channel: str, + runner: list[str], + models: list[str], + lite: bool, +) -> dict[str, object]: + return { + "backend": backend, + "channel": channel, + "runner": list(runner), + "models": list(models), + "lite": lite, + } + + +def create_validation_plan( + event_name: str, + models_csv: str = "", + lite: bool = False, +) -> dict[str, list[dict[str, object]]]: + if event_name not in SUPPORTED_EVENTS: + raise ValueError(f"Unsupported workflow event: {event_name}") + + models = _parse_models(models_csv) + if event_name != "workflow_dispatch" and (models or lite): + raise ValueError("Manual model inputs are valid only for workflow_dispatch") + if models and lite: + raise ValueError("Explicit models and lite mode are mutually exclusive") + + if event_name in {"pull_request", "merge_group"}: + return { + "include": [ + _row( + "vulkan", + "", + LARGE_VULKAN_RUNNER, + [K2_SMALL, K2_MEDIUM, K2_LARGE], + False, + ), + _row("rocm", "stable", SMALL_ROCM_RUNNER, [K2_SMALL], False), + _row("rocm", "nightly", SMALL_ROCM_RUNNER, [K2_SMALL], False), + ] + } + + runner_vulkan = SMALL_VULKAN_RUNNER if lite else LARGE_VULKAN_RUNNER + runner_rocm = SMALL_ROCM_RUNNER if lite else LARGE_ROCM_RUNNER + return { + "include": [ + _row("vulkan", "", runner_vulkan, models, lite), + _row("rocm", "stable", runner_rocm, models, lite), + _row("rocm", "nightly", runner_rocm, models, lite), + ] + } + + +def write_github_output( + output_path: Path, + plan: dict[str, list[dict[str, object]]], +) -> None: + matrix = json.dumps(plan, separators=(",", ":")) + with output_path.open("a", encoding="utf-8") as output: + output.write(f"matrix={matrix}\n") + + +def _bool_argument(value: str) -> bool: + normalized = value.strip().lower() + if normalized == "true": + return True + if normalized in {"", "false"}: + return False + raise argparse.ArgumentTypeError("expected true or false") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--event", required=True) + parser.add_argument("--models", default="") + parser.add_argument("--lite", default=False, type=_bool_argument) + args = parser.parse_args() + + output_value = os.environ.get("GITHUB_OUTPUT") + if not output_value: + parser.error("GITHUB_OUTPUT is required") + + plan = create_validation_plan(args.event, args.models, args.lite) + write_github_output(Path(output_value), plan) + print(json.dumps(plan, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/utils/validation_model_selection.py b/test/utils/validation_model_selection.py index 78a7fc8cf3..28e1c462be 100644 --- a/test/utils/validation_model_selection.py +++ b/test/utils/validation_model_selection.py @@ -62,6 +62,10 @@ def select_llamacpp_models( models = list(catalog) if requested_model_ids: + if lite: + raise ModelSelectionError( + "Explicit models and lite mode are mutually exclusive" + ) if builtin_model_resolver is None: raise ModelSelectionError( "Explicit model selection requires a built-in model resolver" @@ -80,7 +84,6 @@ def select_llamacpp_models( "model catalog" ) model = dict(model) - model["id"] = requested_model_id model["load_id"] = canonical_model_id recipe = model.get("recipe") if recipe != "llamacpp": @@ -88,6 +91,11 @@ def select_llamacpp_models( f"Model '{requested_model_id}' uses recipe '{recipe}', " "expected 'llamacpp'" ) + labels = model.get("labels") + if not isinstance(labels, list) or "chat" not in labels: + raise ModelSelectionError( + f"Model '{requested_model_id}' does not support chat validation" + ) _require_pullable_checkpoint(model) selected.append(model) return selected @@ -104,7 +112,4 @@ def select_llamacpp_models( if lite: selected = [min(selected, key=_model_size)] - for model in selected: - _require_pullable_checkpoint(model) - return selected diff --git a/test/validate_llamacpp.py b/test/validate_llamacpp.py index cbf1b902eb..b484ab4269 100644 --- a/test/validate_llamacpp.py +++ b/test/validate_llamacpp.py @@ -43,6 +43,35 @@ CHAT_PROMPT = [ {"role": "user", "content": "What is 2+2? Reply in one sentence."}, ] +IFM_CONTROL_PREFIXES = ("", start) + if end < 0: + end = min(start + 79, len(candidate) - 1) + return candidate[start : end + 1], path + elif isinstance(candidate, dict): + items = list(candidate.items()) + for key, nested_value in reversed(items): + pending.append((f"{path}.{key}", nested_value)) + elif isinstance(candidate, (list, tuple)): + for index in range(len(candidate) - 1, -1, -1): + pending.append((f"{path}[{index}]", candidate[index])) + return None def collect_server_logs(output_dir): @@ -245,6 +274,16 @@ def test_model(base_url, model_name, backend, max_tokens=50): return False, f"HTTP {chat_resp.status_code}: {chat_body}", {} message = chat_body["choices"][0]["message"] + marker = find_raw_ifm_control_marker(message, "message") + if marker: + marker_text, marker_path = marker + return ( + False, + f"Raw IFM control marker {marker_text!r} found in " + f"response field {marker_path}", + {}, + ) + content = message.get("content") or "" reasoning = message.get("reasoning_content") or "" combined = content + reasoning @@ -279,6 +318,53 @@ def test_model(base_url, model_name, backend, max_tokens=50): print(f" Warning: failed to unload {model_name}: {exc}", flush=True) +def validate_model_lifecycle( + base_url, + model_name, + backend, + reload_after_unload=False, +): + """Validate one model and optionally validate a second load after unload.""" + result = test_model(base_url, model_name, backend) + if not reload_after_unload or not result[0]: + return result + + try: + health_response, health_body = request_json( + "GET", + f"{base_url}/health", + timeout=TIMEOUT_DEFAULT, + ) + except requests.RequestException as exc: + return False, f"Could not verify unload before reload: {exc}", result[2] + + if health_response.status_code != 200: + return ( + False, + f"Could not verify unload before reload: HTTP " + f"{health_response.status_code} - {health_body}", + result[2], + ) + + if not isinstance(health_body, dict): + return False, "Could not verify unload: invalid health response", result[2] + loaded_models = health_body.get("all_models_loaded") + if not isinstance(loaded_models, list) or not all( + isinstance(model, dict) and isinstance(model.get("model_name"), str) + for model in loaded_models + ): + return False, "Could not verify unload: invalid model inventory", result[2] + + aliases = {model_name} + if model_name.startswith("builtin."): + aliases.add(model_name.removeprefix("builtin.")) + if any(model["model_name"] in aliases for model in loaded_models): + return False, f"Model '{model_name}' is still loaded after unload", result[2] + + print(" Reloading model after unload...", flush=True) + return test_model(base_url, model_name, backend) + + def main(): parser = argparse.ArgumentParser( description="Validate a llama.cpp backend against selected Lemonade models" @@ -334,15 +420,6 @@ def main(): require_running_server(base_url, args.port) - if args.channel: - set_rocm_channel(base_url, args.channel) - - print("Unloading all models for clean state...", flush=True) - try: - unload_all_models(port=args.port) - except requests.RequestException as exc: - print(f"Warning: failed to unload pre-existing models: {exc}", flush=True) - catalog = get_model_catalog(base_url) if not args.model else [] try: selected_models = select_llamacpp_models( @@ -357,6 +434,15 @@ def main(): print(f"ERROR: {exc}", file=sys.stderr, flush=True) sys.exit(1) + if args.channel: + set_rocm_channel(base_url, args.channel) + + print("Unloading all models for clean state...", flush=True) + try: + unload_all_models(port=args.port) + except requests.RequestException as exc: + print(f"Warning: failed to unload pre-existing models: {exc}", flush=True) + selection_mode = "explicitly selected" if args.model else "hot" print( f"Found {len(selected_models)} {selection_mode} llamacpp models:", @@ -382,8 +468,11 @@ def main(): for model in selected_models: model_name = model["id"] print(f"\nTesting: {model_name}", flush=True) - success, response_text, stats = test_model( - base_url, model.get("load_id", model_name), args.backend + success, response_text, stats = validate_model_lifecycle( + base_url, + model.get("load_id", model_name), + args.backend, + reload_after_unload=bool(args.model), ) result = { "model": model_name, From 6e8e1cf4a8739e21a15deb1dc34f3dc9c8791650 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:45:21 -0700 Subject: [PATCH 03/14] fix: harden K2 validation delivery --- .../actions/prepare-debian-build/action.yml | 13 +- .github/scripts/publish_llamacpp_update.sh | 195 +++++++ .github/scripts/resolve_debian_version.sh | 29 + .github/workflows/docs_and_style.yml | 4 + ...lamacpp.yml => validate_llamacpp_core.yml} | 322 +++-------- .../workflows/validate_llamacpp_manual.yml | 29 + .../workflows/validate_llamacpp_schedule.yml | 255 +++++++++ test/test_debian_version.py | 203 +++++++ test/test_llamacpp_release_manifest.py | 146 +++++ test/test_llamacpp_validation_evidence.py | 173 ++++++ test/test_publish_llamacpp_update.py | 537 ++++++++++++++++++ test/test_validate_llamacpp_selection.py | 21 + test/test_validate_llamacpp_workflow_plan.py | 274 ++++++++- test/utils/llamacpp_release_manifest.py | 203 +++++++ test/utils/llamacpp_validation_evidence.py | 195 +++++++ test/utils/llamacpp_validation_plan.py | 75 ++- test/validate_llamacpp.py | 2 +- 17 files changed, 2402 insertions(+), 274 deletions(-) create mode 100755 .github/scripts/publish_llamacpp_update.sh create mode 100755 .github/scripts/resolve_debian_version.sh rename .github/workflows/{validate_llamacpp.yml => validate_llamacpp_core.yml} (71%) create mode 100644 .github/workflows/validate_llamacpp_manual.yml create mode 100644 .github/workflows/validate_llamacpp_schedule.yml create mode 100644 test/test_debian_version.py create mode 100644 test/test_llamacpp_release_manifest.py create mode 100644 test/test_llamacpp_validation_evidence.py create mode 100644 test/test_publish_llamacpp_update.py create mode 100644 test/utils/llamacpp_release_manifest.py create mode 100644 test/utils/llamacpp_validation_evidence.py diff --git a/.github/actions/prepare-debian-build/action.yml b/.github/actions/prepare-debian-build/action.yml index 207fa4af35..0adda25e1c 100644 --- a/.github/actions/prepare-debian-build/action.yml +++ b/.github/actions/prepare-debian-build/action.yml @@ -1,5 +1,5 @@ name: Prepare Debian Build -description: Extract version from git and create Debian packaging metadata from template +description: Resolve the Debian version and create packaging metadata from template inputs: release: @@ -17,14 +17,15 @@ outputs: runs: using: composite steps: - - name: Get version from git describe + - name: Resolve Debian version id: get_version shell: bash run: | - VERSION=$(git describe --tags --always) - # Strip leading 'v' for Debian version compatibility - DEB_VERSION=$(echo "${VERSION}" | sed 's/^v//') - echo "version=${DEB_VERSION}" >> $GITHUB_OUTPUT + DEB_VERSION=$( + "${GITHUB_ACTION_PATH}/../../scripts/resolve_debian_version.sh" \ + "${{ inputs.release }}" + ) + echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" - name: Create Debian packaging metadata shell: bash diff --git a/.github/scripts/publish_llamacpp_update.sh b/.github/scripts/publish_llamacpp_update.sh new file mode 100755 index 0000000000..4a9e760899 --- /dev/null +++ b/.github/scripts/publish_llamacpp_update.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +set -euo pipefail + +versions_path=${VERSIONS_PATH:-src/cpp/resources/backend_versions.json} +body_file=${PR_BODY_FILE:-pr_body.md} +base_branch=${BASE_BRANCH:-main} +repository=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required} +repository_owner=${GITHUB_REPOSITORY_OWNER:?GITHUB_REPOSITORY_OWNER is required} +validated_base_sha=${VALIDATED_BASE_SHA:?VALIDATED_BASE_SHA is required} +llamacpp_release=${LLAMACPP_RELEASE:?LLAMACPP_RELEASE is required} +lemonade_release=${LLAMACPP_LEMONADE_RELEASE:?LLAMACPP_LEMONADE_RELEASE is required} +rocm_release=${LLAMACPP_ROCM_RELEASE:?LLAMACPP_ROCM_RELEASE is required} + +release_pattern='^b[0-9]+$' +for release in "$llamacpp_release" "$lemonade_release" "$rocm_release"; do + if [[ ! "$release" =~ $release_pattern ]]; then + echo "Invalid llama.cpp release tag: ${release}" >&2 + exit 1 + fi +done + +local_base_sha=$(git rev-parse HEAD) +if [[ "$local_base_sha" != "$validated_base_sha" ]]; then + echo "Local HEAD does not match the validated base SHA." >&2 + exit 1 +fi + +if [[ $(git rev-parse --is-shallow-repository) == "true" ]]; then + git fetch --unshallow --no-tags origin "refs/heads/${base_branch}" +else + git fetch --no-tags origin "refs/heads/${base_branch}" +fi +fetched_base_sha=$(git rev-parse FETCH_HEAD) +if [[ "$fetched_base_sha" != "$validated_base_sha" ]]; then + echo "The ${base_branch} branch advanced after validation; refusing publication." >&2 + exit 1 +fi + +fresh_llamacpp_release=$(gh api repos/ggml-org/llama.cpp/releases \ + --jq '[.[] | select(.draft | not) | .tag_name | select(test("^b[0-9]+$"))][0] // empty') +fresh_rocm_release=$(gh api \ + repos/lemonade-sdk/llamacpp-rocm/releases/latest --jq '.tag_name') +fresh_lemonade_release=$(gh api \ + repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name') + +if [[ "$fresh_llamacpp_release" != "$llamacpp_release" || + "$fresh_rocm_release" != "$rocm_release" || + "$fresh_lemonade_release" != "$lemonade_release" ]]; then + echo "The candidate releases changed during validation; refusing publication." >&2 + exit 1 +fi + +has_update=true +if git diff --quiet HEAD -- "$versions_path"; then + has_update=false + branch="" +else + versions_blob=$(git hash-object "$versions_path") + branch="auto/llamacpp-update-${llamacpp_release}-${lemonade_release}-${rocm_release}-${validated_base_sha}-${versions_blob}" +fi + +open_pr_records=$(gh api "repos/${repository}/pulls" \ + --method GET \ + -f state=open \ + -f per_page=100 \ + --paginate \ + --jq '.[] | select(.head.ref | startswith("auto/llamacpp-update-")) | [.number, .head.ref, (.head.repo.full_name // ""), .user.login, .base.ref] | @tsv') + +open_pr="" +stale_prs=() +while IFS=$'\t' read -r number head_ref head_repository author base_ref; do + if [[ -z "$number" ]]; then + continue + fi + + trusted=false + if [[ "$head_repository" == "$repository" && + "$author" == "github-actions[bot]" && + "$base_ref" == "$base_branch" ]]; then + trusted=true + fi + + if [[ "$has_update" == "true" && + "$head_ref" == "$branch" && + "$head_repository" == "$repository" ]]; then + if [[ "$trusted" != "true" ]]; then + echo "Found an untrusted open pull request for ${repository_owner}:${branch}." >&2 + exit 1 + fi + if [[ -n "$open_pr" ]]; then + echo "Multiple open pull requests use ${repository_owner}:${branch}; refusing publication." >&2 + exit 1 + fi + open_pr=$number + elif [[ "$trusted" == "true" ]]; then + stale_prs+=("$number") + fi +done <<< "$open_pr_records" + +assert_base_is_current() { + local remote_base_sha + remote_base_sha=$(git ls-remote --heads origin "refs/heads/${base_branch}" | + awk 'NR == 1 { print $1 }') + if [[ -z "$remote_base_sha" || "$remote_base_sha" != "$validated_base_sha" ]]; then + echo "The ${base_branch} branch advanced after validation; refusing publication." >&2 + exit 1 + fi +} + +assert_base_is_current +for stale_pr in "${stale_prs[@]}"; do + gh api "repos/${repository}/pulls/${stale_pr}" \ + --method PATCH \ + -f state=closed >/dev/null + echo "Closed superseded pull request #${stale_pr}." +done + +if [[ "$has_update" != "true" ]]; then + echo "backend_versions.json is unchanged; nothing to update." + exit 0 +fi + +title="Update llama.cpp to ${llamacpp_release}" +remote_oid=$(git ls-remote --heads origin "refs/heads/$branch" | awk 'NR == 1 { print $1 }') +remote_tree="" +if [[ -n "$remote_oid" ]]; then + git fetch --no-tags origin "refs/heads/$branch" + fetched_oid=$(git rev-parse FETCH_HEAD) + if [[ "$fetched_oid" != "$remote_oid" ]]; then + echo "The publication branch changed while it was inspected." >&2 + exit 1 + fi + remote_tree=$(git rev-parse "${remote_oid}^{tree}") + if ! merge_base=$(git merge-base "$validated_base_sha" "$remote_oid"); then + echo "Publication branch ${branch} has unrelated history." >&2 + exit 1 + fi + if [[ "$merge_base" != "$validated_base_sha" ]]; then + echo "Publication branch ${branch} is not based on the validated commit." >&2 + exit 1 + fi + unexpected_path=$(git diff --name-only "$merge_base" "$remote_oid" -- | + awk -v allowed="$versions_path" '$0 != allowed { print; exit }') + if [[ -n "$unexpected_path" ]]; then + echo "Publication branch ${branch} changes unexpected path ${unexpected_path}." >&2 + exit 1 + fi +elif [[ -n "$open_pr" ]]; then + echo "Trusted pull request #${open_pr} has no publication branch." >&2 + exit 1 +fi + +git config user.name "github-actions[bot]" +git config user.email "github-actions[bot]@users.noreply.github.com" +git checkout -b "$branch" +git add "$versions_path" +git commit -m \ + "Update llama.cpp to ${llamacpp_release}, rocm-stable to ${lemonade_release}, rocm-nightly to ${rocm_release}" +desired_tree=$(git rev-parse 'HEAD^{tree}') + +if [[ -n "$remote_oid" && -z "$open_pr" && "$remote_tree" != "$desired_tree" ]]; then + echo "Existing publication branch does not match the desired update." >&2 + exit 1 +fi +if [[ -n "$remote_oid" && -n "$open_pr" && "$remote_tree" != "$desired_tree" ]]; then + echo "Trusted pull request #${open_pr} does not match the desired update." >&2 + exit 1 +fi + +assert_base_is_current +if [[ -n "$remote_oid" ]]; then + lease="refs/heads/${branch}:${remote_oid}" + push_source="$remote_oid" +else + lease="refs/heads/${branch}:" + push_source=HEAD +fi +git push --force-with-lease="$lease" origin \ + "${push_source}:refs/heads/$branch" + +if [[ -n "$open_pr" ]]; then + gh pr edit "$open_pr" \ + --repo "$repository" \ + --title "$title" \ + --body-file "$body_file" + echo "Refreshed pull request #${open_pr}." +else + gh pr create \ + --repo "$repository" \ + --title "$title" \ + --body-file "$body_file" \ + --base "$base_branch" \ + --head "$branch" +fi diff --git a/.github/scripts/resolve_debian_version.sh b/.github/scripts/resolve_debian_version.sh new file mode 100755 index 0000000000..a398a929a7 --- /dev/null +++ b/.github/scripts/resolve_debian_version.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -euo pipefail + +release=${1:?release is required} + +if git describe --tags --abbrev=0 >/dev/null 2>&1; then + version=$(git describe --tags --always) +else + if [[ ! -f CMakeLists.txt ]]; then + echo "Could not extract version from CMakeLists.txt" >&2 + exit 1 + fi + + version=$(sed -n \ + 's/^project(lemon_cpp VERSION \([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p' \ + CMakeLists.txt) + if [[ -z "$version" ]]; then + echo "Could not extract version from CMakeLists.txt" >&2 + exit 1 + fi + + commit_count=$(git rev-list --count HEAD) + version="${version}+git${commit_count}.g$(git rev-parse --short HEAD)" +fi + +debian_version=${version#v} +dpkg --validate-version "${debian_version}~${release}" +printf '%s\n' "$debian_version" diff --git a/.github/workflows/docs_and_style.yml b/.github/workflows/docs_and_style.yml index cdcfda761b..7c63f589d4 100644 --- a/.github/workflows/docs_and_style.yml +++ b/.github/workflows/docs_and_style.yml @@ -80,7 +80,11 @@ jobs: run: >- python -m unittest test.test_comment_slop + test.test_debian_version + test.test_llamacpp_release_manifest test.test_llamacpp_release_assets + test.test_llamacpp_validation_evidence + test.test_publish_llamacpp_update test.test_validate_llamacpp_selection test.test_validate_llamacpp_workflow_plan -v diff --git a/.github/workflows/validate_llamacpp.yml b/.github/workflows/validate_llamacpp_core.yml similarity index 71% rename from .github/workflows/validate_llamacpp.yml rename to .github/workflows/validate_llamacpp_core.yml index 31915b81c5..025b3c2616 100644 --- a/.github/workflows/validate_llamacpp.yml +++ b/.github/workflows/validate_llamacpp_core.yml @@ -1,7 +1,7 @@ -name: Validate New llama.cpp Release +name: Validate llama.cpp on: - workflow_dispatch: + workflow_call: inputs: lite: description: 'Run in lite mode (smallest model only)' @@ -12,25 +12,62 @@ on: type: string required: false default: '' + secrets: + HUGGINGFACE_ACCESS_TOKEN: + required: false + outputs: + promotion_eligible: + value: ${{ jobs.plan.outputs.promotion_eligible }} + validation_matrix: + value: ${{ jobs.plan.outputs.matrix }} + llamacpp_release: + value: ${{ jobs.get-latest-releases.outputs.llamacpp_release }} + llamacpp_rocm_release: + value: ${{ jobs.get-latest-releases.outputs.llamacpp_rocm_release }} + llamacpp_lemonade_release: + value: ${{ jobs.get-latest-releases.outputs.llamacpp_lemonade_release }} + release_asset_manifest: + value: ${{ jobs.verify-release-assets.outputs.release_asset_manifest }} + ggml_update_backends: + value: ${{ jobs.verify-release-assets.outputs.ggml_update_backends }} + lemonade_update_backends: + value: ${{ jobs.verify-release-assets.outputs.lemonade_update_backends }} + rocm_update_backends: + value: ${{ jobs.verify-release-assets.outputs.rocm_update_backends }} + ggml_missing_count: + value: ${{ jobs.verify-release-assets.outputs.ggml_missing_count }} + rocm_nightly_missing_count: + value: ${{ jobs.verify-release-assets.outputs.rocm_nightly_missing_count }} + rocm_stable_missing_count: + value: ${{ jobs.verify-release-assets.outputs.rocm_stable_missing_count }} + cuda_missing_count: + value: ${{ jobs.verify-release-assets.outputs.cuda_missing_count }} + vulkan_available: + value: ${{ jobs.verify-release-assets.outputs.vulkan_available }} + cpu_available: + value: ${{ jobs.verify-release-assets.outputs.cpu_available }} + metal_available: + value: ${{ jobs.verify-release-assets.outputs.metal_available }} + rocm_nightly_available: + value: ${{ jobs.verify-release-assets.outputs.rocm_nightly_available }} + rocm_stable_available: + value: ${{ jobs.verify-release-assets.outputs.rocm_stable_available }} + cuda_available: + value: ${{ jobs.verify-release-assets.outputs.cuda_available }} pull_request: - types: [opened, synchronize, reopened, labeled] + types: [opened, synchronize, reopened, labeled, unlabeled, closed] merge_group: - schedule: - # Sunday at 12:00 PM ET (17:00 UTC, 16:00 UTC during EDT) - - cron: "0 16 * * 0" permissions: - contents: write - pull-requests: write + contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: LEMONADE_CI_MODE: "True" PYTHONIOENCODING: utf-8 - HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} jobs: plan: @@ -38,6 +75,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.plan.outputs.matrix }} + promotion_eligible: ${{ steps.plan.outputs.promotion_eligible }} steps: - uses: actions/checkout@v5 - name: Build validation matrix @@ -126,6 +164,7 @@ jobs: rocm_nightly_available: ${{ steps.assets.outputs.rocm_nightly_available }} rocm_stable_available: ${{ steps.assets.outputs.rocm_stable_available }} cuda_available: ${{ steps.assets.outputs.cuda_available }} + release_asset_manifest: ${{ steps.manifest.outputs.release_asset_manifest }} steps: - uses: actions/checkout@v5 @@ -146,11 +185,15 @@ jobs: done gh api "repos/ggml-org/llama.cpp/releases/tags/${GGML_RELEASE}" \ - --jq '.assets[].name' | sort > ggml_assets.txt + > ggml_release.json gh api "repos/lemonade-sdk/llamacpp-rocm/releases/tags/${ROCM_RELEASE}" \ - --jq '.assets[].name' | sort > rocm_assets.txt + > rocm_release.json gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LEMONADE_RELEASE}" \ - --jq '.assets[].name' | sort > lemonade_assets.txt + > lemonade_release.json + + jq -r '.assets[].name' ggml_release.json | sort > ggml_assets.txt + jq -r '.assets[].name' rocm_release.json | sort > rocm_assets.txt + jq -r '.assets[].name' lemonade_release.json | sort > lemonade_assets.txt echo "ggml-org/llama.cpp assets for ${GGML_RELEASE}:" cat ggml_assets.txt @@ -159,6 +202,19 @@ jobs: echo "lemonade-sdk/llama.cpp assets for ${LEMONADE_RELEASE}:" cat lemonade_assets.txt + - name: Capture release asset manifest + id: manifest + env: + GGML_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} + ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} + LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} + run: >- + python -m test.utils.llamacpp_release_manifest + --release ggml-org/llama.cpp "$GGML_RELEASE" ggml_release.json + --release lemonade-sdk/llamacpp-rocm "$ROCM_RELEASE" rocm_release.json + --release lemonade-sdk/llama.cpp "$LEMONADE_RELEASE" lemonade_release.json + --github-output "$GITHUB_OUTPUT" + - name: Verify expected asset families id: assets env: @@ -256,8 +312,9 @@ jobs: build: name: Build needs: [get-latest-releases, verify-release-assets, plan] - # Label a PR `ci:upgrades` to opt it back in. - if: (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:upgrades')) && always() && !failure() && !cancelled() + # A new push cancels this run and requires the label to be reapplied, so + # runner authorization belongs to one reviewed head rather than the PR. + if: (github.event_name != 'pull_request' || (github.event.action == 'labeled' && github.event.label.name == 'ci:upgrades')) && always() && !failure() && !cancelled() runs-on: windows-latest steps: - name: Prepare Windows runner for long paths and stale workspace caches @@ -634,6 +691,7 @@ jobs: - name: Run validation with lemond shell: PowerShell env: + HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} LLAMACPP_TEST_MODELS: ${{ join(matrix.models, ',') }} LITE_MODE: ${{ matrix.lite }} run: | @@ -798,6 +856,7 @@ jobs: name: validation-results-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }} path: llamacpp_validation_${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}.json retention-days: 30 + if-no-files-found: error - name: Upload server logs if: always() @@ -811,239 +870,6 @@ jobs: - name: Cleanup if: always() uses: ./.github/actions/cleanup-processes-windows - - # ======================================================================== - # Create PR if both validations passed (schedule or workflow_dispatch only) - # ======================================================================== - create-pr: - name: Create update PR - needs: [get-latest-releases, verify-release-assets, validate] - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - if: >- - (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && - needs.validate.result == 'success' && - needs.verify-release-assets.result == 'success' - steps: - - uses: actions/checkout@v5 - - - name: Download validation results - uses: actions/download-artifact@v7 - with: - pattern: validation-results-* - merge-multiple: true - - - name: Update backend_versions.json with verified releases - shell: bash - env: - LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} - LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.ggml_update_backends }} - LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.rocm_update_backends }} - LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.lemonade_update_backends }} - run: | - set -euo pipefail - - python3 - <<'PY' - import json - import os - import re - from pathlib import Path - - RELEASE_RE = re.compile(r"^b[0-9]+$") - - def parse_csv(value): - return [item.strip() for item in (value or "").split(",") if item.strip()] - - def update(section, release, backends): - if not backends: - return [] - if not RELEASE_RE.match(release): - raise SystemExit(f"Invalid llama.cpp release tag: {release}") - changed = [] - for key in backends: - if key not in section: - raise SystemExit(f"Refusing to create missing llamacpp.{key}") - if not isinstance(section[key], str): - raise SystemExit(f"llamacpp.{key} must be a string") - old = section[key] - section[key] = release - changed.append((key, old, release)) - return changed - - path = Path("src/cpp/resources/backend_versions.json") - data = json.loads(path.read_text(encoding="utf-8")) - section = data.get("llamacpp") - if not isinstance(section, dict): - raise SystemExit("backend_versions.json is missing a llamacpp object") - - changes = [] - changes += update(section, os.environ["LLAMACPP_RELEASE"], parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) - changes += update(section, os.environ["LLAMACPP_ROCM_RELEASE"], parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) - changes += update(section, os.environ["LLAMACPP_LEMONADE_RELEASE"], parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) - - path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") - - print("Updated llamacpp backend versions:") - if changes: - for key, old, new in changes: - print(f" llamacpp.{key}: {old} -> {new}") - else: - print(" none") - PY - - - name: Generate PR body - shell: bash - env: - LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} - LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.ggml_update_backends }} - LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.rocm_update_backends }} - LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.lemonade_update_backends }} - VULKAN_AVAILABLE: ${{ needs.verify-release-assets.outputs.vulkan_available }} - CPU_AVAILABLE: ${{ needs.verify-release-assets.outputs.cpu_available }} - METAL_AVAILABLE: ${{ needs.verify-release-assets.outputs.metal_available }} - ROCM_NIGHTLY_AVAILABLE: ${{ needs.verify-release-assets.outputs.rocm_nightly_available }} - ROCM_STABLE_AVAILABLE: ${{ needs.verify-release-assets.outputs.rocm_stable_available }} - CUDA_AVAILABLE: ${{ needs.verify-release-assets.outputs.cuda_available }} - GGML_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.ggml_missing_count }} - ROCM_NIGHTLY_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.rocm_nightly_missing_count }} - ROCM_STABLE_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.rocm_stable_missing_count }} - CUDA_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.cuda_missing_count }} - run: | - python3 <<'PYEOF' - import json - import os - - def parse_csv(value): - return [item.strip() for item in (value or "").split(",") if item.strip()] - - def yes(name): - return os.environ.get(name, "false") == "true" - - def pin_action(backend, release, enabled): - return release if enabled else "unchanged" - - body_lines = [] - llamacpp = os.environ.get("LLAMACPP_RELEASE", "unknown") - rocm_nightly = os.environ.get("LLAMACPP_ROCM_RELEASE", "unknown") - lemonade_release = os.environ.get("LLAMACPP_LEMONADE_RELEASE", "unknown") - ggml_updates = set(parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) - rocm_updates = set(parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) - lemonade_updates = set(parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) - - pin_plan = { - "vulkan": pin_action("vulkan", llamacpp, "vulkan" in ggml_updates), - "cpu": pin_action("cpu", llamacpp, "cpu" in ggml_updates), - "metal": pin_action("metal", llamacpp, "metal" in ggml_updates), - "rocm-nightly": pin_action("rocm-nightly", rocm_nightly, "rocm-nightly" in rocm_updates), - "rocm-stable": pin_action("rocm-stable", lemonade_release, "rocm-stable" in lemonade_updates), - "cuda": pin_action("cuda", lemonade_release, "cuda" in lemonade_updates), - } - - body_lines.append("## Auto-update llama.cpp backends") - body_lines.append("") - body_lines.append("This PR updates only backend pins whose expected release assets were found.") - body_lines.append("") - body_lines.append(f"- **ggml-org/llama.cpp candidate**: `{llamacpp}`") - body_lines.append(f"- **lemonade-sdk/llamacpp-rocm candidate**: `{rocm_nightly}`") - body_lines.append(f"- **lemonade-sdk/llama.cpp candidate**: `{lemonade_release}`") - body_lines.append("") - body_lines.append("| Backend | Pin action | Asset check |") - body_lines.append("|---|---|---|") - body_lines.append(f"| `vulkan` | `{pin_plan['vulkan']}` | {'complete' if yes('VULKAN_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `cpu` | `{pin_plan['cpu']}` | {'complete' if yes('CPU_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `metal` | `{pin_plan['metal']}` | {'complete' if yes('METAL_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `rocm-nightly` | `{pin_plan['rocm-nightly']}` | {'complete' if yes('ROCM_NIGHTLY_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `rocm-stable` | `{pin_plan['rocm-stable']}` | {'complete' if yes('ROCM_STABLE_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `cuda` | `{pin_plan['cuda']}` | {'complete' if yes('CUDA_AVAILABLE') else 'incomplete'} |") - body_lines.append("") - body_lines.append("Missing asset counts:") - body_lines.append(f"- ggml-org/llama.cpp: `{os.environ.get('GGML_MISSING_COUNT', 'unknown')}`") - body_lines.append(f"- lemonade-sdk/llamacpp-rocm rocm-nightly: `{os.environ.get('ROCM_NIGHTLY_MISSING_COUNT', 'unknown')}`") - body_lines.append(f"- lemonade-sdk/llama.cpp rocm-stable: `{os.environ.get('ROCM_STABLE_MISSING_COUNT', 'unknown')}`") - body_lines.append(f"- lemonade-sdk/llama.cpp cuda: `{os.environ.get('CUDA_MISSING_COUNT', 'unknown')}`") - body_lines.append("") - body_lines.append("CUDA remains asset-gated only until a CUDA self-hosted runner is enabled.") - body_lines.append("") - body_lines.append("## Validation Results") - body_lines.append("") - body_lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |") - body_lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|") - - for backend in ["vulkan", "rocm-stable", "rocm-nightly"]: - # Labels match the artifact names produced by the validate job - results_file = f"llamacpp_validation_{backend}.json" - if not os.path.isfile(results_file): - body_lines.append(f"| _{backend}: no results_ | | | | | | |") - continue - with open(results_file, "r") as f: - results = json.load(f) - for r in results: - status = "PASS" if r["pass"] else "FAIL" - # Truncate response for table readability - resp = str(r.get("response", ""))[:80].replace("|", "\\|").replace("\n", " ") - ttft = r.get("time_to_first_token", "N/A") - if isinstance(ttft, float): - ttft = f"{ttft:.3f}s" - tps = r.get("tokens_per_second", "N/A") - if isinstance(tps, float): - tps = f"{tps:.1f}" - body_lines.append( - f"| {r['model']} ({backend}) | {status} | {resp} | " - f"{r.get('input_tokens', 'N/A')} | {r.get('output_tokens', 'N/A')} | " - f"{ttft} | {tps} |" - ) - - body_lines.append("") - body_lines.append("---") - body_lines.append("*Auto-generated by validate_llamacpp workflow*") - - body = "\n".join(body_lines) - with open("pr_body.md", "w") as f: - f.write(body) - PYEOF - - - name: Create Pull Request - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} - run: | - BRANCH="auto/llamacpp-update-${LLAMACPP_RELEASE}-${LLAMACPP_LEMONADE_RELEASE}-${LLAMACPP_ROCM_RELEASE}" - - # Check if a PR already exists for this update - EXISTING_PR=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || echo "") - if [ -n "$EXISTING_PR" ]; then - echo "PR #$EXISTING_PR already exists for branch $BRANCH. Skipping." - exit 0 - fi - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Skip if backend_versions.json is already up to date - if git diff --quiet src/cpp/resources/backend_versions.json; then - echo "backend_versions.json is unchanged — nothing to update." - exit 0 - fi - - git checkout -b "$BRANCH" - git add src/cpp/resources/backend_versions.json - git commit -m "Update llama.cpp to ${LLAMACPP_RELEASE}, rocm-stable to ${LLAMACPP_LEMONADE_RELEASE}, rocm-nightly to ${LLAMACPP_ROCM_RELEASE}" - git push origin "$BRANCH" - - gh pr create \ - --title "Update llama.cpp to ${LLAMACPP_RELEASE}" \ - --body-file pr_body.md \ - --base main \ - --head "$BRANCH" - # Gate job that ensures: 1. in the merge queue, all jobs in `needs:` ran # successfully (otherwise the merge is blocked), and 2. these jobs do not need # to run on ordinary pull request pushes. diff --git a/.github/workflows/validate_llamacpp_manual.yml b/.github/workflows/validate_llamacpp_manual.yml new file mode 100644 index 0000000000..53c5f87013 --- /dev/null +++ b/.github/workflows/validate_llamacpp_manual.yml @@ -0,0 +1,29 @@ +name: Validate llama.cpp Manually + +on: + workflow_dispatch: + inputs: + lite: + description: 'Run in lite mode (smallest model only)' + type: boolean + default: false + models: + description: 'Optional comma-separated model IDs (empty uses hot models)' + type: string + required: false + default: '' + +permissions: + contents: read + +jobs: + validate: + name: Run manual validation + permissions: + contents: read + uses: ./.github/workflows/validate_llamacpp_core.yml + with: + lite: ${{ inputs.lite }} + models: ${{ inputs.models }} + secrets: + HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} diff --git a/.github/workflows/validate_llamacpp_schedule.yml b/.github/workflows/validate_llamacpp_schedule.yml new file mode 100644 index 0000000000..cd0e976f59 --- /dev/null +++ b/.github/workflows/validate_llamacpp_schedule.yml @@ -0,0 +1,255 @@ +name: Update llama.cpp Releases + +on: + schedule: + - cron: "0 16 * * 0" + +permissions: + contents: read + +concurrency: + group: llamacpp-auto-update + cancel-in-progress: false + +jobs: + validate: + name: Validate release candidates + permissions: + contents: read + uses: ./.github/workflows/validate_llamacpp_core.yml + secrets: + HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} + + publish: + name: Publish validated update + needs: validate + if: >- + needs.validate.result == 'success' && + needs.validate.outputs.promotion_eligible == 'true' + runs-on: ubuntu-latest + concurrency: + group: llamacpp-auto-update-publication + cancel-in-progress: false + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Download validation results + uses: actions/download-artifact@v7 + with: + pattern: validation-results-* + merge-multiple: true + + - name: Validate complete validation evidence + env: + EXPECTED_VALIDATION_MATRIX: ${{ needs.validate.outputs.validation_matrix }} + run: >- + python -m test.utils.llamacpp_validation_evidence + --directory . + --expected-matrix-json "$EXPECTED_VALIDATION_MATRIX" + + - name: Update backend_versions.json with verified releases + shell: bash + env: + LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} + LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.validate.outputs.ggml_update_backends }} + LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.validate.outputs.rocm_update_backends }} + LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.validate.outputs.lemonade_update_backends }} + run: | + set -euo pipefail + + python3 - <<'PY' + import json + import os + import re + from pathlib import Path + + RELEASE_RE = re.compile(r"^b[0-9]+$") + + def parse_csv(value): + return [item.strip() for item in (value or "").split(",") if item.strip()] + + def update(section, release, backends): + if not backends: + return [] + if not RELEASE_RE.match(release): + raise SystemExit(f"Invalid llama.cpp release tag: {release}") + changed = [] + for key in backends: + if key not in section: + raise SystemExit(f"Refusing to create missing llamacpp.{key}") + if not isinstance(section[key], str): + raise SystemExit(f"llamacpp.{key} must be a string") + old = section[key] + section[key] = release + changed.append((key, old, release)) + return changed + + path = Path("src/cpp/resources/backend_versions.json") + data = json.loads(path.read_text(encoding="utf-8")) + section = data.get("llamacpp") + if not isinstance(section, dict): + raise SystemExit("backend_versions.json is missing a llamacpp object") + + changes = [] + changes += update(section, os.environ["LLAMACPP_RELEASE"], parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) + changes += update(section, os.environ["LLAMACPP_ROCM_RELEASE"], parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) + changes += update(section, os.environ["LLAMACPP_LEMONADE_RELEASE"], parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) + + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + print("Updated llamacpp backend versions:") + if changes: + for key, old, new in changes: + print(f" llamacpp.{key}: {old} -> {new}") + else: + print(" none") + PY + + - name: Generate PR body + shell: bash + env: + LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} + LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.validate.outputs.ggml_update_backends }} + LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.validate.outputs.rocm_update_backends }} + LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.validate.outputs.lemonade_update_backends }} + VULKAN_AVAILABLE: ${{ needs.validate.outputs.vulkan_available }} + CPU_AVAILABLE: ${{ needs.validate.outputs.cpu_available }} + METAL_AVAILABLE: ${{ needs.validate.outputs.metal_available }} + ROCM_NIGHTLY_AVAILABLE: ${{ needs.validate.outputs.rocm_nightly_available }} + ROCM_STABLE_AVAILABLE: ${{ needs.validate.outputs.rocm_stable_available }} + CUDA_AVAILABLE: ${{ needs.validate.outputs.cuda_available }} + GGML_MISSING_COUNT: ${{ needs.validate.outputs.ggml_missing_count }} + ROCM_NIGHTLY_MISSING_COUNT: ${{ needs.validate.outputs.rocm_nightly_missing_count }} + ROCM_STABLE_MISSING_COUNT: ${{ needs.validate.outputs.rocm_stable_missing_count }} + CUDA_MISSING_COUNT: ${{ needs.validate.outputs.cuda_missing_count }} + run: | + python3 <<'PYEOF' + import json + import os + + def parse_csv(value): + return [item.strip() for item in (value or "").split(",") if item.strip()] + + def yes(name): + return os.environ.get(name, "false") == "true" + + def pin_action(backend, release, enabled): + return release if enabled else "unchanged" + + body_lines = [] + llamacpp = os.environ.get("LLAMACPP_RELEASE", "unknown") + rocm_nightly = os.environ.get("LLAMACPP_ROCM_RELEASE", "unknown") + lemonade_release = os.environ.get("LLAMACPP_LEMONADE_RELEASE", "unknown") + ggml_updates = set(parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) + rocm_updates = set(parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) + lemonade_updates = set(parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) + + pin_plan = { + "vulkan": pin_action("vulkan", llamacpp, "vulkan" in ggml_updates), + "cpu": pin_action("cpu", llamacpp, "cpu" in ggml_updates), + "metal": pin_action("metal", llamacpp, "metal" in ggml_updates), + "rocm-nightly": pin_action("rocm-nightly", rocm_nightly, "rocm-nightly" in rocm_updates), + "rocm-stable": pin_action("rocm-stable", lemonade_release, "rocm-stable" in lemonade_updates), + "cuda": pin_action("cuda", lemonade_release, "cuda" in lemonade_updates), + } + + body_lines.append("## Auto-update llama.cpp backends") + body_lines.append("") + body_lines.append("This PR updates only backend pins whose expected release assets were found.") + body_lines.append("") + body_lines.append(f"- **ggml-org/llama.cpp candidate**: `{llamacpp}`") + body_lines.append(f"- **lemonade-sdk/llamacpp-rocm candidate**: `{rocm_nightly}`") + body_lines.append(f"- **lemonade-sdk/llama.cpp candidate**: `{lemonade_release}`") + body_lines.append("") + body_lines.append("| Backend | Pin action | Asset check |") + body_lines.append("|---|---|---|") + body_lines.append(f"| `vulkan` | `{pin_plan['vulkan']}` | {'complete' if yes('VULKAN_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `cpu` | `{pin_plan['cpu']}` | {'complete' if yes('CPU_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `metal` | `{pin_plan['metal']}` | {'complete' if yes('METAL_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `rocm-nightly` | `{pin_plan['rocm-nightly']}` | {'complete' if yes('ROCM_NIGHTLY_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `rocm-stable` | `{pin_plan['rocm-stable']}` | {'complete' if yes('ROCM_STABLE_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `cuda` | `{pin_plan['cuda']}` | {'complete' if yes('CUDA_AVAILABLE') else 'incomplete'} |") + body_lines.append("") + body_lines.append("Missing asset counts:") + body_lines.append(f"- ggml-org/llama.cpp: `{os.environ.get('GGML_MISSING_COUNT', 'unknown')}`") + body_lines.append(f"- lemonade-sdk/llamacpp-rocm rocm-nightly: `{os.environ.get('ROCM_NIGHTLY_MISSING_COUNT', 'unknown')}`") + body_lines.append(f"- lemonade-sdk/llama.cpp rocm-stable: `{os.environ.get('ROCM_STABLE_MISSING_COUNT', 'unknown')}`") + body_lines.append(f"- lemonade-sdk/llama.cpp cuda: `{os.environ.get('CUDA_MISSING_COUNT', 'unknown')}`") + body_lines.append("") + body_lines.append("CUDA remains asset-gated only until a CUDA self-hosted runner is enabled.") + body_lines.append("") + body_lines.append("## Validation Results") + body_lines.append("") + body_lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |") + body_lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|") + + for backend in ["vulkan", "rocm-stable", "rocm-nightly"]: + results_file = f"llamacpp_validation_{backend}.json" + with open(results_file, "r") as f: + results = json.load(f) + for r in results: + status = "PASS" if r["pass"] else "FAIL" + # Truncate response for table readability + resp = str(r.get("response", ""))[:80].replace("|", "\\|").replace("\n", " ") + ttft = r.get("time_to_first_token", "N/A") + if isinstance(ttft, float): + ttft = f"{ttft:.3f}s" + tps = r.get("tokens_per_second", "N/A") + if isinstance(tps, float): + tps = f"{tps:.1f}" + body_lines.append( + f"| {r['model']} ({backend}) | {status} | {resp} | " + f"{r.get('input_tokens', 'N/A')} | {r.get('output_tokens', 'N/A')} | " + f"{ttft} | {tps} |" + ) + + body_lines.append("") + body_lines.append("---") + body_lines.append("*Auto-generated by validate_llamacpp workflow*") + + body = "\n".join(body_lines) + with open("pr_body.md", "w") as f: + f.write(body) + PYEOF + + - name: Verify release asset manifest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXPECTED_RELEASE_ASSET_MANIFEST: ${{ needs.validate.outputs.release_asset_manifest }} + LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} + run: | + set -euo pipefail + + gh api "repos/ggml-org/llama.cpp/releases/tags/${LLAMACPP_RELEASE}" \ + > publish_ggml_release.json + gh api "repos/lemonade-sdk/llamacpp-rocm/releases/tags/${LLAMACPP_ROCM_RELEASE}" \ + > publish_rocm_release.json + gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LLAMACPP_LEMONADE_RELEASE}" \ + > publish_lemonade_release.json + + python -m test.utils.llamacpp_release_manifest \ + --release ggml-org/llama.cpp "$LLAMACPP_RELEASE" publish_ggml_release.json \ + --release lemonade-sdk/llamacpp-rocm "$LLAMACPP_ROCM_RELEASE" publish_rocm_release.json \ + --release lemonade-sdk/llama.cpp "$LLAMACPP_LEMONADE_RELEASE" publish_lemonade_release.json \ + --expected-json "$EXPECTED_RELEASE_ASSET_MANIFEST" + + - name: Publish update pull request + env: + BASE_BRANCH: ${{ github.event.repository.default_branch }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} + VALIDATED_BASE_SHA: ${{ github.sha }} + run: bash .github/scripts/publish_llamacpp_update.sh diff --git a/test/test_debian_version.py b/test/test_debian_version.py new file mode 100644 index 0000000000..b4f5e42e0f --- /dev/null +++ b/test/test_debian_version.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Regression tests for Debian package version resolution.""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RESOLVER = ROOT / ".github" / "scripts" / "resolve_debian_version.sh" + + +class DebianVersionTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.repo = Path(self.temporary_directory.name) / "repo" + self.repo.mkdir() + self.git_env = os.environ.copy() + self.git_env.update( + { + "GIT_AUTHOR_DATE": "2001-01-01T00:00:00+0000", + "GIT_AUTHOR_EMAIL": "test@example.com", + "GIT_AUTHOR_NAME": "Test Author", + "GIT_COMMITTER_DATE": "2001-01-01T00:00:00+0000", + "GIT_COMMITTER_EMAIL": "test@example.com", + "GIT_COMMITTER_NAME": "Test Committer", + } + ) + self.run_git("init", "--quiet") + + def run_git(self, *args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=self.repo, + env=self.git_env, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + def write_cmake_version(self, contents: str) -> None: + (self.repo / "CMakeLists.txt").write_text(contents, encoding="utf-8") + + def commit_all(self, message: str) -> None: + self.run_git("add", "--all") + self.run_git("commit", "--quiet", "--allow-empty", "--message", message) + + def move_to_sha_prefix(self, prefix_kind: str) -> str: + for attempt in range(256): + self.run_git( + "commit", + "--quiet", + "--allow-empty", + "--message", + f"sha-prefix-{prefix_kind}-{attempt}", + ) + short_sha = self.run_git("rev-parse", "--short", "HEAD") + if prefix_kind == "alpha" and short_sha[0] in "abcdef": + return short_sha + if prefix_kind == "numeric" and short_sha[0].isdigit(): + return short_sha + self.fail(f"Could not create a {prefix_kind}-leading commit SHA") + + def run_resolver( + self, *, dpkg_exit: int = 0 + ) -> tuple[subprocess.CompletedProcess, Path]: + fake_bin = Path(self.temporary_directory.name) / "fake-bin" + fake_bin.mkdir(exist_ok=True) + validation_log = Path(self.temporary_directory.name) / "dpkg-arguments" + fake_dpkg = fake_bin / "dpkg" + fake_dpkg.write_text( + "#!/bin/sh\n" + 'printf "%s\\n" "$@" > "$DPKG_VALIDATION_LOG"\n' + 'exit "$DPKG_EXIT"\n', + encoding="utf-8", + ) + fake_dpkg.chmod(fake_dpkg.stat().st_mode | stat.S_IXUSR) + + env = os.environ.copy() + env.update( + { + "DPKG_EXIT": str(dpkg_exit), + "DPKG_VALIDATION_LOG": str(validation_log), + "PATH": f"{fake_bin}{os.pathsep}{env['PATH']}", + } + ) + result = subprocess.run( + ["bash", str(RESOLVER), "24.04"], + cwd=self.repo, + env=env, + check=False, + capture_output=True, + text=True, + ) + return result, validation_log + + def assert_tagless_version(self, prefix_kind: str) -> None: + self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") + self.commit_all("initial") + short_sha = self.move_to_sha_prefix(prefix_kind) + commit_count = int(self.run_git("rev-list", "--count", "HEAD")) + + result, validation_log = self.run_resolver() + + expected = f"11.9.0+git{commit_count}.g{short_sha}" + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), expected) + self.assertEqual( + validation_log.read_text(encoding="utf-8").splitlines(), + ["--validate-version", f"{expected}~24.04"], + ) + + def test_tagless_alpha_leading_sha_uses_cmake_version_prefix(self) -> None: + self.assert_tagless_version("alpha") + + def test_tagless_numeric_leading_sha_uses_cmake_version_prefix(self) -> None: + self.assert_tagless_version("numeric") + + def test_tagless_versions_increase_with_commit_ancestry(self) -> None: + self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") + self.commit_all("parent") + parent_sha = self.run_git("rev-parse", "--short", "HEAD") + parent_count = int(self.run_git("rev-list", "--count", "HEAD")) + parent_result, _validation_log = self.run_resolver() + + self.run_git("commit", "--quiet", "--allow-empty", "--message", "child") + child_sha = self.run_git("rev-parse", "--short", "HEAD") + child_count = int(self.run_git("rev-list", "--count", "HEAD")) + child_result, _validation_log = self.run_resolver() + + parent_version = f"11.9.0+git{parent_count}.g{parent_sha}" + child_version = f"11.9.0+git{child_count}.g{child_sha}" + self.assertEqual(parent_result.returncode, 0, parent_result.stderr) + self.assertEqual(child_result.returncode, 0, child_result.stderr) + self.assertEqual(parent_result.stdout.strip(), parent_version) + self.assertEqual(child_result.stdout.strip(), child_version) + self.assertGreater(child_count, parent_count) + + dpkg = shutil.which("dpkg") + if dpkg: + subprocess.run( + [dpkg, "--compare-versions", parent_version, "lt", child_version], + check=True, + ) + subprocess.run( + [dpkg, "--compare-versions", "11.9.0", "lt", parent_version], + check=True, + ) + + def test_reachable_tag_preserves_git_describe_version(self) -> None: + self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") + self.commit_all("tagged release") + self.run_git("tag", "--no-sign", "v4.5.6") + self.run_git("commit", "--quiet", "--allow-empty", "--message", "after tag") + described = self.run_git("describe", "--tags", "--always") + + result, validation_log = self.run_resolver() + + expected = described.removeprefix("v") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), expected) + self.assertEqual( + validation_log.read_text(encoding="utf-8").splitlines(), + ["--validate-version", f"{expected}~24.04"], + ) + + def test_tagless_repo_rejects_missing_or_malformed_cmake_version(self) -> None: + for name, cmake_contents in ( + ("missing", None), + ("malformed", "project(lemon_cpp VERSION next)\n"), + ): + with self.subTest(name=name): + if cmake_contents is None: + (self.repo / "README.md").write_text("test\n", encoding="utf-8") + else: + self.write_cmake_version(cmake_contents) + self.commit_all(name) + + result, validation_log = self.run_resolver() + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertFalse(validation_log.exists()) + + def test_dpkg_validation_failure_is_propagated(self) -> None: + self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") + self.commit_all("initial") + + result, validation_log = self.run_resolver(dpkg_exit=42) + + self.assertEqual(result.returncode, 42) + self.assertEqual(result.stdout, "") + self.assertTrue(validation_log.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_llamacpp_release_manifest.py b/test/test_llamacpp_release_manifest.py new file mode 100644 index 0000000000..29975d96a0 --- /dev/null +++ b/test/test_llamacpp_release_manifest.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Tests for immutable llama.cpp release-asset manifests.""" + +from __future__ import annotations + +import copy +import json +import unittest + +from test.utils import llamacpp_release_manifest as manifest + + +def release_payload(tag: str, release_id: int, asset_seed: int) -> dict: + return { + "id": release_id, + "tag_name": tag, + "assets": [ + { + "name": "z-last.zip", + "id": asset_seed + 1, + "size": 200, + "digest": "sha256:" + "B" * 64, + "updated_at": "2026-09-05T12:34:56Z", + }, + { + "name": "a-first.zip", + "id": asset_seed, + "size": 100, + "digest": "sha256:" + "a" * 64, + "updated_at": "2026-09-05T12:30:00Z", + }, + ], + } + + +class LlamaCppReleaseManifestTests(unittest.TestCase): + def setUp(self) -> None: + self.ggml = release_payload("b1234", 10, 100) + self.rocm = release_payload("b1235", 20, 200) + + def test_manifest_is_canonical_across_release_and_asset_order(self) -> None: + first = manifest.build_release_asset_manifest( + [ + ("lemonade-sdk/llamacpp-rocm", "b1235", self.rocm), + ("ggml-org/llama.cpp", "b1234", self.ggml), + ] + ) + reversed_assets = copy.deepcopy(self.ggml) + reversed_assets["assets"].reverse() + second = manifest.build_release_asset_manifest( + [ + ("ggml-org/llama.cpp", "b1234", reversed_assets), + ("lemonade-sdk/llamacpp-rocm", "b1235", self.rocm), + ] + ) + + self.assertEqual(first, second) + parsed = json.loads(first) + self.assertEqual(parsed["schema_version"], 1) + self.assertEqual( + [release["repository"] for release in parsed["releases"]], + ["ggml-org/llama.cpp", "lemonade-sdk/llamacpp-rocm"], + ) + self.assertEqual( + [asset["name"] for asset in parsed["releases"][0]["assets"]], + ["a-first.zip", "z-last.zip"], + ) + self.assertEqual( + parsed["releases"][0]["assets"][1]["digest"], + "sha256:" + "b" * 64, + ) + + def test_every_asset_requires_a_sha256_digest(self) -> None: + for digest in (None, "", "md5:" + "a" * 32, "sha256:abc"): + with self.subTest(digest=digest): + payload = copy.deepcopy(self.ggml) + payload["assets"][0]["digest"] = digest + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "sha256 digest", + ): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", payload)] + ) + + def test_release_and_asset_identity_fields_are_required(self) -> None: + cases = ( + ("release id", lambda payload: payload.pop("id")), + ("asset name", lambda payload: payload["assets"][0].pop("name")), + ("asset id", lambda payload: payload["assets"][0].pop("id")), + ("asset size", lambda payload: payload["assets"][0].pop("size")), + ( + "asset updated_at", + lambda payload: payload["assets"][0].pop("updated_at"), + ), + ) + for expected_error, mutate in cases: + with self.subTest(expected_error=expected_error): + payload = copy.deepcopy(self.ggml) + mutate(payload) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", payload)] + ) + + def test_tag_mismatch_and_duplicate_assets_fail_closed(self) -> None: + with self.assertRaisesRegex(manifest.ReleaseManifestError, "tag mismatch"): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b9999", self.ggml)] + ) + + payload = copy.deepcopy(self.ggml) + payload["assets"][1]["id"] = payload["assets"][0]["id"] + with self.assertRaisesRegex( + manifest.ReleaseManifestError, "duplicate asset id" + ): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", payload)] + ) + + def test_comparison_rejects_any_asset_metadata_change(self) -> None: + expected = manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", self.ggml)] + ) + changed = copy.deepcopy(self.ggml) + changed["assets"][0]["updated_at"] = "2026-09-05T12:35:00Z" + actual = manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", changed)] + ) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "changed since validation", + ): + manifest.require_exact_manifest_match(expected, actual) + + manifest.require_exact_manifest_match(expected, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_llamacpp_validation_evidence.py b/test/test_llamacpp_validation_evidence.py new file mode 100644 index 0000000000..b7d0766ec7 --- /dev/null +++ b/test/test_llamacpp_validation_evidence.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Tests for scheduled llama.cpp validation evidence checks.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from test.utils import llamacpp_validation_evidence as evidence + +K2_SMALL = "K2-Horizon-0.9B-GGUF" + + +def expected_matrix(models: list[str] | None = None) -> dict: + planned_models = models or [K2_SMALL] + return { + "include": [ + { + "backend": "vulkan", + "channel": "", + "models": [], + "expected_models": planned_models, + }, + { + "backend": "rocm", + "channel": "stable", + "models": [], + "expected_models": planned_models, + }, + { + "backend": "rocm", + "channel": "nightly", + "models": [], + "expected_models": planned_models, + }, + ] + } + + +def passing_result(model: str = "K2-Horizon-0.9B-GGUF") -> dict: + return { + "model": model, + "pass": True, + "response": "Four.", + "input_tokens": 10, + "output_tokens": 2, + "time_to_first_token": 0.5, + "tokens_per_second": 4.0, + } + + +class LlamaCppValidationEvidenceTests(unittest.TestCase): + def write_expected_results(self, root: Path) -> None: + for filename in evidence.EXPECTED_RESULT_FILES: + (root / filename).write_text( + json.dumps([passing_result()]), + encoding="utf-8", + ) + + def test_all_expected_nonempty_passing_results_are_accepted(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + + results = evidence.load_and_validate_results(root, expected_matrix()) + + self.assertEqual(set(results), set(evidence.EXPECTED_RESULT_FILES)) + self.assertTrue(all(len(records) == 1 for records in results.values())) + + def test_missing_empty_and_invalid_json_files_fail_closed(self) -> None: + cases = { + "missing": None, + "empty": "", + "invalid": "not json", + "empty list": "[]", + } + for name, replacement in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + target = root / evidence.EXPECTED_RESULT_FILES[0] + if replacement is None: + target.unlink() + else: + target.write_text(replacement, encoding="utf-8") + + with self.assertRaises(evidence.ValidationEvidenceError): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_failed_or_malformed_records_fail_closed(self) -> None: + cases = { + "failed": [{**passing_result(), "pass": False}], + "non_boolean pass": [{**passing_result(), "pass": 1}], + "invalid metric": [{**passing_result(), "tokens_per_second": "unexpected"}], + "negative token count": [{**passing_result(), "input_tokens": -1}], + "missing field": [ + { + key: value + for key, value in passing_result().items() + if key != "response" + } + ], + "non_object": ["PASS"], + "duplicate model": [passing_result(), passing_result()], + } + for name, replacement in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + (root / evidence.EXPECTED_RESULT_FILES[0]).write_text( + json.dumps(replacement), + encoding="utf-8", + ) + + with self.assertRaises(evidence.ValidationEvidenceError): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_runtime_models_must_exactly_match_each_planned_lane(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + target = root / "llamacpp_validation_rocm-stable.json" + target.write_text( + json.dumps([passing_result("some-other-model")]), + encoding="utf-8", + ) + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "do not match planned models", + ): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_every_planned_lane_must_include_k2_small(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for filename in evidence.EXPECTED_RESULT_FILES: + (root / filename).write_text( + json.dumps([passing_result("some-other-model")]), + encoding="utf-8", + ) + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + K2_SMALL, + ): + evidence.load_and_validate_results( + root, + expected_matrix(["some-other-model"]), + ) + + def test_inconsistent_planned_lane_sets_fail_closed(self) -> None: + matrix = expected_matrix() + matrix["include"][2]["expected_models"] = [K2_SMALL, "another-model"] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + (root / "llamacpp_validation_rocm-nightly.json").write_text( + json.dumps([passing_result(K2_SMALL), passing_result("another-model")]), + encoding="utf-8", + ) + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "same planned model set", + ): + evidence.load_and_validate_results(root, matrix) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_publish_llamacpp_update.py b/test/test_publish_llamacpp_update.py new file mode 100644 index 0000000000..8605de0eba --- /dev/null +++ b/test/test_publish_llamacpp_update.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python3 +"""Regression tests for llama.cpp update publication recovery.""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PUBLISHER = ROOT / ".github" / "scripts" / "publish_llamacpp_update.sh" +VERSIONS_PATH = Path("src/cpp/resources/backend_versions.json") + + +class PublishLlamaCppUpdateTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.temp_root = Path(self.temporary_directory.name) + self.repo = self.temp_root / "repo" + self.remote = self.temp_root / "remote.git" + self.fake_bin = self.temp_root / "fake-bin" + self.event_log = self.temp_root / "events.log" + self.real_git = shutil.which("git") + if not self.real_git: + self.skipTest("git is required") + + self.run_command([self.real_git, "init", "--quiet", "--bare", self.remote]) + self.repo.mkdir() + self.run_git("init", "--quiet") + self.run_git("config", "user.name", "Test Author") + self.run_git("config", "user.email", "test@example.com") + versions = self.repo / VERSIONS_PATH + versions.parent.mkdir(parents=True) + versions.write_text( + '{"llamacpp":{"vulkan":"b1","cpu":"b1"}}\n', encoding="utf-8" + ) + (self.repo / "pr_body.md").write_text("initial body\n", encoding="utf-8") + self.run_git("add", "--all") + self.run_git("commit", "--quiet", "--message", "base") + self.run_git("branch", "-M", "main") + self.run_git("remote", "add", "origin", str(self.remote)) + self.run_git("push", "--quiet", "--set-upstream", "origin", "main") + self.install_command_wrappers() + + @property + def branch_name(self) -> str: + base_sha = self.run_git("rev-parse", "HEAD") + versions_blob = self.run_git("hash-object", str(VERSIONS_PATH)) + return f"auto/llamacpp-update-b100-b200-b300-{base_sha}-{versions_blob}" + + def run_command( + self, + command: list[str], + *, + cwd: Path | None = None, + env: dict[str, str] | None = None, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + env=env, + check=check, + capture_output=True, + text=True, + ) + + def run_git(self, *args: str) -> str: + return self.run_command( + [self.real_git, *args], + cwd=self.repo, + ).stdout.strip() + + def install_command_wrappers(self) -> None: + self.fake_bin.mkdir() + git_wrapper = self.fake_bin / "git" + git_wrapper.write_text( + "#!/bin/sh\n" + "printf 'git' >> \"$EVENT_LOG\"\n" + 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' + "printf '\\n' >> \"$EVENT_LOG\"\n" + 'exec "$REAL_GIT" "$@"\n', + encoding="utf-8", + ) + git_wrapper.chmod(git_wrapper.stat().st_mode | stat.S_IXUSR) + + gh_wrapper = self.fake_bin / "gh" + gh_wrapper.write_text( + "#!/bin/sh\n" + "printf 'gh' >> \"$EVENT_LOG\"\n" + 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' + "printf '\\n' >> \"$EVENT_LOG\"\n" + 'if [ "$1" = api ]; then\n' + ' case "$2" in\n' + " repos/ggml-org/llama.cpp/releases) " + "printf '%s\\n' \"$FRESH_GGML_RELEASE\" ;;\n" + " repos/lemonade-sdk/llamacpp-rocm/releases/latest) " + "printf '%s\\n' \"$FRESH_ROCM_RELEASE\" ;;\n" + " repos/lemonade-sdk/llama.cpp/releases/latest) " + "printf '%s\\n' \"$FRESH_LEMONADE_RELEASE\" ;;\n" + " repos/LLM360/lemonade/pulls) " + "printf '%s\\n' \"$OPEN_PR_RECORDS\" ;;\n" + " esac\n" + "fi\n" + "exit 0\n", + encoding="utf-8", + ) + gh_wrapper.chmod(gh_wrapper.stat().st_mode | stat.S_IXUSR) + + def publisher_env( + self, + *, + open_pr_records: str = "", + fresh_ggml_release: str = "b100", + validated_base_sha: str | None = None, + ) -> dict[str, str]: + env = os.environ.copy() + env.update( + { + "EVENT_LOG": str(self.event_log), + "FRESH_GGML_RELEASE": fresh_ggml_release, + "FRESH_LEMONADE_RELEASE": "b200", + "FRESH_ROCM_RELEASE": "b300", + "GITHUB_REPOSITORY": "LLM360/lemonade", + "GITHUB_REPOSITORY_OWNER": "LLM360", + "LLAMACPP_LEMONADE_RELEASE": "b200", + "LLAMACPP_RELEASE": "b100", + "LLAMACPP_ROCM_RELEASE": "b300", + "OPEN_PR_RECORDS": open_pr_records, + "PATH": f"{self.fake_bin}{os.pathsep}{env['PATH']}", + "PR_BODY_FILE": "pr_body.md", + "REAL_GIT": self.real_git, + "VALIDATED_BASE_SHA": validated_base_sha + or self.run_git("rev-parse", "HEAD"), + } + ) + return env + + def run_publisher( + self, + *, + open_pr_records: str = "", + fresh_ggml_release: str = "b100", + validated_base_sha: str | None = None, + ) -> subprocess.CompletedProcess[str]: + return self.run_command( + ["bash", str(PUBLISHER)], + cwd=self.repo, + env=self.publisher_env( + open_pr_records=open_pr_records, + fresh_ggml_release=fresh_ggml_release, + validated_base_sha=validated_base_sha, + ), + check=False, + ) + + def change_versions(self) -> None: + (self.repo / VERSIONS_PATH).write_text( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n', + encoding="utf-8", + ) + + def trusted_open_pr_record(self, number: int = 17) -> str: + return ( + f"{number}\t{self.branch_name}\tLLM360/lemonade\t" + "github-actions[bot]\tmain" + ) + + def create_remote_update_branch( + self, + versions_contents: str, + *, + unrelated_contents: str | None = None, + ) -> tuple[str, str, str]: + base_oid = self.run_git("rev-parse", "HEAD") + (self.repo / VERSIONS_PATH).write_text(versions_contents, encoding="utf-8") + if unrelated_contents is not None: + (self.repo / "unrelated.txt").write_text( + unrelated_contents, encoding="utf-8" + ) + branch_name = self.branch_name + self.run_git("add", "--all") + self.run_git("commit", "--quiet", "--message", "automated update") + branch_oid = self.run_git("rev-parse", "HEAD") + self.run_git( + "push", + "--quiet", + "origin", + f"HEAD:refs/heads/{branch_name}", + ) + self.run_git("reset", "--quiet", "--hard", base_oid) + return base_oid, branch_name, branch_oid + + def events(self) -> list[str]: + if not self.event_log.exists(): + return [] + return self.event_log.read_text(encoding="utf-8").splitlines() + + def test_unchanged_versions_reconcile_before_exiting(self) -> None: + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher(open_pr_records=stale_record) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("nothing to update", result.stdout) + events = self.events() + self.assertTrue( + any( + event.startswith("gh ") + and "<-f> " in event + for event in events + ) + ) + self.assertFalse(any(event.startswith("git ") for event in events)) + + def test_untrusted_stale_pull_request_is_not_closed(self) -> None: + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\toctocat\tmain" + ) + + result = self.run_publisher(open_pr_records=stale_record) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse( + any("" in event for event in self.events()) + ) + + def test_wrong_base_stale_pull_request_is_not_closed(self) -> None: + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\trelease" + ) + + result = self.run_publisher(open_pr_records=stale_record) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse( + any("" in event for event in self.events()) + ) + + def test_stale_candidate_fails_before_query_or_push(self) -> None: + self.change_versions() + + result = self.run_publisher(fresh_ggml_release="b101") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("candidate releases changed", result.stderr) + events = self.events() + self.assertFalse( + any("" in event for event in events) + ) + self.assertFalse(any(event.startswith("git ") for event in events)) + + def test_mismatched_validated_base_is_rejected_before_publication(self) -> None: + self.change_versions() + + result = self.run_publisher(validated_base_sha="0" * 40) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("validated base", result.stderr.lower()) + self.assertFalse( + any("" in event for event in self.events()) + ) + + def test_remote_base_advance_is_rejected_before_publication(self) -> None: + validated_base = self.run_git("rev-parse", "HEAD") + (self.repo / "main.txt").write_text("new main content\n", encoding="utf-8") + self.run_git("add", "main.txt") + self.run_git("commit", "--quiet", "--message", "advance main") + self.run_git("push", "--quiet", "origin", "main") + self.run_git("reset", "--quiet", "--hard", validated_base) + self.change_versions() + + result = self.run_publisher(validated_base_sha=validated_base) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("advanced after validation", result.stderr) + self.assertFalse( + any("" in event for event in self.events()) + ) + + def test_trusted_open_pr_is_refreshed_on_the_same_validated_base(self) -> None: + _base_oid, branch_name, observed_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + self.change_versions() + + result = self.run_publisher( + open_pr_records=self.trusted_open_pr_record(), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + events = self.events() + query_index = next( + index + for index, event in enumerate(events) + if "" in event + ) + push_index = next( + index + for index, event in enumerate(events) + if event.startswith("git ") + ) + edit_index = next( + index + for index, event in enumerate(events) + if event.startswith("gh ") + ) + self.assertLess(query_index, push_index) + self.assertLess(push_index, edit_index) + self.assertIn("", events[query_index]) + self.assertNotIn("", events[query_index]) + self.assertIn(".base.ref", events[query_index]) + self.assertIn( + f"<--force-with-lease=refs/heads/{branch_name}:{observed_oid}>", + events[push_index], + ) + self.assertIn( + f"<{observed_oid}:refs/heads/{branch_name}>", + events[push_index], + ) + self.assertNotIn(f"", events[push_index]) + self.assertIn("<17>", events[edit_index]) + self.assertIn("<--repo> ", events[edit_index]) + self.assertIn("<--body-file> ", events[edit_index]) + self.assertFalse(any(event.startswith("gh ") for event in events)) + + def test_trusted_open_pr_with_unrelated_changes_is_not_overwritten(self) -> None: + _base_oid, branch_name, _branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n', + unrelated_contents="do not overwrite\n", + ) + self.change_versions() + + record = f"17\t{branch_name}\tLLM360/lemonade\tgithub-actions[bot]\tmain" + result = self.run_publisher(open_pr_records=record) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("changes unexpected path unrelated.txt", result.stderr) + self.assertFalse(any(event.startswith("git ") for event in self.events())) + + def test_untrusted_open_pr_does_not_authorize_branch_overwrite(self) -> None: + _base_oid, branch_name, branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + self.change_versions() + untrusted_record = f"17\t{branch_name}\tLLM360/lemonade\toctocat\tmain" + + result = self.run_publisher(open_pr_records=untrusted_record) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("untrusted open pull request", result.stderr) + self.assertFalse(any(event.startswith("git ") for event in self.events())) + + def test_cross_repository_same_named_pr_cannot_block_publication(self) -> None: + _base_oid, branch_name, branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + self.change_versions() + cross_repo_record = ( + f"17\t{branch_name}\tattacker/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher(open_pr_records=cross_repo_record) + + self.assertEqual(result.returncode, 0, result.stderr) + push = next(event for event in self.events() if event.startswith("git ")) + self.assertIn( + f"<--force-with-lease=refs/heads/{branch_name}:{branch_oid}>", + push, + ) + self.assertIn(f"<{branch_oid}:refs/heads/{branch_name}>", push) + self.assertNotIn(f"", push) + create = next( + event for event in self.events() if event.startswith("gh ") + ) + self.assertIn("<--repo> ", create) + self.assertIn(f"<--head> <{branch_name}>", create) + self.assertNotIn(f"", create) + + def test_wrong_base_pull_request_is_not_trusted(self) -> None: + _base_oid, branch_name, branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + self.change_versions() + wrong_base_record = ( + f"17\t{branch_name}\tLLM360/lemonade\t" "github-actions[bot]\trelease" + ) + + result = self.run_publisher(open_pr_records=wrong_base_record) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("untrusted open pull request", result.stderr) + events = self.events() + self.assertFalse(any(event.startswith("git ") for event in events)) + + def test_evolved_orphan_uses_a_new_content_addressed_branch(self) -> None: + _base_oid, old_branch, _branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b1"}}\n' + ) + self.change_versions() + new_branch = self.branch_name + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotEqual(old_branch, new_branch) + push = next(event for event in self.events() if event.startswith("git ")) + self.assertIn(f"", push) + self.assertTrue( + any(event.startswith("gh ") for event in self.events()) + ) + + def test_base_advance_uses_a_new_branch_and_closes_the_old_pull_request( + self, + ) -> None: + _base_oid, old_branch, _branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + (self.repo / "main.txt").write_text("new main content\n", encoding="utf-8") + self.run_git("add", "main.txt") + self.run_git("commit", "--quiet", "--message", "advance main") + self.run_git("push", "--quiet", "origin", "main") + self.change_versions() + new_branch = self.branch_name + old_record = f"17\t{old_branch}\tLLM360/lemonade\tgithub-actions[bot]\tmain" + + result = self.run_publisher(open_pr_records=old_record) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotEqual(old_branch, new_branch) + events = self.events() + self.assertTrue( + any("" in event for event in events) + ) + push = next(event for event in events if event.startswith("git ")) + self.assertIn(f"", push) + + def test_base_advance_does_not_leave_an_orphan_branch_blocker(self) -> None: + _base_oid, old_branch, _branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + (self.repo / "main.txt").write_text("new main content\n", encoding="utf-8") + self.run_git("add", "main.txt") + self.run_git("commit", "--quiet", "--message", "advance main") + self.run_git("push", "--quiet", "origin", "main") + self.change_versions() + new_branch = self.branch_name + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotEqual(old_branch, new_branch) + push = next(event for event in self.events() if event.startswith("git ")) + self.assertIn(f"", push) + + def test_matching_historical_branch_is_reused_for_a_new_pr(self) -> None: + _base_oid, branch_name, branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + self.change_versions() + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + events = self.events() + query = next( + event for event in events if "" in event + ) + self.assertIn("", query) + self.assertNotIn("", query) + push = next(event for event in events if event.startswith("git ")) + self.assertIn( + f"<--force-with-lease=refs/heads/{branch_name}:{branch_oid}>", + push, + ) + self.assertTrue(any(event.startswith("gh ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + + def test_absent_branch_uses_an_explicit_empty_lease(self) -> None: + self.change_versions() + branch_name = self.branch_name + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + push = next(event for event in self.events() if event.startswith("git ")) + self.assertIn( + f"<--force-with-lease=refs/heads/{branch_name}:>", + push, + ) + self.assertTrue( + any(event.startswith("gh ") for event in self.events()) + ) + + def test_shallow_checkout_can_refresh_an_existing_pull_request(self) -> None: + _base_oid, branch_name, _branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + + shallow_repo = self.temp_root / "shallow-repo" + self.run_command( + [ + self.real_git, + "clone", + "--quiet", + "--depth=1", + "--branch", + "main", + f"file://{self.remote}", + shallow_repo, + ] + ) + self.repo = shallow_repo + self.change_versions() + record = f"17\t{branch_name}\tLLM360/lemonade\tgithub-actions[bot]\tmain" + + result = self.run_publisher(open_pr_records=record) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.run_git("rev-parse", "--is-shallow-repository"), "false") + self.assertTrue( + any(event.startswith("gh ") for event in self.events()) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validate_llamacpp_selection.py b/test/test_validate_llamacpp_selection.py index 6fca472551..95cd40c41f 100644 --- a/test/test_validate_llamacpp_selection.py +++ b/test/test_validate_llamacpp_selection.py @@ -505,6 +505,27 @@ def test_model_rejects_raw_ifm_markers_in_response_fields(self) -> None: rf"(?i)raw IFM control marker.*{response_field}", ) + def test_model_rejects_ifm_chat_boundary_tokens(self) -> None: + for token in ("<|ifm|im_start|>", "<|ifm|im_end|>"): + with self.subTest(token=token): + message = {"content": f"answer {token} leaked"} + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat(message), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, error, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + + self.assertFalse(success) + self.assertRegex(error, "(?i)raw IFM control marker.*content") + def test_model_accepts_clean_nested_tool_call_fields(self) -> None: message = { "content": "I will use the lookup tool.", diff --git a/test/test_validate_llamacpp_workflow_plan.py b/test/test_validate_llamacpp_workflow_plan.py index 57b898ff53..cead15c08e 100644 --- a/test/test_validate_llamacpp_workflow_plan.py +++ b/test/test_validate_llamacpp_workflow_plan.py @@ -13,6 +13,11 @@ K2_SMALL = "K2-Horizon-0.9B-GGUF" K2_MEDIUM = "K2-Horizon-3.7B-GGUF" K2_LARGE = "K2-Horizon-7B-GGUF" +ROOT = Path(__file__).resolve().parents[1] +LEGACY_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp.yml" +VALIDATION_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" +MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" +SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" class LlamaCppValidationPlanTests(unittest.TestCase): @@ -34,15 +39,24 @@ def test_pull_request_and_merge_group_require_k2(self) -> None: self.assertTrue(all(not row["lite"] for row in rows)) self.assertIn("128gb", rows[0]["runner"]) - def test_schedule_and_default_dispatch_keep_full_hot_selection(self) -> None: - for event_name in ("schedule", "workflow_dispatch"): - with self.subTest(event_name=event_name): - rows = planning.create_validation_plan(event_name)["include"] + def test_schedule_captures_exact_hot_model_evidence_contract(self) -> None: + rows = planning.create_validation_plan("schedule")["include"] - self.assert_active_lanes(rows) - self.assertTrue(all(row["models"] == [] for row in rows)) - self.assertTrue(all(not row["lite"] for row in rows)) - self.assertTrue(all("128gb" in row["runner"] for row in rows)) + self.assert_active_lanes(rows) + expected = planning.load_hot_llamacpp_model_ids() + self.assertIn(K2_SMALL, expected) + self.assertTrue(all(row["models"] == [] for row in rows)) + self.assertTrue(all(row["expected_models"] == expected for row in rows)) + self.assertTrue(all(not row["lite"] for row in rows)) + self.assertTrue(all("128gb" in row["runner"] for row in rows)) + + def test_default_dispatch_keeps_runtime_hot_selection(self) -> None: + rows = planning.create_validation_plan("workflow_dispatch")["include"] + + self.assert_active_lanes(rows) + self.assertTrue(all(row["models"] == [] for row in rows)) + self.assertTrue(all(not row["lite"] for row in rows)) + self.assertTrue(all("128gb" in row["runner"] for row in rows)) def test_manual_lite_selection_uses_small_runners(self) -> None: rows = planning.create_validation_plan("workflow_dispatch", lite=True)[ @@ -81,19 +95,251 @@ def test_unknown_event_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "Unsupported"): planning.create_validation_plan("push") + def test_only_scheduled_runs_are_promotion_eligible(self) -> None: + cases = ( + ("schedule", "", False, True), + ("workflow_dispatch", "", False, False), + ("workflow_dispatch", K2_SMALL, False, False), + ("workflow_dispatch", "", True, False), + ("pull_request", "", False, False), + ("merge_group", "", False, False), + ) + + for event_name, models_csv, lite, expected in cases: + with self.subTest( + event_name=event_name, + models_csv=models_csv, + lite=lite, + ): + self.assertEqual( + planning.is_promotion_eligible(event_name, models_csv, lite), + expected, + ) + def test_github_output_contains_compact_matrix_json(self) -> None: plan = planning.create_validation_plan("merge_group") with tempfile.TemporaryDirectory() as directory: output_path = Path(directory) / "github-output.txt" - planning.write_github_output(output_path, plan) + planning.write_github_output( + output_path, + plan, + promotion_eligible=False, + ) - key, payload = ( - output_path.read_text(encoding="utf-8").rstrip().split("=", 1) + outputs = dict( + line.split("=", 1) + for line in output_path.read_text(encoding="utf-8").splitlines() ) - self.assertEqual(key, "matrix") - self.assertEqual(json.loads(payload), plan) - self.assertNotIn("\n", payload) + self.assertEqual(json.loads(outputs["matrix"]), plan) + self.assertNotIn("\n", outputs["matrix"]) + self.assertEqual(outputs["promotion_eligible"], "false") + + def test_validation_core_is_read_only_and_has_no_legacy_entrypoint(self) -> None: + self.assertFalse(LEGACY_WORKFLOW.exists()) + + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + validation_triggers = validation.split("permissions:", 1)[0] + workflow_scope = validation.split("jobs:", 1)[0] + plan_job = validation.split(" plan:\n", 1)[1].split( + " get-latest-releases:\n", 1 + )[0] + build_job = validation.split(" build:\n", 1)[1].split(" validate:\n", 1)[0] + validate_job = validation.split(" validate:\n", 1)[1].split( + " validation-gate:\n", 1 + )[0] + validation_step = validate_job.split( + " - name: Run validation with lemond\n", 1 + )[1].split(" - name: Upload results\n", 1)[0] + result_upload = validate_job.split(" - name: Upload results\n", 1)[ + 1 + ].split(" - name: Upload server logs\n", 1)[0] + + self.assertIn(" workflow_call:\n", validation_triggers) + self.assertIn(" pull_request:\n", validation_triggers) + self.assertIn(" merge_group:\n", validation_triggers) + self.assertNotIn(" workflow_dispatch:\n", validation_triggers) + self.assertNotIn(" schedule:\n", validation_triggers) + self.assertIn("permissions:\n contents: read\n", validation) + self.assertNotIn("contents: write", validation) + self.assertNotIn("pull-requests: write", validation) + self.assertNotIn(" create-pr:\n", validation) + self.assertIn(" HUGGINGFACE_ACCESS_TOKEN:\n", validation_triggers) + self.assertNotIn("HF_TOKEN:", workflow_scope) + self.assertNotIn("secrets.HUGGINGFACE_ACCESS_TOKEN", workflow_scope) + self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", plan_job) + self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", build_job) + self.assertEqual(validate_job.count("HF_TOKEN:"), 1) + self.assertIn( + " HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}\n", + validation_step, + ) + self.assertIn(" if-no-files-found: error\n", result_upload) + + def test_release_asset_manifest_is_captured_and_exported(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + release_job = validation.split(" get-latest-releases:\n", 1)[1].split( + " verify-release-assets:\n", 1 + )[0] + asset_job = validation.split(" verify-release-assets:\n", 1)[1].split( + " build:\n", 1 + )[0] + + self.assertIn(" release_asset_manifest:\n", validation) + self.assertIn( + "jobs.verify-release-assets.outputs.release_asset_manifest", + validation, + ) + self.assertIn("release_asset_manifest:", asset_job) + self.assertIn("llamacpp_release_manifest", asset_job) + self.assertIn('--github-output "$GITHUB_OUTPUT"', asset_job) + self.assertIn("jq -r '.assets[].name'", asset_job) + self.assertNotIn("release_asset_manifest", release_job) + + def test_pr_label_revocation_cancels_authorized_validation(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + validation_triggers = validation.split("permissions:", 1)[0] + concurrency = validation.split("concurrency:\n", 1)[1].split("\nenv:\n", 1)[0] + build_job_header = validation.split(" build:\n", 1)[1].split( + " runs-on:", 1 + )[0] + + self.assertIn( + " types: [opened, synchronize, reopened, labeled, unlabeled, closed]\n", + validation_triggers, + ) + self.assertIn("github.event.pull_request.number", concurrency) + self.assertNotIn("github.ref", concurrency) + self.assertIn( + "cancel-in-progress: ${{ github.event_name == 'pull_request' }}", + concurrency, + ) + self.assertIn("github.event.action == 'labeled'", build_job_header) + self.assertIn( + "github.event.label.name == 'ci:upgrades'", + build_job_header, + ) + self.assertNotIn("github.event.pull_request.labels.*.name", build_job_header) + + def test_manual_validation_calls_the_read_only_core(self) -> None: + manual = MANUAL_WORKFLOW.read_text(encoding="utf-8") + manual_triggers = manual.split("permissions:", 1)[0] + + self.assertIn(" workflow_dispatch:\n", manual_triggers) + self.assertNotIn(" schedule:\n", manual_triggers) + self.assertIn("permissions:\n contents: read\n", manual) + self.assertNotIn("contents: write", manual) + self.assertNotIn("pull-requests: write", manual) + self.assertNotIn("secrets: inherit", manual) + self.assertIn( + "HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}", + manual, + ) + self.assertIn( + "uses: ./.github/workflows/validate_llamacpp_core.yml", + manual, + ) + + def test_scheduled_publication_is_isolated_and_serialized(self) -> None: + schedule = SCHEDULE_WORKFLOW.read_text(encoding="utf-8") + schedule_triggers = schedule.split("permissions:", 1)[0] + validation_job = schedule.split(" validate:\n", 1)[1].split(" publish:\n", 1)[ + 0 + ] + publication = schedule.split(" publish:\n", 1)[1] + + self.assertIn(" schedule:\n", schedule_triggers) + self.assertNotIn(" workflow_dispatch:\n", schedule_triggers) + self.assertNotIn(" workflow_call:\n", schedule_triggers) + self.assertNotIn(" pull_request:\n", schedule_triggers) + self.assertNotIn(" merge_group:\n", schedule_triggers) + self.assertIn("group: llamacpp-auto-update\n", schedule) + self.assertIn( + "uses: ./.github/workflows/validate_llamacpp_core.yml", + validation_job, + ) + self.assertIn("permissions:\n contents: read", validation_job) + self.assertNotIn("contents: write", validation_job) + self.assertNotIn("secrets: inherit", schedule) + self.assertIn( + "HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}", + validation_job, + ) + self.assertIn("group: llamacpp-auto-update-publication", publication) + self.assertIn("cancel-in-progress: false", publication) + self.assertIn( + "needs.validate.outputs.promotion_eligible == 'true'", publication + ) + self.assertIn("contents: write", publication) + self.assertIn("pull-requests: write", publication) + self.assertIn("publish_llamacpp_update.sh", publication) + self.assertIn( + " - uses: actions/checkout@v5\n with:\n fetch-depth: 0", + publication, + ) + self.assertIn("VALIDATED_BASE_SHA: ${{ github.sha }}", publication) + self.assertIn( + "BASE_BRANCH: ${{ github.event.repository.default_branch }}", + publication, + ) + evidence_step = publication.split( + " - name: Validate complete validation evidence\n", 1 + )[1].split( + " - name: Update backend_versions.json with verified releases\n", 1 + )[ + 0 + ] + manifest_step = publication.split( + " - name: Verify release asset manifest\n", 1 + )[1].split(" - name: Publish update pull request\n", 1)[0] + self.assertIn("llamacpp_validation_evidence", evidence_step) + self.assertIn("--expected-matrix-json", evidence_step) + self.assertIn( + "needs.validate.outputs.validation_matrix", + evidence_step, + ) + self.assertIn("llamacpp_release_manifest", manifest_step) + self.assertIn( + "needs.validate.outputs.release_asset_manifest", + manifest_step, + ) + self.assertIn("gh api", manifest_step) + self.assertNotIn("\n - name:", manifest_step) + self.assertLess( + publication.index(" - name: Generate PR body\n"), + publication.index(" - name: Verify release asset manifest\n"), + ) + self.assertLess( + publication.index(" - name: Verify release asset manifest\n"), + publication.index(" - name: Publish update pull request\n"), + ) + + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + output_names = ( + "promotion_eligible", + "validation_matrix", + "llamacpp_release", + "llamacpp_rocm_release", + "llamacpp_lemonade_release", + "ggml_update_backends", + "lemonade_update_backends", + "rocm_update_backends", + "ggml_missing_count", + "rocm_nightly_missing_count", + "rocm_stable_missing_count", + "cuda_missing_count", + "vulkan_available", + "cpu_available", + "metal_available", + "rocm_nightly_available", + "rocm_stable_available", + "cuda_available", + "release_asset_manifest", + ) + for output_name in output_names: + with self.subTest(output_name=output_name): + self.assertIn(f" {output_name}:\n", validation) + self.assertIn(f"needs.validate.outputs.{output_name}", schedule) if __name__ == "__main__": diff --git a/test/utils/llamacpp_release_manifest.py b/test/utils/llamacpp_release_manifest.py new file mode 100644 index 0000000000..2e54aed433 --- /dev/null +++ b/test/utils/llamacpp_release_manifest.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Capture and compare canonical GitHub release-asset manifests.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Iterable + +SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-fA-F]{64}$") + + +class ReleaseManifestError(ValueError): + """Raised when release metadata is incomplete, malformed, or changed.""" + + +def _positive_integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ReleaseManifestError(f"{label} must be a positive integer") + return value + + +def _canonical_release( + repository: str, + expected_tag: str, + payload: object, +) -> dict: + if not isinstance(repository, str) or repository.count("/") != 1: + raise ReleaseManifestError(f"invalid repository name: {repository!r}") + if not isinstance(payload, dict): + raise ReleaseManifestError(f"{repository}: release payload must be an object") + + actual_tag = payload.get("tag_name") + if actual_tag != expected_tag: + raise ReleaseManifestError( + f"{repository}: release tag mismatch: expected {expected_tag!r}, " + f"got {actual_tag!r}" + ) + release_id = _positive_integer(payload.get("id"), f"{repository}: release id") + raw_assets = payload.get("assets") + if not isinstance(raw_assets, list) or not raw_assets: + raise ReleaseManifestError(f"{repository}: release assets must be nonempty") + + asset_ids: set[int] = set() + asset_names: set[str] = set() + assets = [] + for index, raw_asset in enumerate(raw_assets): + prefix = f"{repository}: asset {index}" + if not isinstance(raw_asset, dict): + raise ReleaseManifestError(f"{prefix} must be an object") + + name = raw_asset.get("name") + if not isinstance(name, str) or not name: + raise ReleaseManifestError(f"{prefix}: asset name must be nonempty") + asset_id = _positive_integer(raw_asset.get("id"), f"{prefix}: asset id") + size = raw_asset.get("size") + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise ReleaseManifestError( + f"{prefix}: asset size must be a nonnegative integer" + ) + digest = raw_asset.get("digest") + if not isinstance(digest, str) or not SHA256_DIGEST_RE.fullmatch(digest): + raise ReleaseManifestError( + f"{prefix}: asset must have a complete sha256 digest" + ) + updated_at = raw_asset.get("updated_at") + if not isinstance(updated_at, str) or not updated_at: + raise ReleaseManifestError(f"{prefix}: asset updated_at must be nonempty") + + if asset_id in asset_ids: + raise ReleaseManifestError(f"{repository}: duplicate asset id {asset_id}") + if name in asset_names: + raise ReleaseManifestError(f"{repository}: duplicate asset name {name!r}") + asset_ids.add(asset_id) + asset_names.add(name) + assets.append( + { + "digest": digest.lower(), + "id": asset_id, + "name": name, + "size": size, + "updated_at": updated_at, + } + ) + + assets.sort(key=lambda asset: (asset["name"], asset["id"])) + return { + "assets": assets, + "release_id": release_id, + "repository": repository, + "tag_name": expected_tag, + } + + +def build_release_asset_manifest( + releases: Iterable[tuple[str, str, object]], +) -> str: + canonical_releases = [ + _canonical_release(repository, expected_tag, payload) + for repository, expected_tag, payload in releases + ] + if not canonical_releases: + raise ReleaseManifestError("release manifest must contain at least one release") + repositories = [release["repository"] for release in canonical_releases] + if len(repositories) != len(set(repositories)): + raise ReleaseManifestError("release manifest contains duplicate repositories") + canonical_releases.sort(key=lambda release: release["repository"]) + return json.dumps( + {"releases": canonical_releases, "schema_version": 1}, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonicalize_manifest(manifest_json: str) -> str: + try: + document = json.loads(manifest_json) + except (TypeError, json.JSONDecodeError) as exc: + raise ReleaseManifestError( + f"release manifest is not valid JSON: {exc}" + ) from exc + if not isinstance(document, dict) or document.get("schema_version") != 1: + raise ReleaseManifestError("release manifest schema_version must be 1") + releases = document.get("releases") + if not isinstance(releases, list): + raise ReleaseManifestError("release manifest releases must be an array") + + inputs = [] + for release in releases: + if not isinstance(release, dict): + raise ReleaseManifestError("release manifest entry must be an object") + expected_keys = {"assets", "release_id", "repository", "tag_name"} + if set(release) != expected_keys: + raise ReleaseManifestError("release manifest entry has unexpected fields") + inputs.append( + ( + release["repository"], + release["tag_name"], + { + "assets": release["assets"], + "id": release["release_id"], + "tag_name": release["tag_name"], + }, + ) + ) + + canonical = build_release_asset_manifest(inputs) + if json.loads(canonical) != document: + raise ReleaseManifestError("release manifest has unexpected fields") + return canonical + + +def require_exact_manifest_match(expected_json: str, actual_json: str) -> None: + expected = _canonicalize_manifest(expected_json) + actual = _canonicalize_manifest(actual_json) + if expected != actual: + raise ReleaseManifestError( + "release assets changed since validation; refusing publication" + ) + + +def _load_release_specs(specs: list[list[str]]) -> list[tuple[str, str, object]]: + releases = [] + for repository, expected_tag, path_text in specs: + path = Path(path_text) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ReleaseManifestError(f"could not read {path}: {exc}") from exc + releases.append((repository, expected_tag, payload)) + return releases + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--release", + action="append", + nargs=3, + required=True, + metavar=("REPOSITORY", "TAG", "JSON_PATH"), + ) + parser.add_argument("--expected-json") + parser.add_argument("--github-output", type=Path) + args = parser.parse_args() + + try: + actual = build_release_asset_manifest(_load_release_specs(args.release)) + if args.expected_json is not None: + require_exact_manifest_match(args.expected_json, actual) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as output_file: + output_file.write(f"release_asset_manifest={actual}\n") + except ReleaseManifestError as exc: + parser.exit(1, f"ERROR: {exc}\n") + + print(actual) + + +if __name__ == "__main__": + main() diff --git a/test/utils/llamacpp_validation_evidence.py b/test/utils/llamacpp_validation_evidence.py new file mode 100644 index 0000000000..e75d103449 --- /dev/null +++ b/test/utils/llamacpp_validation_evidence.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Validate complete scheduled llama.cpp result evidence.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +EXPECTED_RESULT_FILES = ( + "llamacpp_validation_vulkan.json", + "llamacpp_validation_rocm-stable.json", + "llamacpp_validation_rocm-nightly.json", +) +K2_SMALL = "K2-Horizon-0.9B-GGUF" +REQUIRED_RESULT_FIELDS = { + "model", + "pass", + "response", + "input_tokens", + "output_tokens", + "time_to_first_token", + "tokens_per_second", +} + + +class ValidationEvidenceError(ValueError): + """Raised when scheduled validation evidence is incomplete or failed.""" + + +def _validate_record(filename: str, index: int, record: object) -> dict: + if not isinstance(record, dict): + raise ValidationEvidenceError(f"{filename}: result {index} must be an object") + missing_fields = REQUIRED_RESULT_FIELDS - set(record) + if missing_fields: + raise ValidationEvidenceError( + f"{filename}: result {index} missing fields: " + + ", ".join(sorted(missing_fields)) + ) + model = record["model"] + if not isinstance(model, str) or not model: + raise ValidationEvidenceError( + f"{filename}: result {index} model must be nonempty" + ) + passed = record["pass"] + if not isinstance(passed, bool): + raise ValidationEvidenceError( + f"{filename}: result {index} pass must be boolean" + ) + if not passed: + raise ValidationEvidenceError(f"{filename}: {model} did not pass validation") + response = record["response"] + if not isinstance(response, str) or not response: + raise ValidationEvidenceError( + f"{filename}: result {index} response must be nonempty" + ) + for field in ("input_tokens", "output_tokens"): + value = record[field] + if not ( + value == "N/A" + or (isinstance(value, int) and not isinstance(value, bool) and value >= 0) + ): + raise ValidationEvidenceError( + f"{filename}: result {index} {field} must be nonnegative or N/A" + ) + for field in ("time_to_first_token", "tokens_per_second"): + value = record[field] + if not ( + value == "N/A" + or ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and value >= 0 + ) + ): + raise ValidationEvidenceError( + f"{filename}: result {index} {field} must be nonnegative or N/A" + ) + return record + + +def _expected_models_by_file(expected_matrix: object) -> dict[str, list[str]]: + if not isinstance(expected_matrix, dict): + raise ValidationEvidenceError("validation matrix must be an object") + rows = expected_matrix.get("include") + if not isinstance(rows, list): + raise ValidationEvidenceError("validation matrix include must be an array") + + expected = {} + for index, row in enumerate(rows): + if not isinstance(row, dict): + raise ValidationEvidenceError( + f"validation matrix row {index} must be an object" + ) + backend = row.get("backend") + channel = row.get("channel") + if not isinstance(backend, str) or not isinstance(channel, str): + raise ValidationEvidenceError( + f"validation matrix row {index} backend and channel must be strings" + ) + label = f"{backend}-{channel}" if channel else backend + filename = f"llamacpp_validation_{label}.json" + if filename in expected: + raise ValidationEvidenceError(f"duplicate validation lane: {label}") + if row.get("models") != []: + raise ValidationEvidenceError( + f"scheduled validation lane {label} must keep runtime hot selection" + ) + models = row.get("expected_models") + if ( + not isinstance(models, list) + or not models + or not all(isinstance(model, str) and model for model in models) + or len(models) != len(set(models)) + ): + raise ValidationEvidenceError( + f"validation matrix lane {label} needs unique expected models" + ) + if K2_SMALL not in models: + raise ValidationEvidenceError( + f"validation matrix lane {label} must include {K2_SMALL}" + ) + expected[filename] = models + + if set(expected) != set(EXPECTED_RESULT_FILES): + raise ValidationEvidenceError( + "validation matrix must contain exactly the scheduled validation lanes" + ) + planned_model_lists = list(expected.values()) + if any(models != planned_model_lists[0] for models in planned_model_lists[1:]): + raise ValidationEvidenceError( + "scheduled validation lanes must use the same planned model set" + ) + return expected + + +def load_and_validate_results( + root: Path, + expected_matrix: object, +) -> dict[str, list[dict]]: + expected_models = _expected_models_by_file(expected_matrix) + validated = {} + for filename in EXPECTED_RESULT_FILES: + path = root / filename + if not path.is_file(): + raise ValidationEvidenceError(f"missing validation result: {filename}") + try: + raw = path.read_text(encoding="utf-8") + except OSError as exc: + raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc + if not raw.strip(): + raise ValidationEvidenceError(f"validation result is empty: {filename}") + try: + records = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValidationEvidenceError( + f"validation result is invalid JSON: {filename}: {exc}" + ) from exc + if not isinstance(records, list) or not records: + raise ValidationEvidenceError( + f"validation result must be a nonempty array: {filename}" + ) + checked = [ + _validate_record(filename, index, record) + for index, record in enumerate(records) + ] + models = [record["model"] for record in checked] + if len(models) != len(set(models)): + raise ValidationEvidenceError( + f"validation result contains duplicate models: {filename}" + ) + if models != expected_models[filename]: + raise ValidationEvidenceError( + f"{filename}: result models do not match planned models" + ) + validated[filename] = checked + return validated + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--directory", type=Path, default=Path.cwd()) + parser.add_argument("--expected-matrix-json", required=True) + args = parser.parse_args() + try: + expected_matrix = json.loads(args.expected_matrix_json) + results = load_and_validate_results(args.directory, expected_matrix) + except (json.JSONDecodeError, ValidationEvidenceError) as exc: + parser.exit(1, f"ERROR: {exc}\n") + for filename, records in results.items(): + print(f"{filename}: {len(records)} passing result(s)") + + +if __name__ == "__main__": + main() diff --git a/test/utils/llamacpp_validation_plan.py b/test/utils/llamacpp_validation_plan.py index 8e44ced78e..6ad30c6cf6 100644 --- a/test/utils/llamacpp_validation_plan.py +++ b/test/utils/llamacpp_validation_plan.py @@ -11,6 +11,13 @@ K2_MEDIUM = "K2-Horizon-3.7B-GGUF" K2_LARGE = "K2-Horizon-7B-GGUF" SUPPORTED_EVENTS = {"merge_group", "pull_request", "schedule", "workflow_dispatch"} +MODEL_REGISTRY = ( + Path(__file__).resolve().parents[2] + / "src" + / "cpp" + / "resources" + / "server_models.json" +) LARGE_VULKAN_RUNNER = [ "self-hosted", @@ -46,20 +53,44 @@ def _parse_models(models_csv: str) -> list[str]: return [model.strip() for model in models_csv.split(",") if model.strip()] +def load_hot_llamacpp_model_ids(path: Path = MODEL_REGISTRY) -> list[str]: + try: + registry = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"Could not read model registry {path}: {exc}") from exc + if not isinstance(registry, dict): + raise ValueError(f"Model registry {path} must be an object") + models = sorted( + model_id + for model_id, model in registry.items() + if isinstance(model, dict) + and model.get("recipe") == "llamacpp" + and isinstance(model.get("labels"), list) + and "hot" in model["labels"] + ) + if not models: + raise ValueError(f"Model registry {path} has no hot llama.cpp models") + return models + + def _row( backend: str, channel: str, runner: list[str], models: list[str], lite: bool, + expected_models: list[str] | None = None, ) -> dict[str, object]: - return { + row = { "backend": backend, "channel": channel, "runner": list(runner), "models": list(models), "lite": lite, } + if expected_models is not None: + row["expected_models"] = list(expected_models) + return row def create_validation_plan( @@ -93,22 +124,51 @@ def create_validation_plan( runner_vulkan = SMALL_VULKAN_RUNNER if lite else LARGE_VULKAN_RUNNER runner_rocm = SMALL_ROCM_RUNNER if lite else LARGE_ROCM_RUNNER + expected_models = ( + load_hot_llamacpp_model_ids() if event_name == "schedule" else None + ) return { "include": [ - _row("vulkan", "", runner_vulkan, models, lite), - _row("rocm", "stable", runner_rocm, models, lite), - _row("rocm", "nightly", runner_rocm, models, lite), + _row("vulkan", "", runner_vulkan, models, lite, expected_models), + _row( + "rocm", + "stable", + runner_rocm, + models, + lite, + expected_models, + ), + _row( + "rocm", + "nightly", + runner_rocm, + models, + lite, + expected_models, + ), ] } +def is_promotion_eligible( + event_name: str, + models_csv: str = "", + lite: bool = False, +) -> bool: + create_validation_plan(event_name, models_csv, lite) + return event_name == "schedule" + + def write_github_output( output_path: Path, plan: dict[str, list[dict[str, object]]], + *, + promotion_eligible: bool, ) -> None: matrix = json.dumps(plan, separators=(",", ":")) with output_path.open("a", encoding="utf-8") as output: output.write(f"matrix={matrix}\n") + output.write(f"promotion_eligible={str(promotion_eligible).lower()}\n") def _bool_argument(value: str) -> bool: @@ -132,7 +192,12 @@ def main() -> None: parser.error("GITHUB_OUTPUT is required") plan = create_validation_plan(args.event, args.models, args.lite) - write_github_output(Path(output_value), plan) + promotion_eligible = is_promotion_eligible(args.event, args.models, args.lite) + write_github_output( + Path(output_value), + plan, + promotion_eligible=promotion_eligible, + ) print(json.dumps(plan, indent=2)) diff --git a/test/validate_llamacpp.py b/test/validate_llamacpp.py index b484ab4269..a5dd70cf23 100644 --- a/test/validate_llamacpp.py +++ b/test/validate_llamacpp.py @@ -43,7 +43,7 @@ CHAT_PROMPT = [ {"role": "user", "content": "What is 2+2? Reply in one sentence."}, ] -IFM_CONTROL_PREFIXES = (" Date: Sat, 5 Sep 2026 23:41:40 -0700 Subject: [PATCH 04/14] fix: enforce K2 capability and release gates --- .github/scripts/llamacpp_validation_args.ps1 | 34 +- .github/workflows/docs_and_style.yml | 1 + .github/workflows/validate_llamacpp_core.yml | 239 +++- .github/workflows/validate_llamacpp_pr.yml | 382 ++++++ .../workflows/validate_llamacpp_schedule.yml | 29 +- test/test_llamacpp_capabilities.py | 680 ++++++++++ test/test_llamacpp_release_manifest.py | 18 +- test/test_llamacpp_validation_args.ps1 | 76 ++ test/test_llamacpp_validation_evidence.py | 171 ++- test/test_validate_llamacpp_selection.py | 379 ++++++ test/test_validate_llamacpp_workflow_plan.py | 288 ++++- test/utils/llamacpp_capability_validation.py | 1123 +++++++++++++++++ test/utils/llamacpp_release_manifest.py | 22 +- test/utils/llamacpp_validation_evidence.py | 138 +- test/utils/llamacpp_validation_plan.py | 52 +- test/validate_llamacpp.py | 234 +++- 16 files changed, 3722 insertions(+), 144 deletions(-) create mode 100644 .github/workflows/validate_llamacpp_pr.yml create mode 100644 test/test_llamacpp_capabilities.py create mode 100644 test/utils/llamacpp_capability_validation.py diff --git a/.github/scripts/llamacpp_validation_args.ps1 b/.github/scripts/llamacpp_validation_args.ps1 index 1a567ca37d..1549b39adc 100644 --- a/.github/scripts/llamacpp_validation_args.ps1 +++ b/.github/scripts/llamacpp_validation_args.ps1 @@ -3,7 +3,11 @@ function Get-LlamaCppValidationModelArguments { param( [AllowEmptyString()] [string] $Models = "", - [bool] $Lite = $false + [bool] $Lite = $false, + [AllowEmptyString()] + [string] $CapabilityProfile = "", + [AllowEmptyString()] + [string] $CapabilityModels = "" ) $modelIds = @( @@ -16,6 +20,25 @@ function Get-LlamaCppValidationModelArguments { throw "Explicit models and lite mode are mutually exclusive." } + $capabilityModelIds = @( + $CapabilityModels -split "," | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } + ) + if (($CapabilityProfile -eq "") -ne ($capabilityModelIds.Count -eq 0)) { + throw "Capability profile and capability models must be provided together." + } + if ($CapabilityProfile -and $CapabilityProfile -ne "k2-horizon-v1") { + throw "Unsupported capability profile: $CapabilityProfile" + } + if ($modelIds.Count -gt 0) { + foreach ($modelId in $capabilityModelIds) { + if ($modelId -notin $modelIds) { + throw "Capability model '$modelId' must be present in explicit models." + } + } + } + $validationArguments = @() foreach ($modelId in $modelIds) { $validationArguments += "--model" @@ -26,5 +49,14 @@ function Get-LlamaCppValidationModelArguments { $validationArguments += "--lite" } + if ($CapabilityProfile) { + $validationArguments += "--capability-profile" + $validationArguments += $CapabilityProfile + foreach ($modelId in $capabilityModelIds) { + $validationArguments += "--capability-model" + $validationArguments += $modelId + } + } + return $validationArguments } diff --git a/.github/workflows/docs_and_style.yml b/.github/workflows/docs_and_style.yml index 7c63f589d4..1d6285b45a 100644 --- a/.github/workflows/docs_and_style.yml +++ b/.github/workflows/docs_and_style.yml @@ -81,6 +81,7 @@ jobs: python -m unittest test.test_comment_slop test.test_debian_version + test.test_llamacpp_capabilities test.test_llamacpp_release_manifest test.test_llamacpp_release_assets test.test_llamacpp_validation_evidence diff --git a/.github/workflows/validate_llamacpp_core.yml b/.github/workflows/validate_llamacpp_core.yml index 025b3c2616..ccb97ee19a 100644 --- a/.github/workflows/validate_llamacpp_core.yml +++ b/.github/workflows/validate_llamacpp_core.yml @@ -54,32 +54,96 @@ on: value: ${{ jobs.verify-release-assets.outputs.rocm_stable_available }} cuda_available: value: ${{ jobs.verify-release-assets.outputs.cuda_available }} - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled, closed] - merge_group: - permissions: contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - env: LEMONADE_CI_MODE: "True" PYTHONIOENCODING: utf-8 jobs: + validate-invocation: + name: Validate invocation + runs-on: ubuntu-latest + steps: + - name: Reject untrusted caller events + env: + EVENT_NAME: ${{ github.event_name }} + run: | + set -euo pipefail + + case "$EVENT_NAME" in + merge_group|pull_request_target|schedule|workflow_dispatch) + ;; + *) + echo "::error::Unsupported invocation event: $EVENT_NAME" + exit 1 + ;; + esac + plan: name: Plan validation + needs: validate-invocation runs-on: ubuntu-latest outputs: - matrix: ${{ steps.plan.outputs.matrix }} - promotion_eligible: ${{ steps.plan.outputs.promotion_eligible }} + matrix: ${{ steps.protected-plan.outputs.matrix || steps.branch-plan.outputs.matrix }} + promotion_eligible: ${{ steps.protected-plan.outputs.promotion_eligible || steps.branch-plan.outputs.promotion_eligible }} steps: - uses: actions/checkout@v5 - - name: Build validation matrix - id: plan + if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' + with: + repository: ${{ github.repository }} + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 2 + - name: Build trusted protected-change matrix + id: protected-plan + if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' + run: | + set -euo pipefail + matrix=$(jq -cn '{ + include: [ + { + target: "windows-vulkan", + backend: "vulkan", + channel: "", + runner: ["self-hosted", "Windows", "128gb", "vulkan", "lemon-prod"], + models: [ + "K2-Horizon-0.9B-GGUF", + "K2-Horizon-3.7B-GGUF", + "K2-Horizon-7B-GGUF" + ], + capability_profile: "k2-horizon-v1", + capability_models: ["K2-Horizon-0.9B-GGUF"], + lite: false + }, + { + target: "rocm-stable", + backend: "rocm", + channel: "stable", + runner: ["self-hosted", "Windows", "stx-halo", "rocm", "lemon-prod"], + models: ["K2-Horizon-0.9B-GGUF"], + capability_profile: "k2-horizon-v1", + capability_models: ["K2-Horizon-0.9B-GGUF"], + lite: false + }, + { + target: "rocm-nightly", + backend: "rocm", + channel: "nightly", + runner: ["self-hosted", "Windows", "stx-halo", "rocm", "lemon-prod"], + models: ["K2-Horizon-0.9B-GGUF"], + capability_profile: "k2-horizon-v1", + capability_models: ["K2-Horizon-0.9B-GGUF"], + lite: false + } + ] + }') + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "promotion_eligible=false" >> "$GITHUB_OUTPUT" + - name: Build branch validation matrix + id: branch-plan + if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' env: EVENT_NAME: ${{ github.event_name }} MANUAL_MODELS: ${{ inputs.models || '' }} @@ -98,7 +162,8 @@ jobs: # Only the auto-update runs resolve upstream releases. Everywhere else the # build validates the pins committed in backend_versions.json, so a bad # upstream release can't fail a merge or hide a broken pin. - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: validate-invocation + if: needs.validate-invocation.result == 'success' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest outputs: llamacpp_release: ${{ steps.llamacpp.outputs.release }} @@ -167,6 +232,10 @@ jobs: release_asset_manifest: ${{ steps.manifest.outputs.release_asset_manifest }} steps: - uses: actions/checkout@v5 + with: + repository: ${{ github.repository }} + ref: ${{ github.sha }} + persist-credentials: false - name: Fetch release asset lists env: @@ -191,6 +260,24 @@ jobs: gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LEMONADE_RELEASE}" \ > lemonade_release.json + attach_source_commit() { + local repository="$1" + local tag="$2" + local path="$3" + local source_commit + source_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') + if ! [[ "$source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::Could not resolve ${repository} ${tag} to a commit." + exit 1 + fi + jq --arg source_commit "$source_commit" \ + '.source_commit = $source_commit' "$path" > "${path}.with-source" + mv "${path}.with-source" "$path" + } + attach_source_commit ggml-org/llama.cpp "$GGML_RELEASE" ggml_release.json + attach_source_commit lemonade-sdk/llamacpp-rocm "$ROCM_RELEASE" rocm_release.json + attach_source_commit lemonade-sdk/llama.cpp "$LEMONADE_RELEASE" lemonade_release.json + jq -r '.assets[].name' ggml_release.json | sort > ggml_assets.txt jq -r '.assets[].name' rocm_release.json | sort > rocm_assets.txt jq -r '.assets[].name' lemonade_release.json | sort > lemonade_assets.txt @@ -311,10 +398,8 @@ jobs: build: name: Build - needs: [get-latest-releases, verify-release-assets, plan] - # A new push cancels this run and requires the label to be reapplied, so - # runner authorization belongs to one reviewed head rather than the PR. - if: (github.event_name != 'pull_request' || (github.event.action == 'labeled' && github.event.label.name == 'ci:upgrades')) && always() && !failure() && !cancelled() + needs: [validate-invocation, get-latest-releases, verify-release-assets, plan] + if: always() && !failure() && !cancelled() runs-on: windows-latest steps: - name: Prepare Windows runner for long paths and stale workspace caches @@ -332,9 +417,38 @@ jobs: - uses: actions/checkout@v5 with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.sha }} + persist-credentials: false clean: true fetch-depth: 0 + - name: Verify authorized merge commit + if: github.event_name == 'pull_request_target' + shell: PowerShell + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + $ErrorActionPreference = "Stop" + if ((git rev-parse HEAD^1) -ne $env:BASE_SHA) { + throw "The merge commit does not contain the authorized base commit." + } + if ((git rev-parse HEAD^2) -ne $env:HEAD_SHA) { + throw "The merge commit does not contain the authorized pull request head." + } + + - name: Verify merge-group commit + if: github.event_name == 'merge_group' + shell: PowerShell + env: + MERGE_GROUP_SHA: ${{ github.event.merge_group.head_sha }} + run: | + $ErrorActionPreference = "Stop" + if ((git rev-parse HEAD) -ne $env:MERGE_GROUP_SHA) { + throw "The checkout does not match the authorized merge-group commit." + } + - name: Update backend_versions.json if: needs.get-latest-releases.result == 'success' shell: PowerShell @@ -432,12 +546,12 @@ jobs: # Validate backends on self-hosted runner # ======================================================================== validate: - name: Validate ${{ matrix.backend }}${{ matrix.channel && format(' ({0})', matrix.channel) || '' }} + name: Validate ${{ matrix.target }} needs: [get-latest-releases, build, plan] if: always() && needs.build.result == 'success' runs-on: ${{ matrix.runner }} strategy: - fail-fast: ${{ github.event_name == 'merge_group' }} + fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} steps: - name: Capture Vulkan runner configuration @@ -657,7 +771,37 @@ jobs: - uses: actions/checkout@v5 with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.sha }} + persist-credentials: false clean: true + fetch-depth: 2 + + - name: Verify authorized merge commit + if: github.event_name == 'pull_request_target' + shell: PowerShell + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + $ErrorActionPreference = "Stop" + if ((git rev-parse HEAD^1) -ne $env:BASE_SHA) { + throw "The merge commit does not contain the authorized base commit." + } + if ((git rev-parse HEAD^2) -ne $env:HEAD_SHA) { + throw "The merge commit does not contain the authorized pull request head." + } + + - name: Verify merge-group commit + if: github.event_name == 'merge_group' + shell: PowerShell + env: + MERGE_GROUP_SHA: ${{ github.event.merge_group.head_sha }} + run: | + $ErrorActionPreference = "Stop" + if ((git rev-parse HEAD) -ne $env:MERGE_GROUP_SHA) { + throw "The checkout does not match the authorized merge-group commit." + } - name: Cleanup processes uses: ./.github/actions/cleanup-processes-windows @@ -691,7 +835,9 @@ jobs: - name: Run validation with lemond shell: PowerShell env: - HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} + CAPABILITY_MODELS: ${{ join(matrix.capability_models || fromJSON('[]'), ',') }} + CAPABILITY_PROFILE: ${{ matrix.capability_profile || '' }} + HF_TOKEN: ${{ github.event_name != 'pull_request_target' && github.event_name != 'merge_group' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }} LLAMACPP_TEST_MODELS: ${{ join(matrix.models, ',') }} LITE_MODE: ${{ matrix.lite }} run: | @@ -700,7 +846,7 @@ jobs: $lemondExe = (Resolve-Path "build\Release\lemond.exe").Path $backend = "${{ matrix.backend }}" $channel = "${{ matrix.channel }}" - $label = if ($channel) { "$backend-$channel" } else { $backend } + $label = "${{ matrix.target }}" $logsDir = "server-logs-$label" $cacheDir = Join-Path $PWD "ci-cache-$label" $cachePrefix = [IO.Path]::GetFullPath($cacheDir) @@ -747,6 +893,7 @@ jobs: $proc = $null $validationExitCode = 0 + $cleanupExitCode = 0 try { $proc = Start-Process ` @@ -780,12 +927,18 @@ jobs: . ".github/scripts/llamacpp_validation_args.ps1" $validationArgs += @(Get-LlamaCppValidationModelArguments ` -Models $env:LLAMACPP_TEST_MODELS ` - -Lite ($env:LITE_MODE -eq "true")) + -Lite ($env:LITE_MODE -eq "true") ` + -CapabilityProfile $env:CAPABILITY_PROFILE ` + -CapabilityModels $env:CAPABILITY_MODELS) & $venvPython @validationArgs $validationExitCode = $LASTEXITCODE } finally { if ($null -ne $proc) { + if ($proc.HasExited) { + $cleanupExitCode = 1 + Write-Warning "lemond exited before graceful shutdown." + } try { if (-not $proc.HasExited) { Invoke-WebRequest -Uri "http://127.0.0.1:$port/internal/shutdown" ` @@ -793,17 +946,22 @@ jobs: $null = $proc.WaitForExit(10000) } } catch { + $cleanupExitCode = 1 Write-Host "Graceful shutdown failed; forcing targeted cleanup." ` -ForegroundColor Yellow } if (-not $proc.HasExited) { + $cleanupExitCode = 1 Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue $null = $proc.WaitForExit(5000) } # Catch a llama-server orphan even if its lemond parent exited first. $orphans = @(Get-ValidationLlamaServers) + if ($orphans.Count -gt 0) { + $cleanupExitCode = 1 + } foreach ($orphan in $orphans) { Write-Host "Stopping validation llama-server PID $($orphan.ProcessId)." ` -ForegroundColor Yellow @@ -826,6 +984,7 @@ jobs: if ($proc.HasExited -and $remainingOrphans.Count -eq 0) { Remove-Item $pidFile -Force -ErrorAction SilentlyContinue } else { + $cleanupExitCode = 1 if (-not $proc.HasExited) { Write-Warning "lemond PID $($proc.Id) is still running after cleanup." } @@ -848,13 +1007,16 @@ jobs: if ($validationExitCode -ne 0) { exit $validationExitCode } + if ($cleanupExitCode -ne 0) { + exit $cleanupExitCode + } - name: Upload results if: always() uses: actions/upload-artifact@v7 with: - name: validation-results-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }} - path: llamacpp_validation_${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}.json + name: validation-results-${{ matrix.target }} + path: llamacpp_validation_${{ matrix.target }}.json retention-days: 30 if-no-files-found: error @@ -862,20 +1024,18 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }} - path: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}/ + name: server-logs-${{ matrix.target }} + path: server-logs-${{ matrix.target }}/ retention-days: 30 if-no-files-found: ignore - name: Cleanup if: always() uses: ./.github/actions/cleanup-processes-windows - # Gate job that ensures: 1. in the merge queue, all jobs in `needs:` ran - # successfully (otherwise the merge is blocked), and 2. these jobs do not need - # to run on ordinary pull request pushes. + # Keep a stable terminal job for callers that require one validation result. validation-gate: name: llama.cpp validation - needs: [build, validate, plan] + needs: [validate-invocation, build, validate, plan] if: always() runs-on: ubuntu-latest steps: @@ -883,18 +1043,13 @@ jobs: env: NEEDS: ${{ toJSON(needs) }} run: | - # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...} - # Fail if any job broke. In the merge queue, also fail if any never ran. + set -euo pipefail echo "$NEEDS" - broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS") - if [ -n "$broke" ]; then - echo "FAILED: $broke" + + incomplete=$(jq -r \ + 'to_entries[] | select(.value.result != "success") | .key' \ + <<<"$NEEDS" | paste -sd ',' -) + if [ -n "$incomplete" ]; then + echo "FAILED OR INCOMPLETE: $incomplete" exit 1 fi - if [ "${{ github.event_name }}" = "merge_group" ]; then - absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS") - if [ -n "$absent" ]; then - echo "DID NOT RUN IN MERGE QUEUE: $absent" - exit 1 - fi - fi diff --git a/.github/workflows/validate_llamacpp_pr.yml b/.github/workflows/validate_llamacpp_pr.yml new file mode 100644 index 0000000000..c99895b823 --- /dev/null +++ b/.github/workflows/validate_llamacpp_pr.yml @@ -0,0 +1,382 @@ +name: Validate llama.cpp protected change + +on: + pull_request_target: + types: [synchronize, edited, labeled, unlabeled, closed] + merge_group: + types: [checks_requested] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}-{1}', github.event.pull_request.number, (github.event.action == 'synchronize' || github.event.action == 'closed' || (github.event.action == 'edited' && github.event.changes.base.ref.from) || ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name == 'ci:upgrades')) && 'validation' || github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'merge_group' || github.event.action == 'synchronize' || github.event.action == 'closed' || (github.event.action == 'edited' && github.event.changes.base.ref.from) || (github.event.action == 'unlabeled' && github.event.label.name == 'ci:upgrades') }} + +jobs: + authorize-invocation: + name: Authorize invocation + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + run_validation: ${{ steps.authorize.outputs.run_validation }} + steps: + - name: Verify trusted invocation + id: authorize + env: + ACTOR: ${{ github.actor }} + EVENT_ACTION: ${{ github.event.action || '' }} + EVENT_LABEL: ${{ github.event.label.name || '' }} + EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number || '' }} + run: | + set -euo pipefail + + if [ "$EVENT_NAME" = "merge_group" ]; then + echo "run_validation=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$EVENT_NAME" != "pull_request_target" ]; then + echo "::error::Unsupported invocation event: $EVENT_NAME" + exit 1 + fi + + if [ "$EVENT_ACTION" != "labeled" ] || [ "$EVENT_LABEL" != "ci:upgrades" ]; then + echo "run_validation=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") + current_state=$(jq -r '.state' <<<"$pull_request") + current_authorized=$(jq -r \ + 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") + if [ "$current_state" != "open" ] || [ "$current_authorized" != "true" ]; then + echo "run_validation=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + permission=$(gh api \ + "repos/${GITHUB_REPOSITORY}/collaborators/${ACTOR}/permission" \ + --jq '.permission') + case "$permission" in + admin|maintain|write|push) + echo "run_validation=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "::error::$ACTOR cannot authorize self-hosted validation." + exit 1 + ;; + esac + + revoke-check: + name: Revoke removed authorization + if: >- + github.event_name == 'pull_request_target' && + github.event.action == 'unlabeled' && + github.event.label.name == 'ci:upgrades' && + github.event.pull_request.state == 'open' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + pull-requests: read + steps: + - name: Revoke the authorized merge check + env: + CHECK_NAME: llama.cpp authorized validation v2 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") + current_state=$(jq -r '.state' <<<"$pull_request") + current_authorized=$(jq -r \ + 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") + if [ "$current_state" != "open" ] || [ "$current_authorized" = "true" ]; then + echo "Pull request is closed or authorization was restored; skipping revocation." + exit 0 + fi + + external_prefix="lemonade-llamacpp-pr-${PR_NUMBER}-" + external_id="${external_prefix}revoked-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + check_runs=$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" \ + -f filter=all \ + -F per_page=100) + revoked_id=$(jq -r --arg external_id "$external_id" \ + '[.check_runs[] | select(.external_id == $external_id) | .id] | last // empty' \ + <<<"$check_runs") + + if [ -z "$revoked_id" ]; then + response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f head_sha="$MERGE_SHA" \ + -f external_id="$external_id" \ + -f status=completed \ + -f conclusion=cancelled \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=The ci:upgrades authorization label was removed.") + revoked_id=$(jq -r '.id // empty' <<<"$response") + fi + if ! [[ "$revoked_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid revocation check-run ID." + exit 1 + fi + + prior_check_ids=$(jq -r \ + --arg external_id "$external_id" \ + --arg external_prefix "$external_prefix" \ + '.check_runs[] + | select((.external_id // "") | startswith($external_prefix)) + | select(.external_id != $external_id) + | .id' <<<"$check_runs") + while IFS= read -r prior_check_id; do + [ -n "$prior_check_id" ] || continue + if ! [[ "$prior_check_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid prior check-run ID." + exit 1 + fi + gh api --method PATCH \ + "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f status=completed \ + -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=The ci:upgrades authorization label was removed." >/dev/null + done <<<"$prior_check_ids" + + start-check: + name: Start authorized merge check + needs: authorize-invocation + if: needs.authorize-invocation.outputs.run_validation == 'true' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + outputs: + check_run_id: ${{ steps.create.outputs.check_run_id }} + steps: + - name: Mark validation in progress + id: create + env: + CHECK_NAME: llama.cpp authorized validation v2 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MERGE_SHA: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.event.pull_request.merge_commit_sha }} + RUN_ATTEMPT: ${{ github.run_attempt }} + RUN_ID: ${{ github.run_id }} + TARGET_KEY: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}', github.event.pull_request.number) }} + run: | + set -euo pipefail + EXTERNAL_PREFIX="lemonade-llamacpp-${TARGET_KEY}-" + EXTERNAL_ID="${EXTERNAL_PREFIX}${RUN_ID}-${RUN_ATTEMPT}" + check_runs=$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" \ + -f filter=all \ + -F per_page=100) + check_run_id=$(jq -r --arg external_id "$EXTERNAL_ID" \ + '[.check_runs[] | select(.external_id == $external_id) | .id] | last // empty' \ + <<<"$check_runs") + + if [ -z "$check_run_id" ]; then + response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f head_sha="$MERGE_SHA" \ + -f external_id="$EXTERNAL_ID" \ + -f status=in_progress \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}") + check_run_id=$(jq -r '.id // empty' <<<"$response") + fi + if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid check-run ID." + exit 1 + fi + + prior_check_ids=$(jq -r \ + --arg external_id "$EXTERNAL_ID" \ + --arg external_prefix "$EXTERNAL_PREFIX" \ + '.check_runs[] + | select((.external_id // "") | startswith($external_prefix)) + | select(.external_id != $external_id) + | .id' <<<"$check_runs") + while IFS= read -r prior_check_id; do + [ -n "$prior_check_id" ] || continue + if ! [[ "$prior_check_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid prior check-run ID." + exit 1 + fi + gh api --method PATCH \ + "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f status=completed \ + -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=Superseded by a newer authorized validation run." >/dev/null + done <<<"$prior_check_ids" + + echo "check_run_id=$check_run_id" >> "$GITHUB_OUTPUT" + + validate: + name: Run authorized validation + needs: [authorize-invocation, start-check] + if: >- + needs.authorize-invocation.outputs.run_validation == 'true' && + needs.start-check.result == 'success' + permissions: + contents: read + uses: ./.github/workflows/validate_llamacpp_core.yml + + report: + name: Report authorized validation check + needs: [authorize-invocation, start-check, validate] + if: >- + always() && + (github.event_name == 'merge_group' || + (github.event.action == 'labeled' && + github.event.label.name == 'ci:upgrades')) + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + pull-requests: read + steps: + - name: Assess authorized validation + id: assess + env: + AUTHORIZATION_RESULT: ${{ needs.authorize-invocation.result }} + RUN_VALIDATION: ${{ needs.authorize-invocation.outputs.run_validation }} + START_CHECK_RESULT: ${{ needs.start-check.result }} + VALIDATION_RESULT: ${{ needs.validate.result }} + run: | + set -euo pipefail + + conclusion=failure + passed=false + summary="Invocation authorization failed." + if [ "$AUTHORIZATION_RESULT" = "success" ] && [ "$RUN_VALIDATION" = "true" ]; then + if [ "$START_CHECK_RESULT" != "success" ]; then + summary="Could not start the authorized merge validation check." + elif [ "$VALIDATION_RESULT" = "success" ]; then + conclusion=success + passed=true + summary="All authorized llama.cpp validation jobs passed." + else + summary="Authorized validation result: $VALIDATION_RESULT." + fi + fi + + { + echo "conclusion=$conclusion" + echo "passed=$passed" + echo "summary=$summary" + } >> "$GITHUB_OUTPUT" + + - name: Report result on the authorized merge commit + if: >- + always() && + (github.event_name == 'merge_group' || + (github.event.action == 'labeled' && + github.event.label.name == 'ci:upgrades')) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} + CHECK_NAME: llama.cpp authorized validation v2 + CHECK_RUN_ID: ${{ needs.start-check.outputs.check_run_id }} + CONCLUSION: ${{ steps.assess.outputs.conclusion || 'failure' }} + EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} + MERGE_SHA: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.event.pull_request.merge_commit_sha }} + PR_NUMBER: ${{ github.event.pull_request.number || '' }} + RUN_ATTEMPT: ${{ github.run_attempt }} + RUN_ID: ${{ github.run_id }} + SUMMARY: ${{ steps.assess.outputs.summary || 'Validation assessment failed.' }} + TARGET_KEY: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}', github.event.pull_request.number) }} + run: | + set -euo pipefail + EXTERNAL_PREFIX="lemonade-llamacpp-${TARGET_KEY}-" + EXTERNAL_ID="${EXTERNAL_PREFIX}${RUN_ID}-${RUN_ATTEMPT}" + stale=false + if [ "$CONCLUSION" = "success" ] && [ "$EVENT_NAME" = "pull_request_target" ]; then + pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") + current_state=$(jq -r '.state' <<<"$pull_request") + current_base=$(jq -r '.base.sha' <<<"$pull_request") + current_head=$(jq -r '.head.sha' <<<"$pull_request") + current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") + current_authorized=$(jq -r \ + 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") + if [ "$current_state" != "open" ] || \ + [ "$current_authorized" != "true" ] || \ + [ "$current_base" != "$BASE_SHA" ] || \ + [ "$current_head" != "$HEAD_SHA" ] || \ + [ "$current_merge" != "$MERGE_SHA" ]; then + stale=true + CONCLUSION=failure + SUMMARY="The pull request authorization or merge identity changed during validation." + fi + fi + + if ! [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then + check_runs=$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" \ + -f filter=all \ + -F per_page=100) + CHECK_RUN_ID=$(jq -r --arg external_id "$EXTERNAL_ID" \ + '[.check_runs[] | select(.external_id == $external_id) | .id] | last // empty' \ + <<<"$check_runs") + fi + + if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then + gh api --method PATCH \ + "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f status=completed \ + -f conclusion="$CONCLUSION" \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=$SUMMARY" >/dev/null + else + gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f head_sha="$MERGE_SHA" \ + -f external_id="$EXTERNAL_ID" \ + -f status=completed \ + -f conclusion="$CONCLUSION" \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=$SUMMARY" >/dev/null + fi + + if [ "$stale" = "true" ]; then + exit 1 + fi + + - name: Enforce result + if: steps.assess.outputs.passed != 'true' + run: | + echo "${{ steps.assess.outputs.summary || 'Validation assessment failed.' }}" + exit 1 diff --git a/.github/workflows/validate_llamacpp_schedule.yml b/.github/workflows/validate_llamacpp_schedule.yml index cd0e976f59..884be484be 100644 --- a/.github/workflows/validate_llamacpp_schedule.yml +++ b/.github/workflows/validate_llamacpp_schedule.yml @@ -131,6 +131,7 @@ jobs: ROCM_NIGHTLY_MISSING_COUNT: ${{ needs.validate.outputs.rocm_nightly_missing_count }} ROCM_STABLE_MISSING_COUNT: ${{ needs.validate.outputs.rocm_stable_missing_count }} CUDA_MISSING_COUNT: ${{ needs.validate.outputs.cuda_missing_count }} + VALIDATION_MATRIX: ${{ needs.validate.outputs.validation_matrix }} run: | python3 <<'PYEOF' import json @@ -185,15 +186,15 @@ jobs: body_lines.append(f"- lemonade-sdk/llama.cpp rocm-stable: `{os.environ.get('ROCM_STABLE_MISSING_COUNT', 'unknown')}`") body_lines.append(f"- lemonade-sdk/llama.cpp cuda: `{os.environ.get('CUDA_MISSING_COUNT', 'unknown')}`") body_lines.append("") - body_lines.append("CUDA remains asset-gated only until a CUDA self-hosted runner is enabled.") - body_lines.append("") body_lines.append("## Validation Results") body_lines.append("") body_lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |") body_lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|") - for backend in ["vulkan", "rocm-stable", "rocm-nightly"]: - results_file = f"llamacpp_validation_{backend}.json" + validation_matrix = json.loads(os.environ["VALIDATION_MATRIX"]) + for lane in validation_matrix["include"]: + target = lane["target"] + results_file = f"llamacpp_validation_{target}.json" with open(results_file, "r") as f: results = json.load(f) for r in results: @@ -207,7 +208,7 @@ jobs: if isinstance(tps, float): tps = f"{tps:.1f}" body_lines.append( - f"| {r['model']} ({backend}) | {status} | {resp} | " + f"| {r['model']} ({target}) | {status} | {resp} | " f"{r.get('input_tokens', 'N/A')} | {r.get('output_tokens', 'N/A')} | " f"{ttft} | {tps} |" ) @@ -238,6 +239,24 @@ jobs: gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LLAMACPP_LEMONADE_RELEASE}" \ > publish_lemonade_release.json + attach_source_commit() { + local repository="$1" + local tag="$2" + local path="$3" + local source_commit + source_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') + if ! [[ "$source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::Could not resolve ${repository} ${tag} to a commit." + exit 1 + fi + jq --arg source_commit "$source_commit" \ + '.source_commit = $source_commit' "$path" > "${path}.with-source" + mv "${path}.with-source" "$path" + } + attach_source_commit ggml-org/llama.cpp "$LLAMACPP_RELEASE" publish_ggml_release.json + attach_source_commit lemonade-sdk/llamacpp-rocm "$LLAMACPP_ROCM_RELEASE" publish_rocm_release.json + attach_source_commit lemonade-sdk/llama.cpp "$LLAMACPP_LEMONADE_RELEASE" publish_lemonade_release.json + python -m test.utils.llamacpp_release_manifest \ --release ggml-org/llama.cpp "$LLAMACPP_RELEASE" publish_ggml_release.json \ --release lemonade-sdk/llamacpp-rocm "$LLAMACPP_ROCM_RELEASE" publish_rocm_release.json \ diff --git a/test/test_llamacpp_capabilities.py b/test/test_llamacpp_capabilities.py new file mode 100644 index 0000000000..2ebee6f32f --- /dev/null +++ b/test/test_llamacpp_capabilities.py @@ -0,0 +1,680 @@ +#!/usr/bin/env python3 +"""Tests for the llama.cpp capability validation profile.""" + +from __future__ import annotations + +import copy +import json +import types +import unittest + +from test.utils import llamacpp_capability_validation as capabilities + +MODEL = "builtin.K2-Horizon-0.9B-GGUF" + + +class CapabilityHarness: + def __init__(self) -> None: + self.requests = [] + self.reasoning_marker = None + self.omit_stream_done = False + self.early_stream_done = False + self.response_model = MODEL + self.stream_response_models = [MODEL, MODEL] + self.stream_reasoning = "high reasoning" + self.default_tool_response_is_text = False + self.stop_tool_call = False + self.tool_reasoning = False + self.tool_arguments = json.dumps( + capabilities.EXPECTED_TOOL_ARGUMENTS, + separators=(",", ":"), + ) + + @staticmethod + def response(status_code=200): + return types.SimpleNamespace(status_code=status_code) + + def openai(self, message, finish_reason="stop"): + return { + "id": "chatcmpl-test", + "model": self.response_model, + "choices": [ + { + "index": 0, + "finish_reason": finish_reason, + "message": {"role": "assistant", **message}, + } + ], + } + + def tool_call(self): + return { + "id": "call_weather", + "type": "function", + "function": { + "name": capabilities.TOOL_NAME, + "arguments": self.tool_arguments, + }, + } + + def stream_chunk(self, index, choices): + chunk = {"choices": choices} + model = self.stream_response_models[index] + if model is not None: + chunk["model"] = model + return chunk + + def request_json(self, method, url, timeout, **kwargs): + self.requests.append((method, url, kwargs["json"])) + payload = kwargs["json"] + if url.endswith("/api/chat"): + message = { + "role": "assistant", + "thinking": "Seventeen plus twenty-five is forty-two.", + "content": "42", + } + if self.stop_tool_call: + message["tool_calls"] = [self.tool_call()] + return self.response(), { + "model": self.response_model, + "done": True, + "done_reason": "stop", + "message": message, + } + if url.endswith("/v1/messages?beta=true"): + content = [{"type": "text", "text": "ANTHROPIC-OK"}] + if self.stop_tool_call: + content.append( + { + "type": "tool_use", + "id": "call_weather", + "name": capabilities.TOOL_NAME, + "input": capabilities.EXPECTED_TOOL_ARGUMENTS, + } + ) + return self.response(), { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": self.response_model, + "stop_reason": "end_turn", + "content": content, + "usage": {"input_tokens": 8, "output_tokens": 3}, + } + + kwargs_value = payload.get("chat_template_kwargs", {}) + if payload.get("tool_choice") == "none": + return self.response(), self.openai({"content": "K2-SUNNY-731"}) + if payload.get("tools"): + if ( + "tool_call_format" not in kwargs_value + and self.default_tool_response_is_text + ): + return self.response(), self.openai({"content": "Not a tool call."}) + message = {"content": None, "tool_calls": [self.tool_call()]} + if self.tool_reasoning: + message["reasoning_content"] = "thinking was supposed to be disabled" + return self.response(), self.openai( + message, + "tool_calls", + ) + if "reasoning_effort" in kwargs_value: + effort = kwargs_value["reasoning_effort"] + reasoning = f"{effort} reasoning" + if self.reasoning_marker == effort: + reasoning += " " + message = {"content": "42", "reasoning_content": reasoning} + if self.stop_tool_call: + message["tool_calls"] = [self.tool_call()] + return self.response(), self.openai(message) + message = {"content": "42"} + if self.stop_tool_call: + message["tool_calls"] = [self.tool_call()] + return self.response(), self.openai(message) + + def request_stream(self, method, url, timeout, **kwargs): + self.requests.append((method, url, kwargs["json"])) + payload = kwargs["json"] + if payload.get("tools"): + events = [ + "data: " + + json.dumps( + self.stream_chunk( + 0, + [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_stream", + "type": "function", + "function": { + "name": "lookup_", + "arguments": '{"city":"Paris","unit":"', + }, + } + ] + }, + } + ], + ), + separators=(",", ":"), + ), + "data: " + + json.dumps( + self.stream_chunk( + 1, + [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "function": { + "name": "weather", + "arguments": ( + 'celsius","days":2,' + '"include_hourly":false}' + ), + }, + } + ] + }, + "finish_reason": "tool_calls", + } + ], + ), + separators=(",", ":"), + ), + ] + elif "reasoning_effort" in payload.get("chat_template_kwargs", {}): + events = [ + "data: " + + json.dumps( + self.stream_chunk( + 0, + [ + { + "index": 0, + "delta": {"reasoning_content": self.stream_reasoning}, + } + ], + ), + separators=(",", ":"), + ), + "data: " + + json.dumps( + self.stream_chunk( + 1, + [ + { + "index": 0, + "delta": {"content": "42"}, + "finish_reason": "stop", + } + ], + ), + separators=(",", ":"), + ), + ] + else: + events = [ + ": keep-alive", + "data: " + + json.dumps( + self.stream_chunk( + 0, + [{"index": 0, "delta": {"content": "4"}}], + ), + separators=(",", ":"), + ), + "data: " + + json.dumps( + self.stream_chunk( + 1, + [ + { + "index": 0, + "delta": {"content": "2"}, + "finish_reason": "stop", + } + ], + ), + separators=(",", ":"), + ), + 'data: {"choices":[],"usage":{"completion_tokens":2}}', + ] + if self.early_stream_done: + events.insert(len(events) - 1, "data: [DONE]") + elif not self.omit_stream_done: + events.append("data: [DONE]") + return self.response(), events + + +class LlamaCppCapabilityTests(unittest.TestCase): + def run_profile(self, harness=None): + active = harness or CapabilityHarness() + matrix, passed, summary = capabilities.validate_capabilities( + capabilities.K2_HORIZON_PROFILE, + base_url="http://localhost:13305/api/v1", + model=MODEL, + request_json=active.request_json, + request_stream=active.request_stream, + timeout=60, + ) + return active, matrix, passed, summary + + def test_complete_profile_exercises_the_exact_contract(self) -> None: + harness, matrix, passed, summary = self.run_profile() + + self.assertTrue(passed, summary) + self.assertTrue(matrix["pass"]) + self.assertEqual(matrix["profile"], capabilities.K2_HORIZON_PROFILE) + self.assertEqual(matrix["model"], MODEL) + self.assertEqual( + [case["id"] for case in matrix["cases"]], + list(capabilities.K2_HORIZON_CASE_IDS), + ) + capabilities.validate_capability_matrix_evidence( + matrix, + capabilities.K2_HORIZON_PROFILE, + MODEL, + ) + + openai_payloads = [ + payload for _method, url, payload in harness.requests if "/api/v1/" in url + ] + self.assertEqual( + [ + payload["chat_template_kwargs"]["reasoning_effort"] + for payload in openai_payloads + if "reasoning_effort" in payload.get("chat_template_kwargs", {}) + ], + ["high", "high", "medium", "low"], + ) + self.assertTrue( + all( + payload["max_completion_tokens"] == 3072 + for payload in openai_payloads + if "reasoning_effort" in payload.get("chat_template_kwargs", {}) + ) + ) + self.assertEqual( + [ + payload["chat_template_kwargs"]["tool_call_format"] + for payload in openai_payloads + if payload.get("tools") + and payload.get("tool_choice") == "required" + and "tool_call_format" in payload.get("chat_template_kwargs", {}) + ], + ["json", "xml", "xml_typed", "xml"], + ) + default_tool_payloads = [ + payload + for payload in openai_payloads + if payload.get("tools") + and payload.get("tool_choice") == "required" + and "tool_call_format" not in payload.get("chat_template_kwargs", {}) + ] + self.assertEqual(len(default_tool_payloads), 1) + self.assertEqual( + default_tool_payloads[0]["chat_template_kwargs"], + {"enable_thinking": False}, + ) + followup = next( + payload + for payload in openai_payloads + if payload.get("tool_choice") == "none" + ) + assistant = followup["messages"][1] + tool_result = followup["messages"][2] + self.assertEqual(assistant["tool_calls"][0]["id"], "call_weather") + self.assertEqual(tool_result["tool_call_id"], "call_weather") + self.assertIn("K2-SUNNY-731", tool_result["content"]) + ollama_payload = next( + payload + for _method, url, payload in harness.requests + if url.endswith("/api/chat") + ) + self.assertEqual(ollama_payload["options"]["num_predict"], 3072) + + def test_reasoning_marker_fails_the_named_case_and_matrix(self) -> None: + harness = CapabilityHarness() + harness.reasoning_marker = "medium" + + _harness, matrix, passed, summary = self.run_profile(harness) + + self.assertFalse(passed) + self.assertFalse(matrix["pass"]) + failed = [case for case in matrix["cases"] if not case["pass"]] + self.assertEqual([case["id"] for case in failed], ["openai_reasoning_medium"]) + self.assertRegex(summary, "medium.*IFM|IFM.*medium") + with self.assertRaises(capabilities.CapabilityValidationError): + capabilities.validate_capability_matrix_evidence( + matrix, + capabilities.K2_HORIZON_PROFILE, + MODEL, + ) + + def test_stream_requires_a_terminal_done_frame(self) -> None: + harness = CapabilityHarness() + harness.omit_stream_done = True + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} + self.assertEqual( + failed_ids, + { + "openai_plain_off_stream", + "openai_reasoning_high_stream", + "openai_tool_stream_xml", + }, + ) + + def test_stream_rejects_done_before_the_final_data_frame(self) -> None: + harness = CapabilityHarness() + harness.early_stream_done = True + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} + self.assertEqual( + failed_ids, + { + "openai_plain_off_stream", + "openai_reasoning_high_stream", + "openai_tool_stream_xml", + }, + ) + + def test_nonstream_responses_are_bound_to_the_requested_model(self) -> None: + harness = CapabilityHarness() + harness.response_model = "builtin.Not-K2" + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} + self.assertTrue( + {case["id"] for case in matrix["cases"] if not case["stream"]}.issubset( + failed_ids + ) + ) + self.assertNotIn("openai_plain_off_stream", failed_ids) + + def test_stream_reasoning_requires_separate_marker_free_content(self) -> None: + for stream_reasoning in ("", "high reasoning"): + with self.subTest(stream_reasoning=stream_reasoning): + harness = CapabilityHarness() + harness.stream_reasoning = stream_reasoning + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + self.assertEqual( + {case["id"] for case in matrix["cases"] if not case["pass"]}, + {"openai_reasoning_high_stream"}, + ) + + def test_stream_accepts_a_stable_backend_model_identity(self) -> None: + harness = CapabilityHarness() + harness.stream_response_models = [ + "K2-Horizon-1B-BF16.gguf", + "K2-Horizon-1B-BF16.gguf", + ] + + _harness, _matrix, passed, summary = self.run_profile(harness) + + self.assertTrue(passed, summary) + + def test_stream_rejects_changed_or_missing_backend_model_identity(self) -> None: + cases = ( + ["K2-Horizon-1B-BF16.gguf", "different.gguf"], + [None, "K2-Horizon-1B-BF16.gguf"], + ["K2-Horizon-1B-BF16.gguf", None], + ["K2-Horizon-1B-BF16.gguf", " "], + ) + for stream_models in cases: + with self.subTest(stream_models=stream_models): + harness = CapabilityHarness() + harness.stream_response_models = stream_models + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + self.assertEqual( + {case["id"] for case in matrix["cases"] if not case["pass"]}, + { + "openai_plain_off_stream", + "openai_reasoning_high_stream", + "openai_tool_stream_xml", + }, + ) + + def test_stream_rejects_duplicate_json_members(self) -> None: + invalid_events = ( + ( + 'data: {"model":"wrong","model":"backend",' + '"choices":[{"index":0,"delta":{"content":"42"},' + '"finish_reason":"stop"}]}', + "model", + ), + ( + 'data: {"model":"backend","choices":[],"choices":' + '[{"index":0,"delta":{"content":"42"},' + '"finish_reason":"stop"}]}', + "choices", + ), + ) + for event, duplicate_key in invalid_events: + with self.subTest(duplicate_key=duplicate_key): + with self.assertRaisesRegex( + capabilities.CapabilityValidationError, + f"repeats key '{duplicate_key}'", + ): + capabilities._decode_openai_stream([event, "data: [DONE]"]) + + def test_stream_rejects_choice_chunks_after_finish_reason(self) -> None: + later_choices = ( + {"index": 0, "delta": {"content": "2"}}, + {"index": 0, "delta": {"reasoning_content": "later"}}, + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_late", + "type": "function", + "function": { + "name": capabilities.TOOL_NAME, + "arguments": "{}", + }, + } + ] + }, + }, + {"index": 0, "delta": {}, "finish_reason": "stop"}, + ) + first = { + "model": "backend", + "choices": [ + { + "index": 0, + "delta": {"content": "4"}, + "finish_reason": "stop", + } + ], + } + for later_choice in later_choices: + with self.subTest(later_choice=later_choice): + later = {"model": "backend", "choices": [later_choice]} + events = [ + f"data: {json.dumps(first)}", + f"data: {json.dumps(later)}", + "data: [DONE]", + ] + + with self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "after a terminal finish_reason", + ): + capabilities._decode_openai_stream(events) + + def test_tool_arguments_require_exact_json_keys_and_types(self) -> None: + arguments = ( + '{"city":"London","city":"Paris","unit":"celsius",' + '"days":2,"include_hourly":false}', + '{"city":"Paris","unit":"celsius","days":2.0,' '"include_hourly":false}', + '{"city":"Paris","unit":"celsius","days":2,' '"include_hourly":0}', + ) + + for tool_arguments in arguments: + with self.subTest(tool_arguments=tool_arguments): + harness = CapabilityHarness() + harness.tool_arguments = tool_arguments + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + failed_ids = { + case["id"] for case in matrix["cases"] if not case["pass"] + } + self.assertTrue( + { + "openai_tool_json", + "openai_tool_xml", + "openai_tool_xml_typed", + }.issubset(failed_ids) + ) + + def test_omitted_tool_format_requires_the_xml_default(self) -> None: + harness = CapabilityHarness() + harness.default_tool_response_is_text = True + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + self.assertEqual( + {case["id"] for case in matrix["cases"] if not case["pass"]}, + {"openai_tool_default_xml"}, + ) + + def test_ordinary_stop_responses_reject_tool_calls(self) -> None: + harness = CapabilityHarness() + harness.stop_tool_call = True + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + self.assertEqual( + {case["id"] for case in matrix["cases"] if not case["pass"]}, + { + "openai_plain_off_nonstream", + "openai_reasoning_high", + "openai_reasoning_medium", + "openai_reasoning_low", + "ollama_thinking", + "anthropic_translation", + }, + ) + + def test_tool_requests_reject_reasoning_when_thinking_is_disabled(self) -> None: + harness = CapabilityHarness() + harness.tool_reasoning = True + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} + self.assertTrue( + { + "openai_tool_json", + "openai_tool_xml", + "openai_tool_xml_typed", + }.issubset(failed_ids) + ) + + def test_evidence_rejects_missing_unknown_and_non_boolean_cases(self) -> None: + _harness, matrix, passed, _summary = self.run_profile() + self.assertTrue(passed) + + mutations = [] + missing = copy.deepcopy(matrix) + missing["cases"].pop() + mutations.append(missing) + unknown = copy.deepcopy(matrix) + unknown["cases"][0]["id"] = "unknown" + mutations.append(unknown) + non_boolean = copy.deepcopy(matrix) + non_boolean["cases"][0]["pass"] = 1 + mutations.append(non_boolean) + extra = copy.deepcopy(matrix) + extra["unexpected"] = True + mutations.append(extra) + wrong_protocol = copy.deepcopy(matrix) + wrong_protocol["cases"][0]["protocol"] = "ollama" + mutations.append(wrong_protocol) + wrong_stream = copy.deepcopy(matrix) + wrong_stream["cases"][0]["stream"] = True + mutations.append(wrong_stream) + wrong_finish = copy.deepcopy(matrix) + wrong_finish["cases"][0]["finish_reason"] = "tool_calls" + mutations.append(wrong_finish) + stray_reasoning = copy.deepcopy(matrix) + stray_reasoning["cases"][0]["reasoning_chars"] = 1 + mutations.append(stray_reasoning) + stray_tool = copy.deepcopy(matrix) + stray_tool["cases"][0]["tool_name"] = capabilities.TOOL_NAME + mutations.append(stray_tool) + float_integer = copy.deepcopy(matrix) + float_tool_case = next( + case for case in float_integer["cases"] if case["id"] == "openai_tool_json" + ) + float_tool_case["tool_arguments"]["days"] = 2.0 + mutations.append(float_integer) + integer_boolean = copy.deepcopy(matrix) + boolean_tool_case = next( + case + for case in integer_boolean["cases"] + if case["id"] == "openai_tool_json" + ) + boolean_tool_case["tool_arguments"]["include_hourly"] = 0 + mutations.append(integer_boolean) + + for mutation in mutations: + with self.subTest(mutation=mutation): + with self.assertRaises(capabilities.CapabilityValidationError): + capabilities.validate_capability_matrix_evidence( + mutation, + capabilities.K2_HORIZON_PROFILE, + MODEL, + ) + + def test_unknown_profile_fails_without_requests(self) -> None: + harness = CapabilityHarness() + + with self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "Unsupported capability profile", + ): + capabilities.validate_capabilities( + "unknown", + base_url="http://localhost:13305/api/v1", + model=MODEL, + request_json=harness.request_json, + request_stream=harness.request_stream, + timeout=60, + ) + + self.assertEqual(harness.requests, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_llamacpp_release_manifest.py b/test/test_llamacpp_release_manifest.py index 29975d96a0..e25b5bf8d6 100644 --- a/test/test_llamacpp_release_manifest.py +++ b/test/test_llamacpp_release_manifest.py @@ -14,6 +14,7 @@ def release_payload(tag: str, release_id: int, asset_seed: int) -> dict: return { "id": release_id, "tag_name": tag, + "source_commit": f"{release_id:040x}", "assets": [ { "name": "z-last.zip", @@ -56,7 +57,7 @@ def test_manifest_is_canonical_across_release_and_asset_order(self) -> None: self.assertEqual(first, second) parsed = json.loads(first) - self.assertEqual(parsed["schema_version"], 1) + self.assertEqual(parsed["schema_version"], 2) self.assertEqual( [release["repository"] for release in parsed["releases"]], ["ggml-org/llama.cpp", "lemonade-sdk/llamacpp-rocm"], @@ -87,6 +88,7 @@ def test_every_asset_requires_a_sha256_digest(self) -> None: def test_release_and_asset_identity_fields_are_required(self) -> None: cases = ( ("release id", lambda payload: payload.pop("id")), + ("source commit", lambda payload: payload.pop("source_commit")), ("asset name", lambda payload: payload["assets"][0].pop("name")), ("asset id", lambda payload: payload["assets"][0].pop("id")), ("asset size", lambda payload: payload["assets"][0].pop("size")), @@ -108,6 +110,20 @@ def test_release_and_asset_identity_fields_are_required(self) -> None: [("ggml-org/llama.cpp", "b1234", payload)] ) + def test_source_commit_must_be_a_full_git_oid(self) -> None: + for source_commit in (None, "", "abc123", "g" * 40): + with self.subTest(source_commit=source_commit): + payload = copy.deepcopy(self.ggml) + payload["source_commit"] = source_commit + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "source commit", + ): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", payload)] + ) + def test_tag_mismatch_and_duplicate_assets_fail_closed(self) -> None: with self.assertRaisesRegex(manifest.ReleaseManifestError, "tag mismatch"): manifest.build_release_asset_manifest( diff --git a/test/test_llamacpp_validation_args.ps1 b/test/test_llamacpp_validation_args.ps1 index cff27a553b..8731c6a7e7 100644 --- a/test/test_llamacpp_validation_args.ps1 +++ b/test/test_llamacpp_validation_args.ps1 @@ -25,6 +25,42 @@ Assert-Arguments ` -Expected @() ` -Case "default mode" +Assert-Arguments ` + -Actual @( + Get-LlamaCppValidationModelArguments ` + -Models "" ` + -Lite $false ` + -CapabilityProfile "k2-horizon-v1" ` + -CapabilityModels "K2-Horizon-0.9B-GGUF" + ) ` + -Expected @( + "--capability-profile", + "k2-horizon-v1", + "--capability-model", + "K2-Horizon-0.9B-GGUF" + ) ` + -Case "hot selection capability profile" + +Assert-Arguments ` + -Actual @( + Get-LlamaCppValidationModelArguments ` + -Models "K2-Horizon-0.9B-GGUF,K2-Horizon-3.7B-GGUF" ` + -Lite $false ` + -CapabilityProfile "k2-horizon-v1" ` + -CapabilityModels "K2-Horizon-0.9B-GGUF" + ) ` + -Expected @( + "--model", + "K2-Horizon-0.9B-GGUF", + "--model", + "K2-Horizon-3.7B-GGUF", + "--capability-profile", + "k2-horizon-v1", + "--capability-model", + "K2-Horizon-0.9B-GGUF" + ) ` + -Case "capability profile" + Assert-Arguments ` -Actual @( Get-LlamaCppValidationModelArguments ` @@ -74,3 +110,43 @@ if ($null -eq $conflict) { if ($conflict.Exception.Message -notmatch "mutually exclusive") { throw "Conflict error must explain that explicit models and lite mode are mutually exclusive." } + +$missingProfile = $null +try { + Get-LlamaCppValidationModelArguments ` + -Models "K2-Horizon-0.9B-GGUF" ` + -CapabilityModels "K2-Horizon-0.9B-GGUF" | Out-Null +} catch { + $missingProfile = $_ +} + +if ($null -eq $missingProfile) { + throw "Capability models without a profile must be rejected." +} + +$missingModel = $null +try { + Get-LlamaCppValidationModelArguments ` + -Models "K2-Horizon-0.9B-GGUF" ` + -CapabilityProfile "k2-horizon-v1" | Out-Null +} catch { + $missingModel = $_ +} + +if ($null -eq $missingModel) { + throw "A capability profile without capability models must be rejected." +} + +$unselectedModel = $null +try { + Get-LlamaCppValidationModelArguments ` + -Models "K2-Horizon-0.9B-GGUF" ` + -CapabilityProfile "k2-horizon-v1" ` + -CapabilityModels "K2-Horizon-3.7B-GGUF" | Out-Null +} catch { + $unselectedModel = $_ +} + +if ($null -eq $unselectedModel) { + throw "Capability models outside the explicit selection must be rejected." +} diff --git a/test/test_llamacpp_validation_evidence.py b/test/test_llamacpp_validation_evidence.py index b7d0766ec7..4970cf10e6 100644 --- a/test/test_llamacpp_validation_evidence.py +++ b/test/test_llamacpp_validation_evidence.py @@ -8,6 +8,7 @@ import unittest from pathlib import Path +from test.utils import llamacpp_capability_validation as capabilities from test.utils import llamacpp_validation_evidence as evidence K2_SMALL = "K2-Horizon-0.9B-GGUF" @@ -18,29 +19,106 @@ def expected_matrix(models: list[str] | None = None) -> dict: return { "include": [ { + "target": "windows-vulkan", "backend": "vulkan", "channel": "", "models": [], "expected_models": planned_models, + "capability_profile": capabilities.K2_HORIZON_PROFILE, + "capability_models": [K2_SMALL], }, { + "target": "rocm-stable", "backend": "rocm", "channel": "stable", "models": [], "expected_models": planned_models, + "capability_profile": capabilities.K2_HORIZON_PROFILE, + "capability_models": [K2_SMALL], }, { + "target": "rocm-nightly", "backend": "rocm", "channel": "nightly", "models": [], "expected_models": planned_models, + "capability_profile": capabilities.K2_HORIZON_PROFILE, + "capability_models": [K2_SMALL], }, ] } -def passing_result(model: str = "K2-Horizon-0.9B-GGUF") -> dict: +def passing_capability_matrix(model: str = K2_SMALL) -> dict: + cases = [] + for case_id in capabilities.K2_HORIZON_CASE_IDS: + protocol = "openai" + if case_id == "ollama_thinking": + protocol = "ollama" + elif case_id == "anthropic_translation": + protocol = "anthropic" + stream = case_id in { + "openai_plain_off_stream", + "openai_reasoning_high_stream", + "openai_tool_stream_xml", + } + tool_case = case_id in { + "openai_tool_json", + "openai_tool_xml", + "openai_tool_xml_typed", + "openai_tool_default_xml", + "openai_tool_stream_xml", + } + reasoning_case = case_id.startswith("openai_reasoning_") or case_id == ( + "ollama_thinking" + ) + content_case = case_id in { + "openai_plain_off_nonstream", + "openai_plain_off_stream", + "openai_tool_result_followup", + "anthropic_translation", + "ollama_thinking", + } or case_id.startswith("openai_reasoning_") + finish_reason = "stop" + if tool_case: + finish_reason = "tool_calls" + elif case_id == "anthropic_translation": + finish_reason = "end_turn" + cases.append( + { + "id": case_id, + "protocol": protocol, + "stream": stream, + "pass": True, + "status_code": 200, + "finish_reason": finish_reason, + "content_chars": 2 if content_case else 0, + "reasoning_chars": 8 if reasoning_case else 0, + "tool_call_count": 1 if tool_case else 0, + "terminal_frame": True, + "tool_name": capabilities.TOOL_NAME if tool_case else None, + "tool_arguments": ( + capabilities.EXPECTED_TOOL_ARGUMENTS if tool_case else None + ), + "parity_with": ( + "openai_tool_xml" + if case_id in {"openai_tool_default_xml", "openai_tool_stream_xml"} + else None + ), + "raw_ifm_marker": None, + "error": None, + } + ) return { + "profile": capabilities.K2_HORIZON_PROFILE, + "model": f"builtin.{model.removeprefix('builtin.')}", + "pass": True, + "cases": cases, + } + + +def passing_result(model: str = K2_SMALL, *, capability: bool | None = None) -> dict: + result = { "model": model, "pass": True, "response": "Four.", @@ -49,6 +127,9 @@ def passing_result(model: str = "K2-Horizon-0.9B-GGUF") -> dict: "time_to_first_token": 0.5, "tokens_per_second": 4.0, } + if capability if capability is not None else model == K2_SMALL: + result["capability_matrix"] = passing_capability_matrix(model) + return result class LlamaCppValidationEvidenceTests(unittest.TestCase): @@ -93,6 +174,7 @@ def test_failed_or_malformed_records_fail_closed(self) -> None: cases = { "failed": [{**passing_result(), "pass": False}], "non_boolean pass": [{**passing_result(), "pass": 1}], + "blank response": [{**passing_result(), "response": " \t"}], "invalid metric": [{**passing_result(), "tokens_per_second": "unexpected"}], "negative token count": [{**passing_result(), "input_tokens": -1}], "missing field": [ @@ -168,6 +250,93 @@ def test_inconsistent_planned_lane_sets_fail_closed(self) -> None: ): evidence.load_and_validate_results(root, matrix) + def test_capability_evidence_is_required_and_validated(self) -> None: + mutations = {} + missing = passing_result() + missing.pop("capability_matrix") + mutations["missing"] = missing + failed = passing_result() + failed["capability_matrix"]["pass"] = False + mutations["failed"] = failed + marker = passing_result() + marker["capability_matrix"]["cases"][0]["raw_ifm_marker"] = "" + mutations["marker"] = marker + wrong_model = passing_result() + wrong_model["capability_matrix"]["model"] = "builtin.Other-Llama" + mutations["wrong model"] = wrong_model + + for name, result in mutations.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + (root / evidence.EXPECTED_RESULT_FILES[0]).write_text( + json.dumps([result]), + encoding="utf-8", + ) + + with self.assertRaises(evidence.ValidationEvidenceError): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_duplicate_json_keys_in_evidence_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + target = root / evidence.EXPECTED_RESULT_FILES[0] + raw = json.dumps([passing_result()]) + raw = raw.replace( + '"city": "Paris"', + '"city": "London", "city": "Paris"', + 1, + ) + target.write_text(raw, encoding="utf-8") + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "duplicate JSON key", + ): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_non_designated_model_cannot_attach_capability_evidence(self) -> None: + other_model = "Other-Llama" + matrix = expected_matrix([K2_SMALL, other_model]) + records = [ + passing_result(), + passing_result(other_model, capability=True), + ] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for filename in evidence.EXPECTED_RESULT_FILES: + (root / filename).write_text(json.dumps(records), encoding="utf-8") + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "unexpected capability matrix", + ): + evidence.load_and_validate_results(root, matrix) + + def test_capability_plan_fields_are_paired_and_model_bound(self) -> None: + mutations = {} + missing_profile = expected_matrix() + missing_profile["include"][0].pop("capability_profile") + mutations["missing profile"] = missing_profile + missing_models = expected_matrix() + missing_models["include"][0].pop("capability_models") + mutations["missing models"] = missing_models + unselected = expected_matrix() + unselected["include"][0]["capability_models"] = ["Other-Llama"] + mutations["unselected model"] = unselected + unknown_profile = expected_matrix() + unknown_profile["include"][0]["capability_profile"] = "unknown" + mutations["unknown profile"] = unknown_profile + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + for name, matrix in mutations.items(): + with self.subTest(name=name): + with self.assertRaises(evidence.ValidationEvidenceError): + evidence.load_and_validate_results(root, matrix) + if __name__ == "__main__": unittest.main() diff --git a/test/test_validate_llamacpp_selection.py b/test/test_validate_llamacpp_selection.py index 95cd40c41f..15af57565b 100644 --- a/test/test_validate_llamacpp_selection.py +++ b/test/test_validate_llamacpp_selection.py @@ -15,6 +15,7 @@ from pathlib import Path from unittest import mock +from test.utils import llamacpp_capability_validation as capabilities from test.utils import validation_model_selection as selection MODEL_REGISTRY = ( @@ -52,6 +53,7 @@ def load_validation_module(): stubs = { "requests": requests_module, "utils": utils_package, + "utils.llamacpp_capability_validation": capabilities, "utils.server_base": server_base_module, "utils.test_models": test_models_module, "utils.validation_model_selection": selection, @@ -396,6 +398,29 @@ class LlamaCppValidationRuntimeTests(unittest.TestCase): def response(status_code=200): return types.SimpleNamespace(status_code=status_code) + def test_request_json_rejects_duplicate_response_members(self) -> None: + response_text = '{"model":"wrong","model":"right"}' + response = types.SimpleNamespace( + status_code=200, + content=response_text.encode("utf-8"), + text=response_text, + json=lambda **kwargs: json.loads(response_text, **kwargs), + ) + + with mock.patch.object( + VALIDATION.requests, + "request", + return_value=response, + create=True, + ): + _response, body = VALIDATION.request_json( + "GET", + "http://localhost:13305/api/v1/health", + timeout=1, + ) + + self.assertEqual(body, {"raw_text": response_text}) + def request_json_for_chat(self, chat_message, operations=None): loaded = False @@ -557,6 +582,184 @@ def test_model_accepts_clean_nested_tool_call_fields(self) -> None: self.assertTrue(success) self.assertEqual(response_text, message["content"]) + def test_plain_chat_requires_visible_final_content(self) -> None: + message = { + "content": "", + "reasoning_content": "The answer should be four.", + } + + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat(message), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, error, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + + self.assertFalse(success) + self.assertIn("visible final content", error) + + def test_model_runs_requested_capability_profile_and_records_evidence(self) -> None: + matrix = { + "profile": capabilities.K2_HORIZON_PROFILE, + "model": "builtin.Test-Llama", + "pass": True, + "cases": [], + } + evidence = {} + + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat({"content": "The answer is 4."}), + ) as request, + mock.patch.object( + VALIDATION, + "validate_capabilities", + return_value=(matrix, True, "12/12 capability cases passed"), + ) as validate, + contextlib.redirect_stdout(io.StringIO()), + ): + success, _response, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + capability_profile=capabilities.K2_HORIZON_PROFILE, + capability_evidence=evidence, + ) + + self.assertTrue(success) + self.assertEqual(evidence, matrix) + validate.assert_called_once_with( + capabilities.K2_HORIZON_PROFILE, + base_url="http://localhost:13305/api/v1", + model="builtin.Test-Llama", + request_json=request, + request_stream=VALIDATION.request_stream, + timeout=VALIDATION.TIMEOUT_INFERENCE, + ) + + def test_capability_failure_fails_model_and_preserves_matrix(self) -> None: + matrix = { + "profile": capabilities.K2_HORIZON_PROFILE, + "model": "builtin.Test-Llama", + "pass": False, + "cases": [{"id": "openai_reasoning_low", "pass": False}], + } + evidence = {} + + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat({"content": "The answer is 4."}), + ), + mock.patch.object( + VALIDATION, + "validate_capabilities", + return_value=(matrix, False, "reasoning low failed"), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, error, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + capability_profile=capabilities.K2_HORIZON_PROFILE, + capability_evidence=evidence, + ) + + self.assertFalse(success) + self.assertIn("reasoning low failed", error) + self.assertEqual(evidence, matrix) + + def test_model_requires_health_to_confirm_the_requested_backend(self) -> None: + cases = ( + (503, {}, "health inventory returned HTTP 503"), + (200, {}, "missing all_models_loaded"), + (200, {"all_models_loaded": "invalid"}, "invalid model inventory"), + (200, {"all_models_loaded": []}, "is absent from the loaded inventory"), + ( + 200, + {"all_models_loaded": [{"model_name": "builtin.Test-Llama"}]}, + "missing recipe_options", + ), + ( + 200, + { + "all_models_loaded": [ + { + "model_name": "builtin.Test-Llama", + "recipe_options": "invalid", + } + ] + }, + "invalid recipe_options", + ), + ( + 200, + { + "all_models_loaded": [ + { + "model_name": "builtin.Test-Llama", + "recipe_options": {}, + } + ] + }, + "missing llamacpp_backend", + ), + ( + 200, + { + "all_models_loaded": [ + { + "model_name": "builtin.Test-Llama", + "recipe_options": {"llamacpp_backend": "cpu"}, + } + ] + }, + "backend 'cpu' instead of 'vulkan'", + ), + ) + + for status_code, health, expected_error in cases: + with self.subTest(health=health, status_code=status_code): + + def request_json(method, url, timeout, **_kwargs): + del method, timeout + operation = url.rsplit("/", maxsplit=1)[-1] + if operation == "load": + return self.response(), {} + if operation == "health": + return self.response(status_code), health + if operation == "unload": + return self.response(), {} + self.fail(f"Unexpected request after invalid health: {operation}") + + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=request_json, + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, error, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + + self.assertFalse(success) + self.assertIn(expected_error, error) + def run_validation(self, explicit): operations = [] model = { @@ -664,6 +867,43 @@ def test_reload_requires_a_valid_loaded_model_inventory(self) -> None: self.assertRegex(error, "Could not verify unload") test_model.assert_called_once() + def test_reload_smoke_does_not_repeat_the_capability_profile(self) -> None: + evidence = {} + health = {"all_models_loaded": []} + + with ( + mock.patch.object( + VALIDATION, + "test_model", + side_effect=[(True, "first", {}), (True, "second", {})], + ) as test_model, + mock.patch.object( + VALIDATION, + "request_json", + return_value=(self.response(), health), + ), + ): + result = VALIDATION.validate_model_lifecycle( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + reload_after_unload=True, + capability_profile=capabilities.K2_HORIZON_PROFILE, + capability_evidence=evidence, + ) + + self.assertTrue(result[0]) + self.assertEqual(test_model.call_count, 2) + first_call, second_call = test_model.call_args_list + self.assertEqual( + first_call.kwargs, + { + "capability_profile": capabilities.K2_HORIZON_PROFILE, + "capability_evidence": evidence, + }, + ) + self.assertEqual(second_call.kwargs, {}) + class LlamaCppValidationSideEffectTests(unittest.TestCase): def test_invalid_explicit_model_is_rejected_before_server_mutation(self) -> None: @@ -692,6 +932,145 @@ def test_invalid_explicit_model_is_rejected_before_server_mutation(self) -> None set_channel.assert_not_called() unload_models.assert_not_called() + def test_capability_options_must_be_paired(self) -> None: + cases = ( + ["--capability-profile", capabilities.K2_HORIZON_PROFILE], + ["--capability-model", "Test-Llama"], + ) + + for extra_args in cases: + with self.subTest(extra_args=extra_args): + argv = [ + "validate_llamacpp.py", + "--backend", + "rocm", + "--channel", + "nightly", + *extra_args, + ] + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object(VALIDATION, "set_rocm_channel") as set_channel, + mock.patch.object(VALIDATION, "unload_all_models") as unload_models, + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit), + ): + VALIDATION.main() + + set_channel.assert_not_called() + unload_models.assert_not_called() + + def test_capability_profile_can_target_a_selected_hot_model(self) -> None: + model = { + "id": "K2-Horizon-0.9B-GGUF", + "recipe": "llamacpp", + "labels": ["chat", "hot"], + "checkpoint": "IFM/example:model.gguf", + } + matrix = { + "profile": capabilities.K2_HORIZON_PROFILE, + "model": "K2-Horizon-0.9B-GGUF", + "pass": True, + "cases": [], + } + + def validate_lifecycle( + _base_url, + _model_name, + _backend, + reload_after_unload=False, + *, + capability_profile="", + capability_evidence=None, + ): + self.assertFalse(reload_after_unload) + self.assertEqual(capability_profile, capabilities.K2_HORIZON_PROFILE) + capability_evidence.update(matrix) + return True, "Four.", {} + + with tempfile.TemporaryDirectory() as temp_dir: + output_path = Path(temp_dir) / "results.json" + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--skip-install", + "--capability-profile", + capabilities.K2_HORIZON_PROFILE, + "--capability-model", + model["id"], + "--output", + str(output_path), + ] + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object( + VALIDATION, "get_model_catalog", return_value=[model] + ), + mock.patch.object(VALIDATION, "unload_all_models"), + mock.patch.object( + VALIDATION, + "validate_model_lifecycle", + side_effect=validate_lifecycle, + ) as lifecycle, + contextlib.redirect_stdout(io.StringIO()), + ): + VALIDATION.main() + + records = json.loads(output_path.read_text(encoding="utf-8")) + + lifecycle.assert_called_once() + self.assertEqual(records[0]["capability_matrix"], matrix) + + def test_capability_model_must_be_selected_and_unique(self) -> None: + cases = ( + ["--capability-model", "Other-Llama"], + [ + "--capability-model", + "Test-Llama", + "--capability-model", + "Test-Llama", + ], + ) + model = { + "id": "Test-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/Test-Llama:model.gguf", + } + + for capability_args in cases: + with self.subTest(capability_args=capability_args): + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--model", + "Test-Llama", + "--capability-profile", + capabilities.K2_HORIZON_PROFILE, + *capability_args, + ] + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object( + VALIDATION, + "get_builtin_model", + return_value=model, + ), + mock.patch.object(VALIDATION, "install_backend") as install, + mock.patch.object(VALIDATION, "unload_all_models") as unload_models, + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit), + ): + VALIDATION.main() + + install.assert_not_called() + unload_models.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/test/test_validate_llamacpp_workflow_plan.py b/test/test_validate_llamacpp_workflow_plan.py index cead15c08e..fee0bd3bfe 100644 --- a/test/test_validate_llamacpp_workflow_plan.py +++ b/test/test_validate_llamacpp_workflow_plan.py @@ -16,8 +16,10 @@ ROOT = Path(__file__).resolve().parents[1] LEGACY_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp.yml" VALIDATION_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" +PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" +DOCS_AND_STYLE_WORKFLOW = ROOT / ".github" / "workflows" / "docs_and_style.yml" class LlamaCppValidationPlanTests(unittest.TestCase): @@ -26,6 +28,10 @@ def assert_active_lanes(self, rows): [(row["backend"], row["channel"]) for row in rows], [("vulkan", ""), ("rocm", "stable"), ("rocm", "nightly")], ) + self.assertEqual( + [row["target"] for row in rows], + ["windows-vulkan", "rocm-stable", "rocm-nightly"], + ) def test_pull_request_and_merge_group_require_k2(self) -> None: for event_name in ("pull_request", "merge_group"): @@ -38,6 +44,12 @@ def test_pull_request_and_merge_group_require_k2(self) -> None: self.assertEqual(rows[2]["models"], [K2_SMALL]) self.assertTrue(all(not row["lite"] for row in rows)) self.assertIn("128gb", rows[0]["runner"]) + self.assertTrue( + all(row["capability_profile"] == "k2-horizon-v1" for row in rows) + ) + self.assertTrue( + all(row["capability_models"] == [K2_SMALL] for row in rows) + ) def test_schedule_captures_exact_hot_model_evidence_contract(self) -> None: rows = planning.create_validation_plan("schedule")["include"] @@ -49,6 +61,10 @@ def test_schedule_captures_exact_hot_model_evidence_contract(self) -> None: self.assertTrue(all(row["expected_models"] == expected for row in rows)) self.assertTrue(all(not row["lite"] for row in rows)) self.assertTrue(all("128gb" in row["runner"] for row in rows)) + self.assertTrue( + all(row["capability_profile"] == "k2-horizon-v1" for row in rows) + ) + self.assertTrue(all(row["capability_models"] == [K2_SMALL] for row in rows)) def test_default_dispatch_keeps_runtime_hot_selection(self) -> None: rows = planning.create_validation_plan("workflow_dispatch")["include"] @@ -57,6 +73,8 @@ def test_default_dispatch_keeps_runtime_hot_selection(self) -> None: self.assertTrue(all(row["models"] == [] for row in rows)) self.assertTrue(all(not row["lite"] for row in rows)) self.assertTrue(all("128gb" in row["runner"] for row in rows)) + self.assertTrue(all("capability_profile" not in row for row in rows)) + self.assertTrue(all("capability_models" not in row for row in rows)) def test_manual_lite_selection_uses_small_runners(self) -> None: rows = planning.create_validation_plan("workflow_dispatch", lite=True)[ @@ -156,13 +174,16 @@ def test_validation_core_is_read_only_and_has_no_legacy_entrypoint(self) -> None ].split(" - name: Upload server logs\n", 1)[0] self.assertIn(" workflow_call:\n", validation_triggers) - self.assertIn(" pull_request:\n", validation_triggers) - self.assertIn(" merge_group:\n", validation_triggers) + self.assertNotIn(" pull_request_target:\n", validation_triggers) + self.assertNotIn(" pull_request:\n", validation_triggers) + self.assertNotIn(" merge_group:\n", validation_triggers) self.assertNotIn(" workflow_dispatch:\n", validation_triggers) self.assertNotIn(" schedule:\n", validation_triggers) self.assertIn("permissions:\n contents: read\n", validation) self.assertNotIn("contents: write", validation) self.assertNotIn("pull-requests: write", validation) + self.assertNotIn("checks: write", validation) + self.assertNotIn("concurrency:", workflow_scope) self.assertNotIn(" create-pr:\n", validation) self.assertIn(" HUGGINGFACE_ACCESS_TOKEN:\n", validation_triggers) self.assertNotIn("HF_TOKEN:", workflow_scope) @@ -171,11 +192,25 @@ def test_validation_core_is_read_only_and_has_no_legacy_entrypoint(self) -> None self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", build_job) self.assertEqual(validate_job.count("HF_TOKEN:"), 1) self.assertIn( + " HF_TOKEN: ${{ github.event_name != 'pull_request_target' && github.event_name != 'merge_group' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }}\n", + validation_step, + ) + self.assertNotIn( " HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}\n", validation_step, ) + self.assertIn(" $cleanupExitCode = 0\n", validation_step) + self.assertIn(" $cleanupExitCode = 1\n", validation_step) + self.assertIn(" if ($cleanupExitCode -ne 0) {\n", validation_step) self.assertIn(" if-no-files-found: error\n", result_upload) + def test_focused_ci_runs_the_capability_contract_tests(self) -> None: + workflow = DOCS_AND_STYLE_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn("test.test_llamacpp_capabilities", workflow) + self.assertIn("test.test_llamacpp_validation_evidence", workflow) + self.assertIn("test.test_validate_llamacpp_selection", workflow) + def test_release_asset_manifest_is_captured_and_exported(self) -> None: validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") release_job = validation.split(" get-latest-releases:\n", 1)[1].split( @@ -194,32 +229,251 @@ def test_release_asset_manifest_is_captured_and_exported(self) -> None: self.assertIn("llamacpp_release_manifest", asset_job) self.assertIn('--github-output "$GITHUB_OUTPUT"', asset_job) self.assertIn("jq -r '.assets[].name'", asset_job) + self.assertIn("repos/${repository}/commits/${tag}", asset_job) + self.assertIn("source_commit", asset_job) self.assertNotIn("release_asset_manifest", release_job) - def test_pr_label_revocation_cancels_authorized_validation(self) -> None: + def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: + pull_request = PR_WORKFLOW.read_text(encoding="utf-8") validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - validation_triggers = validation.split("permissions:", 1)[0] - concurrency = validation.split("concurrency:\n", 1)[1].split("\nenv:\n", 1)[0] + pull_request_triggers = pull_request.split("permissions:", 1)[0] + concurrency = pull_request.split("concurrency:\n", 1)[1].split("\njobs:\n", 1)[ + 0 + ] + authorization_job = pull_request.split(" authorize-invocation:\n", 1)[1].split( + " start-check:\n", 1 + )[0] + start_check_job = pull_request.split(" start-check:\n", 1)[1].split( + " validate:\n", 1 + )[0] + validation_job = pull_request.split(" validate:\n", 1)[1].split( + " report:\n", 1 + )[0] + report_job = pull_request.split(" report:\n", 1)[1] + report_step = report_job.split( + " - name: Report result on the authorized merge commit\n", 1 + )[1].split(" - name: Enforce result\n", 1)[0] + + self.assertIn(" pull_request_target:\n", pull_request_triggers) + self.assertIn(" merge_group:\n", pull_request_triggers) + self.assertNotIn(" pull_request:\n", pull_request_triggers) + self.assertNotIn(" workflow_call:\n", pull_request_triggers) + self.assertIn( + " types: [synchronize, edited, labeled, unlabeled, closed]\n", + pull_request_triggers, + ) + self.assertIn("github.event.pull_request.number", concurrency) + self.assertNotIn("github.ref", concurrency) + self.assertIn("github.run_id", concurrency) + self.assertIn("github.event.action == 'synchronize'", concurrency) + self.assertIn("github.event.action == 'edited'", concurrency) + self.assertIn("github.event.changes.base.ref.from", concurrency) + self.assertIn("github.event.action == 'closed'", concurrency) + self.assertIn("github.event.action == 'unlabeled'", concurrency) + self.assertIn("github.event.label.name == 'ci:upgrades'", concurrency) + self.assertNotIn("cancel-in-progress: true", concurrency) + + self.assertIn("github.event.action", authorization_job) + self.assertIn('if [ "$EVENT_NAME" = "merge_group" ]', authorization_job) + self.assertIn("github.event.label.name", authorization_job) + self.assertIn("github.actor", authorization_job) + self.assertIn("collaborators/${ACTOR}/permission", authorization_job) + self.assertIn("admin|maintain|write|push", authorization_job) + self.assertIn( + 'echo "run_validation=true" >> "$GITHUB_OUTPUT"', authorization_job + ) + self.assertNotIn("actions/checkout", authorization_job) + + self.assertIn("needs: authorize-invocation", start_check_job) + self.assertIn("checks: write", start_check_job) + self.assertIn("github.event.pull_request.merge_commit_sha", start_check_job) + self.assertIn("github.event.merge_group.head_sha", start_check_job) + self.assertNotIn("github.event.pull_request.head.sha", start_check_job) + self.assertIn('head_sha="$MERGE_SHA"', start_check_job) + self.assertIn("CHECK_NAME: llama.cpp authorized validation v2", start_check_job) + self.assertIn('name="$CHECK_NAME"', start_check_job) + self.assertIn('external_id="$EXTERNAL_ID"', start_check_job) + self.assertIn("github.run_id", start_check_job) + self.assertIn("github.run_attempt", start_check_job) + self.assertIn("status=in_progress", start_check_job) + self.assertIn("check_run_id", start_check_job) + + self.assertIn("needs: [authorize-invocation, start-check]", validation_job) + self.assertIn( + "needs.authorize-invocation.outputs.run_validation == 'true'", + validation_job, + ) + self.assertIn( + "uses: ./.github/workflows/validate_llamacpp_core.yml", validation_job + ) + self.assertIn("permissions:\n contents: read", validation_job) + self.assertNotIn("secrets:", validation_job) + self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", pull_request) + + self.assertIn( + "needs: [authorize-invocation, start-check, validate]", report_job + ) + self.assertIn("checks: write", report_job) + self.assertIn("repos/${GITHUB_REPOSITORY}/check-runs", report_job) + self.assertIn("github.event.pull_request.head.sha", report_job) + self.assertIn("needs.start-check.outputs.check_run_id", report_job) + self.assertIn("commits/${MERGE_SHA}/check-runs", report_step) + self.assertIn(".external_id", report_step) + self.assertIn("--arg external_id", report_step) + self.assertIn('head_sha="$MERGE_SHA"', report_step) + self.assertIn("CHECK_NAME: llama.cpp authorized validation v2", report_job) + self.assertIn('name="$CHECK_NAME"', report_step) + self.assertIn('external_id="$EXTERNAL_ID"', report_step) + self.assertNotIn('head_sha="$HEAD_SHA"', report_step) + self.assertNotIn('-f name="llama.cpp validation"', pull_request) + self.assertNotIn(" name: llama.cpp validation\n", pull_request) + self.assertIn("--method PATCH", report_step) + self.assertIn("check-runs/${CHECK_RUN_ID}", report_step) + self.assertIn("pull-requests: read", report_job) + self.assertIn("repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", report_job) + self.assertIn("github.event.pull_request.base.sha", report_job) + self.assertIn("github.event.pull_request.merge_commit_sha", report_job) + self.assertIn("github.event.merge_group.head_sha", report_job) + self.assertIn(".base.sha", report_job) + self.assertIn(".head.sha", report_job) + self.assertIn(".merge_commit_sha", report_job) + self.assertLess( + report_step.index("repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}"), + report_step.index("--method PATCH"), + ) + self.assertIn("github.event.action == 'labeled'", report_job) + self.assertIn("github.event.label.name == 'ci:upgrades'", report_job) + self.assertIn("needs.validate.result", report_job) + self.assertIn("github.event.action == 'labeled'", report_step) + self.assertIn("github.event.label.name == 'ci:upgrades'", report_step) + + self.assertIn(" validate-invocation:\n", validation) + invocation_job = validation.split(" validate-invocation:\n", 1)[1].split( + " plan:\n", 1 + )[0] + plan_job = validation.split(" plan:\n", 1)[1].split( + " get-latest-releases:\n", 1 + )[0] build_job_header = validation.split(" build:\n", 1)[1].split( " runs-on:", 1 )[0] + validation_gate = validation.split(" validation-gate:\n", 1)[1] self.assertIn( - " types: [opened, synchronize, reopened, labeled, unlabeled, closed]\n", - validation_triggers, + "merge_group|pull_request_target|schedule|workflow_dispatch", + invocation_job, ) - self.assertIn("github.event.pull_request.number", concurrency) - self.assertNotIn("github.ref", concurrency) + self.assertIn("Unsupported invocation event", invocation_job) + self.assertNotIn("actions/checkout", invocation_job) + self.assertIn("needs: validate-invocation", plan_job) + self.assertIn("validate-invocation", build_job_header) + self.assertIn( + "needs: [validate-invocation, build, validate, plan]", validation_gate + ) + self.assertNotIn("checks: write", validation_gate) + self.assertNotIn("check-runs", validation_gate) + + merge_repository = "github.repository" + merge_sha = "github.event.pull_request.merge_commit_sha" + self.assertGreaterEqual(validation.count(merge_repository), 4) + self.assertGreaterEqual(validation.count(merge_sha), 2) + self.assertGreaterEqual( + validation.count("github.event.merge_group.head_sha"), 4 + ) + self.assertIn("github.event.pull_request.base.sha", validation) + self.assertIn("github.event.pull_request.head.sha", validation) + self.assertEqual(validation.count("persist-credentials: false"), 4) + + def test_authorization_removal_revokes_the_merge_check(self) -> None: + pull_request = PR_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn(" revoke-check:\n", pull_request) + revoke_job = pull_request.split(" revoke-check:\n", 1)[1].split( + " start-check:\n", 1 + )[0] + + self.assertIn("github.event.action == 'unlabeled'", revoke_job) + self.assertIn("github.event.label.name == 'ci:upgrades'", revoke_job) + self.assertIn("github.event.pull_request.state == 'open'", revoke_job) + self.assertIn("checks: write", revoke_job) + self.assertIn("pull-requests: read", revoke_job) + self.assertIn("repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", revoke_job) + self.assertIn("current_state", revoke_job) + self.assertIn("current_authorized", revoke_job) self.assertIn( - "cancel-in-progress: ${{ github.event_name == 'pull_request' }}", - concurrency, + 'if [ "$current_state" != "open" ] || ' + '[ "$current_authorized" = "true" ]', + revoke_job, ) - self.assertIn("github.event.action == 'labeled'", build_job_header) + self.assertIn("github.event.pull_request.merge_commit_sha", revoke_job) + self.assertIn("commits/${MERGE_SHA}/check-runs", revoke_job) + self.assertIn("llama.cpp authorized validation v2", revoke_job) + self.assertIn("conclusion=cancelled", revoke_job) + self.assertIn("--method PATCH", revoke_job) + self.assertIn("--method POST", revoke_job) + self.assertNotIn("actions/checkout", revoke_job) + + def test_closed_pull_requests_cannot_start_protected_validation(self) -> None: + pull_request = PR_WORKFLOW.read_text(encoding="utf-8") + authorization_job = pull_request.split(" authorize-invocation:\n", 1)[1].split( + " revoke-check:\n", 1 + )[0] + + self.assertIn("current_state=$(jq -r '.state'", authorization_job) + self.assertIn('if [ "$current_state" != "open" ]', authorization_job) + state_guard = authorization_job.index('if [ "$current_state" != "open" ]') + authorization_output = authorization_job.index( + 'echo "run_validation=true" >> "$GITHUB_OUTPUT"', + state_guard, + ) + self.assertLess(state_guard, authorization_output) + + def test_protected_change_runner_matrix_is_defined_by_trusted_yaml(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + plan_job = validation.split(" plan:\n", 1)[1].split( + " get-latest-releases:\n", 1 + )[0] + + self.assertIn(" - name: Build trusted protected-change matrix\n", plan_job) + trusted_step = plan_job.split( + " - name: Build trusted protected-change matrix\n", 1 + )[1].split(" - name: Build branch validation matrix\n", 1)[0] + branch_step = plan_job.split( + " - name: Build branch validation matrix\n", 1 + )[1] + + self.assertIn("github.event_name == 'pull_request_target'", trusted_step) + self.assertIn("github.event_name == 'merge_group'", trusted_step) + self.assertIn('"lemon-prod"', trusted_step) + self.assertIn('"K2-Horizon-0.9B-GGUF"', trusted_step) + self.assertIn('"K2-Horizon-3.7B-GGUF"', trusted_step) + self.assertIn('"K2-Horizon-7B-GGUF"', trusted_step) + self.assertIn('target: "windows-vulkan"', trusted_step) + self.assertEqual(trusted_step.count('capability_profile: "k2-horizon-v1"'), 3) + self.assertEqual( + trusted_step.count('capability_models: ["K2-Horizon-0.9B-GGUF"]'), + 3, + ) + self.assertNotIn("python -m test.utils", trusted_step) + self.assertIn("github.event_name != 'pull_request_target'", branch_step) + self.assertIn("github.event_name != 'merge_group'", branch_step) + self.assertIn("python -m test.utils.llamacpp_validation_plan", branch_step) + self.assertIn("steps.protected-plan.outputs.matrix", plan_job) + self.assertIn("steps.branch-plan.outputs.matrix", plan_job) + + validate_job = validation.split(" validate:\n", 1)[1].split( + " validation-gate:\n", 1 + )[0] + self.assertIn(' $label = "${{ matrix.target }}"\n', validate_job) self.assertIn( - "github.event.label.name == 'ci:upgrades'", - build_job_header, + " CAPABILITY_PROFILE: ${{ matrix.capability_profile || '' }}\n", + validate_job, ) - self.assertNotIn("github.event.pull_request.labels.*.name", build_job_header) + self.assertIn(" CAPABILITY_MODELS:", validate_job) + self.assertIn("-CapabilityProfile $env:CAPABILITY_PROFILE", validate_job) + self.assertIn("-CapabilityModels $env:CAPABILITY_MODELS", validate_job) + self.assertIn("validation-results-${{ matrix.target }}", validate_job) + self.assertIn("llamacpp_validation_${{ matrix.target }}.json", validate_job) def test_manual_validation_calls_the_read_only_core(self) -> None: manual = MANUAL_WORKFLOW.read_text(encoding="utf-8") @@ -229,6 +483,7 @@ def test_manual_validation_calls_the_read_only_core(self) -> None: self.assertNotIn(" schedule:\n", manual_triggers) self.assertIn("permissions:\n contents: read\n", manual) self.assertNotIn("contents: write", manual) + self.assertNotIn("checks: write", manual) self.assertNotIn("pull-requests: write", manual) self.assertNotIn("secrets: inherit", manual) self.assertIn( @@ -260,6 +515,7 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: ) self.assertIn("permissions:\n contents: read", validation_job) self.assertNotIn("contents: write", validation_job) + self.assertNotIn("checks: write", validation_job) self.assertNotIn("secrets: inherit", schedule) self.assertIn( "HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}", @@ -304,6 +560,8 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: manifest_step, ) self.assertIn("gh api", manifest_step) + self.assertIn("repos/${repository}/commits/${tag}", manifest_step) + self.assertIn("source_commit", manifest_step) self.assertNotIn("\n - name:", manifest_step) self.assertLess( publication.index(" - name: Generate PR body\n"), diff --git a/test/utils/llamacpp_capability_validation.py b/test/utils/llamacpp_capability_validation.py new file mode 100644 index 0000000000..29f2601e70 --- /dev/null +++ b/test/utils/llamacpp_capability_validation.py @@ -0,0 +1,1123 @@ +"""Fail-closed API capability validation for llama.cpp models.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Iterable +from typing import Any + +K2_HORIZON_PROFILE = "k2-horizon-v1" +K2_HORIZON_CASE_IDS = ( + "openai_plain_off_nonstream", + "openai_plain_off_stream", + "openai_reasoning_high", + "openai_reasoning_high_stream", + "openai_reasoning_medium", + "openai_reasoning_low", + "openai_tool_json", + "openai_tool_xml", + "openai_tool_xml_typed", + "openai_tool_default_xml", + "openai_tool_stream_xml", + "openai_tool_result_followup", + "ollama_thinking", + "anthropic_translation", +) +CASE_CONTRACTS = { + "openai_plain_off_nonstream": ("openai", False, "stop", True, False, 0, None), + "openai_plain_off_stream": ("openai", True, "stop", True, False, 0, None), + "openai_reasoning_high": ("openai", False, "stop", True, True, 0, None), + "openai_reasoning_high_stream": ( + "openai", + True, + "stop", + True, + True, + 0, + None, + ), + "openai_reasoning_medium": ("openai", False, "stop", True, True, 0, None), + "openai_reasoning_low": ("openai", False, "stop", True, True, 0, None), + "openai_tool_json": ("openai", False, "tool_calls", False, False, 1, None), + "openai_tool_xml": ("openai", False, "tool_calls", False, False, 1, None), + "openai_tool_xml_typed": ( + "openai", + False, + "tool_calls", + False, + False, + 1, + None, + ), + "openai_tool_default_xml": ( + "openai", + False, + "tool_calls", + False, + False, + 1, + "openai_tool_xml", + ), + "openai_tool_stream_xml": ( + "openai", + True, + "tool_calls", + False, + False, + 1, + "openai_tool_xml", + ), + "openai_tool_result_followup": ( + "openai", + False, + "stop", + True, + False, + 0, + None, + ), + "ollama_thinking": ("ollama", False, "stop", True, True, 0, None), + "anthropic_translation": ( + "anthropic", + False, + "end_turn", + True, + False, + 0, + None, + ), +} +TOOL_NAME = "lookup_weather" +EXPECTED_TOOL_ARGUMENTS = { + "city": "Paris", + "unit": "celsius", + "days": 2, + "include_hourly": False, +} +IFM_CONTROL_PREFIXES = ("", start) + if end < 0: + end = min(start + 79, len(candidate) - 1) + return candidate[start : end + 1], path + elif isinstance(candidate, dict): + for key, nested_value in reversed(list(candidate.items())): + pending.append((f"{path}.{key}", nested_value)) + elif isinstance(candidate, (list, tuple)): + for index in range(len(candidate) - 1, -1, -1): + pending.append((f"{path}[{index}]", candidate[index])) + return None + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise CapabilityValidationError(message) + + +def _marker_free(value: object, field: str = "response") -> None: + marker = find_raw_ifm_control_marker(value, field) + if marker: + marker_text, marker_path = marker + raise CapabilityValidationError( + f"raw IFM control marker {marker_text!r} in {marker_path}" + ) + + +def _status_code(response: object) -> int: + status_code = getattr(response, "status_code", None) + _require( + isinstance(status_code, int) and not isinstance(status_code, bool), + "response has no integer status code", + ) + return status_code + + +def _canonical_model_id(model: str) -> str: + return model.removeprefix("builtin.") + + +def _require_response_model(body: dict, expected_model: str, field: str) -> None: + response_model = body.get("model") + _require( + isinstance(response_model, str) and bool(response_model), + f"{field} model is missing", + ) + _require( + _canonical_model_id(response_model) == _canonical_model_id(expected_model), + f"{field} model {response_model!r} does not match {expected_model!r}", + ) + + +def reject_duplicate_json_object(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise CapabilityValidationError(f"JSON object repeats key {key!r}") + result[key] = value + return result + + +def _validate_tool_arguments(arguments: object) -> dict: + _require(isinstance(arguments, dict), "tool arguments must be an object") + _require( + set(arguments) == set(EXPECTED_TOOL_ARGUMENTS), + "tool argument keys do not match request", + ) + for field in ("city", "unit"): + _require( + type(arguments[field]) is str + and arguments[field] == EXPECTED_TOOL_ARGUMENTS[field], + f"tool argument {field} has the wrong value or type", + ) + _require( + type(arguments["days"]) is int + and arguments["days"] == EXPECTED_TOOL_ARGUMENTS["days"], + "tool argument days has the wrong value or type", + ) + _require( + type(arguments["include_hourly"]) is bool + and arguments["include_hourly"] is EXPECTED_TOOL_ARGUMENTS["include_hourly"], + "tool argument include_hourly has the wrong value or type", + ) + return arguments + + +def _decode_tool_arguments(arguments_text: str) -> dict: + try: + arguments = json.loads( + arguments_text, + object_pairs_hook=reject_duplicate_json_object, + ) + except json.JSONDecodeError as exc: + raise CapabilityValidationError( + f"tool arguments are invalid JSON: {exc}" + ) from exc + return _validate_tool_arguments(arguments) + + +def _post_json( + request_json: Callable[..., tuple[object, object]], + url: str, + timeout: int, + payload: dict, +) -> tuple[dict, int]: + response, body = request_json("POST", url, timeout=timeout, json=payload) + status_code = _status_code(response) + _require(status_code == 200, f"HTTP {status_code}: {body}") + _require(isinstance(body, dict), "response body must be an object") + _marker_free(body) + expected_model = payload.get("model") + _require( + isinstance(expected_model, str) and expected_model, "request model is missing" + ) + _require_response_model(body, expected_model, "response") + return body, status_code + + +def _openai_message(body: dict, finish_reason: str) -> dict: + choices = body.get("choices") + _require(isinstance(choices, list) and len(choices) == 1, "expected one choice") + choice = choices[0] + _require(isinstance(choice, dict), "choice must be an object") + _require( + choice.get("finish_reason") == finish_reason, + f"expected finish_reason {finish_reason!r}", + ) + message = choice.get("message") + _require(isinstance(message, dict), "choice.message must be an object") + _require( + message.get("role") in (None, "assistant"), "message role is not assistant" + ) + _marker_free(message, "message") + return message + + +def _visible_content(message: dict, token: str | None = None) -> str: + content = message.get("content") + _require(isinstance(content, str) and bool(content.strip()), "content is blank") + if token is not None: + _require(token in content, f"content does not contain {token!r}") + return content + + +def _reasoning_content(message: dict) -> str: + reasoning = message.get("reasoning_content") + _require( + isinstance(reasoning, str) and bool(reasoning.strip()), + "reasoning_content is blank", + ) + return reasoning + + +def _require_no_reasoning(message: dict) -> None: + reasoning = message.get("reasoning_content") + _require( + reasoning is None or (isinstance(reasoning, str) and not reasoning.strip()), + "thinking-disabled response contains reasoning_content", + ) + + +def _require_no_tool_calls(message: dict) -> None: + tool_calls = message.get("tool_calls") + _require( + tool_calls is None or (isinstance(tool_calls, list) and not tool_calls), + "ordinary response contains tool calls", + ) + + +def _parse_tool_call(message: dict) -> tuple[dict, dict]: + content = message.get("content") + _require( + content is None or (isinstance(content, str) and not content.strip()), + "tool response contains unstructured visible content", + ) + calls = message.get("tool_calls") + _require(isinstance(calls, list) and len(calls) == 1, "expected one tool call") + call = calls[0] + _require(isinstance(call, dict), "tool call must be an object") + _require(isinstance(call.get("id"), str) and call["id"], "tool call id is missing") + _require(call.get("type") == "function", "tool call type must be function") + function = call.get("function") + _require(isinstance(function, dict), "tool call function must be an object") + _require(function.get("name") == TOOL_NAME, "unexpected tool function name") + arguments_text = function.get("arguments") + _require(isinstance(arguments_text, str), "tool arguments must be a JSON string") + arguments = _decode_tool_arguments(arguments_text) + return call, arguments + + +def _base_openai_payload(model: str, *, stream: bool, max_tokens: int) -> dict: + return { + "model": model, + "messages": [{"role": "user", "content": ARITHMETIC_PROMPT}], + "max_completion_tokens": max_tokens, + "temperature": 0, + "top_p": 1, + "seed": 42, + "stream": stream, + } + + +def _tool_payload(model: str, tool_format: str | None, *, stream: bool) -> dict: + payload = { + "model": model, + "messages": [{"role": "user", "content": TOOL_PROMPT}], + "tools": [TOOL_DEFINITION], + "tool_choice": "required", + "max_completion_tokens": 256, + "temperature": 0, + "top_p": 1, + "seed": 42, + "stream": stream, + "chat_template_kwargs": {"enable_thinking": False}, + } + if tool_format is not None: + payload["chat_template_kwargs"]["tool_call_format"] = tool_format + return payload + + +def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: + content_parts: list[str] = [] + reasoning_parts: list[str] = [] + tools: dict[int, dict[str, Any]] = {} + finish_reason = None + stream_model = None + saw_choice = False + saw_done = False + + for event_index, event in enumerate(events): + if isinstance(event, bytes): + try: + event = event.decode("utf-8") + except UnicodeDecodeError as exc: + raise CapabilityValidationError( + f"stream event {event_index} is not UTF-8" + ) from exc + if isinstance(event, str): + line = event.strip() + if not line or line.startswith(":"): + continue + _require( + not saw_done, "stream contains data after the terminal [DONE] frame" + ) + _marker_free(line, f"stream[{event_index}]") + _require(line.startswith("data:"), "stream line is not an SSE data frame") + data = line[len("data:") :].strip() + if data == "[DONE]": + _require(not saw_done, "stream contains duplicate [DONE] frames") + _require( + isinstance(finish_reason, str), + "stream ended before a terminal finish_reason", + ) + saw_done = True + continue + try: + chunk = json.loads( + data, + object_pairs_hook=reject_duplicate_json_object, + ) + except json.JSONDecodeError as exc: + raise CapabilityValidationError( + f"stream event {event_index} contains invalid JSON: {exc}" + ) from exc + else: + _require( + not saw_done, "stream contains data after the terminal [DONE] frame" + ) + chunk = event + _require(isinstance(chunk, dict), "stream chunk must be an object") + _marker_free(chunk, f"stream[{event_index}]") + _require( + "error" not in chunk, f"stream returned an error: {chunk.get('error')}" + ) + choices = chunk.get("choices") + _require(isinstance(choices, list), "stream choices must be an array") + if not choices: + continue + _require( + finish_reason is None, + "stream contains a choice chunk after a terminal finish_reason", + ) + _require( + len(choices) == 1 and isinstance(choices[0], dict), "invalid stream choice" + ) + response_model = chunk.get("model") + _require( + isinstance(response_model, str) and bool(response_model.strip()), + f"stream[{event_index}] model must be a nonempty string", + ) + _require( + stream_model in (None, response_model), + f"stream model changed from {stream_model!r} to {response_model!r}", + ) + stream_model = response_model + saw_choice = True + choice = choices[0] + delta = choice.get("delta", {}) + _require(isinstance(delta, dict), "stream delta must be an object") + for key, destination in ( + ("content", content_parts), + ("reasoning_content", reasoning_parts), + ): + value = delta.get(key) + if value is not None: + _require(isinstance(value, str), f"stream {key} must be a string") + destination.append(value) + + tool_deltas = delta.get("tool_calls", []) + _require(isinstance(tool_deltas, list), "stream tool_calls must be an array") + for tool_delta in tool_deltas: + _require( + isinstance(tool_delta, dict), "stream tool delta must be an object" + ) + index = tool_delta.get("index") + _require( + isinstance(index, int) and not isinstance(index, bool) and index >= 0, + "stream tool delta has an invalid index", + ) + target = tools.setdefault( + index, + {"id": None, "type": None, "name": "", "arguments": ""}, + ) + for key in ("id", "type"): + value = tool_delta.get(key) + if value is not None: + _require( + isinstance(value, str) and value, + f"stream tool {key} is invalid", + ) + _require( + target[key] in (None, value), + f"stream tool {key} changed between chunks", + ) + target[key] = value + function = tool_delta.get("function") + if function is not None: + _require(isinstance(function, dict), "stream tool function is invalid") + name = function.get("name") + if name is not None: + _require(isinstance(name, str), "stream tool name must be a string") + target["name"] += name + arguments = function.get("arguments") + if arguments is not None: + _require( + isinstance(arguments, str), + "stream tool arguments must be a string", + ) + target["arguments"] += arguments + + current_finish = choice.get("finish_reason") + if current_finish is not None: + _require(isinstance(current_finish, str), "finish_reason must be a string") + _require( + finish_reason in (None, current_finish), + "finish_reason changed between chunks", + ) + finish_reason = current_finish + + _require(saw_choice, "stream contains no choice chunks") + _require(saw_done, "stream is missing the terminal [DONE] frame") + _require(isinstance(finish_reason, str), "stream has no terminal finish_reason") + tool_calls = [] + for index in sorted(tools): + tool = tools[index] + tool_calls.append( + { + "id": tool["id"], + "type": tool["type"], + "function": { + "name": tool["name"], + "arguments": tool["arguments"], + }, + } + ) + message = { + "role": "assistant", + "content": "".join(content_parts), + "reasoning_content": "".join(reasoning_parts), + "tool_calls": tool_calls, + } + _marker_free(message, "assembled_message") + return message, finish_reason, saw_done + + +def _post_openai_stream( + request_stream: Callable[..., tuple[object, Iterable[object]]], + url: str, + timeout: int, + payload: dict, +) -> tuple[dict, str, int, bool]: + response, events = request_stream("POST", url, timeout=timeout, json=payload) + status_code = _status_code(response) + _require(status_code == 200, f"stream returned HTTP {status_code}") + expected_model = payload.get("model") + _require( + isinstance(expected_model, str) and expected_model, "request model is missing" + ) + message, finish_reason, terminal = _decode_openai_stream(events) + return message, finish_reason, status_code, terminal + + +def _new_case(case_id: str, protocol: str, stream: bool) -> dict: + return { + "id": case_id, + "protocol": protocol, + "stream": stream, + "pass": False, + "status_code": 0, + "finish_reason": None, + "content_chars": 0, + "reasoning_chars": 0, + "tool_call_count": 0, + "terminal_frame": not stream, + "tool_name": None, + "tool_arguments": None, + "parity_with": None, + "raw_ifm_marker": None, + "error": None, + } + + +def _run_case( + cases: list[dict], + case_id: str, + protocol: str, + stream: bool, + action: Callable[[], tuple[dict, object]], +): + case = _new_case(case_id, protocol, stream) + artifact = None + try: + values, artifact = action() + _require(isinstance(values, dict), "case metrics must be an object") + unknown = set(values) - CASE_FIELDS + _require(not unknown, f"case returned unknown metrics: {sorted(unknown)}") + case.update(values) + case["pass"] = True + except Exception as exc: # pylint: disable=broad-exception-caught + marker = find_raw_ifm_control_marker(str(exc), "error") + if marker: + case["raw_ifm_marker"] = marker[0] + case["error"] = str(exc)[:512] or exc.__class__.__name__ + cases.append(case) + return artifact + + +def _openai_plain_case(request_json, url, timeout, payload): + body, status = _post_json(request_json, url, timeout, payload) + message = _openai_message(body, "stop") + content = _visible_content(message, "42") + _require_no_reasoning(message) + _require_no_tool_calls(message) + return { + "status_code": status, + "finish_reason": "stop", + "content_chars": len(content), + }, message + + +def _openai_plain_stream_case(request_stream, url, timeout, payload): + message, finish, status, terminal = _post_openai_stream( + request_stream, url, timeout, payload + ) + _require(finish == "stop", "plain stream did not finish with stop") + content = _visible_content(message, "42") + _require_no_reasoning(message) + _require_no_tool_calls(message) + return { + "status_code": status, + "finish_reason": finish, + "content_chars": len(content), + "terminal_frame": terminal, + }, message + + +def _openai_reasoning_case(request_json, url, timeout, payload): + body, status = _post_json(request_json, url, timeout, payload) + message = _openai_message(body, "stop") + content = _visible_content(message, "42") + reasoning = _reasoning_content(message) + _require_no_tool_calls(message) + return { + "status_code": status, + "finish_reason": "stop", + "content_chars": len(content), + "reasoning_chars": len(reasoning), + }, message + + +def _openai_reasoning_stream_case(request_stream, url, timeout, payload): + message, finish, status, terminal = _post_openai_stream( + request_stream, url, timeout, payload + ) + _require(finish == "stop", "reasoning stream did not finish with stop") + content = _visible_content(message, "42") + reasoning = _reasoning_content(message) + _require_no_tool_calls(message) + return { + "status_code": status, + "finish_reason": finish, + "content_chars": len(content), + "reasoning_chars": len(reasoning), + "terminal_frame": terminal, + }, message + + +def _openai_tool_case(request_json, url, timeout, payload): + body, status = _post_json(request_json, url, timeout, payload) + message = _openai_message(body, "tool_calls") + _require_no_reasoning(message) + call, arguments = _parse_tool_call(message) + return { + "status_code": status, + "finish_reason": "tool_calls", + "tool_call_count": 1, + "tool_name": TOOL_NAME, + "tool_arguments": arguments, + }, call + + +def _openai_tool_parity_case( + request_json, + url, + timeout, + payload, + expected_call, + parity_with, +): + _require(isinstance(expected_call, dict), f"{parity_with} tool case did not pass") + values, call = _openai_tool_case(request_json, url, timeout, payload) + expected_function = expected_call.get("function", {}) + _require( + call["function"]["name"] == expected_function.get("name") + and _decode_tool_arguments(call["function"]["arguments"]) + == _decode_tool_arguments(expected_function.get("arguments", "null")), + f"tool call differs from {parity_with}", + ) + values["parity_with"] = parity_with + return values, call + + +def _openai_tool_stream_case(request_stream, url, timeout, payload, expected_call): + _require( + isinstance(expected_call, dict), "non-streaming XML tool case did not pass" + ) + message, finish, status, terminal = _post_openai_stream( + request_stream, url, timeout, payload + ) + _require(finish == "tool_calls", "tool stream did not finish with tool_calls") + _require_no_reasoning(message) + call, arguments = _parse_tool_call(message) + expected_function = expected_call.get("function", {}) + _require( + call["function"]["name"] == expected_function.get("name") + and arguments + == _decode_tool_arguments(expected_function.get("arguments", "null")), + "streaming tool call differs from non-streaming XML result", + ) + return { + "status_code": status, + "finish_reason": finish, + "tool_call_count": 1, + "terminal_frame": terminal, + "tool_name": TOOL_NAME, + "tool_arguments": arguments, + "parity_with": "openai_tool_xml", + }, call + + +def _openai_followup_case(request_json, url, timeout, model, call): + _require(isinstance(call, dict), "non-streaming XML tool case did not pass") + payload = { + "model": model, + "messages": [ + {"role": "user", "content": TOOL_PROMPT}, + {"role": "assistant", "content": None, "tool_calls": [call]}, + { + "role": "tool", + "tool_call_id": call["id"], + "content": json.dumps( + { + "validation_token": "K2-SUNNY-731", + "forecast": "sunny", + }, + separators=(",", ":"), + ), + }, + { + "role": "user", + "content": ( + "Return only the validation_token from the tool result. " + "Do not call a tool." + ), + }, + ], + "tools": [TOOL_DEFINITION], + "tool_choice": "none", + "max_completion_tokens": 128, + "temperature": 0, + "top_p": 1, + "seed": 42, + "stream": False, + "chat_template_kwargs": {"enable_thinking": False}, + } + body, status = _post_json(request_json, url, timeout, payload) + message = _openai_message(body, "stop") + content = _visible_content(message, "K2-SUNNY-731") + _require_no_tool_calls(message) + _require_no_reasoning(message) + return { + "status_code": status, + "finish_reason": "stop", + "content_chars": len(content), + }, message + + +def _ollama_case(request_json, root_url, timeout, model): + payload = { + "model": model, + "messages": [{"role": "user", "content": ARITHMETIC_PROMPT}], + "think": True, + "stream": False, + "options": { + "num_predict": 3072, + "temperature": 0, + "top_p": 1, + "seed": 42, + }, + } + body, status = _post_json(request_json, f"{root_url}/api/chat", timeout, payload) + _require(body.get("done") is True, "Ollama response is not complete") + _require(body.get("done_reason") == "stop", "Ollama response did not stop") + message = body.get("message") + _require(isinstance(message, dict), "Ollama message must be an object") + _require(message.get("role") == "assistant", "Ollama role is not assistant") + content = _visible_content(message, "42") + thinking = message.get("thinking") + _require(isinstance(thinking, str) and thinking.strip(), "Ollama thinking is blank") + _require_no_tool_calls(message) + return { + "status_code": status, + "finish_reason": "stop", + "content_chars": len(content), + "reasoning_chars": len(thinking), + }, message + + +def _anthropic_case(request_json, root_url, timeout, model): + payload = { + "model": model, + "system": [{"type": "text", "text": "Be concise."}], + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "Reply with ANTHROPIC-OK."}], + } + ], + "max_tokens": 128, + "temperature": 0, + "stream": False, + "thinking": {"type": "disabled"}, + } + body, status = _post_json( + request_json, f"{root_url}/v1/messages?beta=true", timeout, payload + ) + _require(body.get("type") == "message", "Anthropic type is not message") + _require(body.get("role") == "assistant", "Anthropic role is not assistant") + _require(isinstance(body.get("id"), str) and body["id"], "Anthropic id is missing") + _require( + isinstance(body.get("model"), str) and body["model"], + "Anthropic model is missing", + ) + _require(body.get("stop_reason") == "end_turn", "Anthropic response did not end") + blocks = body.get("content") + _require(isinstance(blocks, list) and blocks, "Anthropic content is empty") + _require( + not any( + isinstance(block, dict) and block.get("type") == "tool_use" + for block in blocks + ), + "ordinary Anthropic response contains tool use", + ) + text = "".join( + block.get("text", "") + for block in blocks + if isinstance(block, dict) and block.get("type") == "text" + ) + _require("ANTHROPIC-OK" in text, "Anthropic text does not contain sentinel") + usage = body.get("usage") + _require(isinstance(usage, dict), "Anthropic usage is missing") + for field in ("input_tokens", "output_tokens"): + value = usage.get(field) + _require( + isinstance(value, int) and not isinstance(value, bool) and value >= 0, + f"Anthropic {field} is invalid", + ) + return { + "status_code": status, + "finish_reason": "end_turn", + "content_chars": len(text), + }, body + + +def validate_capabilities( + profile: str, + *, + base_url: str, + model: str, + request_json: Callable[..., tuple[object, object]], + request_stream: Callable[..., tuple[object, Iterable[object]]], + timeout: int, +) -> tuple[dict, bool, str]: + """Run a deterministic capability profile and return compact evidence.""" + if profile != K2_HORIZON_PROFILE: + raise CapabilityValidationError(f"Unsupported capability profile: {profile}") + if not base_url.endswith("/api/v1"): + raise CapabilityValidationError("base_url must end with /api/v1") + if not isinstance(model, str) or not model: + raise CapabilityValidationError("model must be a nonempty string") + + root_url = base_url[: -len("/api/v1")] + openai_url = f"{base_url}/chat/completions" + cases: list[dict] = [] + + off_payload = _base_openai_payload(model, stream=False, max_tokens=256) + off_payload["chat_template_kwargs"] = {"enable_thinking": False} + _run_case( + cases, + "openai_plain_off_nonstream", + "openai", + False, + lambda: _openai_plain_case(request_json, openai_url, timeout, off_payload), + ) + + off_stream_payload = _base_openai_payload(model, stream=True, max_tokens=256) + off_stream_payload["stream_options"] = {"include_usage": True} + off_stream_payload["chat_template_kwargs"] = {"enable_thinking": False} + _run_case( + cases, + "openai_plain_off_stream", + "openai", + True, + lambda: _openai_plain_stream_case( + request_stream, openai_url, timeout, off_stream_payload + ), + ) + + for effort in ("high", "medium", "low"): + payload = _base_openai_payload(model, stream=False, max_tokens=3072) + payload["chat_template_kwargs"] = {"reasoning_effort": effort} + _run_case( + cases, + f"openai_reasoning_{effort}", + "openai", + False, + lambda payload=payload: _openai_reasoning_case( + request_json, openai_url, timeout, payload + ), + ) + if effort == "high": + stream_payload = _base_openai_payload( + model, + stream=True, + max_tokens=3072, + ) + stream_payload["stream_options"] = {"include_usage": True} + stream_payload["chat_template_kwargs"] = {"reasoning_effort": effort} + _run_case( + cases, + "openai_reasoning_high_stream", + "openai", + True, + lambda: _openai_reasoning_stream_case( + request_stream, + openai_url, + timeout, + stream_payload, + ), + ) + + xml_call = None + for tool_format in ("json", "xml", "xml_typed"): + call = _run_case( + cases, + f"openai_tool_{tool_format}", + "openai", + False, + lambda tool_format=tool_format: _openai_tool_case( + request_json, + openai_url, + timeout, + _tool_payload(model, tool_format, stream=False), + ), + ) + if tool_format == "xml": + xml_call = call + + _run_case( + cases, + "openai_tool_default_xml", + "openai", + False, + lambda: _openai_tool_parity_case( + request_json, + openai_url, + timeout, + _tool_payload(model, None, stream=False), + xml_call, + "openai_tool_xml", + ), + ) + + _run_case( + cases, + "openai_tool_stream_xml", + "openai", + True, + lambda: _openai_tool_stream_case( + request_stream, + openai_url, + timeout, + _tool_payload(model, "xml", stream=True), + xml_call, + ), + ) + _run_case( + cases, + "openai_tool_result_followup", + "openai", + False, + lambda: _openai_followup_case( + request_json, openai_url, timeout, model, xml_call + ), + ) + _run_case( + cases, + "ollama_thinking", + "ollama", + False, + lambda: _ollama_case(request_json, root_url, timeout, model), + ) + _run_case( + cases, + "anthropic_translation", + "anthropic", + False, + lambda: _anthropic_case(request_json, root_url, timeout, model), + ) + + passed = all(case["pass"] for case in cases) + matrix = { + "profile": profile, + "model": model, + "pass": passed, + "cases": cases, + } + if passed: + summary = f"{len(cases)}/{len(cases)} capability cases passed" + else: + failed = [case["id"] for case in cases if not case["pass"]] + details = "; ".join( + f"{case['id']}: {case['error']}" for case in cases if not case["pass"] + ) + summary = f"Failed capability cases {', '.join(failed)}: {details}" + return matrix, passed, summary[:1024] + + +def validate_capability_matrix_evidence( + value: object, + expected_profile: str, + expected_model: str, +) -> None: + """Validate one compact capability evidence envelope.""" + _require( + expected_profile == K2_HORIZON_PROFILE, + f"Unsupported capability profile: {expected_profile}", + ) + _require(isinstance(value, dict), "capability matrix must be an object") + _require( + set(value) == {"profile", "model", "pass", "cases"}, + "capability matrix has unexpected fields", + ) + _require(value["profile"] == expected_profile, "capability profile mismatch") + _require( + isinstance(expected_model, str) and expected_model, "expected model is missing" + ) + _require_response_model(value, expected_model, "capability matrix") + _require(value["pass"] is True, "capability matrix did not pass") + cases = value["cases"] + _require(isinstance(cases, list), "capability cases must be an array") + _require( + [case.get("id") if isinstance(case, dict) else None for case in cases] + == list(K2_HORIZON_CASE_IDS), + "capability cases do not match the profile contract", + ) + + for case in cases: + _require(set(case) == CASE_FIELDS, f"{case['id']}: unexpected case fields") + _require(case["pass"] is True, f"{case['id']}: case did not pass") + _require(case["error"] is None, f"{case['id']}: case has an error") + _require( + case["raw_ifm_marker"] is None, + f"{case['id']}: raw IFM marker was recorded", + ) + _require( + type(case["status_code"]) is int and case["status_code"] == 200, + f"{case['id']}: HTTP status is not 200", + ) + _require( + case["terminal_frame"] is True, + f"{case['id']}: response is not terminal", + ) + for field in ("content_chars", "reasoning_chars", "tool_call_count"): + value_field = case[field] + _require( + isinstance(value_field, int) + and not isinstance(value_field, bool) + and value_field >= 0, + f"{case['id']}: {field} is invalid", + ) + ( + protocol, + stream, + finish_reason, + has_content, + has_reasoning, + tool_call_count, + parity_with, + ) = CASE_CONTRACTS[case["id"]] + _require(case["protocol"] == protocol, f"{case['id']}: protocol mismatch") + _require(case["stream"] is stream, f"{case['id']}: stream flag mismatch") + _require( + case["finish_reason"] == finish_reason, + f"{case['id']}: finish reason mismatch", + ) + _require( + (case["content_chars"] > 0) is has_content, + f"{case['id']}: content contract mismatch", + ) + _require( + (case["reasoning_chars"] > 0) is has_reasoning, + f"{case['id']}: reasoning contract mismatch", + ) + _require( + case["tool_call_count"] == tool_call_count, + f"{case['id']}: tool-call count mismatch", + ) + _require( + case["parity_with"] == parity_with, + f"{case['id']}: parity binding mismatch", + ) + if tool_call_count: + _require( + case["tool_name"] == TOOL_NAME, f"{case['id']}: tool name mismatch" + ) + _validate_tool_arguments(case["tool_arguments"]) + else: + _require(case["tool_name"] is None, f"{case['id']}: unexpected tool name") + _require( + case["tool_arguments"] is None, + f"{case['id']}: unexpected tool arguments", + ) diff --git a/test/utils/llamacpp_release_manifest.py b/test/utils/llamacpp_release_manifest.py index 2e54aed433..4fe52094b4 100644 --- a/test/utils/llamacpp_release_manifest.py +++ b/test/utils/llamacpp_release_manifest.py @@ -10,6 +10,7 @@ from typing import Iterable SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-fA-F]{64}$") +GIT_OID_RE = re.compile(r"^[0-9a-fA-F]{40}$") class ReleaseManifestError(ValueError): @@ -39,6 +40,11 @@ def _canonical_release( f"got {actual_tag!r}" ) release_id = _positive_integer(payload.get("id"), f"{repository}: release id") + source_commit = payload.get("source_commit") + if not isinstance(source_commit, str) or not GIT_OID_RE.fullmatch(source_commit): + raise ReleaseManifestError( + f"{repository}: source commit must be a full 40-character Git OID" + ) raw_assets = payload.get("assets") if not isinstance(raw_assets, list) or not raw_assets: raise ReleaseManifestError(f"{repository}: release assets must be nonempty") @@ -90,6 +96,7 @@ def _canonical_release( "assets": assets, "release_id": release_id, "repository": repository, + "source_commit": source_commit.lower(), "tag_name": expected_tag, } @@ -108,7 +115,7 @@ def build_release_asset_manifest( raise ReleaseManifestError("release manifest contains duplicate repositories") canonical_releases.sort(key=lambda release: release["repository"]) return json.dumps( - {"releases": canonical_releases, "schema_version": 1}, + {"releases": canonical_releases, "schema_version": 2}, separators=(",", ":"), sort_keys=True, ) @@ -121,8 +128,8 @@ def _canonicalize_manifest(manifest_json: str) -> str: raise ReleaseManifestError( f"release manifest is not valid JSON: {exc}" ) from exc - if not isinstance(document, dict) or document.get("schema_version") != 1: - raise ReleaseManifestError("release manifest schema_version must be 1") + if not isinstance(document, dict) or document.get("schema_version") != 2: + raise ReleaseManifestError("release manifest schema_version must be 2") releases = document.get("releases") if not isinstance(releases, list): raise ReleaseManifestError("release manifest releases must be an array") @@ -131,7 +138,13 @@ def _canonicalize_manifest(manifest_json: str) -> str: for release in releases: if not isinstance(release, dict): raise ReleaseManifestError("release manifest entry must be an object") - expected_keys = {"assets", "release_id", "repository", "tag_name"} + expected_keys = { + "assets", + "release_id", + "repository", + "source_commit", + "tag_name", + } if set(release) != expected_keys: raise ReleaseManifestError("release manifest entry has unexpected fields") inputs.append( @@ -141,6 +154,7 @@ def _canonicalize_manifest(manifest_json: str) -> str: { "assets": release["assets"], "id": release["release_id"], + "source_commit": release["source_commit"], "tag_name": release["tag_name"], }, ) diff --git a/test/utils/llamacpp_validation_evidence.py b/test/utils/llamacpp_validation_evidence.py index e75d103449..24e52184bb 100644 --- a/test/utils/llamacpp_validation_evidence.py +++ b/test/utils/llamacpp_validation_evidence.py @@ -5,10 +5,24 @@ import argparse import json +import re from pathlib import Path +if __package__: + from .llamacpp_capability_validation import ( + K2_HORIZON_PROFILE, + CapabilityValidationError, + validate_capability_matrix_evidence, + ) +else: + from llamacpp_capability_validation import ( # type: ignore[import-not-found] + K2_HORIZON_PROFILE, + CapabilityValidationError, + validate_capability_matrix_evidence, + ) + EXPECTED_RESULT_FILES = ( - "llamacpp_validation_vulkan.json", + "llamacpp_validation_windows-vulkan.json", "llamacpp_validation_rocm-stable.json", "llamacpp_validation_rocm-nightly.json", ) @@ -28,6 +42,15 @@ class ValidationEvidenceError(ValueError): """Raised when scheduled validation evidence is incomplete or failed.""" +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ValidationEvidenceError(f"duplicate JSON key: {key}") + result[key] = value + return result + + def _validate_record(filename: str, index: int, record: object) -> dict: if not isinstance(record, dict): raise ValidationEvidenceError(f"{filename}: result {index} must be an object") @@ -50,7 +73,7 @@ def _validate_record(filename: str, index: int, record: object) -> dict: if not passed: raise ValidationEvidenceError(f"{filename}: {model} did not pass validation") response = record["response"] - if not isinstance(response, str) or not response: + if not isinstance(response, str) or not response.strip(): raise ValidationEvidenceError( f"{filename}: result {index} response must be nonempty" ) @@ -79,7 +102,11 @@ def _validate_record(filename: str, index: int, record: object) -> dict: return record -def _expected_models_by_file(expected_matrix: object) -> dict[str, list[str]]: +def _canonical_model_id(model_id: str) -> str: + return model_id.removeprefix("builtin.") + + +def _expected_models_by_file(expected_matrix: object) -> dict[str, dict]: if not isinstance(expected_matrix, dict): raise ValidationEvidenceError("validation matrix must be an object") rows = expected_matrix.get("include") @@ -94,17 +121,21 @@ def _expected_models_by_file(expected_matrix: object) -> dict[str, list[str]]: ) backend = row.get("backend") channel = row.get("channel") + target = row.get("target") if not isinstance(backend, str) or not isinstance(channel, str): raise ValidationEvidenceError( f"validation matrix row {index} backend and channel must be strings" ) - label = f"{backend}-{channel}" if channel else backend - filename = f"llamacpp_validation_{label}.json" + if not isinstance(target, str) or not re.fullmatch(r"[a-z0-9-]+", target): + raise ValidationEvidenceError( + f"validation matrix row {index} target must be a safe nonempty label" + ) + filename = f"llamacpp_validation_{target}.json" if filename in expected: - raise ValidationEvidenceError(f"duplicate validation lane: {label}") + raise ValidationEvidenceError(f"duplicate validation lane: {target}") if row.get("models") != []: raise ValidationEvidenceError( - f"scheduled validation lane {label} must keep runtime hot selection" + f"scheduled validation lane {target} must keep runtime hot selection" ) models = row.get("expected_models") if ( @@ -114,23 +145,70 @@ def _expected_models_by_file(expected_matrix: object) -> dict[str, list[str]]: or len(models) != len(set(models)) ): raise ValidationEvidenceError( - f"validation matrix lane {label} needs unique expected models" + f"validation matrix lane {target} needs unique expected models" ) if K2_SMALL not in models: raise ValidationEvidenceError( - f"validation matrix lane {label} must include {K2_SMALL}" + f"validation matrix lane {target} must include {K2_SMALL}" + ) + has_capability_profile = "capability_profile" in row + has_capability_models = "capability_models" in row + if has_capability_profile != has_capability_models: + raise ValidationEvidenceError( + f"validation matrix lane {target} must pair capability profile and models" + ) + if not has_capability_profile: + raise ValidationEvidenceError( + f"scheduled validation lane {target} needs capability evidence" + ) + capability_profile = row["capability_profile"] + capability_models = row["capability_models"] + if capability_profile != K2_HORIZON_PROFILE: + raise ValidationEvidenceError( + f"validation matrix lane {target} has unsupported capability profile" + ) + if ( + not isinstance(capability_models, list) + or not capability_models + or not all(isinstance(model, str) and model for model in capability_models) + or len(capability_models) != len(set(capability_models)) + ): + raise ValidationEvidenceError( + f"validation matrix lane {target} needs unique capability models" + ) + if any(model not in models for model in capability_models): + raise ValidationEvidenceError( + f"validation matrix lane {target} has an unselected capability model" + ) + if K2_SMALL not in capability_models: + raise ValidationEvidenceError( + f"validation matrix lane {target} must validate {K2_SMALL} capabilities" ) - expected[filename] = models + expected[filename] = { + "models": models, + "capability_profile": capability_profile, + "capability_models": capability_models, + } if set(expected) != set(EXPECTED_RESULT_FILES): raise ValidationEvidenceError( "validation matrix must contain exactly the scheduled validation lanes" ) - planned_model_lists = list(expected.values()) + planned_model_lists = [lane["models"] for lane in expected.values()] if any(models != planned_model_lists[0] for models in planned_model_lists[1:]): raise ValidationEvidenceError( "scheduled validation lanes must use the same planned model set" ) + planned_capability_models = [ + lane["capability_models"] for lane in expected.values() + ] + if any( + models != planned_capability_models[0] + for models in planned_capability_models[1:] + ): + raise ValidationEvidenceError( + "scheduled validation lanes must use the same capability model set" + ) return expected @@ -138,7 +216,7 @@ def load_and_validate_results( root: Path, expected_matrix: object, ) -> dict[str, list[dict]]: - expected_models = _expected_models_by_file(expected_matrix) + expected_lanes = _expected_models_by_file(expected_matrix) validated = {} for filename in EXPECTED_RESULT_FILES: path = root / filename @@ -151,7 +229,7 @@ def load_and_validate_results( if not raw.strip(): raise ValidationEvidenceError(f"validation result is empty: {filename}") try: - records = json.loads(raw) + records = json.loads(raw, object_pairs_hook=_unique_json_object) except json.JSONDecodeError as exc: raise ValidationEvidenceError( f"validation result is invalid JSON: {filename}: {exc}" @@ -169,10 +247,37 @@ def load_and_validate_results( raise ValidationEvidenceError( f"validation result contains duplicate models: {filename}" ) - if models != expected_models[filename]: + expected_lane = expected_lanes[filename] + if models != expected_lane["models"]: raise ValidationEvidenceError( f"{filename}: result models do not match planned models" ) + capability_models = { + _canonical_model_id(model) for model in expected_lane["capability_models"] + } + for record in checked: + model_id = _canonical_model_id(record["model"]) + if model_id not in capability_models: + if "capability_matrix" in record: + raise ValidationEvidenceError( + f"{filename}: {record['model']} has an unexpected capability matrix" + ) + continue + if "capability_matrix" not in record: + raise ValidationEvidenceError( + f"{filename}: {record['model']} is missing capability evidence" + ) + capability_matrix = record["capability_matrix"] + try: + validate_capability_matrix_evidence( + capability_matrix, + expected_lane["capability_profile"], + record["model"], + ) + except CapabilityValidationError as exc: + raise ValidationEvidenceError( + f"{filename}: {record['model']} capability evidence is invalid: {exc}" + ) from exc validated[filename] = checked return validated @@ -183,7 +288,10 @@ def main() -> None: parser.add_argument("--expected-matrix-json", required=True) args = parser.parse_args() try: - expected_matrix = json.loads(args.expected_matrix_json) + expected_matrix = json.loads( + args.expected_matrix_json, + object_pairs_hook=_unique_json_object, + ) results = load_and_validate_results(args.directory, expected_matrix) except (json.JSONDecodeError, ValidationEvidenceError) as exc: parser.exit(1, f"ERROR: {exc}\n") diff --git a/test/utils/llamacpp_validation_plan.py b/test/utils/llamacpp_validation_plan.py index 6ad30c6cf6..da0d5c5eb1 100644 --- a/test/utils/llamacpp_validation_plan.py +++ b/test/utils/llamacpp_validation_plan.py @@ -74,14 +74,18 @@ def load_hot_llamacpp_model_ids(path: Path = MODEL_REGISTRY) -> list[str]: def _row( + target: str, backend: str, channel: str, runner: list[str], models: list[str], lite: bool, expected_models: list[str] | None = None, + capability_profile: str = "", + capability_models: list[str] | None = None, ) -> dict[str, object]: row = { + "target": target, "backend": backend, "channel": channel, "runner": list(runner), @@ -90,6 +94,9 @@ def _row( } if expected_models is not None: row["expected_models"] = list(expected_models) + if capability_profile: + row["capability_profile"] = capability_profile + row["capability_models"] = list(capability_models or []) return row @@ -111,14 +118,35 @@ def create_validation_plan( return { "include": [ _row( + "windows-vulkan", "vulkan", "", LARGE_VULKAN_RUNNER, [K2_SMALL, K2_MEDIUM, K2_LARGE], False, + capability_profile="k2-horizon-v1", + capability_models=[K2_SMALL], + ), + _row( + "rocm-stable", + "rocm", + "stable", + SMALL_ROCM_RUNNER, + [K2_SMALL], + False, + capability_profile="k2-horizon-v1", + capability_models=[K2_SMALL], + ), + _row( + "rocm-nightly", + "rocm", + "nightly", + SMALL_ROCM_RUNNER, + [K2_SMALL], + False, + capability_profile="k2-horizon-v1", + capability_models=[K2_SMALL], ), - _row("rocm", "stable", SMALL_ROCM_RUNNER, [K2_SMALL], False), - _row("rocm", "nightly", SMALL_ROCM_RUNNER, [K2_SMALL], False), ] } @@ -127,24 +155,42 @@ def create_validation_plan( expected_models = ( load_hot_llamacpp_model_ids() if event_name == "schedule" else None ) + capability_profile = "k2-horizon-v1" if event_name == "schedule" else "" + capability_models = [K2_SMALL] if event_name == "schedule" else None return { "include": [ - _row("vulkan", "", runner_vulkan, models, lite, expected_models), _row( + "windows-vulkan", + "vulkan", + "", + runner_vulkan, + models, + lite, + expected_models, + capability_profile, + capability_models, + ), + _row( + "rocm-stable", "rocm", "stable", runner_rocm, models, lite, expected_models, + capability_profile, + capability_models, ), _row( + "rocm-nightly", "rocm", "nightly", runner_rocm, models, lite, expected_models, + capability_profile, + capability_models, ), ] } diff --git a/test/validate_llamacpp.py b/test/validate_llamacpp.py index a5dd70cf23..5211f14c5b 100644 --- a/test/validate_llamacpp.py +++ b/test/validate_llamacpp.py @@ -14,7 +14,8 @@ llama.cpp models or all models with recipe `llamacpp` and label `hot` 2. Installs the requested llamacpp backend via `POST /api/v1/install` 3. For each selected model, loads it with the requested backend, sends a - `chat/completions` request, queries `/api/v1/stats`, and unloads it + `chat/completions` request, optionally runs an API capability profile, + queries `/api/v1/stats`, and unloads it 4. Outputs a JSON results file for CI consumption """ @@ -29,6 +30,12 @@ import requests +from utils.llamacpp_capability_validation import ( + K2_HORIZON_PROFILE, + find_raw_ifm_control_marker, + reject_duplicate_json_object, + validate_capabilities, +) from utils.server_base import _auth_headers, unload_all_models, wait_for_server from utils.test_models import PORT, TIMEOUT_DEFAULT from utils.validation_model_selection import ( @@ -39,39 +46,10 @@ TIMEOUT_HEALTH = 60 TIMEOUT_INFERENCE = 1800 # 30 minutes; large models may need 60+ GB download -VALIDATION_CTX_SIZE = 4096 +VALIDATION_CTX_SIZE = 8192 CHAT_PROMPT = [ {"role": "user", "content": "What is 2+2? Reply in one sentence."}, ] -IFM_CONTROL_PREFIXES = ("", start) - if end < 0: - end = min(start + 79, len(candidate) - 1) - return candidate[start : end + 1], path - elif isinstance(candidate, dict): - items = list(candidate.items()) - for key, nested_value in reversed(items): - pending.append((f"{path}.{key}", nested_value)) - elif isinstance(candidate, (list, tuple)): - for index in range(len(candidate) - 1, -1, -1): - pending.append((f"{path}[{index}]", candidate[index])) - return None def collect_server_logs(output_dir): @@ -105,12 +83,28 @@ def request_json(method, url, timeout, **kwargs): body = {} if response.content: try: - body = response.json() + body = response.json(object_pairs_hook=reject_duplicate_json_object) except ValueError: body = {"raw_text": response.text} return response, body +def request_stream(method, url, timeout, **kwargs): + """Perform a streaming HTTP request and return decoded response lines.""" + auth_headers = _auth_headers() + headers = {**(kwargs.get("headers") or {}), **auth_headers} + request_kwargs = {**kwargs, "headers": headers, "stream": True} + response = requests.request(method, url, timeout=timeout, **request_kwargs) + + def lines(): + try: + yield from response.iter_lines(decode_unicode=True) + finally: + response.close() + + return response, lines() + + def require_running_server(base_url, port): """Wait for a running server and confirm the health endpoint responds.""" wait_for_server(port=port, timeout=TIMEOUT_HEALTH) @@ -216,7 +210,15 @@ def unload_model(base_url, model_name=None): ) -def test_model(base_url, model_name, backend, max_tokens=50): +def test_model( + base_url, + model_name, + backend, + max_tokens=50, + *, + capability_profile="", + capability_evidence=None, +): """Send a chat/completions request and return (success, response_text, stats).""" print(f" Loading model: {model_name} (backend={backend})", flush=True) try: @@ -242,22 +244,62 @@ def test_model(base_url, model_name, backend, max_tokens=50): f"{base_url}/health", timeout=TIMEOUT_DEFAULT, ) - if health_resp.status_code == 200: - loaded = { - model["model_name"]: model - for model in health_body.get("all_models_loaded", []) - } - loaded_model = loaded.get(model_name, {}) - if not loaded_model and model_name.startswith("builtin."): - loaded_model = loaded.get(model_name.removeprefix("builtin."), {}) - recipe_options = loaded_model.get("recipe_options", {}) - actual_backend = recipe_options.get("llamacpp_backend") - if actual_backend and actual_backend != backend: - return ( - False, - f"Model loaded with backend '{actual_backend}' instead of '{backend}'", - {}, - ) + if health_resp.status_code != 200: + return ( + False, + f"Loaded-model health inventory returned HTTP " + f"{health_resp.status_code}: {health_body}", + {}, + ) + if not isinstance(health_body, dict): + return False, "Loaded-model health response is not an object", {} + if "all_models_loaded" not in health_body: + return ( + False, + "Loaded-model health response is missing all_models_loaded", + {}, + ) + loaded_models = health_body["all_models_loaded"] + if not isinstance(loaded_models, list) or not all( + isinstance(model, dict) and isinstance(model.get("model_name"), str) + for model in loaded_models + ): + return ( + False, + "Loaded-model health response has an invalid model inventory", + {}, + ) + + aliases = {model_name} + if model_name.startswith("builtin."): + aliases.add(model_name.removeprefix("builtin.")) + else: + aliases.add(f"builtin.{model_name}") + loaded_model = next( + (model for model in loaded_models if model["model_name"] in aliases), + None, + ) + if loaded_model is None: + return ( + False, + f"Model '{model_name}' is absent from the loaded inventory", + {}, + ) + + if "recipe_options" not in loaded_model: + return False, f"Loaded model '{model_name}' is missing recipe_options", {} + recipe_options = loaded_model["recipe_options"] + if not isinstance(recipe_options, dict): + return False, f"Loaded model '{model_name}' has invalid recipe_options", {} + actual_backend = recipe_options.get("llamacpp_backend") + if not isinstance(actual_backend, str) or not actual_backend: + return False, f"Loaded model '{model_name}' is missing llamacpp_backend", {} + if actual_backend != backend: + return ( + False, + f"Model loaded with backend '{actual_backend}' instead of '{backend}'", + {}, + ) print(" Sending chat/completions request...", flush=True) chat_resp, chat_body = request_json( @@ -284,11 +326,26 @@ def test_model(base_url, model_name, backend, max_tokens=50): {}, ) - content = message.get("content") or "" - reasoning = message.get("reasoning_content") or "" - combined = content + reasoning - if not combined: - return False, "Empty response (no content or reasoning_content)", {} + content = message.get("content") + if not isinstance(content, str) or not content.strip(): + return False, "Plain chat response has no visible final content", {} + + if capability_profile: + if not isinstance(capability_evidence, dict): + return False, "Capability evidence output is required", {} + matrix, capabilities_passed, capability_summary = validate_capabilities( + capability_profile, + base_url=base_url, + model=model_name, + request_json=request_json, + request_stream=request_stream, + timeout=TIMEOUT_INFERENCE, + ) + capability_evidence.clear() + capability_evidence.update(matrix) + if not capabilities_passed: + return False, f"Capability validation failed: {capability_summary}", {} + print(f" Capabilities: {capability_summary}", flush=True) stats = {} stats_resp, stats_body = request_json( @@ -305,8 +362,7 @@ def test_model(base_url, model_name, backend, max_tokens=50): flush=True, ) - response_text = content if content else f"[reasoning] {reasoning}" - return True, response_text, stats + return True, content, stats except (KeyError, IndexError, TypeError, ValueError) as exc: return False, f"Bad response format: {exc}", {} except requests.RequestException as exc: @@ -323,9 +379,18 @@ def validate_model_lifecycle( model_name, backend, reload_after_unload=False, + *, + capability_profile="", + capability_evidence=None, ): """Validate one model and optionally validate a second load after unload.""" - result = test_model(base_url, model_name, backend) + capability_args = {} + if capability_profile: + capability_args = { + "capability_profile": capability_profile, + "capability_evidence": capability_evidence, + } + result = test_model(base_url, model_name, backend, **capability_args) if not reload_after_unload or not result[0]: return result @@ -358,6 +423,8 @@ def validate_model_lifecycle( aliases = {model_name} if model_name.startswith("builtin."): aliases.add(model_name.removeprefix("builtin.")) + else: + aliases.add(f"builtin.{model_name}") if any(model["model_name"] in aliases for model in loaded_models): return False, f"Model '{model_name}' is still loaded after unload", result[2] @@ -409,9 +476,34 @@ def main(): default=None, help="Directory to collect server log files into (for CI artifact upload)", ) + parser.add_argument( + "--capability-profile", + choices=[K2_HORIZON_PROFILE], + default="", + help="Optional API capability contract to validate", + ) + parser.add_argument( + "--capability-model", + action="append", + default=[], + help="Selected validation model that must run the capability profile", + ) add_model_selection_arguments(parser) args = parser.parse_args() + has_capability_profile = bool(args.capability_profile) + has_capability_models = bool(args.capability_model) + if has_capability_profile != has_capability_models: + parser.error( + "--capability-profile and --capability-model must be supplied together" + ) + + canonical_capability_models = [ + model_id.removeprefix("builtin.") for model_id in args.capability_model + ] + if len(set(canonical_capability_models)) != len(canonical_capability_models): + parser.error("--capability-model values must be unique") + # Label used for output filenames and artifact names label = f"{args.backend}-{args.channel}" if args.channel else args.backend @@ -434,6 +526,24 @@ def main(): print(f"ERROR: {exc}", file=sys.stderr, flush=True) sys.exit(1) + selected_model_ids = { + candidate.removeprefix("builtin.") + for model in selected_models + for candidate in (model["id"], model.get("load_id", model["id"])) + } + missing_capability_models = sorted( + set(canonical_capability_models) - selected_model_ids + ) + if missing_capability_models: + print( + "ERROR: Capability models are absent from the selected validation set: " + + ", ".join(missing_capability_models), + file=sys.stderr, + flush=True, + ) + sys.exit(1) + capability_model_ids = set(canonical_capability_models) + if args.channel: set_rocm_channel(base_url, args.channel) @@ -468,11 +578,19 @@ def main(): for model in selected_models: model_name = model["id"] print(f"\nTesting: {model_name}", flush=True) + runs_capability_profile = ( + model_name.removeprefix("builtin.") in capability_model_ids + ) + capability_evidence = {} if runs_capability_profile else None success, response_text, stats = validate_model_lifecycle( base_url, model.get("load_id", model_name), args.backend, reload_after_unload=bool(args.model), + capability_profile=( + args.capability_profile if runs_capability_profile else "" + ), + capability_evidence=capability_evidence, ) result = { "model": model_name, @@ -483,6 +601,8 @@ def main(): "time_to_first_token": stats.get("time_to_first_token", "N/A"), "tokens_per_second": stats.get("tokens_per_second", "N/A"), } + if runs_capability_profile: + result["capability_matrix"] = capability_evidence results.append(result) status = "PASS" if success else "FAIL" print(f" Result: {status}", flush=True) From 7817bd3119e6cad0ea1c87f4736b79cf7e897402 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Sun, 6 Sep 2026 02:55:35 -0700 Subject: [PATCH 05/14] fix: harden llama.cpp release provenance --- .../capture_llamacpp_release_manifest.sh | 152 ++++ .github/scripts/publish_llamacpp_update.sh | 145 +++- .../scripts/verify_llamacpp_pin_manifest.sh | 72 ++ .github/workflows/docs_and_style.yml | 1 + .github/workflows/validate_llamacpp_core.yml | 159 +++- .../workflows/validate_llamacpp_schedule.yml | 41 +- docs/guide/configuration/llamacpp.md | 5 +- test/test_llamacpp_release_manifest.py | 797 +++++++++++++++++- test/test_publish_llamacpp_update.py | 436 +++++++++- test/test_validate_llamacpp_selection.py | 36 +- test/test_validate_llamacpp_workflow_plan.py | 135 ++- test/test_verify_llamacpp_pin_manifest.py | 255 ++++++ test/utils/llamacpp_release_manifest.py | 774 ++++++++++++++++- test/validate_llamacpp.py | 1 + 14 files changed, 2883 insertions(+), 126 deletions(-) create mode 100755 .github/scripts/capture_llamacpp_release_manifest.sh create mode 100755 .github/scripts/verify_llamacpp_pin_manifest.sh create mode 100644 test/test_verify_llamacpp_pin_manifest.py diff --git a/.github/scripts/capture_llamacpp_release_manifest.sh b/.github/scripts/capture_llamacpp_release_manifest.sh new file mode 100755 index 0000000000..de96faf087 --- /dev/null +++ b/.github/scripts/capture_llamacpp_release_manifest.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 EXPECTED_MANIFEST_PATH SNAPSHOT_DIRECTORY" >&2 + exit 2 +fi + +expected_manifest_path=$1 +snapshot_directory=$2 +manifest_tool=${LLAMACPP_RELEASE_MANIFEST_TOOL:-test/utils/llamacpp_release_manifest.py} +trusted_source_repository=ggml-org/llama.cpp +source_manifest_max_bytes=65536 +source_manifest_directory=$(mktemp -d) + +mkdir -p "$snapshot_directory" + +canonical_expected_path="${snapshot_directory}/expected.json" +python "$manifest_tool" \ + --materialize-manifest "$canonical_expected_path" \ + --manifest-json "$(<"$expected_manifest_path")" >/dev/null + +source_repository=$(jq -er '.releases[0].source_repository' "$canonical_expected_path") +upstream_reference=$(jq -er '.releases[0].upstream_reference' "$canonical_expected_path") +upstream_reference_head=$(jq -er \ + '.releases[0].upstream_reference_head' "$canonical_expected_path") +if [[ "$source_repository" != "$trusted_source_repository" ]]; then + echo "Manifest source repository must be ${trusted_source_repository}." >&2 + exit 1 +fi + +tag_for_repository() { + jq -er --arg repository "$1" \ + '.releases | map(select(.repository == $repository)) + | if length == 1 then .[0].tag_name else error("missing release") end' \ + "$canonical_expected_path" +} + +ggml_release=$(tag_for_repository ggml-org/llama.cpp) +rocm_release=$(tag_for_repository lemonade-sdk/llamacpp-rocm) +lemonade_release=$(tag_for_repository lemonade-sdk/llama.cpp) + +release_pattern='^b[0-9]+$' +for release in "$ggml_release" "$rocm_release" "$lemonade_release"; do + if [[ ! "$release" =~ $release_pattern ]]; then + echo "Manifest release tag must match bNNNN: ${release}" >&2 + exit 1 + fi +done + +source_default_branch=$(gh api "repos/${trusted_source_repository}" \ + --jq '.default_branch') +if [[ -z "$source_default_branch" ]]; then + echo "Could not resolve the trusted source repository default branch." >&2 + exit 1 +fi +trusted_upstream_reference="refs/heads/${source_default_branch}" +if [[ "$upstream_reference" != "$trusted_upstream_reference" ]]; then + echo "Manifest upstream reference is not the trusted default branch." >&2 + exit 1 +fi +live_upstream_head=$(gh api \ + "repos/${trusted_source_repository}/commits/${source_default_branch}" --jq '.sha') +if ! [[ "$live_upstream_head" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "Could not resolve the trusted upstream reference to a full commit." >&2 + exit 1 +fi + +anchor_comparison_path="${snapshot_directory}/upstream-anchor-comparison.json" +gh api \ + "repos/${trusted_source_repository}/compare/${upstream_reference_head}...${live_upstream_head}" \ + >"$anchor_comparison_path" +python "$manifest_tool" \ + --require-upstream-ancestry "$anchor_comparison_path" \ + "$trusted_source_repository" "$upstream_reference_head" "$live_upstream_head" + +ggml_path="${snapshot_directory}/ggml.json" +rocm_path="${snapshot_directory}/rocm.json" +lemonade_path="${snapshot_directory}/lemonade.json" + +gh api "repos/ggml-org/llama.cpp/releases/tags/${ggml_release}" >"$ggml_path" +gh api "repos/lemonade-sdk/llamacpp-rocm/releases/tags/${rocm_release}" >"$rocm_path" +gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${lemonade_release}" \ + >"$lemonade_path" + +attach_release_claims() { + local repository=$1 + local tag=$2 + local path=$3 + local release_tag_commit publisher_claim_type + local publisher_claimed_source_commit comparison_path + local source_manifest_asset_id source_manifest_path + + release_tag_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') + if ! [[ "$release_tag_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "Could not resolve ${repository} ${tag} to a release tag commit." >&2 + exit 1 + fi + if [[ "$repository" == "$source_repository" ]]; then + publisher_claim_type="source-release-tag" + publisher_claimed_source_commit=$release_tag_commit + else + publisher_claim_type="immutable-source-manifest" + source_manifest_asset_id=$(python "$manifest_tool" \ + --source-manifest-asset-id "$path" "$repository") + source_manifest_path="${source_manifest_directory}/source-${source_manifest_asset_id}.json" + gh api "repos/${repository}/releases/assets/${source_manifest_asset_id}" \ + -H "Accept: application/octet-stream" | + head -c "$((source_manifest_max_bytes + 1))" >"$source_manifest_path" + publisher_claimed_source_commit=$(python "$manifest_tool" \ + --validate-source-manifest "$source_manifest_path" "$path" \ + "$repository" "$tag" "$release_tag_commit") + fi + if ! [[ "$publisher_claimed_source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "${repository} ${tag} did not bind one full upstream source commit." >&2 + exit 1 + fi + comparison_path="${path}.upstream-comparison.json" + gh api \ + "repos/${source_repository}/compare/${publisher_claimed_source_commit}...${upstream_reference_head}" \ + >"$comparison_path" + python "$manifest_tool" \ + --require-upstream-ancestry "$comparison_path" "$source_repository" \ + "$publisher_claimed_source_commit" "$upstream_reference_head" + jq \ + --arg publisher_claim_type "$publisher_claim_type" \ + --arg publisher_claimed_source_commit "$publisher_claimed_source_commit" \ + --arg release_tag_commit "$release_tag_commit" \ + --arg source_repository "$source_repository" \ + --arg upstream_reference "$upstream_reference" \ + --arg upstream_reference_head "$upstream_reference_head" \ + '.publisher_claim_type = $publisher_claim_type + | .publisher_claimed_source_commit = $publisher_claimed_source_commit + | .release_tag_commit = $release_tag_commit + | .source_repository = $source_repository + | .upstream_reference = $upstream_reference + | .upstream_reference_head = $upstream_reference_head + | del(.source_commit)' \ + "$path" >"${path}.with-source" + mv "${path}.with-source" "$path" +} + +attach_release_claims ggml-org/llama.cpp "$ggml_release" "$ggml_path" +attach_release_claims lemonade-sdk/llamacpp-rocm "$rocm_release" "$rocm_path" +attach_release_claims lemonade-sdk/llama.cpp "$lemonade_release" "$lemonade_path" + +python "$manifest_tool" \ + --release ggml-org/llama.cpp "$ggml_release" "$ggml_path" \ + --release lemonade-sdk/llamacpp-rocm "$rocm_release" "$rocm_path" \ + --release lemonade-sdk/llama.cpp "$lemonade_release" "$lemonade_path" \ + --expected-json "$(<"$canonical_expected_path")" diff --git a/.github/scripts/publish_llamacpp_update.sh b/.github/scripts/publish_llamacpp_update.sh index 4a9e760899..143b731e35 100755 --- a/.github/scripts/publish_llamacpp_update.sh +++ b/.github/scripts/publish_llamacpp_update.sh @@ -3,6 +3,7 @@ set -euo pipefail versions_path=${VERSIONS_PATH:-src/cpp/resources/backend_versions.json} +manifest_path=${RELEASE_MANIFEST_PATH:-.github/llamacpp_release_manifest.json} body_file=${PR_BODY_FILE:-pr_body.md} base_branch=${BASE_BRANCH:-main} repository=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required} @@ -11,6 +12,40 @@ validated_base_sha=${VALIDATED_BASE_SHA:?VALIDATED_BASE_SHA is required} llamacpp_release=${LLAMACPP_RELEASE:?LLAMACPP_RELEASE is required} lemonade_release=${LLAMACPP_LEMONADE_RELEASE:?LLAMACPP_LEMONADE_RELEASE is required} rocm_release=${LLAMACPP_ROCM_RELEASE:?LLAMACPP_ROCM_RELEASE is required} +expected_release_manifest=${EXPECTED_RELEASE_ASSET_MANIFEST:?EXPECTED_RELEASE_ASSET_MANIFEST is required} + +script_directory=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repository_root=$(cd -- "${script_directory}/../.." && pwd) +manifest_tool="${repository_root}/test/utils/llamacpp_release_manifest.py" +capture_tool="${script_directory}/capture_llamacpp_release_manifest.sh" +snapshot_root=$(mktemp -d) + +manifest_digest=$(python "$manifest_tool" \ + --materialize-manifest "$manifest_path" \ + --manifest-json "$expected_release_manifest" \ + --append-summary-to "$body_file") + +manifest_tag() { + jq -er --arg repository "$1" \ + '.releases | map(select(.repository == $repository)) + | if length == 1 then .[0].tag_name else error("missing release") end' \ + "$manifest_path" +} + +if [[ $(manifest_tag ggml-org/llama.cpp) != "$llamacpp_release" || + $(manifest_tag lemonade-sdk/llamacpp-rocm) != "$rocm_release" || + $(manifest_tag lemonade-sdk/llama.cpp) != "$lemonade_release" ]]; then + echo "The release manifest does not match the validated candidate tags." >&2 + exit 1 +fi + +snapshot_number=0 +recheck_release_manifest() { + snapshot_number=$((snapshot_number + 1)) + LLAMACPP_RELEASE_MANIFEST_TOOL="$manifest_tool" \ + bash "$capture_tool" "$manifest_path" \ + "${snapshot_root}/snapshot-${snapshot_number}" >/dev/null +} release_pattern='^b[0-9]+$' for release in "$llamacpp_release" "$lemonade_release" "$rocm_release"; do @@ -51,13 +86,15 @@ if [[ "$fresh_llamacpp_release" != "$llamacpp_release" || exit 1 fi +recheck_release_manifest + has_update=true if git diff --quiet HEAD -- "$versions_path"; then has_update=false branch="" else versions_blob=$(git hash-object "$versions_path") - branch="auto/llamacpp-update-${llamacpp_release}-${lemonade_release}-${rocm_release}-${validated_base_sha}-${versions_blob}" + branch="auto/llamacpp-update-${llamacpp_release}-${lemonade_release}-${rocm_release}-${validated_base_sha}-${versions_blob}-${manifest_digest}" fi open_pr_records=$(gh api "repos/${repository}/pulls" \ @@ -69,6 +106,7 @@ open_pr_records=$(gh api "repos/${repository}/pulls" \ open_pr="" stale_prs=() +publication_actor='github-actions[bot]' while IFS=$'\t' read -r number head_ref head_repository author base_ref; do if [[ -z "$number" ]]; then continue @@ -76,7 +114,7 @@ while IFS=$'\t' read -r number head_ref head_repository author base_ref; do trusted=false if [[ "$head_repository" == "$repository" && - "$author" == "github-actions[bot]" && + "$author" == "$publication_actor" && "$base_ref" == "$base_branch" ]]; then trusted=true fi @@ -94,7 +132,7 @@ while IFS=$'\t' read -r number head_ref head_repository author base_ref; do fi open_pr=$number elif [[ "$trusted" == "true" ]]; then - stale_prs+=("$number") + stale_prs+=("${number}"$'\t'"${head_ref}") fi done <<< "$open_pr_records" @@ -108,19 +146,73 @@ assert_base_is_current() { fi } -assert_base_is_current -for stale_pr in "${stale_prs[@]}"; do - gh api "repos/${repository}/pulls/${stale_pr}" \ - --method PATCH \ - -f state=closed >/dev/null - echo "Closed superseded pull request #${stale_pr}." -done +assert_publication_branch() { + local expected_oid=$1 + local observed_oid + observed_oid=$(git ls-remote --heads origin "refs/heads/${branch}" | + awk 'NR == 1 { print $1 }') + if [[ -z "$observed_oid" || "$observed_oid" != "$expected_oid" ]]; then + echo "The publication branch changed after push." >&2 + exit 1 + fi +} + +require_pull_request_identity() { + local number=$1 + local expected_head_ref=$2 + local expected_head_sha=${3:-} + local expected_state=${4:-open} + local expected_base_sha=${5:-} + local pull_request_json + + if ! [[ "$number" =~ ^[1-9][0-9]*$ ]]; then + echo "Invalid pull request number: ${number}" >&2 + exit 1 + fi + pull_request_json=$(gh api "repos/${repository}/pulls/${number}") + if ! jq -e \ + --arg state "$expected_state" \ + --arg repository "$repository" \ + --arg head_ref "$expected_head_ref" \ + --arg head_sha "$expected_head_sha" \ + --arg actor "$publication_actor" \ + --arg base_ref "$base_branch" \ + --arg base_sha "$expected_base_sha" \ + '.state == $state and + .head.repo.full_name == $repository and + .head.ref == $head_ref and + ($head_sha == "" or .head.sha == $head_sha) and + .user.login == $actor and + .base.ref == $base_ref and + ($base_sha == "" or .base.sha == $base_sha)' \ + <<< "$pull_request_json" >/dev/null; then + echo "Pull request #${number} identity changed before publication." >&2 + exit 1 + fi +} + +close_stale_prs() { + local stale_record stale_pr stale_head_ref + for stale_record in "${stale_prs[@]}"; do + IFS=$'\t' read -r stale_pr stale_head_ref <<< "$stale_record" + require_pull_request_identity "$stale_pr" "$stale_head_ref" + gh api "repos/${repository}/pulls/${stale_pr}" \ + --method PATCH \ + -f state=closed >/dev/null + require_pull_request_identity "$stale_pr" "$stale_head_ref" "" closed + echo "Closed superseded pull request #${stale_pr}." + done +} if [[ "$has_update" != "true" ]]; then + assert_base_is_current + recheck_release_manifest + close_stale_prs echo "backend_versions.json is unchanged; nothing to update." exit 0 fi +assert_base_is_current title="Update llama.cpp to ${llamacpp_release}" remote_oid=$(git ls-remote --heads origin "refs/heads/$branch" | awk 'NR == 1 { print $1 }') remote_tree="" @@ -141,7 +233,8 @@ if [[ -n "$remote_oid" ]]; then exit 1 fi unexpected_path=$(git diff --name-only "$merge_base" "$remote_oid" -- | - awk -v allowed="$versions_path" '$0 != allowed { print; exit }') + awk -v versions="$versions_path" -v manifest="$manifest_path" \ + '$0 != versions && $0 != manifest { print; exit }') if [[ -n "$unexpected_path" ]]; then echo "Publication branch ${branch} changes unexpected path ${unexpected_path}." >&2 exit 1 @@ -154,9 +247,11 @@ fi git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git checkout -b "$branch" -git add "$versions_path" -git commit -m \ - "Update llama.cpp to ${llamacpp_release}, rocm-stable to ${lemonade_release}, rocm-nightly to ${rocm_release}" +recheck_release_manifest +git add "$versions_path" "$manifest_path" +git commit \ + -m "Update llama.cpp to ${llamacpp_release}, rocm-stable to ${lemonade_release}, rocm-nightly to ${rocm_release}" \ + -m "LlamaCpp-Release-Manifest-SHA256: ${manifest_digest}" desired_tree=$(git rev-parse 'HEAD^{tree}') if [[ -n "$remote_oid" && -z "$open_pr" && "$remote_tree" != "$desired_tree" ]]; then @@ -169,27 +264,45 @@ if [[ -n "$remote_oid" && -n "$open_pr" && "$remote_tree" != "$desired_tree" ]]; fi assert_base_is_current +recheck_release_manifest if [[ -n "$remote_oid" ]]; then lease="refs/heads/${branch}:${remote_oid}" push_source="$remote_oid" + published_oid=$remote_oid else lease="refs/heads/${branch}:" push_source=HEAD + published_oid=$(git rev-parse HEAD) fi git push --force-with-lease="$lease" origin \ "${push_source}:refs/heads/$branch" +assert_base_is_current +assert_publication_branch "$published_oid" if [[ -n "$open_pr" ]]; then + require_pull_request_identity \ + "$open_pr" "$branch" "$published_oid" open "$validated_base_sha" gh pr edit "$open_pr" \ --repo "$repository" \ --title "$title" \ --body-file "$body_file" + current_pr=$open_pr echo "Refreshed pull request #${open_pr}." else - gh pr create \ + created_pr_url=$(gh pr create \ --repo "$repository" \ --title "$title" \ --body-file "$body_file" \ --base "$base_branch" \ - --head "$branch" + --head "$branch") + created_pr_url=${created_pr_url%/} + current_pr=${created_pr_url##*/} fi + +require_pull_request_identity \ + "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" +close_stale_prs +assert_base_is_current +assert_publication_branch "$published_oid" +require_pull_request_identity \ + "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" diff --git a/.github/scripts/verify_llamacpp_pin_manifest.sh b/.github/scripts/verify_llamacpp_pin_manifest.sh new file mode 100755 index 0000000000..f9847e92c6 --- /dev/null +++ b/.github/scripts/verify_llamacpp_pin_manifest.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 4 ]]; then + echo "Usage: $0 TRUSTED_REPOSITORY CANDIDATE_REPOSITORY BASE_SHA CANDIDATE_SHA" >&2 + exit 2 +fi + +trusted_repository=$1 +candidate_repository=$2 +base_sha=$3 +candidate_sha=$4 +oid_pattern='^[0-9a-fA-F]{40}$' + +if ! [[ "$base_sha" =~ $oid_pattern && "$candidate_sha" =~ $oid_pattern ]]; then + echo "Protected validation commits must be full Git OIDs." >&2 + exit 1 +fi +if [[ $(git -C "$trusted_repository" rev-parse HEAD) != "$base_sha" ]]; then + echo "Trusted checkout does not match the protected base commit." >&2 + exit 1 +fi +if [[ $(git -C "$candidate_repository" rev-parse HEAD) != "$candidate_sha" ]]; then + echo "Candidate checkout does not match the protected merge commit." >&2 + exit 1 +fi + +script_directory=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repository_root=$(cd -- "${script_directory}/../.." && pwd) +manifest_tool="${repository_root}/test/utils/llamacpp_release_manifest.py" +capture_tool="${script_directory}/capture_llamacpp_release_manifest.sh" +snapshot_directory=$(mktemp -d) +base_versions="${snapshot_directory}/base-versions.json" +candidate_versions="${snapshot_directory}/candidate-versions.json" +candidate_manifest="${snapshot_directory}/candidate-manifest.json" +manifest_path=.github/llamacpp_release_manifest.json + +git -C "$trusted_repository" show \ + "${base_sha}:src/cpp/resources/backend_versions.json" >"$base_versions" +git -C "$candidate_repository" show \ + "${candidate_sha}:src/cpp/resources/backend_versions.json" >"$candidate_versions" + +changes=$(python "$manifest_tool" \ + --managed-pin-changes "$base_versions" "$candidate_versions") +if [[ "$changes" == '{}' ]]; then + base_manifest_entry=$(git -C "$trusted_repository" ls-tree \ + "$base_sha" -- "$manifest_path") + candidate_manifest_entry=$(git -C "$candidate_repository" ls-tree \ + "$candidate_sha" -- "$manifest_path") + if [[ "$base_manifest_entry" != "$candidate_manifest_entry" ]]; then + echo "Release manifest changes require changed managed llama.cpp pins." >&2 + exit 1 + fi + echo "No changed managed llama.cpp pins; no release manifest is required." + exit 0 +fi + +if ! git -C "$candidate_repository" show \ + "${candidate_sha}:${manifest_path}" \ + >"$candidate_manifest"; then + echo "Changed managed llama.cpp pins require a committed release manifest." >&2 + exit 1 +fi +python "$manifest_tool" \ + --require-managed-pin-manifest \ + "$base_versions" "$candidate_versions" "$candidate_manifest" >/dev/null +LLAMACPP_RELEASE_MANIFEST_TOOL="$manifest_tool" \ + bash "$capture_tool" "$candidate_manifest" "${snapshot_directory}/live" \ + >/dev/null + +echo "Verified publisher source claims and asset metadata for changed managed llama.cpp pins." diff --git a/.github/workflows/docs_and_style.yml b/.github/workflows/docs_and_style.yml index 1d6285b45a..e53df44b48 100644 --- a/.github/workflows/docs_and_style.yml +++ b/.github/workflows/docs_and_style.yml @@ -88,6 +88,7 @@ jobs: test.test_publish_llamacpp_update test.test_validate_llamacpp_selection test.test_validate_llamacpp_workflow_plan + test.test_verify_llamacpp_pin_manifest -v - name: Run PowerShell validation argument tests shell: pwsh diff --git a/.github/workflows/validate_llamacpp_core.yml b/.github/workflows/validate_llamacpp_core.yml index ccb97ee19a..4c906e2c05 100644 --- a/.github/workflows/validate_llamacpp_core.yml +++ b/.github/workflows/validate_llamacpp_core.yml @@ -260,23 +260,82 @@ jobs: gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LEMONADE_RELEASE}" \ > lemonade_release.json - attach_source_commit() { + source_repository=ggml-org/llama.cpp + source_manifest_max_bytes=65536 + source_manifest_directory=$(mktemp -d) + source_default_branch=$(gh api "repos/${source_repository}" --jq '.default_branch') + if [ -z "$source_default_branch" ]; then + echo "::error::Could not resolve the ${source_repository} default branch." + exit 1 + fi + upstream_reference="refs/heads/${source_default_branch}" + upstream_reference_head=$(gh api \ + "repos/${source_repository}/commits/${source_default_branch}" --jq '.sha') + if ! [[ "$upstream_reference_head" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::Could not resolve ${upstream_reference} to a full commit." + exit 1 + fi + + attach_release_claims() { local repository="$1" local tag="$2" local path="$3" - local source_commit - source_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') - if ! [[ "$source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::Could not resolve ${repository} ${tag} to a commit." + local release_tag_commit publisher_claim_type + local publisher_claimed_source_commit comparison_path + local source_manifest_asset_id source_manifest_path + release_tag_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') + if ! [[ "$release_tag_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::Could not resolve ${repository} ${tag} to a release tag commit." exit 1 fi - jq --arg source_commit "$source_commit" \ - '.source_commit = $source_commit' "$path" > "${path}.with-source" + if [ "$repository" = "$source_repository" ]; then + publisher_claim_type="source-release-tag" + publisher_claimed_source_commit="$release_tag_commit" + else + publisher_claim_type="immutable-source-manifest" + source_manifest_asset_id=$(python -m test.utils.llamacpp_release_manifest \ + --source-manifest-asset-id "$path" "$repository") + source_manifest_path="${source_manifest_directory}/source-${source_manifest_asset_id}.json" + gh api "repos/${repository}/releases/assets/${source_manifest_asset_id}" \ + -H "Accept: application/octet-stream" | + head -c "$((source_manifest_max_bytes + 1))" > "$source_manifest_path" + publisher_claimed_source_commit=$(python -m test.utils.llamacpp_release_manifest \ + --validate-source-manifest "$source_manifest_path" "$path" \ + "$repository" "$tag" "$release_tag_commit") + fi + if ! [[ "$publisher_claimed_source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::${repository} ${tag} did not bind one full upstream source commit." + exit 1 + fi + comparison_path="${path}.upstream-comparison.json" + gh api \ + "repos/${source_repository}/compare/${publisher_claimed_source_commit}...${upstream_reference_head}" \ + > "$comparison_path" + python -m test.utils.llamacpp_release_manifest \ + --require-upstream-ancestry "$comparison_path" "$source_repository" \ + "$publisher_claimed_source_commit" "$upstream_reference_head" + jq \ + --arg publisher_claim_type "$publisher_claim_type" \ + --arg publisher_claimed_source_commit "$publisher_claimed_source_commit" \ + --arg release_tag_commit "$release_tag_commit" \ + --arg source_repository "$source_repository" \ + --arg upstream_reference "$upstream_reference" \ + --arg upstream_reference_head "$upstream_reference_head" \ + '.publisher_claim_type = $publisher_claim_type + | .publisher_claimed_source_commit = $publisher_claimed_source_commit + | .release_tag_commit = $release_tag_commit + | .source_repository = $source_repository + | .upstream_reference = $upstream_reference + | .upstream_reference_head = $upstream_reference_head + | del(.source_commit)' \ + "$path" > "${path}.with-source" mv "${path}.with-source" "$path" } - attach_source_commit ggml-org/llama.cpp "$GGML_RELEASE" ggml_release.json - attach_source_commit lemonade-sdk/llamacpp-rocm "$ROCM_RELEASE" rocm_release.json - attach_source_commit lemonade-sdk/llama.cpp "$LEMONADE_RELEASE" lemonade_release.json + attach_release_claims ggml-org/llama.cpp "$GGML_RELEASE" ggml_release.json + attach_release_claims lemonade-sdk/llamacpp-rocm \ + "$ROCM_RELEASE" rocm_release.json + attach_release_claims lemonade-sdk/llama.cpp \ + "$LEMONADE_RELEASE" lemonade_release.json jq -r '.assets[].name' ggml_release.json | sort > ggml_assets.txt jq -r '.assets[].name' rocm_release.json | sort > rocm_assets.txt @@ -396,9 +455,43 @@ jobs: print("Eligible lemonade-sdk/llama.cpp backend updates: " + (values["lemonade_update_backends"] or "none")) PY + verify-protected-release-manifest: + name: Verify protected release manifest + needs: validate-invocation + if: >- + needs.validate-invocation.result == 'success' && + (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') + runs-on: ubuntu-latest + steps: + - name: Check out the protected candidate commit + uses: actions/checkout@v5 + with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + path: candidate + persist-credentials: false + fetch-depth: 1 + - name: Check out trusted validation tools + uses: actions/checkout@v5 + with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + path: trusted + persist-credentials: false + fetch-depth: 1 + - name: Verify managed llama.cpp pins + env: + BASE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + CANDIDATE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + bash trusted/.github/scripts/verify_llamacpp_pin_manifest.sh + "$GITHUB_WORKSPACE/trusted" "$GITHUB_WORKSPACE/candidate" + "$BASE_SHA" "$CANDIDATE_SHA" + build: name: Build - needs: [validate-invocation, get-latest-releases, verify-release-assets, plan] + needs: [validate-invocation, get-latest-releases, verify-release-assets, verify-protected-release-manifest, plan] if: always() && !failure() && !cancelled() runs-on: windows-latest steps: @@ -1032,10 +1125,52 @@ jobs: - name: Cleanup if: always() uses: ./.github/actions/cleanup-processes-windows + + reverify-protected-release-manifest: + name: Reverify protected release manifest + needs: [validate, verify-protected-release-manifest] + if: >- + always() && needs.validate.result == 'success' && + (needs.verify-protected-release-manifest.result == 'success' || + needs.verify-protected-release-manifest.result == 'skipped') + runs-on: ubuntu-latest + steps: + - name: No protected manifest recheck is needed + if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' + run: echo "This invocation does not review a protected pin change." + - name: Check out the protected candidate commit + if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' + uses: actions/checkout@v5 + with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + path: candidate + persist-credentials: false + fetch-depth: 1 + - name: Check out trusted validation tools + if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' + uses: actions/checkout@v5 + with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + path: trusted + persist-credentials: false + fetch-depth: 1 + - name: Reverify managed llama.cpp pins + if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' + env: + BASE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + CANDIDATE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + bash trusted/.github/scripts/verify_llamacpp_pin_manifest.sh + "$GITHUB_WORKSPACE/trusted" "$GITHUB_WORKSPACE/candidate" + "$BASE_SHA" "$CANDIDATE_SHA" + # Keep a stable terminal job for callers that require one validation result. validation-gate: name: llama.cpp validation - needs: [validate-invocation, build, validate, plan] + needs: [validate-invocation, build, validate, reverify-protected-release-manifest, plan] if: always() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/validate_llamacpp_schedule.yml b/.github/workflows/validate_llamacpp_schedule.yml index 884be484be..ad5b9261f1 100644 --- a/.github/workflows/validate_llamacpp_schedule.yml +++ b/.github/workflows/validate_llamacpp_schedule.yml @@ -52,7 +52,7 @@ jobs: --directory . --expected-matrix-json "$EXPECTED_VALIDATION_MATRIX" - - name: Update backend_versions.json with verified releases + - name: Update backend_versions.json from release manifest shell: bash env: LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} @@ -226,47 +226,22 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} EXPECTED_RELEASE_ASSET_MANIFEST: ${{ needs.validate.outputs.release_asset_manifest }} - LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} run: | set -euo pipefail - gh api "repos/ggml-org/llama.cpp/releases/tags/${LLAMACPP_RELEASE}" \ - > publish_ggml_release.json - gh api "repos/lemonade-sdk/llamacpp-rocm/releases/tags/${LLAMACPP_ROCM_RELEASE}" \ - > publish_rocm_release.json - gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LLAMACPP_LEMONADE_RELEASE}" \ - > publish_lemonade_release.json - - attach_source_commit() { - local repository="$1" - local tag="$2" - local path="$3" - local source_commit - source_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') - if ! [[ "$source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::Could not resolve ${repository} ${tag} to a commit." - exit 1 - fi - jq --arg source_commit "$source_commit" \ - '.source_commit = $source_commit' "$path" > "${path}.with-source" - mv "${path}.with-source" "$path" - } - attach_source_commit ggml-org/llama.cpp "$LLAMACPP_RELEASE" publish_ggml_release.json - attach_source_commit lemonade-sdk/llamacpp-rocm "$LLAMACPP_ROCM_RELEASE" publish_rocm_release.json - attach_source_commit lemonade-sdk/llama.cpp "$LLAMACPP_LEMONADE_RELEASE" publish_lemonade_release.json - + snapshot_directory=$(mktemp -d) + expected_path="${snapshot_directory}/expected-input.json" python -m test.utils.llamacpp_release_manifest \ - --release ggml-org/llama.cpp "$LLAMACPP_RELEASE" publish_ggml_release.json \ - --release lemonade-sdk/llamacpp-rocm "$LLAMACPP_ROCM_RELEASE" publish_rocm_release.json \ - --release lemonade-sdk/llama.cpp "$LLAMACPP_LEMONADE_RELEASE" publish_lemonade_release.json \ - --expected-json "$EXPECTED_RELEASE_ASSET_MANIFEST" + --materialize-manifest "$expected_path" \ + --manifest-json "$EXPECTED_RELEASE_ASSET_MANIFEST" >/dev/null + bash .github/scripts/capture_llamacpp_release_manifest.sh \ + "$expected_path" "${snapshot_directory}/live" >/dev/null - name: Publish update pull request env: BASE_BRANCH: ${{ github.event.repository.default_branch }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXPECTED_RELEASE_ASSET_MANIFEST: ${{ needs.validate.outputs.release_asset_manifest }} LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} diff --git a/docs/guide/configuration/llamacpp.md b/docs/guide/configuration/llamacpp.md index 5874a3df8d..4857cb54b9 100644 --- a/docs/guide/configuration/llamacpp.md +++ b/docs/guide/configuration/llamacpp.md @@ -23,7 +23,8 @@ These are caller-supplied starting points. Lemonade does not install per-model s These model IDs must ship only with managed llama.cpp binaries built from accepted K2-Horizon support on current upstream llama.cpp. The catalog entries are staged while that upstream port, the coordinated backend pins, and packaged runtime validation are completed. Do not -publish them with the current managed pins or substitute the obsolete IFM development branch. +publish them with the current managed pins or substitute the IFM development branch for a +managed release. After compatible managed binaries are pinned, start a model with a bounded context: @@ -61,7 +62,7 @@ print(message.content) The upstream parser and packaged-runtime gate must prove high, medium, low, and disabled reasoning plus the `json`, `xml`, and `xml_typed` tool-call formats before publication. The -obsolete IFM development branch does not pass that complete matrix: medium and low reasoning +IFM development branch does not pass that complete matrix: medium and low reasoning can expose IFM control tags, and the non-default tool formats can cause parser errors. To disable thinking after the compatible managed release ships, set `chat_template_kwargs.enable_thinking` to `false`. Use a large output allowance for long diff --git a/test/test_llamacpp_release_manifest.py b/test/test_llamacpp_release_manifest.py index e25b5bf8d6..a1f60bea94 100644 --- a/test/test_llamacpp_release_manifest.py +++ b/test/test_llamacpp_release_manifest.py @@ -1,43 +1,397 @@ #!/usr/bin/env python3 -"""Tests for immutable llama.cpp release-asset manifests.""" +"""Tests for canonical llama.cpp release-asset manifests.""" from __future__ import annotations import copy +import hashlib import json +import tempfile import unittest +from pathlib import Path from test.utils import llamacpp_release_manifest as manifest -def release_payload(tag: str, release_id: int, asset_seed: int) -> dict: +def release_payload( + tag: str, + release_id: int, + asset_seed: int, + *, + repository: str = "ggml-org/llama.cpp", + publisher_claim_type: str = "source-release-tag", + publisher_claimed_source_commit: str | None = None, +) -> dict: + release_tag_commit = f"{release_id:040x}" + claimed_commit = publisher_claimed_source_commit or release_tag_commit + assets = [ + { + "name": "z-last.zip", + "id": asset_seed + 1, + "size": 200, + "digest": "sha256:" + "B" * 64, + "state": "uploaded", + "updated_at": "2026-09-05T12:34:56Z", + }, + { + "name": "a-first.zip", + "id": asset_seed, + "size": 100, + "digest": "sha256:" + "a" * 64, + "state": "uploaded", + "updated_at": "2026-09-05T12:30:00Z", + }, + ] + if publisher_claim_type == "immutable-source-manifest": + assets.append( + { + "name": ".llamacpp-source.json", + "id": asset_seed + 2, + "size": 512, + "digest": "sha256:" + "c" * 64, + "state": "uploaded", + "updated_at": "2026-09-05T12:36:00Z", + } + ) return { + "body": "release notes", + "draft": False, "id": release_id, + "immutable": True, + "publisher_claim_type": publisher_claim_type, + "publisher_claimed_source_commit": claimed_commit, + "release_tag_commit": release_tag_commit, + "source_repository": repository, "tag_name": tag, - "source_commit": f"{release_id:040x}", - "assets": [ - { - "name": "z-last.zip", - "id": asset_seed + 1, - "size": 200, - "digest": "sha256:" + "B" * 64, - "updated_at": "2026-09-05T12:34:56Z", - }, - { - "name": "a-first.zip", - "id": asset_seed, - "size": 100, - "digest": "sha256:" + "a" * 64, - "updated_at": "2026-09-05T12:30:00Z", - }, - ], + "upstream_reference": "refs/heads/master", + "upstream_reference_head": "d" * 40, + "assets": assets, } class LlamaCppReleaseManifestTests(unittest.TestCase): def setUp(self) -> None: self.ggml = release_payload("b1234", 10, 100) - self.rocm = release_payload("b1235", 20, 200) + self.rocm = release_payload( + "b1235", + 20, + 200, + publisher_claim_type="immutable-source-manifest", + ) + + def _immutable_source_manifest_fixture(self) -> tuple[dict, bytes, str]: + release = copy.deepcopy(self.ggml) + release["draft"] = False + release["tag_name"] = "b4321" + release["release_tag_commit"] = "b" * 40 + for asset in release["assets"]: + asset["state"] = "uploaded" + asset["digest"] = asset["digest"].lower() + source_commit = "c" * 40 + document = { + "assets": sorted( + ( + { + "digest": asset["digest"].lower(), + "name": asset["name"], + "size": asset["size"], + } + for asset in release["assets"] + ), + key=lambda asset: asset["name"], + ), + "release_repository": "lemonade-sdk/llama.cpp", + "release_tag": "b4321", + "release_tag_commit": "b" * 40, + "schema_version": 1, + "source_commit": source_commit, + "source_repository": "ggml-org/llama.cpp", + } + evidence = json.dumps( + document, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + release["assets"].append( + { + "digest": "sha256:" + hashlib.sha256(evidence).hexdigest(), + "id": 999, + "name": ".llamacpp-source.json", + "size": len(evidence), + "state": "uploaded", + "updated_at": "2026-09-05T12:40:00Z", + } + ) + return release, evidence, source_commit + + @staticmethod + def _bind_source_evidence(release: dict, evidence: bytes) -> None: + source_asset = next( + asset + for asset in release["assets"] + if asset["name"] == manifest.SOURCE_MANIFEST_ASSET_NAME + ) + source_asset["size"] = len(evidence) + source_asset["digest"] = "sha256:" + hashlib.sha256(evidence).hexdigest() + + def test_immutable_source_manifest_binds_release_source_and_assets(self) -> None: + release, evidence, source_commit = self._immutable_source_manifest_fixture() + + asset = manifest.locate_immutable_source_manifest_asset( + release, + "lemonade-sdk/llama.cpp", + ) + self.assertEqual(asset["id"], 999) + self.assertEqual( + manifest.validate_immutable_source_manifest( + evidence, + release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ), + source_commit, + ) + + def test_immutable_source_manifest_rejects_unbound_or_ambiguous_input(self) -> None: + release, evidence, _source_commit = self._immutable_source_manifest_fixture() + document = json.loads(evidence) + + document_cases = ( + ("release repository", "release_repository", "attacker/llama.cpp"), + ("release tag", "release_tag", "b9999"), + ("release tag commit", "release_tag_commit", "d" * 40), + ("source repository", "source_repository", "attacker/llama.cpp"), + ("source commit", "source_commit", "abc123"), + ("asset metadata", "assets", document["assets"][:-1]), + ) + for expected_error, field, value in document_cases: + with self.subTest(field=field): + changed = copy.deepcopy(document) + changed[field] = value + changed_bytes = json.dumps( + changed, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + changed_release = copy.deepcopy(release) + self._bind_source_evidence(changed_release, changed_bytes) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.validate_immutable_source_manifest( + changed_bytes, + changed_release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ) + + duplicate_key_evidence = ( + evidence[:-1] + b',"source_commit":"' + b"d" * 40 + b'"}' + ) + duplicate_release = copy.deepcopy(release) + self._bind_source_evidence(duplicate_release, duplicate_key_evidence) + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "duplicate JSON member", + ): + manifest.validate_immutable_source_manifest( + duplicate_key_evidence, + duplicate_release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ) + + for mutate, expected_error in ( + (lambda payload: payload["assets"].pop(), "exactly one"), + ( + lambda payload: payload["assets"].append( + copy.deepcopy(payload["assets"][-1]) + ), + "exactly one", + ), + ( + lambda payload: payload["assets"][-1].update(size=0), + "size", + ), + ( + lambda payload: payload["assets"][-1].update(size=65_537), + "size", + ), + ( + lambda payload: payload["assets"][-1].update( + digest="sha256:" + "A" * 64 + ), + "sha256 digest", + ), + ( + lambda payload: payload["assets"][-1].update(state="new"), + "uploaded", + ), + ): + with self.subTest(expected_error=expected_error): + changed_release = copy.deepcopy(release) + mutate(changed_release) + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.locate_immutable_source_manifest_asset( + changed_release, + "lemonade-sdk/llama.cpp", + ) + + def test_immutable_source_manifest_schema_is_strict(self) -> None: + release, evidence, _source_commit = self._immutable_source_manifest_fixture() + document = json.loads(evidence) + cases = [] + + extra_top_level = copy.deepcopy(document) + extra_top_level["unexpected"] = True + cases.append(("unexpected fields", extra_top_level)) + for schema_version in (True, 1.0, 2, "1"): + changed = copy.deepcopy(document) + changed["schema_version"] = schema_version + cases.append(("schema_version", changed)) + extra_asset_field = copy.deepcopy(document) + extra_asset_field["assets"][0]["unexpected"] = True + cases.append(("unexpected fields", extra_asset_field)) + duplicate_asset = copy.deepcopy(document) + duplicate_asset["assets"].append(copy.deepcopy(duplicate_asset["assets"][0])) + cases.append(("duplicate asset", duplicate_asset)) + self_reference = copy.deepcopy(document) + self_reference["assets"][0]["name"] = manifest.SOURCE_MANIFEST_ASSET_NAME + cases.append(("exclude itself", self_reference)) + for expected_error, changed in cases: + with self.subTest(expected_error=expected_error): + changed_bytes = json.dumps( + changed, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + changed_release = copy.deepcopy(release) + self._bind_source_evidence(changed_release, changed_bytes) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.validate_immutable_source_manifest( + changed_bytes, + changed_release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ) + + reversed_document = copy.deepcopy(document) + reversed_document["assets"].reverse() + reversed_evidence = json.dumps( + reversed_document, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + reversed_release = copy.deepcopy(release) + self._bind_source_evidence(reversed_release, reversed_evidence) + self.assertEqual( + manifest.validate_immutable_source_manifest( + reversed_evidence, + reversed_release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ), + document["source_commit"], + ) + + def test_immutable_source_manifest_measures_raw_evidence_before_parsing( + self, + ) -> None: + release, evidence, _source_commit = self._immutable_source_manifest_fixture() + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, "size does not match" + ): + manifest.validate_immutable_source_manifest( + evidence + b" ", + release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ) + + same_size_corruption = b"[" + evidence[1:] + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "sha256 digest does not match", + ): + manifest.validate_immutable_source_manifest( + same_size_corruption, + release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ) + + invalid_utf8 = b"\xff" + invalid_utf8_release = copy.deepcopy(release) + self._bind_source_evidence(invalid_utf8_release, invalid_utf8) + with self.assertRaisesRegex(manifest.ReleaseManifestError, "UTF-8 JSON"): + manifest.validate_immutable_source_manifest( + invalid_utf8, + invalid_utf8_release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ) + + def test_immutable_source_manifest_release_metadata_is_strict(self) -> None: + release, _evidence, _source_commit = self._immutable_source_manifest_fixture() + source_index = next( + index + for index, asset in enumerate(release["assets"]) + if asset["name"] == manifest.SOURCE_MANIFEST_ASSET_NAME + ) + cases = ( + ("release draft", lambda payload: payload.update(draft=True)), + ( + "positive integer", + lambda payload: payload["assets"][source_index].update(id=True), + ), + ( + "duplicate release asset name", + lambda payload: payload["assets"][1].update( + name=payload["assets"][0]["name"] + ), + ), + ( + "must be uploaded", + lambda payload: payload["assets"][0].update(state="new"), + ), + ( + "lowercase sha256 digest", + lambda payload: payload["assets"][0].update( + digest="sha256:" + "A" * 64 + ), + ), + ) + for expected_error, mutate in cases: + with self.subTest(expected_error=expected_error): + changed_release = copy.deepcopy(release) + mutate(changed_release) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.locate_immutable_source_manifest_asset( + changed_release, + "lemonade-sdk/llama.cpp", + ) def test_manifest_is_canonical_across_release_and_asset_order(self) -> None: first = manifest.build_release_asset_manifest( @@ -57,7 +411,7 @@ def test_manifest_is_canonical_across_release_and_asset_order(self) -> None: self.assertEqual(first, second) parsed = json.loads(first) - self.assertEqual(parsed["schema_version"], 2) + self.assertEqual(parsed["schema_version"], 4) self.assertEqual( [release["repository"] for release in parsed["releases"]], ["ggml-org/llama.cpp", "lemonade-sdk/llamacpp-rocm"], @@ -88,7 +442,32 @@ def test_every_asset_requires_a_sha256_digest(self) -> None: def test_release_and_asset_identity_fields_are_required(self) -> None: cases = ( ("release id", lambda payload: payload.pop("id")), - ("source commit", lambda payload: payload.pop("source_commit")), + ("release immutable", lambda payload: payload.pop("immutable")), + ("release draft", lambda payload: payload.pop("draft")), + ( + "release tag commit", + lambda payload: payload.pop("release_tag_commit"), + ), + ( + "source repository", + lambda payload: payload.pop("source_repository"), + ), + ( + "publisher claim type", + lambda payload: payload.pop("publisher_claim_type"), + ), + ( + "publisher-claimed source commit", + lambda payload: payload.pop("publisher_claimed_source_commit"), + ), + ( + "upstream reference", + lambda payload: payload.pop("upstream_reference"), + ), + ( + "upstream reference head", + lambda payload: payload.pop("upstream_reference_head"), + ), ("asset name", lambda payload: payload["assets"][0].pop("name")), ("asset id", lambda payload: payload["assets"][0].pop("id")), ("asset size", lambda payload: payload["assets"][0].pop("size")), @@ -96,6 +475,10 @@ def test_release_and_asset_identity_fields_are_required(self) -> None: "asset updated_at", lambda payload: payload["assets"][0].pop("updated_at"), ), + ( + "asset must be uploaded", + lambda payload: payload["assets"][0].pop("state"), + ), ) for expected_error, mutate in cases: with self.subTest(expected_error=expected_error): @@ -110,20 +493,279 @@ def test_release_and_asset_identity_fields_are_required(self) -> None: [("ggml-org/llama.cpp", "b1234", payload)] ) - def test_source_commit_must_be_a_full_git_oid(self) -> None: - for source_commit in (None, "", "abc123", "g" * 40): - with self.subTest(source_commit=source_commit): + def test_mutable_releases_fail_closed(self) -> None: + for immutable in (False, None, 1, "true"): + with self.subTest(immutable=immutable): + payload = copy.deepcopy(self.ggml) + payload["immutable"] = immutable + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "release immutable must be true", + ): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", payload)] + ) + + def test_draft_releases_fail_closed(self) -> None: + for draft in (True, None, 0, "false"): + with self.subTest(draft=draft): payload = copy.deepcopy(self.ggml) - payload["source_commit"] = source_commit + payload["draft"] = draft with self.assertRaisesRegex( manifest.ReleaseManifestError, - "source commit", + "release draft must be false", ): manifest.build_release_asset_manifest( [("ggml-org/llama.cpp", "b1234", payload)] ) + def test_publisher_claim_commits_must_be_full_git_oids(self) -> None: + for field in ("release_tag_commit", "publisher_claimed_source_commit"): + for commit in (None, "", "abc123", "g" * 40): + with self.subTest(field=field, commit=commit): + payload = copy.deepcopy(self.ggml) + payload[field] = commit + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + ( + "publisher-claimed source commit" + if field == "publisher_claimed_source_commit" + else field.replace("_", " ") + ), + ): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", payload)] + ) + + def test_packaging_commit_is_distinct_from_binary_source_commit(self) -> None: + payload = release_payload( + "b4321", + 30, + 300, + repository="ggml-org/llama.cpp", + publisher_claim_type="immutable-source-manifest", + publisher_claimed_source_commit="f" * 40, + ) + + parsed = json.loads( + manifest.build_release_asset_manifest( + [("lemonade-sdk/llama.cpp", "b4321", payload)] + ) + )["releases"][0] + + self.assertEqual(parsed["release_tag_commit"], f"{30:040x}") + self.assertEqual(parsed["source_repository"], "ggml-org/llama.cpp") + self.assertEqual(parsed["publisher_claimed_source_commit"], "f" * 40) + self.assertNotEqual( + parsed["release_tag_commit"], + parsed["publisher_claimed_source_commit"], + ) + + def test_upstream_release_tag_commit_must_equal_source_commit(self) -> None: + payload = copy.deepcopy(self.ggml) + payload["publisher_claimed_source_commit"] = "f" * 40 + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "release tag commit must equal publisher-claimed source commit", + ): + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", payload)] + ) + + def test_release_note_source_claims_are_rejected(self) -> None: + payload = release_payload( + "b4321", + 30, + 300, + repository="ggml-org/llama.cpp", + publisher_claim_type="immutable-source-manifest", + publisher_claimed_source_commit="f" * 40, + ) + payload["publisher_claim_type"] = "release-note-marker" + payload["body"] = f"**Llama.cpp Source Commit**: {'f' * 40}" + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "publisher claim type", + ): + manifest.build_release_asset_manifest( + [("lemonade-sdk/llama.cpp", "b4321", payload)] + ) + + def test_fork_network_commit_is_not_upstream_ancestry(self) -> None: + claimed_commit = "3" * 40 + upstream_head = "7" * 40 + comparison = { + "base_commit": {"sha": claimed_commit}, + "merge_base_commit": {"sha": "4" * 40}, + "status": "diverged", + "url": ( + "https://api.github.com/repos/ggml-org/llama.cpp/compare/" + f"{claimed_commit}...{upstream_head}" + ), + } + + self.assertTrue( + hasattr(manifest, "require_upstream_ancestry"), + "upstream ancestry validator is required", + ) + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "not reachable", + ): + manifest.require_upstream_ancestry( + comparison, + "ggml-org/llama.cpp", + claimed_commit, + upstream_head, + ) + + def test_untrusted_manifest_anchor_must_reach_live_upstream_head(self) -> None: + manifest_anchor = "3" * 40 + live_upstream_head = "7" * 40 + comparison = { + "base_commit": {"sha": manifest_anchor}, + "merge_base_commit": {"sha": "4" * 40}, + "status": "diverged", + "url": ( + "https://api.github.com/repos/ggml-org/llama.cpp/compare/" + f"{manifest_anchor}...{live_upstream_head}" + ), + } + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "not reachable", + ): + manifest.require_upstream_ancestry( + comparison, + "ggml-org/llama.cpp", + manifest_anchor, + live_upstream_head, + ) + + def test_unchanged_managed_pins_do_not_require_a_manifest(self) -> None: + versions = { + "llamacpp": { + "cpu": "b1", + "cuda": "b2", + "metal": "b1", + "rocm-nightly": "b3", + "rocm-stable": "b2", + "vulkan": "b1", + } + } + + self.assertFalse( + manifest.require_release_manifest_for_managed_pin_changes( + versions, + copy.deepcopy(versions), + None, + ) + ) + + def test_changed_managed_pin_requires_a_matching_release_manifest(self) -> None: + base = { + "llamacpp": { + "cpu": "b1", + "cuda": "b2", + "metal": "b1", + "rocm-nightly": "b3", + "rocm-stable": "b2", + "vulkan": "b1", + } + } + candidate = copy.deepcopy(base) + candidate["llamacpp"]["vulkan"] = "b1234" + release_manifest = manifest.build_release_asset_manifest( + [ + ("ggml-org/llama.cpp", "b1234", self.ggml), + ( + "lemonade-sdk/llamacpp-rocm", + "b3", + release_payload( + "b3", + 20, + 200, + publisher_claim_type="immutable-source-manifest", + ), + ), + ( + "lemonade-sdk/llama.cpp", + "b2", + release_payload( + "b2", + 30, + 300, + publisher_claim_type="immutable-source-manifest", + ), + ), + ] + ) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "changed managed llama.cpp pins", + ): + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + None, + ) + + self.assertTrue( + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + release_manifest, + ) + ) + + mismatched = json.loads(release_manifest) + mismatched["releases"][0]["tag_name"] = "b9999" + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "does not match changed managed llama.cpp pin", + ): + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + json.dumps(mismatched), + ) + + def test_manifest_binds_publisher_claim_and_upstream_reference(self) -> None: + payload = copy.deepcopy(self.rocm) + payload.update( + { + "publisher_claim_type": "immutable-source-manifest", + "upstream_reference": "refs/heads/master", + "upstream_reference_head": "d" * 40, + } + ) + + parsed = json.loads( + manifest.build_release_asset_manifest( + [("lemonade-sdk/llamacpp-rocm", "b1235", payload)] + ) + )["releases"][0] + + self.assertEqual( + parsed.get("publisher_claimed_source_commit"), + payload["publisher_claimed_source_commit"], + ) + self.assertEqual( + parsed.get("publisher_claim_type"), "immutable-source-manifest" + ) + self.assertIs(parsed.get("release_immutable"), True) + self.assertEqual(parsed.get("upstream_reference"), "refs/heads/master") + self.assertEqual(parsed.get("upstream_reference_head"), "d" * 40) + self.assertNotIn("release_body_digest", parsed) + self.assertNotIn("source_commit", parsed) + def test_tag_mismatch_and_duplicate_assets_fail_closed(self) -> None: with self.assertRaisesRegex(manifest.ReleaseManifestError, "tag mismatch"): manifest.build_release_asset_manifest( @@ -157,6 +799,107 @@ def test_comparison_rejects_any_asset_metadata_change(self) -> None: manifest.require_exact_manifest_match(expected, expected) + def test_canonical_manifest_rejects_duplicate_json_members(self) -> None: + canonical = manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", self.ggml)] + ) + duplicate = canonical[:-1] + ',"schema_version":4}' + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "duplicate JSON member", + ): + manifest.canonicalize_release_asset_manifest(duplicate) + + def test_canonical_manifest_requires_integer_schema_version(self) -> None: + canonical = json.loads( + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", self.ggml)] + ) + ) + for schema_version in (True, 4.0, "4"): + with self.subTest(schema_version=schema_version): + changed = copy.deepcopy(canonical) + changed["schema_version"] = schema_version + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "schema_version must be 4", + ): + manifest.canonicalize_release_asset_manifest(json.dumps(changed)) + + def test_comparison_rejects_any_publisher_claim_change(self) -> None: + release = release_payload( + "b1234", + 10, + 100, + publisher_claim_type="immutable-source-manifest", + publisher_claimed_source_commit="f" * 40, + ) + expected = manifest.build_release_asset_manifest( + [("lemonade-sdk/llama.cpp", "b1234", release)] + ) + cases = ( + ("release_tag_commit", "e" * 40), + ("publisher_claimed_source_commit", "e" * 40), + ) + for field, value in cases: + with self.subTest(field=field): + changed = copy.deepcopy(release) + changed[field] = value + actual = manifest.build_release_asset_manifest( + [("lemonade-sdk/llama.cpp", "b1234", changed)] + ) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "changed since validation", + ): + manifest.require_exact_manifest_match(expected, actual) + + def test_release_note_edits_do_not_change_manifest_identity(self) -> None: + expected = manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", self.ggml)] + ) + changed = copy.deepcopy(self.ggml) + changed["body"] = "updated release notes" + + self.assertEqual( + manifest.build_release_asset_manifest( + [("ggml-org/llama.cpp", "b1234", changed)] + ), + expected, + ) + + def test_materialized_manifest_binds_digest_and_publisher_claims(self) -> None: + expected = manifest.build_release_asset_manifest( + [ + ("ggml-org/llama.cpp", "b1234", self.ggml), + ("lemonade-sdk/llamacpp-rocm", "b1235", self.rocm), + ] + ) + expected_digest = hashlib.sha256(expected.encode("utf-8")).hexdigest() + + self.assertTrue( + hasattr(manifest, "materialize_release_asset_manifest"), + "manifest materializer is required", + ) + with tempfile.TemporaryDirectory() as directory: + output_path = Path(directory) / "release-manifest.json" + digest, summary = manifest.materialize_release_asset_manifest( + expected, + output_path, + ) + + self.assertEqual(digest, expected_digest) + self.assertEqual(output_path.read_text(encoding="utf-8"), expected + "\n") + self.assertIn(expected_digest, summary) + self.assertIn("Only immutable GitHub releases are accepted", summary) + self.assertIn("immutable publisher manifest", summary) + self.assertIn("not cryptographic build provenance", summary) + self.assertIn(self.ggml["publisher_claimed_source_commit"], summary) + self.assertIn(self.rocm["publisher_claimed_source_commit"], summary) + if __name__ == "__main__": unittest.main() diff --git a/test/test_publish_llamacpp_update.py b/test/test_publish_llamacpp_update.py index 8605de0eba..21fa79c352 100644 --- a/test/test_publish_llamacpp_update.py +++ b/test/test_publish_llamacpp_update.py @@ -3,6 +3,8 @@ from __future__ import annotations +import hashlib +import json import os import shutil import stat @@ -11,9 +13,12 @@ import unittest from pathlib import Path +from test.utils import llamacpp_release_manifest as release_manifest + ROOT = Path(__file__).resolve().parents[1] PUBLISHER = ROOT / ".github" / "scripts" / "publish_llamacpp_update.sh" VERSIONS_PATH = Path("src/cpp/resources/backend_versions.json") +MANIFEST_PATH = Path(".github/llamacpp_release_manifest.json") class PublishLlamaCppUpdateTests(unittest.TestCase): @@ -25,6 +30,8 @@ def setUp(self) -> None: self.remote = self.temp_root / "remote.git" self.fake_bin = self.temp_root / "fake-bin" self.event_log = self.temp_root / "events.log" + self.snapshot_count = self.temp_root / "snapshot-count.txt" + self.closed_prs = self.temp_root / "closed-prs.txt" self.real_git = shutil.which("git") if not self.real_git: self.skipTest("git is required") @@ -39,6 +46,35 @@ def setUp(self) -> None: versions.write_text( '{"llamacpp":{"vulkan":"b1","cpu":"b1"}}\n', encoding="utf-8" ) + self.upstream_head = "f" * 40 + self.live_upstream_head = "9" * 40 + self.release_payloads = { + "ggml": self.release_payload( + repository="ggml-org/llama.cpp", + tag="b100", + release_id=100, + release_tag_commit="a" * 40, + claimed_source_commit="a" * 40, + ), + "rocm": self.release_payload( + repository="lemonade-sdk/llamacpp-rocm", + tag="b300", + release_id=300, + release_tag_commit="b" * 40, + claimed_source_commit="c" * 40, + ), + "lemonade": self.release_payload( + repository="lemonade-sdk/llama.cpp", + tag="b200", + release_id=200, + release_tag_commit="d" * 40, + claimed_source_commit="e" * 40, + ), + } + self.expected_manifest = self.build_expected_manifest() + manifest_path = self.repo / MANIFEST_PATH + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(self.expected_manifest + "\n", encoding="utf-8") (self.repo / "pr_body.md").write_text("initial body\n", encoding="utf-8") self.run_git("add", "--all") self.run_git("commit", "--quiet", "--message", "base") @@ -51,7 +87,101 @@ def setUp(self) -> None: def branch_name(self) -> str: base_sha = self.run_git("rev-parse", "HEAD") versions_blob = self.run_git("hash-object", str(VERSIONS_PATH)) - return f"auto/llamacpp-update-b100-b200-b300-{base_sha}-{versions_blob}" + manifest_digest = hashlib.sha256(self.expected_manifest.encode()).hexdigest() + return ( + f"auto/llamacpp-update-b100-b200-b300-{base_sha}-{versions_blob}-" + f"{manifest_digest}" + ) + + def release_payload( + self, + *, + repository: str, + tag: str, + release_id: int, + release_tag_commit: str, + claimed_source_commit: str, + ) -> dict: + binary_asset = { + "digest": "sha256:" + f"{release_id:064x}"[-64:], + "id": release_id * 10, + "name": f"asset-{release_id}.zip", + "size": release_id, + "state": "uploaded", + "updated_at": "2026-09-06T12:00:00Z", + } + assets = [binary_asset] + source_manifest = None + if repository != "ggml-org/llama.cpp": + source_manifest = json.dumps( + { + "assets": [ + { + "digest": binary_asset["digest"], + "name": binary_asset["name"], + "size": binary_asset["size"], + } + ], + "release_repository": repository, + "release_tag": tag, + "release_tag_commit": release_tag_commit, + "schema_version": 1, + "source_commit": claimed_source_commit, + "source_repository": "ggml-org/llama.cpp", + }, + separators=(",", ":"), + sort_keys=True, + ) + source_bytes = source_manifest.encode("utf-8") + assets.append( + { + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + "id": release_id * 10 + 1, + "name": ".llamacpp-source.json", + "size": len(source_bytes), + "state": "uploaded", + "updated_at": "2026-09-06T12:01:00Z", + } + ) + return { + "assets": assets, + "body": "release notes", + "draft": False, + "id": release_id, + "immutable": True, + "tag_name": tag, + "_claimed_source_commit": claimed_source_commit, + "_release_tag_commit": release_tag_commit, + "_source_manifest": source_manifest, + } + + def build_expected_manifest(self) -> str: + releases = [] + for key, repository, tag in ( + ("ggml", "ggml-org/llama.cpp", "b100"), + ("rocm", "lemonade-sdk/llamacpp-rocm", "b300"), + ("lemonade", "lemonade-sdk/llama.cpp", "b200"), + ): + payload = dict(self.release_payloads[key]) + claimed_commit = payload.pop("_claimed_source_commit") + release_tag_commit = payload.pop("_release_tag_commit") + payload.pop("_source_manifest") + payload.update( + { + "publisher_claim_type": ( + "source-release-tag" + if repository == "ggml-org/llama.cpp" + else "immutable-source-manifest" + ), + "publisher_claimed_source_commit": claimed_commit, + "release_tag_commit": release_tag_commit, + "source_repository": "ggml-org/llama.cpp", + "upstream_reference": "refs/heads/master", + "upstream_reference_head": self.upstream_head, + } + ) + releases.append((repository, tag, payload)) + return release_manifest.build_release_asset_manifest(releases) def run_command( self, @@ -84,6 +214,13 @@ def install_command_wrappers(self) -> None: "printf 'git' >> \"$EVENT_LOG\"\n" 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' "printf '\\n' >> \"$EVENT_LOG\"\n" + 'if [ "$1" = push ] && ' + '[ "$MUTATE_BRANCH_AFTER_PUSH" = true ]; then\n' + ' "$REAL_GIT" "$@" || exit $?\n' + ' "$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" update-ref ' + '"refs/heads/$EXPECTED_BRANCH" "$VALIDATED_BASE_SHA"\n' + " exit $?\n" + "fi\n" 'exec "$REAL_GIT" "$@"\n', encoding="utf-8", ) @@ -95,16 +232,102 @@ def install_command_wrappers(self) -> None: "printf 'gh' >> \"$EVENT_LOG\"\n" 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' "printf '\\n' >> \"$EVENT_LOG\"\n" + 'if [ "$1" = pr ] && [ "$2" = create ]; then\n' + " printf 'https://github.com/LLM360/lemonade/pull/%s\\n' " + '"$CREATED_PR_NUMBER"\n' + " exit 0\n" + "fi\n" 'if [ "$1" = api ]; then\n' ' case "$2" in\n' + " repos/ggml-org/llama.cpp) printf '%s\\n' master ;;\n" " repos/ggml-org/llama.cpp/releases) " "printf '%s\\n' \"$FRESH_GGML_RELEASE\" ;;\n" " repos/lemonade-sdk/llamacpp-rocm/releases/latest) " "printf '%s\\n' \"$FRESH_ROCM_RELEASE\" ;;\n" " repos/lemonade-sdk/llama.cpp/releases/latest) " "printf '%s\\n' \"$FRESH_LEMONADE_RELEASE\" ;;\n" + " repos/ggml-org/llama.cpp/releases/tags/b100)\n" + " snapshot=0\n" + ' if [ -f "$SNAPSHOT_COUNT_FILE" ]; then ' + 'snapshot=$(cat "$SNAPSHOT_COUNT_FILE"); fi\n' + " snapshot=$((snapshot + 1))\n" + ' printf \'%s\\n\' "$snapshot" > "$SNAPSHOT_COUNT_FILE"\n' + " printf '%s\\n' \"$GGML_RELEASE_PAYLOAD\" ;;\n" + " repos/lemonade-sdk/llamacpp-rocm/releases/tags/b300)\n" + ' if [ "$MUTATE_ON_SNAPSHOT" -gt 0 ] && ' + '[ "$(cat "$SNAPSHOT_COUNT_FILE")" -ge "$MUTATE_ON_SNAPSHOT" ]; then\n' + " printf '%s\\n' \"$MUTATED_ROCM_RELEASE_PAYLOAD\"\n" + " else\n" + " printf '%s\\n' \"$ROCM_RELEASE_PAYLOAD\"\n" + " fi ;;\n" + " repos/lemonade-sdk/llama.cpp/releases/tags/b200) " + "printf '%s\\n' \"$LEMONADE_RELEASE_PAYLOAD\" ;;\n" + " repos/lemonade-sdk/llamacpp-rocm/releases/assets/3001) " + "printf '%s' \"$ROCM_SOURCE_MANIFEST\" ;;\n" + " repos/lemonade-sdk/llama.cpp/releases/assets/2001) " + "printf '%s' \"$LEMONADE_SOURCE_MANIFEST\" ;;\n" + " repos/ggml-org/llama.cpp/commits/b100) " + "printf '%s\\n' \"$GGML_RELEASE_TAG_COMMIT\" ;;\n" + " repos/lemonade-sdk/llamacpp-rocm/commits/b300) " + "printf '%s\\n' \"$ROCM_RELEASE_TAG_COMMIT\" ;;\n" + " repos/lemonade-sdk/llama.cpp/commits/b200) " + "printf '%s\\n' \"$LEMONADE_RELEASE_TAG_COMMIT\" ;;\n" + " repos/ggml-org/llama.cpp/commits/master) " + "printf '%s\\n' \"$LIVE_UPSTREAM_HEAD\" ;;\n" + " repos/ggml-org/llama.cpp/compare/*)\n" + " comparison=${2##*/}; base=${comparison%%...*}; " + "head=${comparison#*...}\n" + " status=ahead; merge_base=$base\n" + ' if [ -n "$DIVERGENT_COMMIT" ] && ' + '[ "$base" = "$DIVERGENT_COMMIT" ]; then\n' + " status=diverged; merge_base=$LIVE_UPSTREAM_HEAD\n" + " fi\n" + " printf " + '\'{"status":"%s","base_commit":{"sha":"%s"},' + '"merge_base_commit":{"sha":"%s"},' + '"url":"https://api.github.com/repos/ggml-org/llama.cpp/compare/%s...%s"}\\n\' ' + '"$status" "$base" "$merge_base" "$base" "$head" ;;\n' " repos/LLM360/lemonade/pulls) " "printf '%s\\n' \"$OPEN_PR_RECORDS\" ;;\n" + " repos/LLM360/lemonade/pulls/[0-9]*)\n" + " number=${2##*/}\n" + ' case " $* " in\n' + ' *" --method PATCH "*" state=closed "*)\n' + ' printf \'%s\\n\' "$number" >> "$CLOSED_PRS_FILE"\n' + " printf '{}\\n'\n" + " exit 0 ;;\n" + " esac\n" + " record=$(printf '%s\\n' \"$OPEN_PR_RECORDS\" | " + "awk -F '\\t' -v number=\"$number\" '$1 == number { print; exit }')\n" + " head_ref=$(printf '%s' \"$record\" | cut -f2)\n" + " head_repository=$(printf '%s' \"$record\" | cut -f3)\n" + " author=$(printf '%s' \"$record\" | cut -f4)\n" + " base_ref=$(printf '%s' \"$record\" | cut -f5)\n" + " head_ref=${head_ref:-$EXPECTED_BRANCH}\n" + " head_repository=${head_repository:-LLM360/lemonade}\n" + " author=${author:-github-actions[bot]}\n" + " base_ref=${base_ref:-main}\n" + " head_ref=${PR_DETAIL_HEAD_REF:-$head_ref}\n" + " head_repository=${PR_DETAIL_HEAD_REPOSITORY:-$head_repository}\n" + " author=${PR_DETAIL_AUTHOR:-$author}\n" + " base_ref=${PR_DETAIL_BASE_REF:-$base_ref}\n" + " state=open\n" + ' if [ -f "$CLOSED_PRS_FILE" ] && ' + 'grep -qx "$number" "$CLOSED_PRS_FILE"; then state=closed; fi\n' + ' head_sha=$("$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" ' + 'show-ref --verify --hash "refs/heads/$head_ref" 2>/dev/null || ' + "printf '%040d' 0)\n" + " head_sha=${PR_DETAIL_HEAD_SHA:-$head_sha}\n" + ' base_sha=$("$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" ' + 'show-ref --verify --hash "refs/heads/$base_ref" 2>/dev/null || ' + "printf '%040d' 0)\n" + " printf " + '\'{"state":"%s","head":{"ref":"%s",' + '"sha":"%s","repo":{"full_name":"%s"}},' + '"user":{"login":"%s"},"base":{"ref":"%s",' + '"sha":"%s"}}\\n\' ' + '"$state" "$head_ref" "$head_sha" "$head_repository" ' + '"$author" "$base_ref" "$base_sha" ;;\n' " esac\n" "fi\n" "exit 0\n", @@ -115,45 +338,108 @@ def install_command_wrappers(self) -> None: def publisher_env( self, *, + divergent_commit: str = "", + include_expected_manifest: bool = True, + mutate_on_snapshot: int = 0, open_pr_records: str = "", fresh_ggml_release: str = "b100", + mutate_branch_after_push: bool = False, + pr_detail_base_ref: str = "", + pr_detail_head_ref: str = "", validated_base_sha: str | None = None, ) -> dict[str, str]: + api_payloads = {} + for name, payload in self.release_payloads.items(): + api_payloads[name] = { + key: value for key, value in payload.items() if not key.startswith("_") + } + mutated_rocm = dict(api_payloads["rocm"]) + mutated_rocm["assets"] = [dict(asset) for asset in mutated_rocm["assets"]] + mutated_rocm["assets"][0]["updated_at"] = "2026-09-06T12:02:00Z" env = os.environ.copy() env.update( { + "DIVERGENT_COMMIT": divergent_commit, + "CLOSED_PRS_FILE": str(self.closed_prs), + "CREATED_PR_NUMBER": "41", "EVENT_LOG": str(self.event_log), + "EXPECTED_BRANCH": self.branch_name, "FRESH_GGML_RELEASE": fresh_ggml_release, "FRESH_LEMONADE_RELEASE": "b200", "FRESH_ROCM_RELEASE": "b300", + "GGML_RELEASE_PAYLOAD": json.dumps(api_payloads["ggml"]), + "GGML_RELEASE_TAG_COMMIT": self.release_payloads["ggml"][ + "_release_tag_commit" + ], "GITHUB_REPOSITORY": "LLM360/lemonade", "GITHUB_REPOSITORY_OWNER": "LLM360", + "LEMONADE_RELEASE_PAYLOAD": json.dumps(api_payloads["lemonade"]), + "LEMONADE_SOURCE_MANIFEST": self.release_payloads["lemonade"][ + "_source_manifest" + ], + "LEMONADE_RELEASE_TAG_COMMIT": self.release_payloads["lemonade"][ + "_release_tag_commit" + ], "LLAMACPP_LEMONADE_RELEASE": "b200", "LLAMACPP_RELEASE": "b100", "LLAMACPP_ROCM_RELEASE": "b300", + "LIVE_UPSTREAM_HEAD": self.live_upstream_head, + "MUTATED_ROCM_RELEASE_PAYLOAD": json.dumps(mutated_rocm), + "MUTATE_ON_SNAPSHOT": str(mutate_on_snapshot), + "MUTATE_BRANCH_AFTER_PUSH": ( + "true" if mutate_branch_after_push else "false" + ), "OPEN_PR_RECORDS": open_pr_records, "PATH": f"{self.fake_bin}{os.pathsep}{env['PATH']}", "PR_BODY_FILE": "pr_body.md", + "PR_DETAIL_AUTHOR": "", + "PR_DETAIL_BASE_REF": pr_detail_base_ref, + "PR_DETAIL_HEAD_REF": pr_detail_head_ref, + "PR_DETAIL_HEAD_REPOSITORY": "", + "PR_DETAIL_HEAD_SHA": "", "REAL_GIT": self.real_git, + "REMOTE_REPOSITORY_PATH": str(self.remote), + "ROCM_RELEASE_PAYLOAD": json.dumps(api_payloads["rocm"]), + "ROCM_SOURCE_MANIFEST": self.release_payloads["rocm"][ + "_source_manifest" + ], + "ROCM_RELEASE_TAG_COMMIT": self.release_payloads["rocm"][ + "_release_tag_commit" + ], + "SNAPSHOT_COUNT_FILE": str(self.snapshot_count), "VALIDATED_BASE_SHA": validated_base_sha or self.run_git("rev-parse", "HEAD"), } ) + if include_expected_manifest: + env["EXPECTED_RELEASE_ASSET_MANIFEST"] = self.expected_manifest return env def run_publisher( self, *, + divergent_commit: str = "", + include_expected_manifest: bool = True, + mutate_on_snapshot: int = 0, open_pr_records: str = "", fresh_ggml_release: str = "b100", + mutate_branch_after_push: bool = False, + pr_detail_base_ref: str = "", + pr_detail_head_ref: str = "", validated_base_sha: str | None = None, ) -> subprocess.CompletedProcess[str]: return self.run_command( ["bash", str(PUBLISHER)], cwd=self.repo, env=self.publisher_env( + divergent_commit=divergent_commit, + include_expected_manifest=include_expected_manifest, + mutate_on_snapshot=mutate_on_snapshot, open_pr_records=open_pr_records, fresh_ggml_release=fresh_ggml_release, + mutate_branch_after_push=mutate_branch_after_push, + pr_detail_base_ref=pr_detail_base_ref, + pr_detail_head_ref=pr_detail_head_ref, validated_base_sha=validated_base_sha, ), check=False, @@ -260,6 +546,107 @@ def test_stale_candidate_fails_before_query_or_push(self) -> None: ) self.assertFalse(any(event.startswith("git ") for event in events)) + def test_missing_release_manifest_fails_before_external_queries(self) -> None: + self.change_versions() + + result = self.run_publisher(include_expected_manifest=False) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("EXPECTED_RELEASE_ASSET_MANIFEST", result.stderr) + self.assertEqual(self.events(), []) + + def test_release_mutation_at_final_recheck_prevents_remote_mutation(self) -> None: + self.change_versions() + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + mutate_on_snapshot=3, + open_pr_records=stale_record, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("changed since validation", result.stderr) + events = self.events() + self.assertFalse(any(event.startswith("git ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertFalse(any("<-f> " in event for event in events)) + + def test_mutable_release_prevents_remote_mutation(self) -> None: + self.change_versions() + self.release_payloads["rocm"]["immutable"] = False + + result = self.run_publisher() + + self.assertNotEqual(result.returncode, 0) + self.assertIn("release immutable must be true", result.stderr) + events = self.events() + self.assertFalse(any(event.startswith("git ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + + def test_corrupt_source_manifest_prevents_remote_mutation(self) -> None: + self.change_versions() + self.release_payloads["rocm"]["_source_manifest"] += " " + + result = self.run_publisher() + + self.assertNotEqual(result.returncode, 0) + self.assertIn("source manifest size does not match", result.stderr) + events = self.events() + self.assertFalse(any(event.startswith("git ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + + def test_fork_network_manifest_anchor_fails_against_live_upstream(self) -> None: + self.change_versions() + + result = self.run_publisher(divergent_commit=self.upstream_head) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("not reachable", result.stderr) + events = self.events() + self.assertTrue( + any( + f"{self.upstream_head}...{self.live_upstream_head}>" in event + for event in events + ) + ) + self.assertFalse(any(event.startswith("git ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + + def test_publication_binds_manifest_to_branch_commit_and_pr_body(self) -> None: + self.change_versions() + (self.repo / MANIFEST_PATH).write_text("tampered\n", encoding="utf-8") + expected_branch = self.branch_name + expected_digest = hashlib.sha256( + self.expected_manifest.encode("utf-8") + ).hexdigest() + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.run_git("branch", "--show-current"), expected_branch) + self.assertEqual( + self.run_git("show", f"HEAD:{MANIFEST_PATH}"), + self.expected_manifest, + ) + self.assertIn( + f"LlamaCpp-Release-Manifest-SHA256: {expected_digest}", + self.run_git("log", "-1", "--pretty=%B"), + ) + body = (self.repo / "pr_body.md").read_text(encoding="utf-8") + self.assertIn(f"Canonical manifest SHA-256: `{expected_digest}`", body) + self.assertIn(self.upstream_head, body) + self.assertIn("`true`", body) + self.assertIn("Only immutable GitHub releases are accepted", body) + self.assertIn("immutable publisher manifest", body) + self.assertIn("not cryptographic build provenance", body) + for payload in self.release_payloads.values(): + self.assertIn(payload["_release_tag_commit"], body) + self.assertIn(payload["_claimed_source_commit"], body) + self.assertGreaterEqual(int(self.snapshot_count.read_text()), 3) + def test_mismatched_validated_base_is_rejected_before_publication(self) -> None: self.change_versions() @@ -334,6 +721,53 @@ def test_trusted_open_pr_is_refreshed_on_the_same_validated_base(self) -> None: self.assertIn("<--body-file> ", events[edit_index]) self.assertFalse(any(event.startswith("gh ") for event in events)) + def test_retargeted_open_pr_fails_before_refresh(self) -> None: + self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + self.change_versions() + + result = self.run_publisher( + open_pr_records=self.trusted_open_pr_record(), + pr_detail_base_ref="release", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("identity changed before publication", result.stderr) + self.assertFalse( + any(event.startswith("gh ") for event in self.events()) + ) + + def test_remote_branch_move_after_push_fails_before_pr_mutation(self) -> None: + self.change_versions() + + result = self.run_publisher(mutate_branch_after_push=True) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("publication branch changed after push", result.stderr) + self.assertFalse( + any( + event.startswith("gh ") + or event.startswith("gh ") + for event in self.events() + ) + ) + + def test_retargeted_stale_pr_is_not_closed(self) -> None: + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + open_pr_records=stale_record, + pr_detail_head_ref="retargeted-branch", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("identity changed before publication", result.stderr) + self.assertFalse(any("<-f> " in event for event in self.events())) + def test_trusted_open_pr_with_unrelated_changes_is_not_overwritten(self) -> None: _base_oid, branch_name, _branch_oid = self.create_remote_update_branch( '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n', diff --git a/test/test_validate_llamacpp_selection.py b/test/test_validate_llamacpp_selection.py index 15af57565b..9c64745644 100644 --- a/test/test_validate_llamacpp_selection.py +++ b/test/test_validate_llamacpp_selection.py @@ -424,7 +424,7 @@ def test_request_json_rejects_duplicate_response_members(self) -> None: def request_json_for_chat(self, chat_message, operations=None): loaded = False - def request_json(method, url, timeout, **_kwargs): + def request_json(method, url, timeout, **kwargs): nonlocal loaded del method, timeout operation = url.rsplit("/", maxsplit=1)[-1] @@ -450,7 +450,12 @@ def request_json(method, url, timeout, **_kwargs): ) } if operation == "completions": - return self.response(), {"choices": [{"message": chat_message}]} + message = ( + chat_message(kwargs["json"]) + if callable(chat_message) + else chat_message + ) + return self.response(), {"choices": [{"message": message}]} if operation == "stats": return self.response(), {"output_tokens": 4} return self.response(), {} @@ -605,6 +610,33 @@ def test_plain_chat_requires_visible_final_content(self) -> None: self.assertFalse(success) self.assertIn("visible final content", error) + def test_plain_smoke_disables_reasoning_to_get_visible_final_content(self) -> None: + def reasoning_model_response(payload): + self.assertEqual(payload["max_completion_tokens"], 50) + if payload.get("chat_template_kwargs") == {"enable_thinking": False}: + return {"content": "The answer is 4."} + return { + "content": "", + "reasoning_content": "The answer should be four.", + } + + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat(reasoning_model_response), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, response, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + + self.assertTrue(success, response) + self.assertEqual(response, "The answer is 4.") + def test_model_runs_requested_capability_profile_and_records_evidence(self) -> None: matrix = { "profile": capabilities.K2_HORIZON_PROFILE, diff --git a/test/test_validate_llamacpp_workflow_plan.py b/test/test_validate_llamacpp_workflow_plan.py index fee0bd3bfe..de300d68a7 100644 --- a/test/test_validate_llamacpp_workflow_plan.py +++ b/test/test_validate_llamacpp_workflow_plan.py @@ -210,6 +210,7 @@ def test_focused_ci_runs_the_capability_contract_tests(self) -> None: self.assertIn("test.test_llamacpp_capabilities", workflow) self.assertIn("test.test_llamacpp_validation_evidence", workflow) self.assertIn("test.test_validate_llamacpp_selection", workflow) + self.assertIn("test.test_verify_llamacpp_pin_manifest", workflow) def test_release_asset_manifest_is_captured_and_exported(self) -> None: validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") @@ -229,10 +230,126 @@ def test_release_asset_manifest_is_captured_and_exported(self) -> None: self.assertIn("llamacpp_release_manifest", asset_job) self.assertIn('--github-output "$GITHUB_OUTPUT"', asset_job) self.assertIn("jq -r '.assets[].name'", asset_job) - self.assertIn("repos/${repository}/commits/${tag}", asset_job) - self.assertIn("source_commit", asset_job) + self.assertIn("attach_release_claims", asset_job) + self.assertIn("--source-manifest-asset-id", asset_job) + self.assertIn("--validate-source-manifest", asset_job) + self.assertIn("Accept: application/octet-stream", asset_job) + self.assertIn("source_manifest_max_bytes=65536", asset_job) + self.assertIn("source_manifest_directory=$(mktemp -d)", asset_job) + self.assertIn( + 'source_manifest_path="${source_manifest_directory}/source-' + '${source_manifest_asset_id}.json"', + asset_job, + ) + self.assertIn('head -c "$((source_manifest_max_bytes + 1))"', asset_job) + self.assertIn('publisher_claim_type="immutable-source-manifest"', asset_job) + self.assertNotIn("--extract-source-commit", asset_job) + self.assertIn("--require-upstream-ancestry", asset_job) + self.assertIn("/compare/", asset_job) + self.assertRegex( + asset_job, + r"attach_release_claims lemonade-sdk/llamacpp-rocm " + r'\s*\\?\s*"\$ROCM_RELEASE" rocm_release\.json', + ) + self.assertRegex( + asset_job, + r"attach_release_claims lemonade-sdk/llama\.cpp " + r'\s*\\?\s*"\$LEMONADE_RELEASE" lemonade_release\.json', + ) + self.assertIn("source_repository=ggml-org/llama.cpp", asset_job) + self.assertIn("upstream_reference_head", asset_job) + self.assertIn("publisher_claimed_source_commit", asset_job) + self.assertIn("publisher_claim_type", asset_job) + self.assertIn("release_tag_commit", asset_job) + self.assertIn("source_repository", asset_job) + self.assertNotIn("attach_source_commit", asset_job) + self.assertNotIn( + 'source_commit=$(gh api "repos/${repository}/commits/${tag}"', + asset_job, + ) self.assertNotIn("release_asset_manifest", release_job) + def test_protected_pin_changes_require_live_committed_manifest(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn(" verify-protected-release-manifest:\n", validation) + manifest_job = validation.split(" verify-protected-release-manifest:\n", 1)[ + 1 + ].split(" build:\n", 1)[0] + build_job = validation.split(" build:\n", 1)[1].split(" validate:\n", 1)[0] + verifier = ( + ROOT / ".github" / "scripts" / "verify_llamacpp_pin_manifest.sh" + ).read_text(encoding="utf-8") + + self.assertIn("pull_request_target", manifest_job) + self.assertIn("merge_group", manifest_job) + self.assertIn("base.sha", manifest_job) + self.assertIn("merge_group.base_sha", manifest_job) + self.assertIn("verify_llamacpp_pin_manifest.sh", manifest_job) + self.assertNotIn("id: managed-pin-changes", manifest_job) + self.assertNotIn("steps.managed-pin-changes.outputs.changed", manifest_job) + self.assertIn(".github/llamacpp_release_manifest.json", verifier) + self.assertIn("capture_llamacpp_release_manifest.sh", verifier) + self.assertIn("changed managed llama.cpp pins", verifier) + self.assertIn("verify-protected-release-manifest", build_job) + self.assertIn(" reverify-protected-release-manifest:\n", validation) + final_manifest_job = validation.split( + " reverify-protected-release-manifest:\n", 1 + )[1].split(" validation-gate:\n", 1)[0] + validation_gate = validation.split(" validation-gate:\n", 1)[1] + self.assertIn("needs: [validate", final_manifest_job) + self.assertIn("verify_llamacpp_pin_manifest.sh", final_manifest_job) + self.assertIn("reverify-protected-release-manifest", validation_gate) + + def test_protected_sidecar_only_changes_fail_closed(self) -> None: + verifier = ( + ROOT / ".github" / "scripts" / "verify_llamacpp_pin_manifest.sh" + ).read_text(encoding="utf-8") + + self.assertIn("manifest_path=.github/llamacpp_release_manifest.json", verifier) + self.assertIn( + 'base_manifest_entry=$(git -C "$trusted_repository" ls-tree', verifier + ) + self.assertIn( + 'candidate_manifest_entry=$(git -C "$candidate_repository" ls-tree', + verifier, + ) + self.assertIn("if [[ \"$changes\" == '{}' ]]", verifier) + self.assertIn( + '[[ "$base_manifest_entry" != "$candidate_manifest_entry" ]]', verifier + ) + self.assertIn( + "Release manifest changes require changed managed llama.cpp pins.", + verifier, + ) + + def test_manifest_recheck_anchors_to_live_trusted_upstream(self) -> None: + capture = ( + ROOT / ".github" / "scripts" / "capture_llamacpp_release_manifest.sh" + ).read_text(encoding="utf-8") + + self.assertIn("trusted_source_repository=ggml-org/llama.cpp", capture) + self.assertIn(".default_branch", capture) + self.assertIn("live_upstream_head", capture) + self.assertIn("upstream-anchor-comparison", capture) + self.assertIn("--source-manifest-asset-id", capture) + self.assertIn("--validate-source-manifest", capture) + self.assertIn("Accept: application/octet-stream", capture) + self.assertIn("source_manifest_max_bytes=65536", capture) + self.assertIn("source_manifest_directory=$(mktemp -d)", capture) + self.assertIn( + 'source_manifest_path="${source_manifest_directory}/source-' + '${source_manifest_asset_id}.json"', + capture, + ) + self.assertIn('head -c "$((source_manifest_max_bytes + 1))"', capture) + self.assertIn('publisher_claim_type="immutable-source-manifest"', capture) + self.assertIn( + "${upstream_reference_head}...${live_upstream_head}", + capture, + ) + self.assertIn("release_pattern='^b[0-9]+$'", capture) + def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: pull_request = PR_WORKFLOW.read_text(encoding="utf-8") validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") @@ -368,7 +485,9 @@ def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: self.assertIn("needs: validate-invocation", plan_job) self.assertIn("validate-invocation", build_job_header) self.assertIn( - "needs: [validate-invocation, build, validate, plan]", validation_gate + "needs: [validate-invocation, build, validate, " + "reverify-protected-release-manifest, plan]", + validation_gate, ) self.assertNotIn("checks: write", validation_gate) self.assertNotIn("check-runs", validation_gate) @@ -382,7 +501,7 @@ def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: ) self.assertIn("github.event.pull_request.base.sha", validation) self.assertIn("github.event.pull_request.head.sha", validation) - self.assertEqual(validation.count("persist-credentials: false"), 4) + self.assertEqual(validation.count("persist-credentials: false"), 8) def test_authorization_removal_revokes_the_merge_check(self) -> None: pull_request = PR_WORKFLOW.read_text(encoding="utf-8") @@ -541,7 +660,7 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: evidence_step = publication.split( " - name: Validate complete validation evidence\n", 1 )[1].split( - " - name: Update backend_versions.json with verified releases\n", 1 + " - name: Update backend_versions.json from release manifest\n", 1 )[ 0 ] @@ -559,9 +678,9 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: "needs.validate.outputs.release_asset_manifest", manifest_step, ) - self.assertIn("gh api", manifest_step) - self.assertIn("repos/${repository}/commits/${tag}", manifest_step) - self.assertIn("source_commit", manifest_step) + self.assertIn("capture_llamacpp_release_manifest.sh", manifest_step) + self.assertIn("--materialize-manifest", manifest_step) + self.assertIn("EXPECTED_RELEASE_ASSET_MANIFEST", publication) self.assertNotIn("\n - name:", manifest_step) self.assertLess( publication.index(" - name: Generate PR body\n"), diff --git a/test/test_verify_llamacpp_pin_manifest.py b/test/test_verify_llamacpp_pin_manifest.py new file mode 100644 index 0000000000..f1ef59a0a3 --- /dev/null +++ b/test/test_verify_llamacpp_pin_manifest.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Behavior tests for trusted llama.cpp pin-manifest verification.""" + +from __future__ import annotations + +import copy +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +from test.utils import llamacpp_release_manifest as manifest + +ROOT = Path(__file__).resolve().parents[1] +VERIFIER = ROOT / ".github" / "scripts" / "verify_llamacpp_pin_manifest.sh" +MANIFEST_TOOL = ROOT / "test" / "utils" / "llamacpp_release_manifest.py" +VERSIONS_PATH = Path("src/cpp/resources/backend_versions.json") +RELEASE_MANIFEST_PATH = Path(".github/llamacpp_release_manifest.json") +MISSING = object() + + +def release_payload(repository: str, tag: str, release_id: int) -> dict: + release_tag_commit = f"{release_id:040x}" + claim_type = ( + "source-release-tag" + if repository == manifest.TRUSTED_SOURCE_REPOSITORY + else "immutable-source-manifest" + ) + assets = [ + { + "digest": "sha256:" + f"{release_id:064x}"[-64:], + "id": release_id * 10, + "name": f"llama-{release_id}.zip", + "size": release_id, + "state": "uploaded", + "updated_at": "2026-09-05T12:30:00Z", + } + ] + if claim_type == "immutable-source-manifest": + assets.append( + { + "digest": "sha256:" + "a" * 64, + "id": release_id * 10 + 1, + "name": manifest.SOURCE_MANIFEST_ASSET_NAME, + "size": 512, + "state": "uploaded", + "updated_at": "2026-09-05T12:31:00Z", + } + ) + return { + "assets": assets, + "draft": False, + "id": release_id, + "immutable": True, + "publisher_claim_type": claim_type, + "publisher_claimed_source_commit": release_tag_commit, + "release_tag_commit": release_tag_commit, + "source_repository": manifest.TRUSTED_SOURCE_REPOSITORY, + "tag_name": tag, + "upstream_reference": "refs/heads/master", + "upstream_reference_head": "f" * 40, + } + + +class VerifyLlamaCppPinManifestTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.directory = Path(self.temporary_directory.name) + self.tool_root = self.directory / "tools" + scripts = self.tool_root / ".github" / "scripts" + utilities = self.tool_root / "test" / "utils" + scripts.mkdir(parents=True) + utilities.mkdir(parents=True) + shutil.copy2(VERIFIER, scripts / VERIFIER.name) + shutil.copy2(MANIFEST_TOOL, utilities / MANIFEST_TOOL.name) + capture = scripts / "capture_llamacpp_release_manifest.sh" + capture.write_text( + "#!/usr/bin/env bash\n" + 'printf \'%s\\n\' "$*" >> "$CAPTURE_LOG"\n' + 'exit "$CAPTURE_EXIT"\n', + encoding="utf-8", + ) + capture.chmod(0o755) + self.verifier = scripts / VERIFIER.name + self.case_number = 0 + self.capture_log = self.directory / "capture-0.log" + self.base_versions = { + "llamacpp": { + "cpu": "b1", + "cuda": "b2", + "metal": "b1", + "rocm-nightly": "b3", + "rocm-stable": "b2", + "vulkan": "b1", + } + } + + def tearDown(self) -> None: + self.temporary_directory.cleanup() + + @staticmethod + def initialize_repository( + path: Path, + versions: dict, + release_manifest: object = MISSING, + ) -> str: + path.mkdir() + subprocess.run(["git", "init", "-q", str(path)], check=True) + subprocess.run( + ["git", "-C", str(path), "config", "user.name", "Test"], check=True + ) + subprocess.run( + ["git", "-C", str(path), "config", "user.email", "test@example.com"], + check=True, + ) + subprocess.run( + ["git", "-C", str(path), "config", "commit.gpgsign", "false"], + check=True, + ) + versions_path = path / VERSIONS_PATH + versions_path.parent.mkdir(parents=True) + versions_path.write_text(json.dumps(versions) + "\n", encoding="utf-8") + if release_manifest is not MISSING: + manifest_path = path / RELEASE_MANIFEST_PATH + manifest_path.parent.mkdir(parents=True) + manifest_path.write_text(str(release_manifest), encoding="utf-8") + subprocess.run(["git", "-C", str(path), "add", "."], check=True) + subprocess.run( + ["git", "-C", str(path), "commit", "-q", "-m", "fixture"], + check=True, + ) + return subprocess.run( + ["git", "-C", str(path), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + def matching_manifest(self, ggml_tag: str = "b9") -> str: + return manifest.build_release_asset_manifest( + [ + ( + "ggml-org/llama.cpp", + ggml_tag, + release_payload("ggml-org/llama.cpp", ggml_tag, 9), + ), + ( + "lemonade-sdk/llama.cpp", + "b2", + release_payload("lemonade-sdk/llama.cpp", "b2", 2), + ), + ( + "lemonade-sdk/llamacpp-rocm", + "b3", + release_payload("lemonade-sdk/llamacpp-rocm", "b3", 3), + ), + ] + ) + + def run_verifier( + self, + candidate_versions: dict, + *, + base_manifest: object = MISSING, + candidate_manifest: object = MISSING, + capture_exit: int = 37, + ) -> subprocess.CompletedProcess[str]: + self.case_number += 1 + trusted = self.directory / f"trusted-{self.case_number}" + candidate = self.directory / f"candidate-{self.case_number}" + self.capture_log = self.directory / f"capture-{self.case_number}.log" + base_sha = self.initialize_repository( + trusted, self.base_versions, base_manifest + ) + candidate_sha = self.initialize_repository( + candidate, candidate_versions, candidate_manifest + ) + env = os.environ.copy() + env.update( + { + "CAPTURE_EXIT": str(capture_exit), + "CAPTURE_LOG": str(self.capture_log), + } + ) + return subprocess.run( + [ + "bash", + str(self.verifier), + str(trusted), + str(candidate), + base_sha, + candidate_sha, + ], + capture_output=True, + check=False, + env=env, + text=True, + ) + + def test_unchanged_pins_skip_capture(self) -> None: + result = self.run_verifier(copy.deepcopy(self.base_versions)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("No changed managed llama.cpp pins", result.stdout) + self.assertFalse(self.capture_log.exists()) + + def test_changed_pin_rejects_missing_and_mismatched_manifests(self) -> None: + candidate = copy.deepcopy(self.base_versions) + candidate["llamacpp"]["vulkan"] = "b9" + + missing = self.run_verifier(candidate) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("require a committed release manifest", missing.stderr) + self.assertFalse(self.capture_log.exists()) + + mismatched = self.run_verifier( + candidate, + candidate_manifest=self.matching_manifest("b8"), + ) + self.assertNotEqual(mismatched.returncode, 0) + self.assertIn("does not match changed managed llama.cpp pin", mismatched.stderr) + self.assertFalse(self.capture_log.exists()) + + def test_sidecar_only_change_is_rejected(self) -> None: + result = self.run_verifier( + copy.deepcopy(self.base_versions), + candidate_manifest="{}\n", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "Release manifest changes require changed managed llama.cpp pins", + result.stderr, + ) + self.assertFalse(self.capture_log.exists()) + + def test_capture_failure_is_propagated(self) -> None: + candidate = copy.deepcopy(self.base_versions) + candidate["llamacpp"]["vulkan"] = "b9" + + result = self.run_verifier( + candidate, + candidate_manifest=self.matching_manifest(), + capture_exit=37, + ) + + self.assertEqual(result.returncode, 37, result.stderr) + self.assertTrue(self.capture_log.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/utils/llamacpp_release_manifest.py b/test/utils/llamacpp_release_manifest.py index 4fe52094b4..3b7be3f57b 100644 --- a/test/utils/llamacpp_release_manifest.py +++ b/test/utils/llamacpp_release_manifest.py @@ -4,25 +4,334 @@ from __future__ import annotations import argparse +import hashlib import json import re from pathlib import Path from typing import Iterable +from urllib.parse import urlparse SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-fA-F]{64}$") +LOWER_SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") GIT_OID_RE = re.compile(r"^[0-9a-fA-F]{40}$") +RELEASE_TAG_RE = re.compile(r"^b[0-9]+$") +SOURCE_MANIFEST_ASSET_NAME = ".llamacpp-source.json" +SOURCE_MANIFEST_MAX_BYTES = 65_536 +TRUSTED_SOURCE_REPOSITORY = "ggml-org/llama.cpp" +PUBLISHER_CLAIM_TYPES = frozenset({"immutable-source-manifest", "source-release-tag"}) +FORK_RELEASE_REPOSITORIES = frozenset( + {"lemonade-sdk/llama.cpp", "lemonade-sdk/llamacpp-rocm"} +) +MANAGED_PIN_REPOSITORIES = { + "cpu": "ggml-org/llama.cpp", + "cuda": "lemonade-sdk/llama.cpp", + "metal": "ggml-org/llama.cpp", + "rocm-nightly": "lemonade-sdk/llamacpp-rocm", + "rocm-stable": "lemonade-sdk/llama.cpp", + "vulkan": "ggml-org/llama.cpp", +} class ReleaseManifestError(ValueError): """Raised when release metadata is incomplete, malformed, or changed.""" +def _is_git_oid(value: object) -> bool: + return isinstance(value, str) and GIT_OID_RE.fullmatch(value) is not None + + +def _reject_duplicate_json_members(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ReleaseManifestError(f"duplicate JSON member: {key}") + result[key] = value + return result + + +def _strict_json_loads(value: str, label: str) -> object: + try: + return json.loads(value, object_pairs_hook=_reject_duplicate_json_members) + except (TypeError, json.JSONDecodeError) as exc: + raise ReleaseManifestError(f"{label} is not valid JSON: {exc}") from exc + + +def require_upstream_ancestry( + comparison: object, + source_repository: str, + publisher_claimed_source_commit: str, + upstream_reference_head: str, +) -> None: + expected_path = ( + f"/repos/{source_repository}/compare/" + f"{publisher_claimed_source_commit}...{upstream_reference_head}" + ) + if not isinstance(comparison, dict): + raise ReleaseManifestError("publisher-claimed source commit is not reachable") + base_commit = comparison.get("base_commit") + merge_base_commit = comparison.get("merge_base_commit") + comparison_url = comparison.get("url") + if ( + not isinstance(source_repository, str) + or source_repository.count("/") != 1 + or not _is_git_oid(publisher_claimed_source_commit) + or not _is_git_oid(upstream_reference_head) + or not isinstance(base_commit, dict) + or not isinstance(merge_base_commit, dict) + or not isinstance(comparison_url, str) + or urlparse(comparison_url).path != expected_path + or base_commit.get("sha", "").lower() != publisher_claimed_source_commit.lower() + or merge_base_commit.get("sha", "").lower() + != publisher_claimed_source_commit.lower() + or comparison.get("status") not in {"ahead", "identical"} + ): + raise ReleaseManifestError("publisher-claimed source commit is not reachable") + + def _positive_integer(value: object, label: str) -> int: if isinstance(value, bool) or not isinstance(value, int) or value <= 0: raise ReleaseManifestError(f"{label} must be a positive integer") return value +def locate_immutable_source_manifest_asset( + release_payload: object, + release_repository: str, +) -> dict: + """Return the single bounded source-manifest asset for a fork release.""" + if release_repository not in FORK_RELEASE_REPOSITORIES: + raise ReleaseManifestError( + f"{release_repository}: immutable source manifests are only valid for " + "managed fork releases" + ) + if not isinstance(release_payload, dict): + raise ReleaseManifestError( + f"{release_repository}: release payload must be an object" + ) + if release_payload.get("immutable") is not True: + raise ReleaseManifestError( + f"{release_repository}: release immutable must be true" + ) + if release_payload.get("draft") is not False: + raise ReleaseManifestError(f"{release_repository}: release draft must be false") + raw_assets = release_payload.get("assets") + if not isinstance(raw_assets, list): + raise ReleaseManifestError( + f"{release_repository}: release assets must be an array" + ) + matches = [ + asset + for asset in raw_assets + if isinstance(asset, dict) and asset.get("name") == SOURCE_MANIFEST_ASSET_NAME + ] + if len(matches) != 1: + raise ReleaseManifestError( + f"{release_repository}: release must contain exactly one " + f"{SOURCE_MANIFEST_ASSET_NAME} asset" + ) + asset = matches[0] + _positive_integer(asset.get("id"), f"{release_repository}: source manifest id") + size = _positive_integer( + asset.get("size"), f"{release_repository}: source manifest size" + ) + if size > SOURCE_MANIFEST_MAX_BYTES: + raise ReleaseManifestError( + f"{release_repository}: source manifest size exceeds " + f"{SOURCE_MANIFEST_MAX_BYTES} bytes" + ) + if asset.get("state") != "uploaded": + raise ReleaseManifestError( + f"{release_repository}: source manifest must be uploaded" + ) + digest = asset.get("digest") + if not isinstance(digest, str) or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None: + raise ReleaseManifestError( + f"{release_repository}: source manifest must have a lowercase sha256 digest" + ) + _source_manifest_asset_bindings(release_payload, release_repository) + return asset + + +def _source_manifest_asset_bindings( + release_payload: object, + release_repository: str, +) -> list[dict]: + if not isinstance(release_payload, dict) or not isinstance( + release_payload.get("assets"), list + ): + raise ReleaseManifestError( + f"{release_repository}: release assets must be an array" + ) + bindings = [] + names: set[str] = set() + for index, asset in enumerate(release_payload["assets"]): + prefix = f"{release_repository}: release asset {index}" + if not isinstance(asset, dict): + raise ReleaseManifestError(f"{prefix} must be an object") + name = asset.get("name") + if not isinstance(name, str) or not name: + raise ReleaseManifestError(f"{prefix} name must be nonempty") + if name in names: + raise ReleaseManifestError( + f"{release_repository}: duplicate release asset name {name!r}" + ) + names.add(name) + if asset.get("state") != "uploaded": + raise ReleaseManifestError(f"{prefix} must be uploaded") + size = asset.get("size") + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise ReleaseManifestError(f"{prefix} size must be nonnegative") + digest = asset.get("digest") + if ( + not isinstance(digest, str) + or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None + ): + raise ReleaseManifestError(f"{prefix} must have a lowercase sha256 digest") + if name != SOURCE_MANIFEST_ASSET_NAME: + bindings.append({"digest": digest, "name": name, "size": size}) + if not bindings: + raise ReleaseManifestError( + f"{release_repository}: source manifest must bind at least one release asset" + ) + return sorted(bindings, key=lambda asset: asset["name"]) + + +def validate_immutable_source_manifest( + evidence: bytes, + release_payload: object, + release_repository: str, + release_tag: str, + release_tag_commit: str, +) -> str: + """Validate one immutable publisher claim and return its upstream source OID.""" + source_asset = locate_immutable_source_manifest_asset( + release_payload, + release_repository, + ) + if not isinstance(evidence, bytes): + raise ReleaseManifestError( + f"{release_repository}: source manifest must be raw bytes" + ) + if len(evidence) != source_asset["size"]: + raise ReleaseManifestError( + f"{release_repository}: source manifest size does not match release metadata" + ) + measured_digest = "sha256:" + hashlib.sha256(evidence).hexdigest() + if measured_digest != source_asset["digest"]: + raise ReleaseManifestError( + f"{release_repository}: source manifest sha256 digest does not match " + "release metadata" + ) + try: + evidence_text = evidence.decode("utf-8") + except UnicodeDecodeError as exc: + raise ReleaseManifestError( + f"{release_repository}: source manifest must be UTF-8 JSON" + ) from exc + document = _strict_json_loads( + evidence_text, + f"{release_repository}: source manifest", + ) + if not isinstance(document, dict): + raise ReleaseManifestError( + f"{release_repository}: source manifest must be an object" + ) + expected_keys = { + "assets", + "release_repository", + "release_tag", + "release_tag_commit", + "schema_version", + "source_commit", + "source_repository", + } + if set(document) != expected_keys: + raise ReleaseManifestError( + f"{release_repository}: source manifest has unexpected fields" + ) + if ( + isinstance(document["schema_version"], bool) + or not isinstance(document["schema_version"], int) + or document["schema_version"] != 1 + ): + raise ReleaseManifestError( + f"{release_repository}: source manifest schema_version must be 1" + ) + if document["release_repository"] != release_repository: + raise ReleaseManifestError( + f"{release_repository}: source manifest release repository mismatch" + ) + if ( + not isinstance(release_tag, str) + or RELEASE_TAG_RE.fullmatch(release_tag) is None + or document["release_tag"] != release_tag + ): + raise ReleaseManifestError( + f"{release_repository}: source manifest release tag mismatch" + ) + if not _is_git_oid(release_tag_commit) or ( + not isinstance(document["release_tag_commit"], str) + or document["release_tag_commit"].lower() != release_tag_commit.lower() + ): + raise ReleaseManifestError( + f"{release_repository}: source manifest release tag commit mismatch" + ) + if document["source_repository"] != TRUSTED_SOURCE_REPOSITORY: + raise ReleaseManifestError( + f"{release_repository}: source manifest source repository must be " + f"{TRUSTED_SOURCE_REPOSITORY}" + ) + source_commit = document["source_commit"] + if not _is_git_oid(source_commit): + raise ReleaseManifestError( + f"{release_repository}: source manifest source commit must be a full " + "40-character Git OID" + ) + raw_bindings = document["assets"] + if not isinstance(raw_bindings, list): + raise ReleaseManifestError( + f"{release_repository}: source manifest assets must be an array" + ) + bindings = [] + names: set[str] = set() + for index, binding in enumerate(raw_bindings): + prefix = f"{release_repository}: source manifest asset {index}" + if not isinstance(binding, dict) or set(binding) != {"digest", "name", "size"}: + raise ReleaseManifestError(f"{prefix} has unexpected fields") + name = binding["name"] + if not isinstance(name, str) or not name: + raise ReleaseManifestError(f"{prefix} name must be nonempty") + if name == SOURCE_MANIFEST_ASSET_NAME: + raise ReleaseManifestError( + f"{release_repository}: source manifest must exclude itself" + ) + if name in names: + raise ReleaseManifestError( + f"{release_repository}: source manifest has duplicate asset {name!r}" + ) + names.add(name) + size = binding["size"] + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise ReleaseManifestError(f"{prefix} size must be nonnegative") + digest = binding["digest"] + if ( + not isinstance(digest, str) + or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None + ): + raise ReleaseManifestError(f"{prefix} must have a lowercase sha256 digest") + bindings.append({"digest": digest, "name": name, "size": size}) + expected_bindings = _source_manifest_asset_bindings( + release_payload, + release_repository, + ) + if sorted(bindings, key=lambda asset: asset["name"]) != expected_bindings: + raise ReleaseManifestError( + f"{release_repository}: source manifest asset metadata does not match " + "the immutable release" + ) + return source_commit.lower() + + def _canonical_release( repository: str, expected_tag: str, @@ -39,11 +348,67 @@ def _canonical_release( f"{repository}: release tag mismatch: expected {expected_tag!r}, " f"got {actual_tag!r}" ) + if ( + not isinstance(expected_tag, str) + or RELEASE_TAG_RE.fullmatch(expected_tag) is None + ): + raise ReleaseManifestError(f"{repository}: release tag must match bNNNN") release_id = _positive_integer(payload.get("id"), f"{repository}: release id") - source_commit = payload.get("source_commit") - if not isinstance(source_commit, str) or not GIT_OID_RE.fullmatch(source_commit): + release_immutable = payload.get("immutable") + if release_immutable is not True: + raise ReleaseManifestError(f"{repository}: release immutable must be true") + release_draft = payload.get("draft") + if release_draft is not False: + raise ReleaseManifestError(f"{repository}: release draft must be false") + release_tag_commit = payload.get("release_tag_commit") + if not _is_git_oid(release_tag_commit): + raise ReleaseManifestError( + f"{repository}: release tag commit must be a full 40-character Git OID" + ) + source_repository = payload.get("source_repository") + if source_repository != TRUSTED_SOURCE_REPOSITORY: + raise ReleaseManifestError( + f"{repository}: source repository must be {TRUSTED_SOURCE_REPOSITORY}" + ) + publisher_claim_type = payload.get("publisher_claim_type") + if publisher_claim_type not in PUBLISHER_CLAIM_TYPES: + raise ReleaseManifestError( + f"{repository}: publisher claim type must identify its source" + ) + publisher_claimed_source_commit = payload.get("publisher_claimed_source_commit") + if not _is_git_oid(publisher_claimed_source_commit): + raise ReleaseManifestError( + f"{repository}: publisher-claimed source commit must be a full " + "40-character Git OID" + ) + upstream_reference = payload.get("upstream_reference") + if not isinstance(upstream_reference, str) or not upstream_reference.startswith( + "refs/heads/" + ): + raise ReleaseManifestError( + f"{repository}: upstream reference must be a branch reference" + ) + upstream_reference_head = payload.get("upstream_reference_head") + if not _is_git_oid(upstream_reference_head): + raise ReleaseManifestError( + f"{repository}: upstream reference head must be a full 40-character Git OID" + ) + if ( + repository == source_repository + and release_tag_commit.lower() != publisher_claimed_source_commit.lower() + ): + raise ReleaseManifestError( + f"{repository}: release tag commit must equal publisher-claimed source commit " + "for a source release" + ) + expected_claim_type = ( + "source-release-tag" + if repository == source_repository + else "immutable-source-manifest" + ) + if publisher_claim_type != expected_claim_type: raise ReleaseManifestError( - f"{repository}: source commit must be a full 40-character Git OID" + f"{repository}: publisher claim type must be {expected_claim_type}" ) raw_assets = payload.get("assets") if not isinstance(raw_assets, list) or not raw_assets: @@ -74,6 +439,9 @@ def _canonical_release( updated_at = raw_asset.get("updated_at") if not isinstance(updated_at, str) or not updated_at: raise ReleaseManifestError(f"{prefix}: asset updated_at must be nonempty") + state = raw_asset.get("state") + if state != "uploaded": + raise ReleaseManifestError(f"{prefix}: asset must be uploaded") if asset_id in asset_ids: raise ReleaseManifestError(f"{repository}: duplicate asset id {asset_id}") @@ -87,17 +455,42 @@ def _canonical_release( "id": asset_id, "name": name, "size": size, + "state": state, "updated_at": updated_at, } ) assets.sort(key=lambda asset: (asset["name"], asset["id"])) + if publisher_claim_type == "immutable-source-manifest": + source_assets = [ + asset for asset in assets if asset["name"] == SOURCE_MANIFEST_ASSET_NAME + ] + if len(source_assets) != 1: + raise ReleaseManifestError( + f"{repository}: immutable source claim requires exactly one " + f"{SOURCE_MANIFEST_ASSET_NAME} asset" + ) + if ( + source_assets[0]["size"] <= 0 + or source_assets[0]["size"] > SOURCE_MANIFEST_MAX_BYTES + or LOWER_SHA256_DIGEST_RE.fullmatch(source_assets[0]["digest"]) is None + ): + raise ReleaseManifestError( + f"{repository}: immutable source manifest metadata is invalid" + ) return { "assets": assets, + "publisher_claim_type": publisher_claim_type, + "publisher_claimed_source_commit": publisher_claimed_source_commit.lower(), + "release_draft": release_draft, "release_id": release_id, + "release_immutable": release_immutable, "repository": repository, - "source_commit": source_commit.lower(), + "release_tag_commit": release_tag_commit.lower(), + "source_repository": source_repository, "tag_name": expected_tag, + "upstream_reference": upstream_reference, + "upstream_reference_head": upstream_reference_head.lower(), } @@ -113,23 +506,35 @@ def build_release_asset_manifest( repositories = [release["repository"] for release in canonical_releases] if len(repositories) != len(set(repositories)): raise ReleaseManifestError("release manifest contains duplicate repositories") + upstream_bindings = { + ( + release["source_repository"], + release["upstream_reference"], + release["upstream_reference_head"], + ) + for release in canonical_releases + } + if len(upstream_bindings) != 1: + raise ReleaseManifestError( + "release manifest must use one immutable upstream reference head" + ) canonical_releases.sort(key=lambda release: release["repository"]) return json.dumps( - {"releases": canonical_releases, "schema_version": 2}, + {"releases": canonical_releases, "schema_version": 4}, separators=(",", ":"), sort_keys=True, ) -def _canonicalize_manifest(manifest_json: str) -> str: - try: - document = json.loads(manifest_json) - except (TypeError, json.JSONDecodeError) as exc: - raise ReleaseManifestError( - f"release manifest is not valid JSON: {exc}" - ) from exc - if not isinstance(document, dict) or document.get("schema_version") != 2: - raise ReleaseManifestError("release manifest schema_version must be 2") +def canonicalize_release_asset_manifest(manifest_json: str) -> str: + document = _strict_json_loads(manifest_json, "release manifest") + if ( + not isinstance(document, dict) + or isinstance(document.get("schema_version"), bool) + or not isinstance(document.get("schema_version"), int) + or document.get("schema_version") != 4 + ): + raise ReleaseManifestError("release manifest schema_version must be 4") releases = document.get("releases") if not isinstance(releases, list): raise ReleaseManifestError("release manifest releases must be an array") @@ -140,10 +545,17 @@ def _canonicalize_manifest(manifest_json: str) -> str: raise ReleaseManifestError("release manifest entry must be an object") expected_keys = { "assets", + "publisher_claim_type", + "publisher_claimed_source_commit", + "release_draft", "release_id", + "release_immutable", "repository", - "source_commit", + "release_tag_commit", + "source_repository", "tag_name", + "upstream_reference", + "upstream_reference_head", } if set(release) != expected_keys: raise ReleaseManifestError("release manifest entry has unexpected fields") @@ -153,9 +565,18 @@ def _canonicalize_manifest(manifest_json: str) -> str: release["tag_name"], { "assets": release["assets"], + "draft": release["release_draft"], "id": release["release_id"], - "source_commit": release["source_commit"], + "immutable": release["release_immutable"], + "publisher_claim_type": release["publisher_claim_type"], + "publisher_claimed_source_commit": release[ + "publisher_claimed_source_commit" + ], + "release_tag_commit": release["release_tag_commit"], + "source_repository": release["source_repository"], "tag_name": release["tag_name"], + "upstream_reference": release["upstream_reference"], + "upstream_reference_head": release["upstream_reference_head"], }, ) ) @@ -167,23 +588,128 @@ def _canonicalize_manifest(manifest_json: str) -> str: def require_exact_manifest_match(expected_json: str, actual_json: str) -> None: - expected = _canonicalize_manifest(expected_json) - actual = _canonicalize_manifest(actual_json) + expected = canonicalize_release_asset_manifest(expected_json) + actual = canonicalize_release_asset_manifest(actual_json) if expected != actual: raise ReleaseManifestError( - "release assets changed since validation; refusing publication" + "release manifest changed since validation; refusing publication" + ) + + +def managed_llamacpp_pin_changes( + base_versions: object, + candidate_versions: object, +) -> dict[str, tuple[str, str]]: + sections = [] + for label, document in ( + ("base", base_versions), + ("candidate", candidate_versions), + ): + if not isinstance(document, dict) or not isinstance( + document.get("llamacpp"), dict + ): + raise ReleaseManifestError( + f"{label} backend versions must contain a llamacpp object" + ) + section = document["llamacpp"] + for backend in MANAGED_PIN_REPOSITORIES: + value = section.get(backend) + if not isinstance(value, str) or RELEASE_TAG_RE.fullmatch(value) is None: + raise ReleaseManifestError( + f"{label} llamacpp.{backend} must be a bNNNN release tag" + ) + sections.append(section) + + base_section, candidate_section = sections + return { + backend: (base_section[backend], candidate_section[backend]) + for backend in MANAGED_PIN_REPOSITORIES + if base_section[backend] != candidate_section[backend] + } + + +def require_release_manifest_for_managed_pin_changes( + base_versions: object, + candidate_versions: object, + manifest_json: str | None, +) -> bool: + changes = managed_llamacpp_pin_changes(base_versions, candidate_versions) + if not changes: + return False + if manifest_json is None: + raise ReleaseManifestError( + "changed managed llama.cpp pins require a committed release manifest" + ) + + canonical = canonicalize_release_asset_manifest(manifest_json) + releases = { + release["repository"]: release for release in json.loads(canonical)["releases"] + } + expected_repositories = set(MANAGED_PIN_REPOSITORIES.values()) + if set(releases) != expected_repositories: + raise ReleaseManifestError( + "changed managed llama.cpp pins require all managed release repositories" + ) + for backend, (_, candidate_tag) in changes.items(): + repository = MANAGED_PIN_REPOSITORIES[backend] + if releases[repository]["tag_name"] != candidate_tag: + raise ReleaseManifestError( + f"release manifest does not match changed managed llama.cpp pin " + f"llamacpp.{backend}={candidate_tag}" + ) + return True + + +def materialize_release_asset_manifest( + manifest_json: str, + output_path: Path, +) -> tuple[str, str]: + canonical = canonicalize_release_asset_manifest(manifest_json) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + document = json.loads(canonical) + summary_lines = [ + "## Release manifest evidence", + "", + ( + "Only immutable GitHub releases are accepted. Fork source commits are " + "bound to the release tag and binary assets by an immutable publisher " + "manifest; this is not cryptographic build provenance." + ), + "", + f"Canonical manifest SHA-256: `{digest}`", + "", + "| Publisher release | Immutable | Release tag commit | Publisher source claim | Upstream boundary |", + "|---|---|---|---|---|", + ] + for release in document["releases"]: + summary_lines.append( + f"| `{release['repository']}@{release['tag_name']}` " + f"| `{str(release['release_immutable']).lower()}` " + f"| `{release['release_tag_commit']}` " + f"| `{release['publisher_claimed_source_commit']}` " + f"(`{release['publisher_claim_type']}`) " + f"| `{release['source_repository']}@{release['upstream_reference_head']}` " + f"(`{release['upstream_reference']}`) |" ) + try: + output_path.write_text(canonical + "\n", encoding="utf-8") + except OSError as exc: + raise ReleaseManifestError(f"could not write {output_path}: {exc}") from exc + return digest, "\n".join(summary_lines) + + +def _load_json_file(path: Path) -> object: + try: + return _strict_json_loads(path.read_text(encoding="utf-8"), str(path)) + except OSError as exc: + raise ReleaseManifestError(f"could not read {path}: {exc}") from exc def _load_release_specs(specs: list[list[str]]) -> list[tuple[str, str, object]]: releases = [] for repository, expected_tag, path_text in specs: path = Path(path_text) - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise ReleaseManifestError(f"could not read {path}: {exc}") from exc - releases.append((repository, expected_tag, payload)) + releases.append((repository, expected_tag, _load_json_file(path))) return releases @@ -193,13 +719,211 @@ def main() -> None: "--release", action="append", nargs=3, - required=True, metavar=("REPOSITORY", "TAG", "JSON_PATH"), ) + parser.add_argument( + "--source-manifest-asset-id", + nargs=2, + metavar=("RELEASE_JSON_PATH", "RELEASE_REPOSITORY"), + ) + parser.add_argument( + "--validate-source-manifest", + nargs=5, + metavar=( + "SOURCE_MANIFEST_PATH", + "RELEASE_JSON_PATH", + "RELEASE_REPOSITORY", + "RELEASE_TAG", + "RELEASE_TAG_COMMIT", + ), + ) + parser.add_argument( + "--require-upstream-ancestry", + nargs=4, + metavar=("JSON_PATH", "REPOSITORY", "CLAIMED_COMMIT", "UPSTREAM_HEAD"), + ) parser.add_argument("--expected-json") parser.add_argument("--github-output", type=Path) + parser.add_argument("--manifest-json") + parser.add_argument("--materialize-manifest", type=Path) + parser.add_argument("--append-summary-to", type=Path) + parser.add_argument( + "--managed-pin-changes", + nargs=2, + type=Path, + metavar=("BASE_VERSIONS", "CANDIDATE_VERSIONS"), + ) + parser.add_argument( + "--require-managed-pin-manifest", + nargs=3, + type=Path, + metavar=("BASE_VERSIONS", "CANDIDATE_VERSIONS", "MANIFEST"), + ) args = parser.parse_args() + source_modes = sum( + mode is not None + for mode in (args.source_manifest_asset_id, args.validate_source_manifest) + ) + if source_modes: + if source_modes != 1 or any( + ( + args.release, + args.require_upstream_ancestry, + args.expected_json is not None, + args.github_output, + args.manifest_json is not None, + args.materialize_manifest, + args.append_summary_to, + args.managed_pin_changes, + args.require_managed_pin_manifest, + ) + ): + parser.error("source-manifest modes cannot be combined with other modes") + try: + if args.source_manifest_asset_id is not None: + release_path, release_repository = args.source_manifest_asset_id + asset = locate_immutable_source_manifest_asset( + _load_json_file(Path(release_path)), + release_repository, + ) + print(asset["id"]) + else: + ( + source_path, + release_path, + release_repository, + release_tag, + release_tag_commit, + ) = args.validate_source_manifest + print( + validate_immutable_source_manifest( + Path(source_path).read_bytes(), + _load_json_file(Path(release_path)), + release_repository, + release_tag, + release_tag_commit, + ) + ) + except (OSError, ReleaseManifestError) as exc: + parser.exit(1, f"ERROR: {exc}\n") + return + + if args.managed_pin_changes: + if any( + ( + args.release, + args.require_upstream_ancestry, + args.expected_json is not None, + args.github_output, + args.manifest_json is not None, + args.materialize_manifest, + args.append_summary_to, + args.require_managed_pin_manifest, + ) + ): + parser.error("--managed-pin-changes cannot be combined with other modes") + try: + base_path, candidate_path = args.managed_pin_changes + changes = managed_llamacpp_pin_changes( + _load_json_file(base_path), + _load_json_file(candidate_path), + ) + except ReleaseManifestError as exc: + parser.exit(1, f"ERROR: {exc}\n") + print( + json.dumps( + { + backend: {"from": old, "to": new} + for backend, (old, new) in changes.items() + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return + if args.require_managed_pin_manifest: + if any( + ( + args.release, + args.require_upstream_ancestry, + args.expected_json is not None, + args.github_output, + args.manifest_json is not None, + args.materialize_manifest, + args.append_summary_to, + ) + ): + parser.error( + "--require-managed-pin-manifest cannot be combined with other modes" + ) + base_path, candidate_path, manifest_path = args.require_managed_pin_manifest + try: + require_release_manifest_for_managed_pin_changes( + _load_json_file(base_path), + _load_json_file(candidate_path), + manifest_path.read_text(encoding="utf-8"), + ) + except (OSError, ReleaseManifestError) as exc: + parser.exit(1, f"ERROR: {exc}\n") + print("changed") + return + + if args.require_upstream_ancestry: + if ( + args.release + or args.expected_json is not None + or args.github_output + or args.manifest_json is not None + or args.materialize_manifest + or args.append_summary_to + or args.managed_pin_changes + or args.require_managed_pin_manifest + ): + parser.error( + "--require-upstream-ancestry cannot be combined with manifest options" + ) + path_text, repository, claimed_commit, upstream_head = ( + args.require_upstream_ancestry + ) + try: + require_upstream_ancestry( + _load_json_file(Path(path_text)), + repository, + claimed_commit, + upstream_head, + ) + except ReleaseManifestError as exc: + parser.exit(1, f"ERROR: {exc}\n") + return + if args.materialize_manifest is not None: + if ( + args.release + or args.expected_json is not None + or args.github_output + or args.manifest_json is None + ): + parser.error( + "--materialize-manifest requires only --manifest-json and optional " + "--append-summary-to" + ) + try: + digest, summary = materialize_release_asset_manifest( + args.manifest_json, + args.materialize_manifest, + ) + if args.append_summary_to is not None: + with args.append_summary_to.open("a", encoding="utf-8") as output_file: + output_file.write(f"\n\n{summary}\n") + except (OSError, ReleaseManifestError) as exc: + parser.exit(1, f"ERROR: {exc}\n") + print(digest) + return + if args.manifest_json is not None or args.append_summary_to is not None: + parser.error("--manifest-json requires --materialize-manifest") + if not args.release: + parser.error("at least one --release is required") + try: actual = build_release_asset_manifest(_load_release_specs(args.release)) if args.expected_json is not None: diff --git a/test/validate_llamacpp.py b/test/validate_llamacpp.py index 5211f14c5b..f5b56e2367 100644 --- a/test/validate_llamacpp.py +++ b/test/validate_llamacpp.py @@ -310,6 +310,7 @@ def test_model( "model": model_name, "messages": CHAT_PROMPT, "max_completion_tokens": max_tokens, + "chat_template_kwargs": {"enable_thinking": False}, }, ) if chat_resp.status_code != 200: From a602f2907acbabc7bc046a45b72a3d940e98a35c Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:22:13 -0700 Subject: [PATCH 06/14] fix: complete K2 validation hardening --- .../actions/prepare-debian-build/action.yml | 13 +- .github/actions/setup-venv/action.yml | 2 +- .../capture_llamacpp_release_manifest.sh | 17 +- .../scripts/llamacpp_required_check_policy.py | 695 +++ .github/scripts/llamacpp_validation_args.ps1 | 62 - .github/scripts/publish_llamacpp_update.sh | 231 +- .github/scripts/resolve_debian_version.sh | 29 - .github/scripts/run_llamacpp_validation.py | 2578 ++++++++++ .../cpp_server_build_test_release.yml | 6 +- .github/workflows/docs_and_style.yml | 17 +- .github/workflows/validate_llamacpp_core.yml | 820 ++- .../workflows/validate_llamacpp_manual.yml | 7 +- .github/workflows/validate_llamacpp_pr.yml | 1441 +++++- .../workflows/validate_llamacpp_schedule.yml | 45 +- CMakeLists.txt | 67 +- docs/dev/testing.md | 47 +- docs/guide/configuration/llamacpp.md | 74 +- src/cpp/include/lemon/streaming_proxy.h | 36 +- src/cpp/include/lemon/utils/process_manager.h | 30 +- .../include/lemon/utils/process_platform.h | 6 +- src/cpp/include/lemon/wrapped_server.h | 13 +- src/cpp/resources/server_models.json | 34 - .../backends/fastflowlm/fastflowlm_models.cpp | 49 +- .../backends/llamacpp/llamacpp_server.cpp | 305 +- .../backends/llamacpp/llamacpp_system_utils.h | 110 + src/cpp/server/mcp_client.cpp | 240 +- src/cpp/server/streaming_proxy.cpp | 367 +- src/cpp/server/system_info.cpp | 40 +- .../server/utils/platform/process_linux.cpp | 197 +- .../server/utils/platform/process_macos.cpp | 613 ++- .../server/utils/platform/process_windows.cpp | 313 +- src/cpp/server/utils/process_manager.cpp | 102 +- src/cpp/server/wrapped_server.cpp | 55 +- test/cpp/test_llamacpp_stream_model.cpp | 892 ++++ test/cpp/test_llamacpp_system_backend.cpp | 89 + test/cpp/test_mcp_client_config.cpp | 124 +- test/cpp/test_mcp_client_process_contract.cpp | 92 + test/cpp/test_process_manager.cpp | 797 +++ test/cpp/test_process_manager_windows.cpp | 549 ++ test/cpp/test_streaming_heartbeat.cpp | 100 + .../llamacpp_validation_artifacts.json | 26 + test/fixtures/llamacpp_validation_models.json | 36 + test/requirements.txt | 3 + test/test_debian_version.py | 203 - test/test_llamacpp_capabilities.py | 304 +- test/test_llamacpp_check_reporting.py | 116 + test/test_llamacpp_pr_file_classification.py | 301 ++ test/test_llamacpp_release_assets.py | 191 +- test/test_llamacpp_release_manifest.py | 504 +- test/test_llamacpp_system_backend.py | 287 +- test/test_llamacpp_validation_args.ps1 | 152 - test/test_llamacpp_validation_artifacts.py | 314 ++ test/test_llamacpp_validation_evidence.py | 417 +- test/test_process_inheritance_contract.py | 299 ++ test/test_publish_llamacpp_update.py | 1099 +++- test/test_run_llamacpp_validation.py | 4466 +++++++++++++++++ test/test_validate_llamacpp_action_pins.py | 281 ++ test/test_validate_llamacpp_selection.py | 2195 +++++++- ...est_validate_llamacpp_trusted_execution.py | 275 + test/test_validate_llamacpp_workflow_plan.py | 4327 +++++++++++++++- test/test_validation_model_catalog.py | 178 + test/test_verify_llamacpp_pin_manifest.py | 231 +- test/utils/llamacpp_capability_validation.py | 161 +- test/utils/llamacpp_release_assets.py | 144 +- test/utils/llamacpp_release_manifest.py | 324 +- test/utils/llamacpp_validation_artifacts.py | 448 ++ test/utils/llamacpp_validation_evidence.py | 361 +- test/utils/llamacpp_validation_plan.py | 381 +- test/utils/server_base.py | 16 +- test/utils/validation_model_catalog.py | 130 + test/utils/validation_model_selection.py | 18 +- test/validate_llamacpp.py | 905 +++- 72 files changed, 28041 insertions(+), 2356 deletions(-) create mode 100644 .github/scripts/llamacpp_required_check_policy.py delete mode 100644 .github/scripts/llamacpp_validation_args.ps1 delete mode 100755 .github/scripts/resolve_debian_version.sh create mode 100644 .github/scripts/run_llamacpp_validation.py create mode 100644 test/cpp/test_llamacpp_stream_model.cpp create mode 100644 test/cpp/test_mcp_client_process_contract.cpp create mode 100644 test/cpp/test_process_manager_windows.cpp create mode 100644 test/fixtures/llamacpp_validation_artifacts.json create mode 100644 test/fixtures/llamacpp_validation_models.json delete mode 100644 test/test_debian_version.py create mode 100644 test/test_llamacpp_check_reporting.py create mode 100644 test/test_llamacpp_pr_file_classification.py delete mode 100644 test/test_llamacpp_validation_args.ps1 create mode 100644 test/test_llamacpp_validation_artifacts.py create mode 100644 test/test_process_inheritance_contract.py create mode 100644 test/test_run_llamacpp_validation.py create mode 100644 test/test_validate_llamacpp_action_pins.py create mode 100644 test/test_validate_llamacpp_trusted_execution.py create mode 100644 test/test_validation_model_catalog.py create mode 100644 test/utils/llamacpp_validation_artifacts.py create mode 100644 test/utils/validation_model_catalog.py diff --git a/.github/actions/prepare-debian-build/action.yml b/.github/actions/prepare-debian-build/action.yml index 0adda25e1c..207fa4af35 100644 --- a/.github/actions/prepare-debian-build/action.yml +++ b/.github/actions/prepare-debian-build/action.yml @@ -1,5 +1,5 @@ name: Prepare Debian Build -description: Resolve the Debian version and create packaging metadata from template +description: Extract version from git and create Debian packaging metadata from template inputs: release: @@ -17,15 +17,14 @@ outputs: runs: using: composite steps: - - name: Resolve Debian version + - name: Get version from git describe id: get_version shell: bash run: | - DEB_VERSION=$( - "${GITHUB_ACTION_PATH}/../../scripts/resolve_debian_version.sh" \ - "${{ inputs.release }}" - ) - echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" + VERSION=$(git describe --tags --always) + # Strip leading 'v' for Debian version compatibility + DEB_VERSION=$(echo "${VERSION}" | sed 's/^v//') + echo "version=${DEB_VERSION}" >> $GITHUB_OUTPUT - name: Create Debian packaging metadata shell: bash diff --git a/.github/actions/setup-venv/action.yml b/.github/actions/setup-venv/action.yml index abf72d075e..ee595a0dda 100644 --- a/.github/actions/setup-venv/action.yml +++ b/.github/actions/setup-venv/action.yml @@ -37,7 +37,7 @@ runs: # For GitHub-hosted runners, use actions/setup-python - name: Setup Python (GitHub-hosted) if: ${{ !contains(runner.name, 'stx') && !contains(runner.name, 'rai') && !contains(runner.name, 'selfhost') }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ inputs.python-version }} diff --git a/.github/scripts/capture_llamacpp_release_manifest.sh b/.github/scripts/capture_llamacpp_release_manifest.sh index de96faf087..f1779ff9bb 100755 --- a/.github/scripts/capture_llamacpp_release_manifest.sh +++ b/.github/scripts/capture_llamacpp_release_manifest.sh @@ -61,7 +61,8 @@ if [[ "$upstream_reference" != "$trusted_upstream_reference" ]]; then exit 1 fi live_upstream_head=$(gh api \ - "repos/${trusted_source_repository}/commits/${source_default_branch}" --jq '.sha') + "repos/${trusted_source_repository}/commits/heads/${source_default_branch}" \ + --jq '.sha') if ! [[ "$live_upstream_head" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "Could not resolve the trusted upstream reference to a full commit." >&2 exit 1 @@ -91,8 +92,10 @@ attach_release_claims() { local release_tag_commit publisher_claim_type local publisher_claimed_source_commit comparison_path local source_manifest_asset_id source_manifest_path + local build_target_attestation_path - release_tag_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') + release_tag_commit=$(gh api \ + "repos/${repository}/commits/tags/${tag}" --jq '.sha') if ! [[ "$release_tag_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "Could not resolve ${repository} ${tag} to a release tag commit." >&2 exit 1 @@ -100,17 +103,21 @@ attach_release_claims() { if [[ "$repository" == "$source_repository" ]]; then publisher_claim_type="source-release-tag" publisher_claimed_source_commit=$release_tag_commit + build_target_attestation_path="${source_manifest_directory}/source-release-targets.json" + printf '[]\n' >"$build_target_attestation_path" else publisher_claim_type="immutable-source-manifest" source_manifest_asset_id=$(python "$manifest_tool" \ --source-manifest-asset-id "$path" "$repository") source_manifest_path="${source_manifest_directory}/source-${source_manifest_asset_id}.json" + build_target_attestation_path="${source_manifest_path}.build-targets" gh api "repos/${repository}/releases/assets/${source_manifest_asset_id}" \ -H "Accept: application/octet-stream" | head -c "$((source_manifest_max_bytes + 1))" >"$source_manifest_path" publisher_claimed_source_commit=$(python "$manifest_tool" \ --validate-source-manifest "$source_manifest_path" "$path" \ - "$repository" "$tag" "$release_tag_commit") + "$repository" "$tag" "$release_tag_commit" \ + --build-target-attestation-output "$build_target_attestation_path") fi if ! [[ "$publisher_claimed_source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "${repository} ${tag} did not bind one full upstream source commit." >&2 @@ -124,13 +131,15 @@ attach_release_claims() { --require-upstream-ancestry "$comparison_path" "$source_repository" \ "$publisher_claimed_source_commit" "$upstream_reference_head" jq \ + --slurpfile build_target_attestations "$build_target_attestation_path" \ --arg publisher_claim_type "$publisher_claim_type" \ --arg publisher_claimed_source_commit "$publisher_claimed_source_commit" \ --arg release_tag_commit "$release_tag_commit" \ --arg source_repository "$source_repository" \ --arg upstream_reference "$upstream_reference" \ --arg upstream_reference_head "$upstream_reference_head" \ - '.publisher_claim_type = $publisher_claim_type + '.build_target_attestations = $build_target_attestations[0] + | .publisher_claim_type = $publisher_claim_type | .publisher_claimed_source_commit = $publisher_claimed_source_commit | .release_tag_commit = $release_tag_commit | .source_repository = $source_repository diff --git a/.github/scripts/llamacpp_required_check_policy.py b/.github/scripts/llamacpp_required_check_policy.py new file mode 100644 index 0000000000..8cfd1539ec --- /dev/null +++ b/.github/scripts/llamacpp_required_check_policy.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +"""Trusted path and lifecycle policy for the llama.cpp required check.""" + +from __future__ import annotations + +import argparse +from datetime import datetime +import json +from pathlib import Path, PurePosixPath +import re +from typing import NamedTuple + +MAX_CHANGED_FILES = 3000 +MAX_COMPARE_COMMITS = 250 +MAX_COMPARE_FILES = 300 +MAX_PAGE_SIZE = 100 +MAX_EVENT_PAGES = 100 +CHANGED_FILE_STATUSES = { + "added", + "changed", + "copied", + "modified", + "removed", + "renamed", + "unchanged", +} +AUTHORIZATION_LABEL = "ci:upgrades" +TRUSTED_PERMISSIONS = {"admin", "maintain", "push", "write"} +AUTHORIZATION_INVALIDATING_EVENTS = { + "automatic_base_change_succeeded", + "base_ref_changed", + "closed", + "head_ref_deleted", + "head_ref_force_pushed", + "head_ref_restored", + "merged", + "reopened", +} +IDENTITY_CHANGING_ACTIONS = {"opened", "reopened", "synchronize"} +REQUIRED_WORKFLOW_ACTIONS = {"opened", "reopened", "synchronize"} +SUPPORTED_ACTIONS = { + "closed", + "edited", + "labeled", + "opened", + "reopened", + "synchronize", + "unlabeled", +} +COMPARE_STATUSES = {"ahead", "behind", "diverged", "identical"} +FULL_OID_PATTERN = re.compile(r"[0-9a-f]{40}") + +NEUTRAL_PREFIXES = ( + "docs/", + "src/app/", + "src/web-app/", +) +PROTECTED_OVERRIDES = frozenset({"src/app/src/renderer/utils/toolDefinitions.json"}) + + +class PolicyDecision(NamedTuple): + mode: str + run_validation: bool + conclusion: str + summary: str + + +class AuthorizationProvenance(NamedTuple): + authorization_event_id: int + authorization_event: str + authorization_actor: str + authorization_actor_id: int + authorization_created_at: str + latest_invalidator_event_id: int + epoch_valid: bool + + +def _validated_path(path: object, field: str) -> str: + if not isinstance(path, str) or not path: + raise ValueError(f"{field} must be a non-empty string") + if "\\" in path or path.startswith("/"): + raise ValueError(f"{field} must be a repository-relative POSIX path") + parsed = PurePosixPath(path) + if any(part in {"", ".", ".."} for part in parsed.parts) or str(parsed) != path: + raise ValueError(f"{field} is not a normalized repository path") + return path + + +def is_protected_path(path: str) -> bool: + normalized = _validated_path(path, "filename") + return normalized in PROTECTED_OVERRIDES or not normalized.startswith( + NEUTRAL_PREFIXES + ) + + +def _classify_file_records(records: list[object], *, source: str) -> bool: + filenames: set[str] = set() + relevant = False + for index, record in enumerate(records): + if not isinstance(record, dict): + raise ValueError(f"{source} file record {index} must be an object") + filename = _validated_path(record.get("filename"), "filename") + if filename in filenames: + raise ValueError(f"duplicate {source} filename: {filename}") + filenames.add(filename) + relevant = relevant or is_protected_path(filename) + + status = record.get("status") + if not isinstance(status, str) or status not in CHANGED_FILE_STATUSES: + raise ValueError(f"{source} file has an invalid status: {filename}") + + has_previous_filename = "previous_filename" in record + if status == "renamed": + if not has_previous_filename: + raise ValueError( + f"renamed file is missing previous_filename: {filename}" + ) + previous = _validated_path(record["previous_filename"], "previous_filename") + relevant = relevant or is_protected_path(previous) + elif has_previous_filename: + raise ValueError( + f"previous_filename is only valid for renamed files: {filename}" + ) + + return relevant + + +def classify_changed_file_pages(pages: object, *, expected_count: int) -> bool: + if isinstance(expected_count, bool) or not isinstance(expected_count, int): + raise ValueError("expected_count must be an integer") + if expected_count < 0 or expected_count > MAX_CHANGED_FILES: + raise ValueError(f"changed_files must be between 0 and {MAX_CHANGED_FILES}") + if not isinstance(pages, list): + raise ValueError("paginated pull-request files must be a JSON array") + if len(pages) > MAX_CHANGED_FILES // MAX_PAGE_SIZE: + raise ValueError("pull-request files exceeded the API pagination limit") + + records: list[object] = [] + for page in pages: + if not isinstance(page, list): + raise ValueError("each pull-request files page must be a JSON array") + if len(page) > MAX_PAGE_SIZE: + raise ValueError("a pull-request files page exceeded per_page=100") + records.extend(page) + + if len(records) != expected_count: + raise ValueError( + "paginated pull-request file count does not match changed_files" + ) + + return _classify_file_records(records, source="pull-request") + + +def _validated_oid(value: object, field: str) -> str: + if not isinstance(value, str) or FULL_OID_PATTERN.fullmatch(value) is None: + raise ValueError(f"{field} must be a lowercase full Git OID") + return value + + +def _validated_count(value: object, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field} must be a nonnegative integer") + return value + + +def _nested_oid(response: dict[str, object], field: str) -> str: + value = response.get(field) + if not isinstance(value, dict): + raise ValueError(f"compare response {field} must be an object") + return _validated_oid(value.get("sha"), f"compare response {field}.sha") + + +def classify_compare_response( + response: object, + *, + expected_base_sha: str, + expected_target_sha: str, + expected_count: int, +) -> bool: + expected_base = _validated_oid(expected_base_sha, "expected base SHA") + expected_target = _validated_oid(expected_target_sha, "expected target SHA") + expected_files = _validated_count(expected_count, "expected file count") + if expected_files > MAX_COMPARE_FILES: + return True + if not isinstance(response, dict): + raise ValueError("compare response must be a JSON object") + + base_commit = _nested_oid(response, "base_commit") + merge_base_commit = _nested_oid(response, "merge_base_commit") + if base_commit != expected_base: + raise ValueError("compare response does not identify the expected base commit") + + status = response.get("status") + if not isinstance(status, str) or status not in COMPARE_STATUSES: + raise ValueError("compare response has an invalid status") + ahead_by = _validated_count(response.get("ahead_by"), "compare response ahead_by") + behind_by = _validated_count( + response.get("behind_by"), "compare response behind_by" + ) + total_commits = _validated_count( + response.get("total_commits"), "compare response total_commits" + ) + if total_commits != ahead_by: + raise ValueError("compare response commit counts are inconsistent") + + expected_relationships = { + "ahead": (True, False), + "behind": (False, True), + "diverged": (True, True), + "identical": (False, False), + } + expects_ahead, expects_behind = expected_relationships[status] + if (ahead_by > 0) != expects_ahead or (behind_by > 0) != expects_behind: + raise ValueError("compare response status and commit counts are inconsistent") + if status in {"ahead", "identical"} and merge_base_commit != expected_base: + raise ValueError("compare response has an unexpected merge-base commit") + if status == "behind" and merge_base_commit != expected_target: + raise ValueError( + "compare response does not identify the expected target commit" + ) + if status == "diverged" and merge_base_commit in {expected_base, expected_target}: + raise ValueError("compare response has an inconsistent divergent merge base") + if (status == "identical") != (expected_base == expected_target): + raise ValueError("compare response has an inconsistent identical target") + + commits = response.get("commits") + if not isinstance(commits, list): + raise ValueError("compare response commits must be a JSON array") + expected_commit_records = min(total_commits, MAX_COMPARE_COMMITS) + if len(commits) != expected_commit_records: + raise ValueError("compare response commit list is incomplete") + commit_oids: set[str] = set() + for index, commit in enumerate(commits): + if not isinstance(commit, dict): + raise ValueError(f"compare response commit {index} must be an object") + commit_oid = _validated_oid( + commit.get("sha"), f"compare response commit {index}.sha" + ) + if commit_oid in commit_oids: + raise ValueError(f"duplicate compare response commit: {commit_oid}") + commit_oids.add(commit_oid) + if status in {"ahead", "diverged"} and commits[-1]["sha"] != expected_target: + raise ValueError( + "compare response does not identify the expected target commit" + ) + + files = response.get("files") + if not isinstance(files, list): + raise ValueError("compare response files must be a JSON array") + if len(files) > MAX_COMPARE_FILES: + raise ValueError("compare response exceeded the file-list limit") + if status in {"behind", "identical"} and files: + raise ValueError("compare response status cannot contain changed files") + if len(files) != expected_files: + return True + return _classify_file_records(files, source="compare response") + + +def _parse_timestamp(value: str, field: str) -> datetime: + try: + timestamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError(f"{field} has an invalid timestamp") from error + if timestamp.tzinfo is None: + raise ValueError(f"{field} has a naive timestamp") + return timestamp + + +def classify_authorization_event_pages( + pages: object, + *, + authorization_after: str | None = None, +) -> AuthorizationProvenance: + if not isinstance(pages, list): + raise ValueError("paginated issue events must be a JSON array") + if len(pages) > MAX_EVENT_PAGES: + raise ValueError("issue events exceeded the trusted pagination limit") + + boundary = ( + _parse_timestamp(authorization_after, "authorization boundary") + if authorization_after is not None + else None + ) + records: list[object] = [] + for page_index, page in enumerate(pages): + if not isinstance(page, list): + raise ValueError("each issue-events page must be a JSON array") + if len(page) > MAX_PAGE_SIZE: + raise ValueError("an issue-events page exceeded per_page=100") + if not page and page_index != len(pages) - 1: + raise ValueError("an intermediate issue-events page was empty") + records.extend(page) + if ( + len(pages) == MAX_EVENT_PAGES + and pages + and isinstance(pages[-1], list) + and len(pages[-1]) == MAX_PAGE_SIZE + ): + raise ValueError("issue events may have exceeded the trusted pagination limit") + + seen_ids: set[int] = set() + seen_node_ids: set[str] = set() + previous_event_id = 0 + previous_order: tuple[datetime, int] | None = None + latest_authorization: tuple[int, dict[str, object]] | None = None + latest_invalidator: tuple[int, int] | None = None + for index, record in enumerate(records): + if not isinstance(record, dict): + raise ValueError(f"issue event record {index} must be an object") + event_id = record.get("id") + node_id = record.get("node_id") + event = record.get("event") + created_at = record.get("created_at") + if ( + isinstance(event_id, bool) + or not isinstance(event_id, int) + or event_id <= 0 + or not isinstance(node_id, str) + or not node_id + or not isinstance(event, str) + or not event + or not isinstance(created_at, str) + ): + raise ValueError(f"issue event record {index} has invalid common fields") + if event_id in seen_ids: + raise ValueError(f"duplicate issue event ID: {event_id}") + if event_id <= previous_event_id: + raise ValueError("issue event IDs are not in strict API order") + if node_id in seen_node_ids: + raise ValueError(f"duplicate issue event node ID: {node_id}") + seen_ids.add(event_id) + previous_event_id = event_id + seen_node_ids.add(node_id) + timestamp = _parse_timestamp( + created_at, + f"issue event record {index} created_at", + ) + order = (timestamp, event_id) + if previous_order is not None and order <= previous_order: + raise ValueError("issue events are not in strict chronological order") + previous_order = order + + if event in {"labeled", "unlabeled"}: + label = record.get("label") + actor = record.get("actor") + if ( + not isinstance(label, dict) + or not isinstance(label.get("name"), str) + or not label["name"] + or not isinstance(actor, dict) + or isinstance(actor.get("id"), bool) + or not isinstance(actor.get("id"), int) + or actor["id"] <= 0 + or not isinstance(actor.get("login"), str) + or not actor["login"] + ): + raise ValueError(f"issue label event {event_id} is malformed") + if label["name"] == AUTHORIZATION_LABEL: + if ( + not isinstance(actor.get("node_id"), str) + or not actor["node_id"] + or actor.get("type") != "User" + or record.get("performed_via_github_app") is not None + or re.fullmatch( + r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})", actor["login"] + ) + is None + ): + raise ValueError( + f"authorization label event {event_id} has an invalid actor" + ) + latest_authorization = (index, record) + + if event in AUTHORIZATION_INVALIDATING_EVENTS: + latest_invalidator = (index, event_id) + + invalidator_id = latest_invalidator[1] if latest_invalidator is not None else 0 + if latest_authorization is None: + return AuthorizationProvenance(0, "", "", 0, "", invalidator_id, False) + + authorization_index, authorization = latest_authorization + actor = authorization["actor"] + assert isinstance(actor, dict) + invalidator_index = latest_invalidator[0] if latest_invalidator is not None else -1 + authorization_created_at = str(authorization["created_at"]) + authorization_timestamp = _parse_timestamp( + authorization_created_at, + "authorization event created_at", + ) + return AuthorizationProvenance( + authorization_event_id=int(authorization["id"]), + authorization_event=str(authorization["event"]), + authorization_actor=str(actor["login"]), + authorization_actor_id=int(actor["id"]), + authorization_created_at=authorization_created_at, + latest_invalidator_event_id=invalidator_id, + epoch_valid=( + authorization["event"] == "labeled" + and authorization_index > invalidator_index + and (boundary is None or authorization_timestamp > boundary) + ), + ) + + +def decide_pull_request_policy( + *, + action: str, + event_label: str, + base_ref_changed: bool, + state: str, + merged: bool, + merge_candidate_available: bool, + relevant: bool, + identity_matches: bool, + authorization_label_present: bool, + actor_permission: str, + has_existing_check: bool, + has_newer_check: bool, + authorization_epoch_valid: bool = True, + authorization_rerun_matches: bool = False, + required_workflow_only: bool = False, +) -> PolicyDecision: + if action not in SUPPORTED_ACTIONS: + raise ValueError(f"unsupported pull_request_target action: {action}") + + if state not in {"open", "closed"}: + raise ValueError(f"unsupported pull request state: {state}") + if merged and state != "closed": + raise ValueError("a merged pull request must be closed") + if merged: + return PolicyDecision( + "merged", + False, + "", + "The pull request merged; the pre-merge gate is left unchanged.", + ) + if state == "closed": + return PolicyDecision( + "closed", + False, + "cancelled", + "The pull request was closed; protected validation was cancelled.", + ) + if action == "labeled" and event_label == AUTHORIZATION_LABEL: + return PolicyDecision( + "awaiting_rerun", + False, + "failure", + "The authorization label was recorded. Re-run all jobs on the " + "matching failed required-workflow run.", + ) + if has_newer_check: + if action in REQUIRED_WORKFLOW_ACTIONS: + return PolicyDecision( + "superseded", + False, + "failure", + "A newer diagnostic decision exists for this candidate. Use the " + "newest failed default-event workflow run for authorization.", + ) + return PolicyDecision( + "preserve", + False, + "", + "A newer exact-commit policy decision superseded this event.", + ) + + authorization_summary = ( + "Protected llama.cpp inputs changed. After the latest required-workflow " + "run fails, a maintainer must apply ci:upgrades and then use Re-run all " + "jobs on that same workflow run. If the label is already present, remove " + "and reapply it after the latest candidate change." + ) + if not identity_matches: + if action in REQUIRED_WORKFLOW_ACTIONS: + return PolicyDecision( + "stale_required", + False, + "failure", + "The required-workflow run no longer identifies the live candidate. " + "Use the newest failed default-event workflow run.", + ) + return PolicyDecision( + "stale", + False, + "", + "A stale event cannot change the live candidate's exact-commit gate.", + ) + if action == "closed": + return PolicyDecision( + "stale", + False, + "", + "A stale lifecycle event cannot change the open pull request's gate.", + ) + + if ( + action in REQUIRED_WORKFLOW_ACTIONS + and identity_matches + and merge_candidate_available + and authorization_label_present + and authorization_epoch_valid + and authorization_rerun_matches + and actor_permission in TRUSTED_PERMISSIONS + ): + return PolicyDecision( + "validate", + True, + "", + "A trusted maintainer authorized protected llama.cpp validation.", + ) + + if not relevant: + return PolicyDecision( + "neutral", + False, + "success", + "No protected llama.cpp validation inputs changed; the gate is neutral.", + ) + + if action == "unlabeled" and event_label == AUTHORIZATION_LABEL: + return PolicyDecision( + "authorization_required", False, "failure", authorization_summary + ) + + if not authorization_label_present: + return PolicyDecision( + "authorization_required", False, "failure", authorization_summary + ) + + if not authorization_epoch_valid or actor_permission not in TRUSTED_PERMISSIONS: + return PolicyDecision( + "authorization_required", False, "failure", authorization_summary + ) + + if not merge_candidate_available: + return PolicyDecision( + "authorization_required", + False, + "failure", + "GitHub has not produced a synthetic test merge commit. Resolve " + "merge conflicts, establish a new default-event workflow run, then " + "apply ci:upgrades and re-run all jobs on that run.", + ) + + invalidated = action in IDENTITY_CHANGING_ACTIONS or ( + action == "edited" and base_ref_changed + ) + if invalidated: + return PolicyDecision( + "authorization_required", False, "failure", authorization_summary + ) + + if required_workflow_only: + return PolicyDecision( + "preserve", + False, + "", + "An unrelated metadata event left the required-workflow gate unchanged.", + ) + + if not has_existing_check: + return PolicyDecision( + "authorization_required", False, "failure", authorization_summary + ) + + return PolicyDecision( + "preserve", + False, + "", + "An unrelated metadata event preserved the exact-commit gate.", + ) + + +def _parse_bool(value: str) -> bool: + if value == "true": + return True + if value == "false": + return False + raise argparse.ArgumentTypeError("expected true or false") + + +def _write_outputs(path: Path, values: dict[str, object]) -> None: + with path.open("a", encoding="utf-8") as output: + for key, value in values.items(): + rendered = str(value).lower() if isinstance(value, bool) else str(value) + if "\n" in rendered or "\r" in rendered: + raise ValueError(f"output {key} must fit on one line") + output.write(f"{key}={rendered}\n") + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + classify = subparsers.add_parser("classify") + classify.add_argument("--pages", type=Path, required=True) + classify.add_argument("--expected-count", type=int, required=True) + classify.add_argument("--github-output", type=Path, required=True) + + classify_compare = subparsers.add_parser("classify-compare") + classify_compare.add_argument("--response", type=Path, required=True) + classify_compare.add_argument("--expected-base-sha", required=True) + classify_compare.add_argument("--expected-target-sha", required=True) + classify_compare.add_argument("--expected-count", type=int, required=True) + classify_compare.add_argument("--github-output", type=Path, required=True) + + provenance = subparsers.add_parser("provenance") + provenance.add_argument("--pages", type=Path, required=True) + provenance.add_argument("--authorization-after") + + decide = subparsers.add_parser("decide") + decide.add_argument("--action", required=True) + decide.add_argument("--event-label", default="") + decide.add_argument("--base-ref-changed", type=_parse_bool, required=True) + decide.add_argument("--state", required=True) + decide.add_argument("--merged", type=_parse_bool, required=True) + decide.add_argument("--merge-candidate-available", type=_parse_bool, required=True) + decide.add_argument("--relevant", type=_parse_bool, required=True) + decide.add_argument("--identity-matches", type=_parse_bool, required=True) + decide.add_argument( + "--authorization-label-present", type=_parse_bool, required=True + ) + decide.add_argument("--authorization-epoch-valid", type=_parse_bool, required=True) + decide.add_argument( + "--authorization-rerun-matches", type=_parse_bool, required=True + ) + decide.add_argument("--actor-permission", default="") + decide.add_argument("--has-existing-check", type=_parse_bool, required=True) + decide.add_argument("--has-newer-check", type=_parse_bool, required=True) + decide.add_argument("--required-workflow-only", type=_parse_bool, required=True) + decide.add_argument("--github-output", type=Path, required=True) + return parser + + +def main() -> int: + args = _build_parser().parse_args() + if args.command == "classify": + pages = json.loads(args.pages.read_text(encoding="utf-8")) + relevant = classify_changed_file_pages( + pages, + expected_count=args.expected_count, + ) + _write_outputs(args.github_output, {"relevant": relevant}) + return 0 + if args.command == "classify-compare": + response = json.loads(args.response.read_text(encoding="utf-8")) + relevant = classify_compare_response( + response, + expected_base_sha=args.expected_base_sha, + expected_target_sha=args.expected_target_sha, + expected_count=args.expected_count, + ) + _write_outputs(args.github_output, {"relevant": relevant}) + return 0 + if args.command == "provenance": + pages = json.loads(args.pages.read_text(encoding="utf-8")) + provenance = classify_authorization_event_pages( + pages, + authorization_after=args.authorization_after, + ) + print(json.dumps(provenance._asdict(), separators=(",", ":"))) + return 0 + + decision = decide_pull_request_policy( + action=args.action, + event_label=args.event_label, + base_ref_changed=args.base_ref_changed, + state=args.state, + merged=args.merged, + merge_candidate_available=args.merge_candidate_available, + relevant=args.relevant, + identity_matches=args.identity_matches, + authorization_label_present=args.authorization_label_present, + authorization_epoch_valid=args.authorization_epoch_valid, + authorization_rerun_matches=args.authorization_rerun_matches, + actor_permission=args.actor_permission, + has_existing_check=args.has_existing_check, + has_newer_check=args.has_newer_check, + required_workflow_only=args.required_workflow_only, + ) + _write_outputs( + args.github_output, + { + "mode": decision.mode, + "run_validation": decision.run_validation, + "conclusion": decision.conclusion, + "summary": decision.summary, + }, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/llamacpp_validation_args.ps1 b/.github/scripts/llamacpp_validation_args.ps1 deleted file mode 100644 index 1549b39adc..0000000000 --- a/.github/scripts/llamacpp_validation_args.ps1 +++ /dev/null @@ -1,62 +0,0 @@ -function Get-LlamaCppValidationModelArguments { - [CmdletBinding()] - param( - [AllowEmptyString()] - [string] $Models = "", - [bool] $Lite = $false, - [AllowEmptyString()] - [string] $CapabilityProfile = "", - [AllowEmptyString()] - [string] $CapabilityModels = "" - ) - - $modelIds = @( - $Models -split "," | - ForEach-Object { $_.Trim() } | - Where-Object { $_ } - ) - - if ($modelIds.Count -gt 0 -and $Lite) { - throw "Explicit models and lite mode are mutually exclusive." - } - - $capabilityModelIds = @( - $CapabilityModels -split "," | - ForEach-Object { $_.Trim() } | - Where-Object { $_ } - ) - if (($CapabilityProfile -eq "") -ne ($capabilityModelIds.Count -eq 0)) { - throw "Capability profile and capability models must be provided together." - } - if ($CapabilityProfile -and $CapabilityProfile -ne "k2-horizon-v1") { - throw "Unsupported capability profile: $CapabilityProfile" - } - if ($modelIds.Count -gt 0) { - foreach ($modelId in $capabilityModelIds) { - if ($modelId -notin $modelIds) { - throw "Capability model '$modelId' must be present in explicit models." - } - } - } - - $validationArguments = @() - foreach ($modelId in $modelIds) { - $validationArguments += "--model" - $validationArguments += $modelId - } - - if ($Lite) { - $validationArguments += "--lite" - } - - if ($CapabilityProfile) { - $validationArguments += "--capability-profile" - $validationArguments += $CapabilityProfile - foreach ($modelId in $capabilityModelIds) { - $validationArguments += "--capability-model" - $validationArguments += $modelId - } - } - - return $validationArguments -} diff --git a/.github/scripts/publish_llamacpp_update.sh b/.github/scripts/publish_llamacpp_update.sh index 143b731e35..f032fbed94 100755 --- a/.github/scripts/publish_llamacpp_update.sh +++ b/.github/scripts/publish_llamacpp_update.sh @@ -13,12 +13,37 @@ llamacpp_release=${LLAMACPP_RELEASE:?LLAMACPP_RELEASE is required} lemonade_release=${LLAMACPP_LEMONADE_RELEASE:?LLAMACPP_LEMONADE_RELEASE is required} rocm_release=${LLAMACPP_ROCM_RELEASE:?LLAMACPP_ROCM_RELEASE is required} expected_release_manifest=${EXPECTED_RELEASE_ASSET_MANIFEST:?EXPECTED_RELEASE_ASSET_MANIFEST is required} +: "${GH_TOKEN:?GH_TOKEN must contain a separately provisioned publication token}" +publication_actor=${EXPECTED_PUBLICATION_ACTOR:?EXPECTED_PUBLICATION_ACTOR is required} + +if ! authenticated_actor=$(gh api graphql \ + -f 'query=query { viewer { login } }' \ + --jq '.data.viewer.login'); then + echo "Could not authenticate the dedicated publication token." >&2 + exit 1 +fi +if [[ -z "$authenticated_actor" || + "$authenticated_actor" != "$publication_actor" ]]; then + echo "The authenticated publication actor does not match EXPECTED_PUBLICATION_ACTOR." >&2 + exit 1 +fi script_directory=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) repository_root=$(cd -- "${script_directory}/../.." && pwd) manifest_tool="${repository_root}/test/utils/llamacpp_release_manifest.py" capture_tool="${script_directory}/capture_llamacpp_release_manifest.sh" snapshot_root=$(mktemp -d) +git_askpass_path="${snapshot_root}/git-askpass.sh" +# The generated helper expands these variables only when Git invokes it. +# shellcheck disable=SC2016 +printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'case "$1" in' \ + ' *Username*) printf "%s\n" x-access-token ;;' \ + ' *Password*) printf "%s\n" "$GH_TOKEN" ;;' \ + ' *) exit 1 ;;' \ + 'esac' > "$git_askpass_path" +chmod 0700 "$git_askpass_path" manifest_digest=$(python "$manifest_tool" \ --materialize-manifest "$manifest_path" \ @@ -47,6 +72,12 @@ recheck_release_manifest() { "${snapshot_root}/snapshot-${snapshot_number}" >/dev/null } +require_managed_pin_manifest() { + python "$manifest_tool" \ + --require-managed-pin-manifest \ + "$base_versions_path" "$versions_path" "$manifest_path" >/dev/null +} + release_pattern='^b[0-9]+$' for release in "$llamacpp_release" "$lemonade_release" "$rocm_release"; do if [[ ! "$release" =~ $release_pattern ]]; then @@ -61,6 +92,54 @@ if [[ "$local_base_sha" != "$validated_base_sha" ]]; then exit 1 fi +base_versions_path="${snapshot_root}/base-backend-versions.json" +git show "${validated_base_sha}:${versions_path}" > "$base_versions_path" +python - "$base_versions_path" "$versions_path" <<'PY' +import json +import re +import sys + +managed_backends = ("cpu", "cuda", "metal", "rocm-nightly", "rocm-stable", "vulkan") +release_pattern = re.compile(r"b[0-9]+") + +try: + with open(sys.argv[1], encoding="utf-8") as base_file: + base_versions = json.load(base_file)["llamacpp"] + with open(sys.argv[2], encoding="utf-8") as candidate_file: + candidate_versions = json.load(candidate_file)["llamacpp"] +except (KeyError, OSError, TypeError, ValueError) as error: + raise SystemExit( + "Scheduled llama.cpp update found invalid llamacpp version mappings." + ) from error + +if not isinstance(base_versions, dict) or not isinstance(candidate_versions, dict): + raise SystemExit( + "Scheduled llama.cpp update found invalid llamacpp version mappings." + ) + +for backend in managed_backends: + if backend not in base_versions or backend not in candidate_versions: + raise SystemExit( + f"Scheduled llama.cpp update changed the managed llamacpp.{backend} pin shape." + ) + base_release = base_versions[backend] + candidate_release = candidate_versions[backend] + if ( + not isinstance(base_release, str) + or release_pattern.fullmatch(base_release) is None + or not isinstance(candidate_release, str) + or release_pattern.fullmatch(candidate_release) is None + ): + raise SystemExit( + f"Scheduled llama.cpp update found an invalid llamacpp.{backend} release tag." + ) + if int(candidate_release[1:]) < int(base_release[1:]): + raise SystemExit( + f"Scheduled llama.cpp update would downgrade llamacpp.{backend} " + f"from {base_release} to {candidate_release}." + ) +PY + if [[ $(git rev-parse --is-shallow-repository) == "true" ]]; then git fetch --unshallow --no-tags origin "refs/heads/${base_branch}" else @@ -72,19 +151,24 @@ if [[ "$fetched_base_sha" != "$validated_base_sha" ]]; then exit 1 fi -fresh_llamacpp_release=$(gh api repos/ggml-org/llama.cpp/releases \ - --jq '[.[] | select(.draft | not) | .tag_name | select(test("^b[0-9]+$"))][0] // empty') -fresh_rocm_release=$(gh api \ - repos/lemonade-sdk/llamacpp-rocm/releases/latest --jq '.tag_name') -fresh_lemonade_release=$(gh api \ - repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name') - -if [[ "$fresh_llamacpp_release" != "$llamacpp_release" || - "$fresh_rocm_release" != "$rocm_release" || - "$fresh_lemonade_release" != "$lemonade_release" ]]; then - echo "The candidate releases changed during validation; refusing publication." >&2 - exit 1 -fi +assert_candidate_releases_current() { + local fresh_llamacpp_release fresh_rocm_release fresh_lemonade_release + fresh_llamacpp_release=$(gh api repos/ggml-org/llama.cpp/releases \ + --jq '[.[] | select(.draft | not) | .tag_name | select(test("^b[0-9]+$"))][0] // empty') + fresh_rocm_release=$(gh api \ + repos/lemonade-sdk/llamacpp-rocm/releases/latest --jq '.tag_name') + fresh_lemonade_release=$(gh api \ + repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name') + + if [[ "$fresh_llamacpp_release" != "$llamacpp_release" || + "$fresh_rocm_release" != "$rocm_release" || + "$fresh_lemonade_release" != "$lemonade_release" ]]; then + echo "The candidate releases changed during validation; refusing publication." >&2 + exit 1 + fi +} + +assert_candidate_releases_current recheck_release_manifest @@ -106,7 +190,6 @@ open_pr_records=$(gh api "repos/${repository}/pulls" \ open_pr="" stale_prs=() -publication_actor='github-actions[bot]' while IFS=$'\t' read -r number head_ref head_repository author base_ref; do if [[ -z "$number" ]]; then continue @@ -170,6 +253,12 @@ require_pull_request_identity() { exit 1 fi pull_request_json=$(gh api "repos/${repository}/pulls/${number}") + if ! pull_request_is_draft=$(jq -r \ + 'if (.draft | type) == "boolean" then .draft else error("invalid draft") end' \ + <<< "$pull_request_json"); then + echo "Pull request #${number} has invalid draft state." >&2 + exit 1 + fi if ! jq -e \ --arg state "$expected_state" \ --arg repository "$repository" \ @@ -191,23 +280,64 @@ require_pull_request_identity() { fi } -close_stale_prs() { +report_stale_prs() { local stale_record stale_pr stale_head_ref for stale_record in "${stale_prs[@]}"; do IFS=$'\t' read -r stale_pr stale_head_ref <<< "$stale_record" - require_pull_request_identity "$stale_pr" "$stale_head_ref" - gh api "repos/${repository}/pulls/${stale_pr}" \ - --method PATCH \ - -f state=closed >/dev/null - require_pull_request_identity "$stale_pr" "$stale_head_ref" "" closed - echo "Closed superseded pull request #${stale_pr}." + echo "Superseded pull request #${stale_pr} (${stale_head_ref}) requires manual cleanup." done } +reconcile_failed_pull_request_creation() { + local number head_ref head_repository author base_ref head_sha records + local matching_pr="" + local record_count=0 + + failed_create_reconciliation=unknown + failed_create_pr="" + if ! records=$(gh api "repos/${repository}/pulls" \ + --method GET \ + -f state=open \ + -f head="${repository_owner}:${branch}" \ + -f base="$base_branch" \ + --paginate \ + --jq '.[] | [.number, .head.ref, (.head.repo.full_name // ""), .user.login, .base.ref, .head.sha] | @tsv'); then + echo "Pull request creation failed, and its result could not be reconciled; preserving ${branch}." >&2 + return + fi + + while IFS=$'\t' read -r number head_ref head_repository author base_ref head_sha; do + if [[ -z "$number" ]]; then + continue + fi + record_count=$((record_count + 1)) + if ! [[ "$number" =~ ^[1-9][0-9]*$ ]] || + [[ "$head_ref" != "$branch" ]] || + [[ "$head_repository" != "$repository" ]] || + [[ "$author" != "$publication_actor" ]] || + [[ "$base_ref" != "$base_branch" ]] || + [[ "$head_sha" != "$published_oid" ]]; then + echo "Pull request creation failed, and its result was ambiguous; preserving ${branch}." >&2 + return + fi + matching_pr=$number + done <<< "$records" + + if [[ "$record_count" -eq 0 ]]; then + failed_create_reconciliation=absent + elif [[ "$record_count" -eq 1 ]]; then + failed_create_reconciliation=adopted + failed_create_pr=$matching_pr + else + echo "Pull request creation failed, and multiple matching pull requests were found; preserving ${branch}." >&2 + fi +} + if [[ "$has_update" != "true" ]]; then assert_base_is_current recheck_release_manifest - close_stale_prs + assert_candidate_releases_current + report_stale_prs echo "backend_versions.json is unchanged; nothing to update." exit 0 fi @@ -244,8 +374,8 @@ elif [[ -n "$open_pr" ]]; then exit 1 fi -git config user.name "github-actions[bot]" -git config user.email "github-actions[bot]@users.noreply.github.com" +git config user.name "$publication_actor" +git config user.email "${publication_actor}@users.noreply.github.com" git checkout -b "$branch" recheck_release_manifest git add "$versions_path" "$manifest_path" @@ -265,6 +395,7 @@ fi assert_base_is_current recheck_release_manifest +require_managed_pin_manifest if [[ -n "$remote_oid" ]]; then lease="refs/heads/${branch}:${remote_oid}" push_source="$remote_oid" @@ -274,7 +405,13 @@ else push_source=HEAD published_oid=$(git rev-parse HEAD) fi -git push --force-with-lease="$lease" origin \ +assert_candidate_releases_current +GIT_ASKPASS="$git_askpass_path" \ + GIT_TERMINAL_PROMPT=0 \ + GIT_CONFIG_COUNT=1 \ + GIT_CONFIG_KEY_0=credential.helper \ + GIT_CONFIG_VALUE_0='' \ + git push --force-with-lease="$lease" origin \ "${push_source}:refs/heads/$branch" assert_base_is_current @@ -289,20 +426,54 @@ if [[ -n "$open_pr" ]]; then current_pr=$open_pr echo "Refreshed pull request #${open_pr}." else - created_pr_url=$(gh pr create \ + if ! created_pr_url=$(gh pr create \ --repo "$repository" \ --title "$title" \ --body-file "$body_file" \ --base "$base_branch" \ - --head "$branch") - created_pr_url=${created_pr_url%/} - current_pr=${created_pr_url##*/} + --head "$branch" \ + --draft); then + assert_candidate_releases_current + reconcile_failed_pull_request_creation + if [[ "$failed_create_reconciliation" == "adopted" ]]; then + current_pr=$failed_create_pr + echo "Recovered pull request #${current_pr} after an ambiguous creation result." + else + if [[ "$failed_create_reconciliation" == "absent" ]]; then + echo "Pull request creation failed; preserving ${branch}." >&2 + fi + exit 1 + fi + else + created_pr_url=${created_pr_url%/} + current_pr=${created_pr_url##*/} + created_draft_pr=true + fi fi require_pull_request_identity \ "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" -close_stale_prs +if [[ "${created_draft_pr:-false}" == "true" && + "$pull_request_is_draft" != "true" ]]; then + echo "New pull request #${current_pr} was not created as a draft." >&2 + exit 1 +fi +assert_base_is_current +assert_publication_branch "$published_oid" +recheck_release_manifest +assert_candidate_releases_current assert_base_is_current assert_publication_branch "$published_oid" require_pull_request_identity \ "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" +if [[ "$pull_request_is_draft" == "true" ]]; then + gh pr ready "$current_pr" --repo "$repository" + echo "Marked pull request #${current_pr} ready for review." +fi +require_pull_request_identity \ + "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" +if [[ "$pull_request_is_draft" != "false" ]]; then + echo "Pull request #${current_pr} remained a draft after promotion." >&2 + exit 1 +fi +report_stale_prs diff --git a/.github/scripts/resolve_debian_version.sh b/.github/scripts/resolve_debian_version.sh deleted file mode 100755 index a398a929a7..0000000000 --- a/.github/scripts/resolve_debian_version.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -release=${1:?release is required} - -if git describe --tags --abbrev=0 >/dev/null 2>&1; then - version=$(git describe --tags --always) -else - if [[ ! -f CMakeLists.txt ]]; then - echo "Could not extract version from CMakeLists.txt" >&2 - exit 1 - fi - - version=$(sed -n \ - 's/^project(lemon_cpp VERSION \([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p' \ - CMakeLists.txt) - if [[ -z "$version" ]]; then - echo "Could not extract version from CMakeLists.txt" >&2 - exit 1 - fi - - commit_count=$(git rev-list --count HEAD) - version="${version}+git${commit_count}.g$(git rev-parse --short HEAD)" -fi - -debian_version=${version#v} -dpkg --validate-version "${debian_version}~${release}" -printf '%s\n' "$debian_version" diff --git a/.github/scripts/run_llamacpp_validation.py b/.github/scripts/run_llamacpp_validation.py new file mode 100644 index 0000000000..7f6d6154db --- /dev/null +++ b/.github/scripts/run_llamacpp_validation.py @@ -0,0 +1,2578 @@ +#!/usr/bin/env python3 +"""Run one llama.cpp validation lane on Windows, Linux, or macOS.""" + +from __future__ import annotations + +import argparse +import ctypes +import os +import re +import signal +import socket +import stat +import subprocess +import sys +import tempfile +import threading +import time +import urllib.request +from pathlib import Path +from typing import NamedTuple + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from test.utils.llamacpp_validation_artifacts import ( + VerifiedValidationArtifact, + verify_validation_artifacts, +) +from test.utils.llamacpp_validation_evidence import ( + ValidationEvidenceError, + load_and_validate_result_file, +) +from test.utils.validation_model_catalog import load_model_catalog + +K2_HORIZON_PROFILE = "k2-horizon-v1" +TARGET_PATTERN = re.compile(r"^[a-z0-9-]+$") +INFERENCE_PROCESS_NAMES = { + "flm", + "lemond", + "lemonade", + "lemonadeserver", + "llama-server", + "moonshine-server", + "ort-server", +} +VALIDATION_MODEL_CATALOG = ROOT / "test/fixtures/llamacpp_validation_models.json" +VALIDATION_ARTIFACT_LOCKS = ROOT / "test/fixtures/llamacpp_validation_artifacts.json" +INHERITED_RUNTIME_ENVIRONMENT_KEYS = { + "COMSPEC", + "CUDA_VISIBLE_DEVICES", + "CURL_CA_BUNDLE", + "GGML_METAL_NO_RESIDENCY", + "GPU_DEVICE_ORDINAL", + "HIP_VISIBLE_DEVICES", + "HSA_OVERRIDE_GFX_VERSION", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NVIDIA_DRIVER_CAPABILITIES", + "NVIDIA_VISIBLE_DEVICES", + "NUMBER_OF_PROCESSORS", + "PATH", + "PATHEXT", + "PROCESSOR_ARCHITECTURE", + "REQUESTS_CA_BUNDLE", + "ROCR_VISIBLE_DEVICES", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TZ", + "VK_ADD_DRIVER_FILES", + "VK_DRIVER_FILES", + "VK_ICD_FILENAMES", + "WINDIR", + "__NV_PRIME_RENDER_OFFLOAD", +} +LOG_PREFIX = r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \[[A-Za-z]+\]" +CHILD_LOG_PREFIX = r"(?:\[\d{1,10}\]\s+)?\d{1,6}\.\d{2}\.\d{3}\.\d{3}\s+[A-Z]\s+" +LOADING_MODEL_MARKER = re.compile( + LOG_PREFIX + r" \(LlamaCpp\) Loading model:\s+(?P\S+)\s*$" +) +BACKEND_MARKER = re.compile( + LOG_PREFIX + r" \(LlamaCpp\) Using LlamaCpp Backend:\s+(?P\S+)\s*$" +) +GGUF_MARKER = re.compile( + LOG_PREFIX + r" \(LlamaCpp\) Using GGUF:\s+(?P.+?\S)\s*$" +) +LOADED_MODEL_MARKER = re.compile( + LOG_PREFIX + + r" \(Process\) " + + CHILD_LOG_PREFIX + + r"srv\s+llama_server:\s+model loaded\s*$" +) +DEVICE_MARKER = re.compile( + LOG_PREFIX + + r" \(Process\) " + + CHILD_LOG_PREFIX + + r"llama_prepare_model_devices:\s+using device\s+(?P\S+).*$" +) +BACKEND_PREFIX = "Using LlamaCpp Backend:" +DEVICE_PREFIX = "llama_prepare_model_devices: using device" +OFFLOAD_MARKER = re.compile( + LOG_PREFIX + + r" \(Process\) " + + CHILD_LOG_PREFIX + + r"load_tensors:\s+offloaded\s+(\d+)/(\d+)\s+layers\s+to\s+GPU\s*$" +) +CPU_BUFFER_MARKER = re.compile( + LOG_PREFIX + + r" \(Process\) " + + CHILD_LOG_PREFIX + + r"load_tensors:\s+CPU(?:_Mapped)?\s+model buffer size\s*=\s*" + + r"(?P[0-9]+(?:\.[0-9]+)?)\s+MiB\s*$" +) +OFFLOAD_PREFIX = "load_tensors: offloaded" +MAX_ATTESTATION_LOG_BYTES = 512 * 1024 * 1024 +MAX_ATTESTATION_LINE_BYTES = 1024 * 1024 +VALIDATION_PHASE_TIMEOUT_SECONDS = 4 * 60 * 60 +PROCESS_WAIT_POLL_SECONDS = 0.2 +EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS = 4.0 +EXCEPTIONAL_TERM_GRACE_SECONDS = 2.0 +DARWIN_EPHEMERAL_RUNNER_CONTEXT_KEY = "LEMONADE_PROTECTED_RUNNER_CONTEXT" +DARWIN_EPHEMERAL_RUNNER_CONTEXT = "github-hosted:macos-latest:macos-metal" +_PR_SET_CHILD_SUBREAPER = 36 +_PR_GET_CHILD_SUBREAPER = 37 +_POSIX_GATE_PROGRAM = """ +import os +import sys + +gate = int(sys.argv[1]) +command = sys.argv[2:] +released = os.read(gate, 1) +os.close(gate) +if released != b"1" or not command: + raise SystemExit(125) +os.execvpe(command[0], command, os.environ) +""" +_WINDOWS_GATE_PROGRAM = """ +import ctypes +import subprocess +import sys + +gate = int(sys.argv[1]) +command = sys.argv[2:] +kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +kernel32.WaitForSingleObject.argtypes = (ctypes.c_void_p, ctypes.c_uint32) +kernel32.WaitForSingleObject.restype = ctypes.c_uint32 +kernel32.CloseHandle.argtypes = (ctypes.c_void_p,) +kernel32.CloseHandle.restype = ctypes.c_int +wait_result = kernel32.WaitForSingleObject(gate, 0xFFFFFFFF) +closed = kernel32.CloseHandle(gate) +if wait_result != 0 or not closed or not command: + raise SystemExit(125) +raise SystemExit(subprocess.call(command)) +""" +ACCELERATOR_DEVICE_PREFIXES = { + "cuda": "CUDA", + "metal": "MTL", + "rocm": "ROCm", + "vulkan": "Vulkan", +} +ATTESTED_BACKENDS = { + "cpu", + "cuda", + "metal", + "rocm-nightly", + "rocm-stable", + "vulkan", +} +_JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 +_JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION = 1 +_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 +_HANDLE_FLAG_INHERIT = 0x00000001 +_LINUX_SUBREAPER_LOCK = threading.Lock() +_LINUX_SUBREAPER_LEASES = 0 +_LINUX_SUBREAPER_ORIGINAL_STATE: bool | None = None + + +class _JobObjectBasicLimitInformation(ctypes.Structure): + _fields_ = ( + ("per_process_user_time_limit", ctypes.c_longlong), + ("per_job_user_time_limit", ctypes.c_longlong), + ("limit_flags", ctypes.c_uint32), + ("minimum_working_set_size", ctypes.c_size_t), + ("maximum_working_set_size", ctypes.c_size_t), + ("active_process_limit", ctypes.c_uint32), + ("affinity", ctypes.c_size_t), + ("priority_class", ctypes.c_uint32), + ("scheduling_class", ctypes.c_uint32), + ) + + +class _JobObjectBasicAccountingInformation(ctypes.Structure): + _fields_ = ( + ("total_user_time", ctypes.c_longlong), + ("total_kernel_time", ctypes.c_longlong), + ("this_period_total_user_time", ctypes.c_longlong), + ("this_period_total_kernel_time", ctypes.c_longlong), + ("total_page_fault_count", ctypes.c_uint32), + ("total_processes", ctypes.c_uint32), + ("active_processes", ctypes.c_uint32), + ("total_terminated_processes", ctypes.c_uint32), + ) + + +class _IoCounters(ctypes.Structure): + _fields_ = ( + ("read_operation_count", ctypes.c_ulonglong), + ("write_operation_count", ctypes.c_ulonglong), + ("other_operation_count", ctypes.c_ulonglong), + ("read_transfer_count", ctypes.c_ulonglong), + ("write_transfer_count", ctypes.c_ulonglong), + ("other_transfer_count", ctypes.c_ulonglong), + ) + + +class _JobObjectExtendedLimitInformation(ctypes.Structure): + _fields_ = ( + ("basic_limit_information", _JobObjectBasicLimitInformation), + ("io_info", _IoCounters), + ("process_memory_limit", ctypes.c_size_t), + ("job_memory_limit", ctypes.c_size_t), + ("peak_process_memory_used", ctypes.c_size_t), + ("peak_job_memory_used", ctypes.c_size_t), + ) + + +class _DirectoryIdentity(NamedTuple): + path: Path + resolved_path: Path + device: int + inode: int + + +class _FileIdentity(NamedTuple): + path: Path + resolved_path: Path + device: int + inode: int + size: int + modified_ns: int + changed_ns: int + + +class ValidationCache(NamedTuple): + temporary_root: _DirectoryIdentity + validation_root: _DirectoryIdentity + cache_directory: _DirectoryIdentity + config_directory: _DirectoryIdentity + hf_cache_directory: _DirectoryIdentity + private_temp_directory: _DirectoryIdentity + results_directory: _DirectoryIdentity + logs_directory: _DirectoryIdentity + + +class _CacheProcessScan(NamedTuple): + processes: list[object] + complete: bool + + +class _ValidationCancelled(RuntimeError): + def __init__(self, signum: int, cleanup_deadline: float): + self.signum = signum + self.cleanup_deadline = cleanup_deadline + super().__init__(signum) + + +_ACTIVE_CANCELLATION_STATE = None + + +class _CancellationSignalGuard: + def __init__(self): + self.signum: int | None = None + self.cleanup_deadline: float | None = None + self._previous_handlers: dict[int, object] = {} + + def _handle_signal(self, signum: int, _frame) -> None: + if self.signum is None: + self.signum = signum + + def _restore_handlers(self) -> None: + for selected_signal, previous_handler in self._previous_handlers.items(): + signal.signal(selected_signal, previous_handler) + self._previous_handlers.clear() + + def __enter__(self): + global _ACTIVE_CANCELLATION_STATE + if _ACTIVE_CANCELLATION_STATE is not None: + raise RuntimeError("Cancellation signal guard is already active") + _ACTIVE_CANCELLATION_STATE = self + if os.name == "nt": + return self + try: + for selected_signal in (signal.SIGINT, signal.SIGTERM): + self._previous_handlers[selected_signal] = signal.getsignal( + selected_signal + ) + signal.signal(selected_signal, self._handle_signal) + except BaseException: + self._restore_handlers() + _ACTIVE_CANCELLATION_STATE = None + raise + return self + + def __exit__(self, _exc_type, _exc_value, _traceback) -> None: + global _ACTIVE_CANCELLATION_STATE + try: + if os.name != "nt": + self._restore_handlers() + finally: + _ACTIVE_CANCELLATION_STATE = None + + +def _raise_if_cancelled() -> None: + cancellation = _ACTIVE_CANCELLATION_STATE + if cancellation is not None and cancellation.signum is not None: + cleanup_deadline = getattr(cancellation, "cleanup_deadline", None) + if cleanup_deadline is None: + cleanup_deadline = time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS + cancellation.cleanup_deadline = cleanup_deadline + raise _ValidationCancelled( + cancellation.signum, + cleanup_deadline, + ) + + +def _cancellation_pending() -> bool: + cancellation = _ACTIVE_CANCELLATION_STATE + return cancellation is not None and cancellation.signum is not None + + +def _windows_last_error() -> int: + getter = getattr(ctypes, "get_last_error", None) + return getter() if getter is not None else 0 + + +def _load_windows_kernel32(): + loader = getattr(ctypes, "WinDLL", None) + if loader is None: + raise RuntimeError("Windows Job Objects are unavailable") + kernel32 = loader("kernel32", use_last_error=True) + handle_type = ctypes.c_void_p + kernel32.CreateJobObjectW.argtypes = (ctypes.c_void_p, ctypes.c_wchar_p) + kernel32.CreateJobObjectW.restype = handle_type + kernel32.SetInformationJobObject.argtypes = ( + handle_type, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_uint32, + ) + kernel32.SetInformationJobObject.restype = ctypes.c_int + kernel32.AssignProcessToJobObject.argtypes = (handle_type, handle_type) + kernel32.AssignProcessToJobObject.restype = ctypes.c_int + kernel32.QueryInformationJobObject.argtypes = ( + handle_type, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_uint32, + ctypes.c_void_p, + ) + kernel32.QueryInformationJobObject.restype = ctypes.c_int + kernel32.CreateEventW.argtypes = ( + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_int, + ctypes.c_wchar_p, + ) + kernel32.CreateEventW.restype = handle_type + kernel32.SetEvent.argtypes = (handle_type,) + kernel32.SetEvent.restype = ctypes.c_int + kernel32.SetHandleInformation.argtypes = ( + handle_type, + ctypes.c_uint32, + ctypes.c_uint32, + ) + kernel32.SetHandleInformation.restype = ctypes.c_int + kernel32.CloseHandle.argtypes = (handle_type,) + kernel32.CloseHandle.restype = ctypes.c_int + return kernel32 + + +class _WindowsKillOnCloseJob: + def __init__(self, api, handle: int): + self._api = api + self._handle = handle + self._close_result: bool | None = None + + @classmethod + def create(cls, api=None): + selected_api = api or _load_windows_kernel32() + handle = selected_api.CreateJobObjectW(None, None) + if not handle: + error = _windows_last_error() + raise OSError(error, "CreateJobObjectW failed") + information = _JobObjectExtendedLimitInformation() + information.basic_limit_information.limit_flags = ( + _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + ) + configured = selected_api.SetInformationJobObject( + handle, + _JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, + ctypes.byref(information), + ctypes.sizeof(information), + ) + if not configured: + error = _windows_last_error() + selected_api.CloseHandle(handle) + raise OSError(error, "SetInformationJobObject failed") + return cls(selected_api, handle) + + @staticmethod + def popen_kwargs() -> dict[str, object]: + return {} + + def start_process(self, command: list[str], **kwargs) -> subprocess.Popen: + if not command: + raise ValueError("Contained command must not be empty") + if self._handle is None: + raise RuntimeError("Windows Job Object is already closed") + if kwargs.get("startupinfo") is not None: + raise ValueError("Contained Windows commands cannot override startupinfo") + if kwargs.get("close_fds") is False: + raise ValueError("Contained Windows commands require close_fds=True") + kwargs.pop("startupinfo", None) + kwargs.pop("close_fds", None) + + gate_handle = self._api.CreateEventW(None, True, False, None) + if not gate_handle: + error = _windows_last_error() + raise OSError(error, "CreateEventW failed") + gate_inheritable = False + process = None + try: + if not self._api.SetHandleInformation( + gate_handle, + _HANDLE_FLAG_INHERIT, + _HANDLE_FLAG_INHERIT, + ): + error = _windows_last_error() + raise OSError(error, "SetHandleInformation failed") + gate_inheritable = True + startup_info = subprocess.STARTUPINFO() + startup_info.lpAttributeList = {"handle_list": [int(gate_handle)]} + process = subprocess.Popen( + [ + sys.executable, + "-I", + "-S", + "-c", + _WINDOWS_GATE_PROGRAM, + str(int(gate_handle)), + *command, + ], + startupinfo=startup_info, + close_fds=True, + **kwargs, + ) + self.assign(process) + if not self._api.SetHandleInformation( + gate_handle, + _HANDLE_FLAG_INHERIT, + 0, + ): + error = _windows_last_error() + raise OSError(error, "SetHandleInformation failed") + gate_inheritable = False + _raise_if_cancelled() + if not self._api.SetEvent(gate_handle): + error = _windows_last_error() + raise OSError(error, "SetEvent failed") + if not self._api.CloseHandle(gate_handle): + error = _windows_last_error() + raise OSError(error, "CloseHandle failed for Windows launch gate") + gate_handle = None + return process + except BaseException as exc: + cleanup_deadline = ( + exc.cleanup_deadline + if isinstance(exc, _ValidationCancelled) + else time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS + ) + if process is not None: + self.abort(cleanup_deadline) + _reap_direct_process(process, cleanup_deadline) + raise + finally: + if gate_handle is not None: + if gate_inheritable: + self._api.SetHandleInformation( + gate_handle, + _HANDLE_FLAG_INHERIT, + 0, + ) + self._api.CloseHandle(gate_handle) + + def assign(self, process: subprocess.Popen) -> None: + process_handle = getattr(process, "_handle", None) + if process_handle is None: + raise RuntimeError("Windows child process handle is unavailable") + if not self._api.AssignProcessToJobObject(self._handle, int(process_handle)): + error = _windows_last_error() + raise OSError(error, "AssignProcessToJobObject failed") + + def close(self) -> bool: + if self._close_result is not None: + return self._close_result + handle = self._handle + self._handle = None + accounting = _JobObjectBasicAccountingInformation() + queried = bool( + self._api.QueryInformationJobObject( + handle, + _JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION, + ctypes.byref(accounting), + ctypes.sizeof(accounting), + None, + ) + ) + closed = bool(self._api.CloseHandle(handle)) + self._close_result = queried and accounting.active_processes == 0 and closed + return self._close_result + + def abort(self, deadline: float) -> bool: + del deadline + return self.close() + + +def _linux_child_subreaper_state() -> bool: + libc = ctypes.CDLL(None, use_errno=True) + state = ctypes.c_int() + if libc.prctl(_PR_GET_CHILD_SUBREAPER, ctypes.byref(state), 0, 0, 0) != 0: + error = ctypes.get_errno() + raise OSError(error, "PR_GET_CHILD_SUBREAPER failed") + return bool(state.value) + + +def _set_linux_child_subreaper(enabled: bool) -> None: + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(_PR_SET_CHILD_SUBREAPER, int(enabled), 0, 0, 0) != 0: + error = ctypes.get_errno() + raise OSError(error, "PR_SET_CHILD_SUBREAPER failed") + + +def _acquire_linux_child_subreaper() -> None: + global _LINUX_SUBREAPER_LEASES + global _LINUX_SUBREAPER_ORIGINAL_STATE + with _LINUX_SUBREAPER_LOCK: + if _LINUX_SUBREAPER_LEASES == 0: + original_state = _linux_child_subreaper_state() + if not original_state: + _set_linux_child_subreaper(True) + _LINUX_SUBREAPER_ORIGINAL_STATE = original_state + _LINUX_SUBREAPER_LEASES += 1 + + +def _release_linux_child_subreaper() -> None: + global _LINUX_SUBREAPER_LEASES + global _LINUX_SUBREAPER_ORIGINAL_STATE + with _LINUX_SUBREAPER_LOCK: + if _LINUX_SUBREAPER_LEASES <= 0: + raise RuntimeError("Linux child-subreaper lease was not acquired") + if _LINUX_SUBREAPER_LEASES == 1 and _LINUX_SUBREAPER_ORIGINAL_STATE is False: + _set_linux_child_subreaper(False) + _LINUX_SUBREAPER_LEASES -= 1 + if _LINUX_SUBREAPER_LEASES == 0: + _LINUX_SUBREAPER_ORIGINAL_STATE = None + + +class _PosixProcessGroupContainment: + def __init__(self, *, allow_untracked_descendants: bool = False): + self._process_groups: list[int] = [] + self._root_processes: dict[tuple[int, float], object] = {} + self._retained_descendants: dict[tuple[int, float], object] = {} + self._tracking_complete = True + self._cleanup_finished = False + self._close_result: bool | None = None + self._linux_runner_process = None + self._linux_subreaper_lease = False + if sys.platform.startswith("linux"): + self._initialize_linux_subreaper() + elif sys.platform == "darwin": + if not allow_untracked_descendants: + raise RuntimeError( + "Darwin cannot contain session-escaping descendants; protected " + "validation requires --allow-darwin-github-hosted-ephemeral-runner " + "on the trusted macos-latest job" + ) + elif not allow_untracked_descendants: + raise RuntimeError( + "Kernel-backed descendant containment is unavailable on this platform" + ) + + @staticmethod + def _process_identity(process) -> tuple[int, float]: + return process.pid, process.create_time() + + def _initialize_linux_subreaper(self) -> None: + if signal.getsignal(signal.SIGCHLD) != signal.SIG_DFL: + raise RuntimeError( + "Linux child-subreaper containment requires the default SIGCHLD handler" + ) + try: + import psutil + except ModuleNotFoundError as exc: + raise RuntimeError( + "Linux child-subreaper containment requires psutil" + ) from exc + try: + runner_process = psutil.Process(os.getpid()) + existing_descendants = runner_process.children(recursive=True) + except (OSError, psutil.Error) as exc: + raise RuntimeError( + "Could not verify an idle process tree before enabling containment" + ) from exc + if existing_descendants: + raise RuntimeError( + "Cannot enable Linux child-subreaper containment with pre-existing " + "runner descendants" + ) + _acquire_linux_child_subreaper() + self._linux_runner_process = runner_process + self._linux_subreaper_lease = True + + @staticmethod + def popen_kwargs() -> dict[str, object]: + return {"start_new_session": True} + + def start_process(self, command: list[str], **kwargs) -> subprocess.Popen: + if not command: + raise ValueError("Contained command must not be empty") + read_gate, write_gate = os.pipe() + process = None + assigned = False + try: + process = subprocess.Popen( + [ + sys.executable, + "-I", + "-S", + "-c", + _POSIX_GATE_PROGRAM, + str(read_gate), + *command, + ], + pass_fds=(read_gate,), + **kwargs, + **self.popen_kwargs(), + ) + os.close(read_gate) + read_gate = -1 + self.assign(process) + assigned = True + _raise_if_cancelled() + os.write(write_gate, b"1") + return process + except BaseException as exc: + cleanup_deadline = ( + exc.cleanup_deadline + if isinstance(exc, _ValidationCancelled) + else time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS + ) + if assigned: + self.abort(cleanup_deadline) + elif process is not None: + _reap_direct_process(process, cleanup_deadline) + raise + finally: + if read_gate >= 0: + os.close(read_gate) + os.close(write_gate) + + def assign(self, process: subprocess.Popen) -> None: + process_group = process.pid + if process_group == os.getpgrp(): + raise RuntimeError( + "Refusing to contain the validation runner process group" + ) + if process_group in self._process_groups: + raise RuntimeError("Process group was assigned more than once") + self._process_groups.append(process_group) + self._remember_root_process(process.pid) + try: + actual_process_group = os.getpgid(process.pid) + except ProcessLookupError: + self.refresh_descendants() + return + if actual_process_group != process_group: + raise RuntimeError("Child process did not start in an isolated session") + self.refresh_descendants() + + def _remember_root_process(self, process_id: int) -> None: + try: + import psutil + except ModuleNotFoundError: + self._tracking_complete = False + return + try: + process = psutil.Process(process_id) + identity = self._process_identity(process) + except (psutil.NoSuchProcess, psutil.ZombieProcess): + return + except (OSError, psutil.Error): + self._tracking_complete = False + return + self._root_processes.setdefault(identity, process) + + def refresh_descendants(self) -> bool: + if ( + not self._root_processes + and not self._retained_descendants + and self._linux_runner_process is None + ): + return self._tracking_complete + try: + import psutil + except ModuleNotFoundError: + self._tracking_complete = False + return False + + complete = True + if self._linux_runner_process is not None: + try: + adopted_processes = self._linux_runner_process.children(recursive=False) + except (OSError, psutil.Error): + adopted_processes = [] + complete = False + root_identities = set(self._root_processes) + for adopted_process in adopted_processes: + try: + identity = self._process_identity(adopted_process) + except (psutil.NoSuchProcess, psutil.ZombieProcess): + continue + except (OSError, psutil.Error): + complete = False + continue + if identity not in root_identities: + self._retained_descendants.setdefault(identity, adopted_process) + discovery_roots = list(self._root_processes.values()) + list( + self._retained_descendants.values() + ) + for root_process in discovery_roots: + try: + descendants = root_process.children(recursive=True) + except (psutil.NoSuchProcess, psutil.ZombieProcess): + continue + except (OSError, psutil.Error): + complete = False + continue + for descendant in descendants: + if descendant.pid == os.getpid(): + complete = False + continue + try: + identity = self._process_identity(descendant) + except (psutil.NoSuchProcess, psutil.ZombieProcess): + continue + except (OSError, psutil.Error): + complete = False + continue + self._retained_descendants.setdefault(identity, descendant) + if self._linux_runner_process is not None: + for identity, descendant in self._retained_descendants.items(): + try: + if descendant.ppid() != os.getpid(): + continue + waited_pid, _status = os.waitpid(descendant.pid, os.WNOHANG) + if waited_pid not in (0, descendant.pid): + complete = False + except ChildProcessError: + try: + if ( + self._process_identity(descendant) == identity + and descendant.is_running() + and descendant.status() + not in {psutil.STATUS_DEAD, psutil.STATUS_ZOMBIE} + ): + complete = False + except (psutil.NoSuchProcess, psutil.ZombieProcess): + pass + except (OSError, psutil.Error): + complete = False + except (psutil.NoSuchProcess, psutil.ZombieProcess): + continue + except (OSError, psutil.Error): + complete = False + if not complete: + self._tracking_complete = False + return complete + + def _release_kernel_tracking(self) -> bool: + if not self._linux_subreaper_lease: + return True + try: + _release_linux_child_subreaper() + except (OSError, RuntimeError): + self._tracking_complete = False + return False + self._linux_subreaper_lease = False + return True + + def _live_retained_descendants(self) -> list[object]: + try: + import psutil + except ModuleNotFoundError: + self._tracking_complete = False + return list(self._retained_descendants.values()) + + dead_statuses = {psutil.STATUS_DEAD, psutil.STATUS_ZOMBIE} + live_processes = [] + for identity, process in self._retained_descendants.items(): + try: + if ( + self._process_identity(process) == identity + and process.is_running() + and process.status() not in dead_statuses + ): + live_processes.append(process) + except (psutil.NoSuchProcess, psutil.ZombieProcess): + continue + except (OSError, psutil.Error): + self._tracking_complete = False + live_processes.append(process) + return live_processes + + def _signal_retained_descendants( + self, + processes: list[object], + selected_signal: int, + ) -> None: + try: + import psutil + except ModuleNotFoundError: + self._tracking_complete = False + return + for process in processes: + try: + process.send_signal(selected_signal) + except (psutil.NoSuchProcess, psutil.ZombieProcess): + continue + except (OSError, psutil.Error): + self._tracking_complete = False + continue + + @staticmethod + def _has_live_members(process_group: int) -> bool: + def probe_process_group() -> bool: + try: + os.killpg(process_group, 0) + except ProcessLookupError: + return False + except OSError: + return True + return True + + try: + import psutil + except ModuleNotFoundError: + return probe_process_group() + + dead_statuses = {psutil.STATUS_DEAD, psutil.STATUS_ZOMBIE} + try: + for process in psutil.process_iter(("pid", "status")): + try: + if ( + process.info["status"] not in dead_statuses + and os.getpgid(process.pid) == process_group + ): + return True + except (OSError, psutil.Error): + continue + except (OSError, psutil.Error): + return probe_process_group() + return False + + def _wait_for_exit( + self, + process_groups: list[int], + timeout: float, + *, + stop_on_cancellation: bool = False, + ) -> bool: + deadline = time.monotonic() + timeout + while True: + if stop_on_cancellation and _cancellation_pending(): + return False + self.refresh_descendants() + if ( + not any(self._has_live_members(group) for group in process_groups) + and not self._live_retained_descendants() + ): + return True + if time.monotonic() >= deadline: + return False + time.sleep(0.05) + + @classmethod + def _signal_groups(cls, process_groups: list[int], selected_signal: int) -> None: + for process_group in process_groups: + if not cls._has_live_members(process_group): + continue + try: + os.killpg(process_group, selected_signal) + except OSError: + continue + + def close(self) -> bool: + if self._cleanup_finished: + return self._close_result + self.refresh_descendants() + live_groups = [ + group for group in self._process_groups if self._has_live_members(group) + ] + live_descendants = self._live_retained_descendants() + had_survivors = bool(live_groups or live_descendants) + self._signal_groups(live_groups, signal.SIGTERM) + self._signal_retained_descendants(live_descendants, signal.SIGTERM) + if not self._wait_for_exit( + live_groups, + 5, + stop_on_cancellation=True, + ): + remaining_groups = [ + group for group in live_groups if self._has_live_members(group) + ] + remaining_descendants = self._live_retained_descendants() + self._signal_groups(remaining_groups, signal.SIGKILL) + self._signal_retained_descendants( + remaining_descendants, + signal.SIGKILL, + ) + self._wait_for_exit( + remaining_groups, + 5, + stop_on_cancellation=True, + ) + self.refresh_descendants() + cleanup_finished = ( + self._tracking_complete + and not any(self._has_live_members(group) for group in self._process_groups) + and not self._live_retained_descendants() + ) + if cleanup_finished and not self._release_kernel_tracking(): + cleanup_finished = False + self._cleanup_finished = cleanup_finished + clean = not had_survivors and cleanup_finished and self._tracking_complete + self._close_result = clean if self._close_result is None else False + return self._close_result + + def abort(self, deadline: float) -> bool: + if self._cleanup_finished: + return self._close_result + self.refresh_descendants() + live_groups = [ + group for group in self._process_groups if self._has_live_members(group) + ] + live_descendants = self._live_retained_descendants() + had_survivors = bool(live_groups or live_descendants) + self._signal_groups(live_groups, signal.SIGTERM) + self._signal_retained_descendants(live_descendants, signal.SIGTERM) + term_timeout = min( + EXCEPTIONAL_TERM_GRACE_SECONDS, + max(0.0, deadline - time.monotonic()), + ) + if not self._wait_for_exit(live_groups, term_timeout): + remaining_groups = [ + group for group in live_groups if self._has_live_members(group) + ] + remaining_descendants = self._live_retained_descendants() + self._signal_groups(remaining_groups, signal.SIGKILL) + self._signal_retained_descendants( + remaining_descendants, + signal.SIGKILL, + ) + kill_timeout = max(0.0, deadline - time.monotonic()) + self._wait_for_exit(remaining_groups, kill_timeout) + self.refresh_descendants() + cleanup_finished = ( + self._tracking_complete + and not any(self._has_live_members(group) for group in self._process_groups) + and not self._live_retained_descendants() + ) + if cleanup_finished and not self._release_kernel_tracking(): + cleanup_finished = False + self._cleanup_finished = cleanup_finished + clean = not had_survivors and cleanup_finished and self._tracking_complete + self._close_result = clean if self._close_result is None else False + return self._close_result + + +def _verified_darwin_ephemeral_runner( + *, + allow_darwin_github_hosted_ephemeral_runner: bool, + backend: str | None, + target: str | None, + environment: dict[str, str] | None = None, +) -> bool: + if sys.platform != "darwin": + return False + selected_environment = os.environ if environment is None else environment + if not allow_darwin_github_hosted_ephemeral_runner: + return False + if backend != "metal" or target != "macos-metal": + raise RuntimeError( + "The Darwin ephemeral-runner exception is restricted to macos-metal" + ) + if ( + selected_environment.get(DARWIN_EPHEMERAL_RUNNER_CONTEXT_KEY) + != DARWIN_EPHEMERAL_RUNNER_CONTEXT + or selected_environment.get("GITHUB_ACTIONS") != "true" + ): + raise RuntimeError( + "The Darwin ephemeral-runner exception requires the trusted " + "github-hosted macos-latest workflow context" + ) + return True + + +def _create_process_containment( + *, + allow_darwin_github_hosted_ephemeral_runner: bool = False, + backend: str | None = None, + target: str | None = None, +): + if os.name == "nt": + return _WindowsKillOnCloseJob.create() + allow_untracked_descendants = _verified_darwin_ephemeral_runner( + allow_darwin_github_hosted_ephemeral_runner=( + allow_darwin_github_hosted_ephemeral_runner + ), + backend=backend, + target=target, + ) + return _PosixProcessGroupContainment( + allow_untracked_descendants=allow_untracked_descendants + ) + + +def _start_contained_process(containment, command: list[str], **kwargs): + start_process = getattr(containment, "start_process", None) + if start_process is not None: + return start_process(command, **kwargs) + process = subprocess.Popen( + command, + **kwargs, + **containment.popen_kwargs(), + ) + containment.assign(process) + return process + + +def _close_process_containment(containment) -> bool: + try: + return containment.close() + except Exception as exc: # pylint: disable=broad-exception-caught + print(f"Process containment cleanup failed: {exc}", file=sys.stderr) + return False + + +def _abort_process_containment(containment, deadline: float) -> bool: + try: + return containment.abort(deadline) + except Exception as exc: # pylint: disable=broad-exception-caught + print(f"Process containment abort failed: {exc}", file=sys.stderr) + return False + + +def _refresh_process_containment(containment) -> bool: + refresh = getattr(containment, "refresh_descendants", None) + return True if refresh is None else bool(refresh()) + + +def _csv_values(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +def build_validation_command( + *, + python_executable: Path, + backend: str, + channel: str, + target: str, + models_csv: str, + lite: bool, + capability_profile: str, + capability_models_csv: str, + port: int, + output_path: Path | None = None, +) -> list[str]: + if not TARGET_PATTERN.fullmatch(target): + raise ValueError( + "target must contain only lowercase letters, digits, and hyphens" + ) + models = _csv_values(models_csv) + capability_models = _csv_values(capability_models_csv) + if models and lite: + raise ValueError("Explicit models and lite mode are mutually exclusive") + if bool(capability_profile) != bool(capability_models): + raise ValueError("Capability profile and capability models must be paired") + if capability_profile and capability_profile != K2_HORIZON_PROFILE: + raise ValueError(f"Unsupported capability profile: {capability_profile}") + if models: + selected_model_ids = {model.removeprefix("builtin.") for model in models} + missing = [ + model + for model in capability_models + if model.removeprefix("builtin.") not in selected_model_ids + ] + if missing: + raise ValueError( + "Capability models must be present in explicit models: " + + ", ".join(missing) + ) + + command = [ + str(python_executable), + str((ROOT / "test" / "validate_llamacpp.py").resolve()), + "--backend", + backend, + "--port", + str(port), + "--output", + str(output_path or f"llamacpp_validation_{target}.json"), + ] + if channel: + command.extend(("--channel", channel)) + for model in models: + command.extend(("--model", model)) + if lite: + command.append("--lite") + if capability_profile: + command.extend(("--capability-profile", capability_profile)) + for model in capability_models: + command.extend(("--capability-model", model)) + return command + + +def build_restart_validation_command( + *, + python_executable: Path, + backend: str, + channel: str, + target: str, + model: str, + port: int, + output_path: Path | None = None, +) -> list[str]: + if not model.strip() or "," in model: + raise ValueError("Restart validation requires exactly one model") + command = build_validation_command( + python_executable=python_executable, + backend=backend, + channel=channel, + target=target, + models_csv=model, + lite=False, + capability_profile="", + capability_models_csv="", + port=port, + output_path=output_path, + ) + if output_path is None: + command[command.index("--output") + 1] = ( + f"llamacpp_restart_validation_{target}.json" + ) + command.append("--skip-install") + return command + + +def build_prepare_validation_command( + *, + python_executable: Path, + backend: str, + channel: str, + target: str, + models_csv: str, + lite: bool, + capability_profile: str, + capability_models_csv: str, + port: int, + output_path: Path | None = None, +) -> list[str]: + command = build_validation_command( + python_executable=python_executable, + backend=backend, + channel=channel, + target=target, + models_csv=models_csv, + lite=lite, + capability_profile=capability_profile, + capability_models_csv=capability_models_csv, + port=port, + output_path=output_path, + ) + command.append("--prepare-only") + return command + + +def _require_unused_port(port: int) -> None: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: + raise RuntimeError(f"Port {port} is already in use") from exc + + +def _require_idle_runner() -> None: + import psutil + + conflicts = [] + for process in psutil.process_iter(("pid", "name")): + try: + name = (process.info.get("name") or "").lower().removesuffix(".exe") + if name in INFERENCE_PROCESS_NAMES: + conflicts.append(f"{name}:{process.pid}") + except psutil.Error: + continue + if conflicts: + raise RuntimeError("Runner is not idle: " + ", ".join(sorted(conflicts))) + + +def _is_link_or_junction(path: Path) -> bool: + if path.is_symlink(): + return True + is_junction = getattr(os.path, "isjunction", None) + if is_junction is not None and is_junction(path): + return True + try: + path_status = path.lstat() + except OSError: + return False + reparse_attribute = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + file_attributes = getattr(path_status, "st_file_attributes", 0) + return bool(reparse_attribute and file_attributes & reparse_attribute) + + +def _capture_directory(path: Path, allowed_root: Path | None) -> _DirectoryIdentity: + if _is_link_or_junction(path): + raise RuntimeError(f"Directory must not be a symlink or junction: {path}") + try: + resolved_path = path.resolve(strict=True) + path_status = path.lstat() + except OSError as exc: + raise RuntimeError(f"Directory does not exist: {path}") from exc + if not stat.S_ISDIR(path_status.st_mode): + raise RuntimeError(f"Path is not a directory: {path}") + if allowed_root is not None: + try: + resolved_path.relative_to(allowed_root) + except ValueError as exc: + raise RuntimeError(f"Directory escapes its allowed root: {path}") from exc + return _DirectoryIdentity( + path=path, + resolved_path=resolved_path, + device=path_status.st_dev, + inode=path_status.st_ino, + ) + + +def create_validation_cache(temporary_root: Path, target: str) -> ValidationCache: + if not TARGET_PATTERN.fullmatch(target): + raise ValueError( + "target must contain only lowercase letters, digits, and hyphens" + ) + temporary_identity = _capture_directory(temporary_root, None) + validation_root = Path( + tempfile.mkdtemp( + prefix=f"llamacpp-validation-{target}-", + dir=temporary_identity.resolved_path, + ) + ) + validation_identity = _capture_directory( + validation_root, temporary_identity.resolved_path + ) + cache_directory = validation_identity.resolved_path / "lemonade-cache" + config_directory = validation_identity.resolved_path / "lemonade-config" + cache_directory.mkdir(mode=0o700) + hf_cache_directory = cache_directory / "hf-hub" + hf_cache_directory.mkdir(mode=0o700) + config_directory.mkdir(mode=0o700) + private_temp_directory = validation_identity.resolved_path / "private-temp" + private_temp_directory.mkdir(mode=0o700) + results_directory = validation_identity.resolved_path / "results" + results_directory.mkdir(mode=0o700) + logs_directory = validation_identity.resolved_path / "logs" + logs_directory.mkdir(mode=0o700) + return ValidationCache( + temporary_root=temporary_identity, + validation_root=validation_identity, + cache_directory=_capture_directory( + cache_directory, validation_identity.resolved_path + ), + config_directory=_capture_directory( + config_directory, validation_identity.resolved_path + ), + hf_cache_directory=_capture_directory( + hf_cache_directory, validation_identity.resolved_path + ), + private_temp_directory=_capture_directory( + private_temp_directory, validation_identity.resolved_path + ), + results_directory=_capture_directory( + results_directory, validation_identity.resolved_path + ), + logs_directory=_capture_directory( + logs_directory, validation_identity.resolved_path + ), + ) + + +def revalidate_validation_cache(cache: ValidationCache) -> None: + current_temporary_root = _capture_directory(cache.temporary_root.path, None) + current_validation_root = _capture_directory( + cache.validation_root.path, + current_temporary_root.resolved_path, + ) + expected_and_current = ( + (cache.temporary_root, current_temporary_root), + (cache.validation_root, current_validation_root), + ( + cache.cache_directory, + _capture_directory( + cache.cache_directory.path, + current_validation_root.resolved_path, + ), + ), + ( + cache.config_directory, + _capture_directory( + cache.config_directory.path, + current_validation_root.resolved_path, + ), + ), + ( + cache.hf_cache_directory, + _capture_directory( + cache.hf_cache_directory.path, + current_validation_root.resolved_path, + ), + ), + ( + cache.private_temp_directory, + _capture_directory( + cache.private_temp_directory.path, + current_validation_root.resolved_path, + ), + ), + ( + cache.results_directory, + _capture_directory( + cache.results_directory.path, + current_validation_root.resolved_path, + ), + ), + ( + cache.logs_directory, + _capture_directory( + cache.logs_directory.path, + current_validation_root.resolved_path, + ), + ), + ) + for expected, current in expected_and_current: + if expected != current: + raise RuntimeError( + f"Validation cache directory changed after creation: {expected.path}" + ) + + +def _file_state(file_status: os.stat_result) -> tuple[int, ...]: + return ( + file_status.st_dev, + file_status.st_ino, + file_status.st_mode, + file_status.st_size, + file_status.st_mtime_ns, + file_status.st_ctime_ns, + ) + + +def _open_binary_no_follow(path: Path): + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + no_follow = getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags | no_follow) + return os.fdopen(descriptor, "rb") + + +def _capture_regular_file( + path: Path, allowed_root: Path, *, require_nonempty: bool +) -> _FileIdentity: + try: + allowed_root = allowed_root.resolve(strict=True) + except OSError as exc: + raise RuntimeError( + f"Validation output root does not exist: {allowed_root}" + ) from exc + if _is_link_or_junction(path): + raise RuntimeError(f"Validation output must not be a link: {path}") + try: + resolved_path = path.resolve(strict=True) + resolved_path.relative_to(allowed_root) + with _open_binary_no_follow(path) as output_file: + file_status = os.fstat(output_file.fileno()) + path_after = path.resolve(strict=True) + path_after.relative_to(allowed_root) + path_status = path_after.stat() + except (OSError, ValueError) as exc: + raise RuntimeError( + f"Validation output is absent or escapes its root: {path}" + ) from exc + if _is_link_or_junction(path): + raise RuntimeError(f"Validation output must not be a link: {path}") + if not stat.S_ISREG(file_status.st_mode) or ( + require_nonempty and file_status.st_size == 0 + ): + qualifier = "nonempty " if require_nonempty else "" + raise RuntimeError( + f"Validation output must be a {qualifier}regular file: {path}" + ) + if path_after != resolved_path or _file_state(file_status) != _file_state( + path_status + ): + raise RuntimeError(f"Validation output changed while inspected: {path}") + return _FileIdentity( + path=path, + resolved_path=resolved_path, + device=file_status.st_dev, + inode=file_status.st_ino, + size=file_status.st_size, + modified_ns=file_status.st_mtime_ns, + changed_ns=file_status.st_ctime_ns, + ) + + +def _revalidate_regular_file( + identity: _FileIdentity, allowed_root: Path, *, require_nonempty: bool +) -> None: + if ( + _capture_regular_file( + identity.path, + allowed_root, + require_nonempty=require_nonempty, + ) + != identity + ): + raise RuntimeError(f"Validation output changed after creation: {identity.path}") + + +def _scan_stable_log( + path: Path, allowed_root: Path, handle_line, byte_budget: int +) -> tuple[_FileIdentity, int]: + try: + allowed_root = allowed_root.resolve(strict=True) + except OSError as exc: + raise RuntimeError( + f"Validation log root does not exist: {allowed_root}" + ) from exc + if _is_link_or_junction(path): + raise RuntimeError(f"Validation log must not be a link: {path}") + try: + resolved_path = path.resolve(strict=True) + resolved_path.relative_to(allowed_root) + with _open_binary_no_follow(path) as log_file: + status_before = os.fstat(log_file.fileno()) + if not stat.S_ISREG(status_before.st_mode): + raise RuntimeError(f"Validation log must be a regular file: {path}") + consumed = 0 + while True: + raw_line = log_file.readline(MAX_ATTESTATION_LINE_BYTES + 1) + if not raw_line: + break + consumed += len(raw_line) + if consumed > byte_budget: + raise RuntimeError( + "Accelerator attestation log exceeds byte budget" + ) + if len(raw_line) > MAX_ATTESTATION_LINE_BYTES or ( + len(raw_line) == MAX_ATTESTATION_LINE_BYTES + and not raw_line.endswith(b"\n") + ): + raise RuntimeError("Accelerator attestation log line is too long") + handle_line(raw_line.decode("utf-8", errors="replace")) + status_after = os.fstat(log_file.fileno()) + path_after = path.resolve(strict=True) + path_after.relative_to(allowed_root) + path_status = path_after.stat() + except RuntimeError: + raise + except (OSError, ValueError) as exc: + raise RuntimeError(f"Could not safely scan validation log: {path}") from exc + if _is_link_or_junction(path): + raise RuntimeError(f"Validation log must not be a link: {path}") + if ( + path_after != resolved_path + or _file_state(status_before) != _file_state(status_after) + or _file_state(status_after) != _file_state(path_status) + ): + raise RuntimeError(f"Validation log changed while scanned: {path}") + identity = _FileIdentity( + path=path, + resolved_path=resolved_path, + device=status_after.st_dev, + inode=status_after.st_ino, + size=status_after.st_size, + modified_ns=status_after.st_mtime_ns, + changed_ns=status_after.st_ctime_ns, + ) + return identity, consumed + + +def require_accelerator_attestation( + log_paths: list[Path], + allowed_root: Path, + phase: str, + expected_artifacts: dict[str, VerifiedValidationArtifact], + expected_backend: str, + *, + selected_models: list[str] | None = None, +) -> list[_FileIdentity]: + cpu_backend = expected_backend == "cpu" + backend_family = expected_backend.split("-", maxsplit=1)[0] + expected_device_prefix = ACCELERATOR_DEVICE_PREFIXES.get(backend_family) + if expected_backend not in ATTESTED_BACKENDS: + raise RuntimeError( + f"{phase} accelerator attestation has unsupported backend: " + f"{expected_backend}" + ) + expected_models = {} + for model_id, artifact in expected_artifacts.items(): + canonical_model_id = model_id.removeprefix("builtin.") + if ( + not isinstance(artifact, VerifiedValidationArtifact) + or artifact.model_id.removeprefix("builtin.") != canonical_model_id + ): + raise RuntimeError( + f"{phase} accelerator attestation has an invalid artifact binding" + ) + expected_models[canonical_model_id] = artifact + canonical_selected_models = [ + model.removeprefix("builtin.") + for model in ( + selected_models if selected_models is not None else list(expected_models) + ) + if isinstance(model, str) and model + ] + if ( + not canonical_selected_models + or len(canonical_selected_models) + != len(selected_models if selected_models is not None else expected_models) + or len(canonical_selected_models) != len(set(canonical_selected_models)) + or not set(expected_models).issubset(canonical_selected_models) + ): + raise RuntimeError( + f"{phase} accelerator attestation has an invalid selected-model binding" + ) + selected_model_ids = set(canonical_selected_models) + attested_models = set() + scanned_identities = [] + total_bytes = 0 + for log_path in log_paths: + current_model = None + current_backend_seen = False + current_identity_seen = False + current_device_seen = False + current_cpu_buffer_seen = False + current_offload = None + + def handle_line(line: str) -> None: + nonlocal current_model, current_identity_seen + nonlocal current_backend_seen, current_device_seen + nonlocal current_cpu_buffer_seen, current_offload + backend_marker = BACKEND_MARKER.search(line) + offload = OFFLOAD_MARKER.search(line) + device = DEVICE_MARKER.search(line) + cpu_buffer = CPU_BUFFER_MARKER.search(line) + if cpu_backend and (DEVICE_PREFIX in line or OFFLOAD_PREFIX in line): + raise RuntimeError( + f"{phase} CPU attestation contains accelerator markers" + ) + if OFFLOAD_PREFIX in line: + if line.count(OFFLOAD_PREFIX) != 1 or offload is None: + raise RuntimeError( + f"{phase} accelerator attestation has malformed offload proof" + ) + offloaded_layers, total_layers = map(int, offload.groups()) + if not 0 < offloaded_layers <= total_layers: + raise RuntimeError( + f"{phase} accelerator attestation has invalid layer offload proof" + ) + + loading = LOADING_MODEL_MARKER.search(line) + if loading is not None: + if current_model is not None: + raise RuntimeError( + f"{phase} accelerator attestation has an incomplete model load" + ) + current_model = loading.group("model").removeprefix("builtin.") + if current_model not in selected_model_ids: + raise RuntimeError( + f"{phase} accelerator attestation has an unexpected model " + f"load: {current_model}" + ) + current_backend_seen = False + current_identity_seen = False + current_device_seen = False + current_cpu_buffer_seen = False + current_offload = None + return + if current_model not in expected_models: + if current_model is None and BACKEND_PREFIX in line: + raise RuntimeError( + f"{phase} accelerator attestation has an unbound backend proof" + ) + if current_model is not None and LOADED_MODEL_MARKER.search(line): + current_model = None + return + if BACKEND_PREFIX in line: + if line.count(BACKEND_PREFIX) != 1 or backend_marker is None: + raise RuntimeError( + f"{phase} accelerator attestation has malformed backend proof" + ) + if current_backend_seen: + raise RuntimeError( + f"{phase} accelerator attestation has duplicate backend proof" + ) + if ( + current_identity_seen + or current_device_seen + or current_cpu_buffer_seen + or current_offload is not None + ): + raise RuntimeError( + f"{phase} accelerator attestation has late backend proof" + ) + actual_backend = backend_marker.group("backend") + if actual_backend != expected_backend: + raise RuntimeError( + f"{phase} accelerator attestation has backend " + f"'{actual_backend}', expected '{expected_backend}'" + ) + current_backend_seen = True + return + gguf = GGUF_MARKER.search(line) + if gguf is not None: + if not current_backend_seen: + raise RuntimeError( + f"{phase} accelerator attestation is missing backend proof" + ) + loaded_path = Path(gguf.group("path")) + try: + if not loaded_path.is_absolute(): + raise ValueError("artifact path is not absolute") + resolved_loaded_path = loaded_path.resolve(strict=True) + except (OSError, ValueError) as exc: + raise RuntimeError( + f"{phase} accelerator attestation has the wrong artifact" + ) from exc + expected_path = expected_models[current_model].resolved_path + if os.path.normcase(os.path.normpath(str(resolved_loaded_path))) != ( + os.path.normcase(os.path.normpath(str(expected_path))) + ): + raise RuntimeError( + f"{phase} accelerator attestation has the wrong artifact" + ) + current_identity_seen = True + if ( + cpu_backend + and cpu_buffer is not None + and current_identity_seen + and float(cpu_buffer.group("size")) > 0 + ): + current_cpu_buffer_seen = True + if current_offload is None: + if device is not None and current_identity_seen: + device_name = device.group("device") + if ( + re.fullmatch( + rf"{re.escape(expected_device_prefix)}[0-9]+", device_name + ) + is None + ): + raise RuntimeError( + f"{phase} accelerator attestation has device " + f"'{device_name}' for the wrong backend" + ) + current_device_seen = True + if offload is not None: + if not current_identity_seen or not current_device_seen: + raise RuntimeError( + f"{phase} accelerator attestation has unbound device/offload proof" + ) + current_offload = tuple(map(int, offload.groups())) + if LOADED_MODEL_MARKER.search(line) is None: + return + if not current_identity_seen: + raise RuntimeError( + f"{phase} accelerator attestation is missing artifact identity" + ) + if not current_backend_seen: + raise RuntimeError( + f"{phase} accelerator attestation is missing backend proof" + ) + if cpu_backend and not current_cpu_buffer_seen: + raise RuntimeError( + f"{phase} CPU attestation is missing a CPU model buffer" + ) + if not cpu_backend and current_offload is None: + raise RuntimeError( + f"{phase} accelerator attestation is missing layer offload proof" + ) + attested_models.add(current_model) + current_model = None + current_backend_seen = False + current_identity_seen = False + current_device_seen = False + current_cpu_buffer_seen = False + current_offload = None + + identity, consumed = _scan_stable_log( + log_path, + allowed_root, + handle_line, + MAX_ATTESTATION_LOG_BYTES - total_bytes, + ) + scanned_identities.append(identity) + total_bytes += consumed + if current_model is not None: + raise RuntimeError( + f"{phase} accelerator attestation has an incomplete model load" + ) + missing_models = sorted(set(expected_models) - attested_models) + if missing_models: + raise RuntimeError( + f"{phase} accelerator attestation is missing models: " + + ", ".join(missing_models) + ) + return scanned_identities + + +def resolve_attestation_backend(backend: str, channel: str) -> str: + if backend == "rocm": + if channel not in {"stable", "nightly"}: + raise RuntimeError( + "ROCm validation requires an exact stable or nightly channel" + ) + return f"rocm-{channel}" + if channel: + raise RuntimeError( + f"{backend} validation must not specify a ROCm release channel" + ) + if backend not in ATTESTED_BACKENDS: + raise RuntimeError(f"Unsupported validation backend: {backend}") + return backend + + +def _emit_workflow_outputs(outputs: dict[str, Path]) -> None: + github_output = os.environ.get("GITHUB_OUTPUT") + if not github_output: + return + with Path(github_output).open("a", encoding="utf-8") as output_file: + for name, path in outputs.items(): + output_file.write(f"{name}={path}\n") + + +def _cache_processes(cache_directory: Path, excluded_pid: int) -> _CacheProcessScan: + try: + import psutil + except ModuleNotFoundError: + return _CacheProcessScan([], False) + + cache_prefix = os.path.normcase(str(cache_directory.resolve()) + os.sep) + matches = [] + try: + for process in psutil.process_iter(("pid", "exe")): + if process.pid == excluded_pid: + continue + try: + executable = process.info.get("exe") + if executable and os.path.normcase( + str(Path(executable).resolve()) + ).startswith(cache_prefix): + matches.append(process) + except (OSError, psutil.Error): + continue + except (OSError, psutil.Error): + return _CacheProcessScan(matches, False) + return _CacheProcessScan(matches, True) + + +def _stop_processes(processes: list[object]) -> bool: + import psutil + + alive = list(processes) + for process in processes: + try: + process.terminate() + except psutil.Error: + pass + alive = _wait_for_psutil_processes(alive, timeout=5) + for process in alive: + try: + process.kill() + except psutil.Error: + pass + alive = _wait_for_psutil_processes(alive, timeout=5) + return not alive + + +def _force_kill_processes(processes: list[object], deadline: float) -> bool: + import psutil + + alive = list(processes) + for process in alive: + try: + process.kill() + except psutil.Error: + pass + reap_timeout = min( + PROCESS_WAIT_POLL_SECONDS, + max(0.0, deadline - time.monotonic()), + ) + try: + _, alive = psutil.wait_procs(alive, timeout=reap_timeout) + except (OSError, psutil.Error): + return False + return not alive + + +def _wait_for_psutil_processes( + processes: list[object], + timeout: float, +) -> list[object]: + import psutil + + deadline = time.monotonic() + timeout + alive = list(processes) + while alive: + _raise_if_cancelled() + remaining = deadline - time.monotonic() + if remaining <= 0: + break + _, alive = psutil.wait_procs( + alive, + timeout=min(PROCESS_WAIT_POLL_SECONDS, remaining), + ) + _raise_if_cancelled() + return alive + + +def _open_local_url(request: urllib.request.Request, timeout: int): + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + return opener.open(request, timeout=timeout) + + +def _open_local_url_with_cancellation( + request: urllib.request.Request, + timeout: int, + containment=None, +) -> None: + outcome: list[BaseException | None] = [] + + def open_url() -> None: + try: + with _open_local_url(request, timeout=timeout): + pass + except BaseException as exc: # pylint: disable=broad-exception-caught + outcome.append(exc) + else: + outcome.append(None) + + worker = threading.Thread(target=open_url, daemon=True) + worker.start() + deadline = time.monotonic() + timeout + while worker.is_alive(): + if containment is not None: + _refresh_process_containment(containment) + _raise_if_cancelled() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("Timed out requesting graceful server shutdown") + worker.join(min(PROCESS_WAIT_POLL_SECONDS, remaining)) + _raise_if_cancelled() + if not outcome: + raise RuntimeError("Graceful server shutdown request returned no result") + if outcome[0] is not None: + raise outcome[0] + + +def _wait_for_process_with_cancellation( + process: subprocess.Popen, + timeout: float, + containment=None, +) -> int: + deadline = time.monotonic() + timeout + while True: + if containment is not None: + _refresh_process_containment(containment) + _raise_if_cancelled() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired( + getattr(process, "args", "process"), timeout + ) + try: + exit_code = process.wait(timeout=min(PROCESS_WAIT_POLL_SECONDS, remaining)) + except subprocess.TimeoutExpired: + continue + _raise_if_cancelled() + return exit_code + + +def _shutdown_server( + process: subprocess.Popen, + port: int, + cache_directory: Path, + containment=None, +) -> bool: + orphan_processes = [] + try: + running_on_entry = process.poll() is None + clean = running_on_entry + if running_on_entry: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/internal/shutdown", + data=b"", + method="POST", + ) + try: + _open_local_url_with_cancellation( + request, + timeout=10, + containment=containment, + ) + if ( + _wait_for_process_with_cancellation( + process, + timeout=10, + containment=containment, + ) + != 0 + ): + clean = False + except (OSError, subprocess.TimeoutExpired): + clean = False + if process.poll() is None: + process.terminate() + try: + _wait_for_process_with_cancellation( + process, + timeout=5, + containment=containment, + ) + except subprocess.TimeoutExpired: + process.kill() + _wait_for_process_with_cancellation( + process, + timeout=5, + containment=containment, + ) + clean = False + + _raise_if_cancelled() + if containment is not None and not _close_process_containment(containment): + clean = False + _raise_if_cancelled() + orphan_scan = _cache_processes(cache_directory, process.pid) + orphan_processes = list(orphan_scan.processes) + if not orphan_scan.complete: + clean = False + if orphan_processes: + clean = False + _stop_processes(orphan_processes) + _raise_if_cancelled() + return clean + except _ValidationCancelled as exc: + cleanup_deadline = exc.cleanup_deadline + cancellation_orphans = list(orphan_processes) + cancellation_scan = _cache_processes(cache_directory, process.pid) + retained_orphan_pids = { + getattr(orphan, "pid", None) for orphan in cancellation_orphans + } + for orphan in cancellation_scan.processes: + if getattr(orphan, "pid", None) not in retained_orphan_pids: + cancellation_orphans.append(orphan) + if cancellation_orphans: + _force_kill_processes(cancellation_orphans, cleanup_deadline) + if containment is not None: + _abort_process_containment(containment, cleanup_deadline) + _reap_direct_process(process, cleanup_deadline) + raise + + +def _reap_direct_process(process: subprocess.Popen, deadline: float) -> None: + try: + if process.poll() is None: + process.kill() + except OSError: + pass + try: + process.wait(timeout=max(0.0, deadline - time.monotonic())) + except (OSError, subprocess.TimeoutExpired): + pass + + +def _wait_for_contained_process( + process: subprocess.Popen, + timeout_seconds: float, + containment, +) -> int: + if timeout_seconds <= 0: + raise ValueError("Validation phase timeout must be positive") + deadline = time.monotonic() + timeout_seconds + while True: + _refresh_process_containment(containment) + _raise_if_cancelled() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError( + f"Validation command timed out after {timeout_seconds:g} seconds" + ) + try: + exit_code = process.wait(timeout=min(PROCESS_WAIT_POLL_SECONDS, remaining)) + except subprocess.TimeoutExpired: + continue + _raise_if_cancelled() + return exit_code + + +def _run_contained_command( + command: list[str], + *, + env: dict[str, str], + containment, + timeout_seconds: float, +) -> int: + process = None + try: + process = _start_contained_process( + containment, + command, + env=env, + ) + return _wait_for_contained_process(process, timeout_seconds, containment) + except BaseException as exc: + cleanup_deadline = ( + exc.cleanup_deadline + if isinstance(exc, _ValidationCancelled) + else time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS + ) + _abort_process_containment(containment, cleanup_deadline) + if process is not None: + _reap_direct_process(process, cleanup_deadline) + if isinstance(exc, _ValidationCancelled): + raise + _raise_if_cancelled() + raise + + +def build_server_environment( + hf_cache_directory: Path, + inherited: dict[str, str] | None = None, + *, + private_temp_directory: Path, + candidate_defaults_path: Path | None = None, + allow_download_credentials: bool = False, +) -> dict[str, str]: + inherited_environment = os.environ if inherited is None else inherited + environment = {} + for key, value in inherited_environment.items(): + normalized_key = key.upper() + if normalized_key in INHERITED_RUNTIME_ENVIRONMENT_KEYS: + environment[normalized_key] = value + if allow_download_credentials and inherited_environment.get("HF_TOKEN"): + environment["HF_TOKEN"] = inherited_environment["HF_TOKEN"] + environment["HF_HUB_CACHE"] = str(hf_cache_directory.resolve()) + environment["LEMONADE_CI_MODE"] = "True" + environment["LLAMA_ARG_LOG_VERBOSITY"] = "4" + environment["NO_PROXY"] = "127.0.0.1" + environment["no_proxy"] = "127.0.0.1" + resolved_private_temp = str(private_temp_directory.resolve()) + environment["TMPDIR"] = resolved_private_temp + environment["TMP"] = resolved_private_temp + environment["TEMP"] = resolved_private_temp + environment["HOME"] = resolved_private_temp + environment["USERPROFILE"] = resolved_private_temp + environment["APPDATA"] = resolved_private_temp + environment["LOCALAPPDATA"] = resolved_private_temp + if candidate_defaults_path is not None: + environment["LEMONADE_DEFAULTS_PATH"] = str(candidate_defaults_path.resolve()) + return environment + + +def resolve_candidate_defaults(lemond: Path) -> Path: + resource_directory = ( + lemond.parent.parent / "resources" + if lemond.suffix.lower() == ".exe" + else lemond.parent / "resources" + ) + defaults_path = resource_directory / "defaults.json" + if _is_link_or_junction(resource_directory): + raise RuntimeError( + f"Candidate defaults directory is not a regular build resource: " + f"{resource_directory}" + ) + try: + resolved_resource_directory = resource_directory.resolve(strict=True) + resolved_defaults = defaults_path.resolve(strict=True) + resolved_defaults.relative_to(resolved_resource_directory) + except (OSError, ValueError) as exc: + raise RuntimeError( + f"Candidate defaults file was not found in build resources: {defaults_path}" + ) from exc + if _is_link_or_junction(defaults_path) or not resolved_defaults.is_file(): + raise RuntimeError( + f"Candidate defaults file is not a regular build resource: {defaults_path}" + ) + return resolved_defaults + + +def _run_validation_process( + *, + lemond: Path, + cache_directory: Path, + config_directory: Path, + command: list[str], + environment: dict[str, str], + port: int, + pid_path: Path, + stdout_path: Path, + stderr_path: Path, + backend: str | None = None, + target: str | None = None, + allow_darwin_github_hosted_ephemeral_runner: bool = False, +) -> tuple[int, bool]: + containment = None + cleanup_succeeded = False + try: + with stdout_path.open("x", encoding="utf-8") as stdout_log, stderr_path.open( + "x", encoding="utf-8" + ) as stderr_log: + containment = _create_process_containment( + allow_darwin_github_hosted_ephemeral_runner=( + allow_darwin_github_hosted_ephemeral_runner + ), + backend=backend, + target=target, + ) + process = _start_contained_process( + containment, + [ + str(lemond), + str(cache_directory.resolve()), + str(config_directory.resolve()), + "--port", + str(port), + "--host", + "127.0.0.1", + ], + stdout=stdout_log, + stderr=stderr_log, + env=environment, + ) + try: + with pid_path.open("x", encoding="ascii") as pid_file: + pid_file.write(f"{process.pid}\n") + print(f"Started lemond PID {process.pid}", flush=True) + time.sleep(0.5) + if process.poll() is not None: + raise RuntimeError( + f"lemond exited before validation started; see {stderr_path}" + ) + exit_code = _run_contained_command( + command, + env=environment, + containment=containment, + timeout_seconds=VALIDATION_PHASE_TIMEOUT_SECONDS, + ) + finally: + cleanup_succeeded = _shutdown_server( + process, + port, + cache_directory, + containment, + ) + finally: + if containment is not None and not _close_process_containment(containment): + cleanup_succeeded = False + _raise_if_cancelled() + if cleanup_succeeded: + pid_path.unlink(missing_ok=True) + return exit_code, cleanup_succeeded + + +def require_validation_result_evidence( + identity: _FileIdentity, + allowed_root: Path, + expected_models: list[str], + *, + capability_profile: str = "", + capability_models: list[str] | None = None, +) -> None: + try: + load_and_validate_result_file( + identity.resolved_path, + expected_models, + capability_profile=capability_profile, + capability_models=capability_models, + ) + except ValidationEvidenceError as exc: + raise RuntimeError(str(exc)) from exc + _revalidate_regular_file(identity, allowed_root, require_nonempty=True) + + +def run_validation(args: argparse.Namespace) -> int: + lemond = args.lemond.resolve() + python_executable = args.python.resolve() + allow_huggingface_download_credentials = bool( + getattr(args, "allow_huggingface_download_credentials", False) + ) + allow_darwin_github_hosted_ephemeral_runner = bool( + getattr(args, "allow_darwin_github_hosted_ephemeral_runner", False) + ) + if not lemond.is_file(): + raise RuntimeError(f"lemond executable not found: {lemond}") + if not python_executable.is_file(): + raise RuntimeError(f"Python executable not found: {python_executable}") + attestation_backend = resolve_attestation_backend(args.backend, args.channel) + + command = build_validation_command( + python_executable=python_executable, + backend=args.backend, + channel=args.channel, + target=args.target, + models_csv=args.models, + lite=args.lite, + capability_profile=args.capability_profile, + capability_models_csv=args.capability_models, + port=args.port, + ) + command.append("--skip-install") + prepare_command = build_prepare_validation_command( + python_executable=python_executable, + backend=args.backend, + channel=args.channel, + target=args.target, + models_csv=args.models, + lite=args.lite, + capability_profile=args.capability_profile, + capability_models_csv=args.capability_models, + port=args.port, + ) + selected_models = _csv_values(args.models) + capability_models = _csv_values(args.capability_models) + restart_models = capability_models or selected_models + if not selected_models: + raise ValueError("Validation requires explicit selected models") + if not restart_models: + raise ValueError( + "A capability model or explicit model is required for restart validation" + ) + artifact_locks = load_model_catalog(VALIDATION_ARTIFACT_LOCKS) + locked_restart_models = [ + model + for model in restart_models + if model.removeprefix("builtin.") in artifact_locks + ] + restart_model_value = (locked_restart_models or restart_models)[0] + restart_command = build_restart_validation_command( + python_executable=python_executable, + backend=args.backend, + channel=args.channel, + target=args.target, + model=restart_model_value, + port=args.port, + ) + _require_idle_runner() + _require_unused_port(args.port) + candidate_defaults_path = resolve_candidate_defaults(lemond) + + temporary_root_value = os.environ.get("RUNNER_TEMP") + if not temporary_root_value: + raise RuntimeError("RUNNER_TEMP is required for validation cache isolation") + validation_cache = create_validation_cache(Path(temporary_root_value), args.target) + cache_directory = validation_cache.cache_directory.resolved_path + config_directory = validation_cache.config_directory.resolved_path + hf_cache_directory = validation_cache.hf_cache_directory.resolved_path + private_temp_directory = validation_cache.private_temp_directory.resolved_path + results_directory = validation_cache.results_directory.resolved_path + logs_directory = validation_cache.logs_directory.resolved_path + validation_result_path = ( + results_directory / f"llamacpp_validation_{args.target}.json" + ) + restart_result_path = ( + results_directory / f"llamacpp_restart_validation_{args.target}.json" + ) + command[command.index("--output") + 1] = str(validation_result_path) + prepare_command[prepare_command.index("--output") + 1] = str(validation_result_path) + restart_command[restart_command.index("--output") + 1] = str(restart_result_path) + prepare_environment = build_server_environment( + hf_cache_directory, + private_temp_directory=private_temp_directory, + candidate_defaults_path=candidate_defaults_path, + allow_download_credentials=allow_huggingface_download_credentials, + ) + environment = build_server_environment( + hf_cache_directory, + private_temp_directory=private_temp_directory, + candidate_defaults_path=candidate_defaults_path, + ) + pid_path = validation_cache.validation_root.resolved_path / "lemond.pid" + prepare_stdout_path = logs_directory / "lemond.prepare.stdout.log" + prepare_stderr_path = logs_directory / "lemond.prepare.stderr.log" + validation_stdout_path = logs_directory / "lemond.stdout.log" + validation_stderr_path = logs_directory / "lemond.stderr.log" + restart_stdout_path = logs_directory / "lemond.restart.stdout.log" + restart_stderr_path = logs_directory / "lemond.restart.stderr.log" + + revalidate_validation_cache(validation_cache) + validation_exit_code, cleanup_succeeded = _run_validation_process( + lemond=lemond, + cache_directory=cache_directory, + config_directory=config_directory, + command=prepare_command, + environment=prepare_environment, + port=args.port, + pid_path=pid_path, + stdout_path=prepare_stdout_path, + stderr_path=prepare_stderr_path, + backend=args.backend, + target=args.target, + allow_darwin_github_hosted_ephemeral_runner=( + allow_darwin_github_hosted_ephemeral_runner + ), + ) + revalidate_validation_cache(validation_cache) + if validation_exit_code: + return validation_exit_code + if not cleanup_succeeded: + print("Prepare process cleanup did not complete", file=sys.stderr) + return 1 + + def verify_selected_artifacts() -> dict[str, VerifiedValidationArtifact]: + revalidate_validation_cache(validation_cache) + verified = verify_validation_artifacts( + hf_cache_directory, + VALIDATION_MODEL_CATALOG, + VALIDATION_ARTIFACT_LOCKS, + selected_models, + ) + if verified: + print( + "Verified immutable validation artifacts: " + ", ".join(verified), + flush=True, + ) + return verified + + verified_artifacts = verify_selected_artifacts() + require_backend_proof = bool(verified_artifacts) + attested_log_identities = [] + _require_unused_port(args.port) + revalidate_validation_cache(validation_cache) + validation_exit_code, cleanup_succeeded = _run_validation_process( + lemond=lemond, + cache_directory=cache_directory, + config_directory=config_directory, + command=command, + environment=environment, + port=args.port, + pid_path=pid_path, + stdout_path=validation_stdout_path, + stderr_path=validation_stderr_path, + backend=args.backend, + target=args.target, + allow_darwin_github_hosted_ephemeral_runner=( + allow_darwin_github_hosted_ephemeral_runner + ), + ) + revalidate_validation_cache(validation_cache) + if validation_exit_code: + return validation_exit_code + if not cleanup_succeeded: + print("Validation process cleanup did not complete", file=sys.stderr) + return 1 + validation_result = _capture_regular_file( + validation_result_path, + results_directory, + require_nonempty=True, + ) + require_validation_result_evidence( + validation_result, + results_directory, + selected_models, + capability_profile=args.capability_profile, + capability_models=capability_models, + ) + verified_artifacts = verify_selected_artifacts() + if require_backend_proof: + attested_log_identities.extend( + require_accelerator_attestation( + [validation_stdout_path, validation_stderr_path], + logs_directory, + "validation", + verified_artifacts, + attestation_backend, + selected_models=selected_models, + ) + ) + _require_unused_port(args.port) + revalidate_validation_cache(validation_cache) + restart_exit_code, restart_cleanup_succeeded = _run_validation_process( + lemond=lemond, + cache_directory=cache_directory, + config_directory=config_directory, + command=restart_command, + environment=environment, + port=args.port, + pid_path=pid_path, + stdout_path=restart_stdout_path, + stderr_path=restart_stderr_path, + backend=args.backend, + target=args.target, + allow_darwin_github_hosted_ephemeral_runner=( + allow_darwin_github_hosted_ephemeral_runner + ), + ) + revalidate_validation_cache(validation_cache) + if restart_exit_code: + return restart_exit_code + if not restart_cleanup_succeeded: + print("Restart validation process cleanup did not complete", file=sys.stderr) + return 1 + restart_result = _capture_regular_file( + restart_result_path, + results_directory, + require_nonempty=True, + ) + require_validation_result_evidence( + restart_result, + results_directory, + [restart_model_value], + ) + verified_artifacts = verify_selected_artifacts() + restart_model = restart_model_value.removeprefix("builtin.") + if require_backend_proof and restart_model in verified_artifacts: + attested_log_identities.extend( + require_accelerator_attestation( + [restart_stdout_path, restart_stderr_path], + logs_directory, + "restart validation", + {restart_model: verified_artifacts[restart_model]}, + attestation_backend, + selected_models=[restart_model_value], + ) + ) + revalidate_validation_cache(validation_cache) + _revalidate_regular_file( + validation_result, + results_directory, + require_nonempty=True, + ) + _revalidate_regular_file( + restart_result, + results_directory, + require_nonempty=True, + ) + log_paths = ( + prepare_stdout_path, + prepare_stderr_path, + validation_stdout_path, + validation_stderr_path, + restart_stdout_path, + restart_stderr_path, + ) + log_identities_by_path = { + identity.path: identity for identity in attested_log_identities + } + for path in log_paths: + if path not in log_identities_by_path: + log_identities_by_path[path] = _capture_regular_file( + path, + logs_directory, + require_nonempty=False, + ) + log_identities = list(log_identities_by_path.values()) + for identity in log_identities: + _revalidate_regular_file(identity, logs_directory, require_nonempty=False) + output_log_paths = { + identity.path: identity.resolved_path for identity in log_identities + } + _emit_workflow_outputs( + { + "validation_result": validation_result.resolved_path, + "restart_result": restart_result.resolved_path, + "prepare_stdout_log": output_log_paths[prepare_stdout_path], + "prepare_stderr_log": output_log_paths[prepare_stderr_path], + "validation_stdout_log": output_log_paths[validation_stdout_path], + "validation_stderr_log": output_log_paths[validation_stderr_path], + "restart_stdout_log": output_log_paths[restart_stdout_path], + "restart_stderr_log": output_log_paths[restart_stderr_path], + } + ) + return 0 + + +def _bool_argument(value: str) -> bool: + normalized = value.strip().lower() + if normalized == "true": + return True + if normalized in {"", "false"}: + return False + raise argparse.ArgumentTypeError("expected true or false") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--lemond", required=True, type=Path) + parser.add_argument("--python", required=True, type=Path) + parser.add_argument( + "--backend", + default=os.environ.get("LLAMACPP_BACKEND"), + choices=("cpu", "cuda", "metal", "rocm", "vulkan"), + ) + parser.add_argument( + "--channel", + default=os.environ.get("LLAMACPP_CHANNEL", ""), + choices=("", "stable", "nightly"), + ) + parser.add_argument("--target", default=os.environ.get("TARGET")) + parser.add_argument("--models", default=os.environ.get("LLAMACPP_TEST_MODELS", "")) + parser.add_argument( + "--lite", + default=_bool_argument(os.environ.get("LITE_MODE", "false")), + type=_bool_argument, + ) + parser.add_argument( + "--capability-profile", default=os.environ.get("CAPABILITY_PROFILE", "") + ) + parser.add_argument( + "--capability-models", default=os.environ.get("CAPABILITY_MODELS", "") + ) + parser.add_argument( + "--allow-huggingface-download-credentials", + action="store_true", + help="allow the prepare phase to inherit the canonical HF_TOKEN value", + ) + parser.add_argument( + "--allow-darwin-github-hosted-ephemeral-runner", + action="store_true", + help=( + "accept github-hosted macos-latest VM disposal as the Darwin " + "descendant-containment boundary" + ), + ) + parser.add_argument("--port", default=13305, type=int) + args = parser.parse_args() + if not args.backend: + parser.error("--backend or LLAMACPP_BACKEND is required") + if not args.target: + parser.error("--target or TARGET is required") + try: + with _CancellationSignalGuard(): + exit_code = run_validation(args) + _raise_if_cancelled() + except _ValidationCancelled as exc: + signal_name = signal.Signals(exc.signum).name + parser.exit( + 128 + exc.signum, + f"ERROR: validation cancelled by {signal_name}\n", + ) + except (OSError, ValueError, RuntimeError) as exc: + parser.exit(1, f"ERROR: {exc}\n") + raise SystemExit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/cpp_server_build_test_release.yml b/.github/workflows/cpp_server_build_test_release.yml index a4f49e7a10..47b6cdfbda 100644 --- a/.github/workflows/cpp_server_build_test_release.yml +++ b/.github/workflows/cpp_server_build_test_release.yml @@ -127,17 +127,17 @@ jobs: if ($failures.Count -gt 0) { exit 1 } Write-Host "Build and packaging successful!" -ForegroundColor Green - - name: Run MCP client unit test on Windows + - name: Run Windows process unit tests shell: PowerShell run: | $ErrorActionPreference = "Stop" # Exercise the Windows-only process, quoting, environment-block, and # pipe/handle paths that Linux and macOS cannot cover. - cmake --build build --config Release --target test_mcp_client_config + cmake --build build --config Release --target test_mcp_client_config test_process_manager_windows if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - ctest --test-dir build -C Release --output-on-failure -R "^mcp_client_config$" + ctest --test-dir build -C Release --output-on-failure -R "^(mcp_client_config|ProcessManagerWindowsTest)$" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Upload Lemonade Server Installers diff --git a/.github/workflows/docs_and_style.yml b/.github/workflows/docs_and_style.yml index e53df44b48..ec0ca82f5a 100644 --- a/.github/workflows/docs_and_style.yml +++ b/.github/workflows/docs_and_style.yml @@ -75,25 +75,30 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.x' - - run: pip install pre-commit + - run: pip install pre-commit psutil requests openai pyyaml + - name: Verify focused worker imports + run: python -c "import test.utils.server_base" - name: Run focused Python unit tests run: >- python -m unittest test.test_comment_slop - test.test_debian_version test.test_llamacpp_capabilities + test.test_llamacpp_check_reporting test.test_llamacpp_release_manifest test.test_llamacpp_release_assets + test.test_llamacpp_validation_artifacts test.test_llamacpp_validation_evidence + test.test_llamacpp_pr_file_classification + test.test_process_inheritance_contract test.test_publish_llamacpp_update + test.test_run_llamacpp_validation + test.test_validation_model_catalog + test.test_validate_llamacpp_action_pins test.test_validate_llamacpp_selection + test.test_validate_llamacpp_trusted_execution test.test_validate_llamacpp_workflow_plan test.test_verify_llamacpp_pin_manifest -v - - name: Run PowerShell validation argument tests - shell: pwsh - run: ./test/test_llamacpp_validation_args.ps1 - - name: Run pre-commit on the changed files # Scoped to what the change touches; running repo-wide would fail on drift that # predates it and is not the contributor's to fix. diff --git a/.github/workflows/validate_llamacpp_core.yml b/.github/workflows/validate_llamacpp_core.yml index 4c906e2c05..3a39b0a9e1 100644 --- a/.github/workflows/validate_llamacpp_core.yml +++ b/.github/workflows/validate_llamacpp_core.yml @@ -3,12 +3,16 @@ name: Validate llama.cpp on: workflow_call: inputs: + authorization_epoch: + description: 'Trusted authorization issue-event ID; zero outside pull requests' + type: string + default: '0' lite: - description: 'Run in lite mode (smallest model only)' + description: 'Run only the K2-Horizon 0.9B smoke model' type: boolean default: false models: - description: 'Optional comma-separated model IDs (empty uses hot models)' + description: 'Additional comma-separated IDs; empty uses hot models + K2 0.9B' type: string required: false default: '' @@ -16,6 +20,8 @@ on: HUGGINGFACE_ACCESS_TOKEN: required: false outputs: + check_run_id: + value: ${{ jobs.validate-invocation.outputs.check_run_id }} promotion_eligible: value: ${{ jobs.plan.outputs.promotion_eligible }} validation_matrix: @@ -58,6 +64,7 @@ permissions: contents: read env: + DEPENDABOT_PR: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.user.login == 'dependabot[bot]' && github.event.pull_request.user.id == 49699333 && github.event.pull_request.user.type == 'Bot' }} LEMONADE_CI_MODE: "True" PYTHONIOENCODING: utf-8 @@ -65,6 +72,11 @@ jobs: validate-invocation: name: Validate invocation runs-on: ubuntu-latest + permissions: + checks: write + contents: read + outputs: + check_run_id: ${{ steps.start-check.outputs.check_run_id }} steps: - name: Reject untrusted caller events env: @@ -81,6 +93,166 @@ jobs: ;; esac + - name: Start authorized merge check + id: start-check + if: (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && env.DEPENDABOT_PR != 'true' + env: + AUTHORIZATION_EPOCH: ${{ inputs.authorization_epoch }} + CHECK_NAME: llama.cpp authorized validation v2 + EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MERGE_SHA: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.event.pull_request.merge_commit_sha }} + RUN_ATTEMPT: ${{ github.run_attempt }} + RUN_ID: ${{ github.run_id }} + RUN_NUMBER: ${{ github.run_number }} + TARGET_KEY: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}', github.event.pull_request.number) }} + run: | + set -euo pipefail + EXTERNAL_PREFIX="lemonade-llamacpp-required-${TARGET_KEY}-" + EXTERNAL_ID="${EXTERNAL_PREFIX}${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}" + if ! [[ "$AUTHORIZATION_EPOCH" =~ ^[0-9]+$ ]] || \ + ! [[ "$MERGE_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::Protected validation requires an exact candidate commit." + exit 1 + fi + if [ "$EVENT_NAME" = "pull_request_target" ] && \ + ! [[ "$AUTHORIZATION_EPOCH" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Pull-request validation requires a live authorization epoch." + exit 1 + fi + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" \ + -f filter=all \ + -F per_page=100) + + IFS=$'\t' read -r newest_number newest_attempt < <( + jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$EXTERNAL_PREFIX" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {epoch: (.epoch | tonumber), number: (.number | tonumber), + attempt: (.attempt | tonumber)}) catch empty] + | sort_by(.number, .attempt) + | (last // {epoch: 0, number: 0, attempt: 0}) + | [.number, .attempt] | @tsv' <<<"$check_runs" + ) + if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ + { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ + [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then + echo "::error::A newer GitHub Actions validation run already owns this exact-commit gate." + exit 1 + fi + + check_run_id=$(jq -r --arg check_name "$CHECK_NAME" \ + --arg external_id "$EXTERNAL_ID" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions" and + .external_id == $external_id) + | .id] | last // empty' <<<"$check_runs") + + if [ -n "$check_run_id" ] && ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid existing check-run ID." + exit 1 + fi + if [ -z "$check_run_id" ]; then + response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f head_sha="$MERGE_SHA" \ + -f external_id="$EXTERNAL_ID" \ + -f status=in_progress \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}") + check_run_id=$(jq -r '.id // empty' <<<"$response") + else + gh api --method PATCH \ + "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f status=in_progress \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >/dev/null + fi + if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid check-run ID." + exit 1 + fi + + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" \ + -f filter=all \ + -F per_page=100) + IFS=$'\t' read -r newest_number newest_attempt < <( + jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$EXTERNAL_PREFIX" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {epoch: (.epoch | tonumber), number: (.number | tonumber), + attempt: (.attempt | tonumber)}) catch empty] + | sort_by(.number, .attempt) + | (last // {epoch: 0, number: 0, attempt: 0}) + | [.number, .attempt] | @tsv' <<<"$check_runs" + ) + if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ + { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ + [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then + gh api --method PATCH \ + "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f status=completed \ + -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=Superseded by a newer policy decision." >/dev/null + echo "::error::A newer GitHub Actions validation run superseded this check." + exit 1 + fi + + prior_check_ids=$(jq -r \ + --arg check_name "$CHECK_NAME" \ + --arg external_prefix "$EXTERNAL_PREFIX" \ + --argjson current_number "$RUN_NUMBER" --argjson current_attempt "$RUN_ATTEMPT" \ + '.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | . as $check + | try (($check.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {epoch: (.epoch | tonumber), number: (.number | tonumber), + attempt: (.attempt | tonumber)}) catch empty + | select(.number < $current_number or + (.number == $current_number and .attempt < $current_attempt)) + | $check.id' <<<"$check_runs") + while IFS= read -r prior_check_id; do + [ -n "$prior_check_id" ] || continue + if ! [[ "$prior_check_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid prior check-run ID." + exit 1 + fi + gh api --method PATCH \ + "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" \ + -f status=completed \ + -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=Superseded by a newer authorized validation run." >/dev/null + done <<<"$prior_check_ids" + + echo "check_run_id=$check_run_id" >> "$GITHUB_OUTPUT" + plan: name: Plan validation needs: validate-invocation @@ -89,7 +261,7 @@ jobs: matrix: ${{ steps.protected-plan.outputs.matrix || steps.branch-plan.outputs.matrix }} promotion_eligible: ${{ steps.protected-plan.outputs.promotion_eligible || steps.branch-plan.outputs.promotion_eligible }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' with: repository: ${{ github.repository }} @@ -107,6 +279,8 @@ jobs: target: "windows-vulkan", backend: "vulkan", channel: "", + managed_pin: "vulkan", + build_platform: "windows", runner: ["self-hosted", "Windows", "128gb", "vulkan", "lemon-prod"], models: [ "K2-Horizon-0.9B-GGUF", @@ -118,9 +292,59 @@ jobs: lite: false }, { - target: "rocm-stable", + target: "linux-vulkan", + backend: "vulkan", + channel: "", + managed_pin: "vulkan", + build_platform: "linux", + runner: ["self-hosted", "Linux", "vulkan", "lemon-prod"], + models: ["K2-Horizon-0.9B-GGUF"], + capability_profile: "k2-horizon-v1", + capability_models: ["K2-Horizon-0.9B-GGUF"], + lite: false + }, + { + target: "linux-cpu", + backend: "cpu", + channel: "", + managed_pin: "cpu", + build_platform: "linux", + runner: ["self-hosted", "Linux", "lemon-prod"], + models: ["K2-Horizon-0.9B-GGUF"], + capability_profile: "k2-horizon-v1", + capability_models: ["K2-Horizon-0.9B-GGUF"], + lite: false + }, + { + target: "macos-metal", + backend: "metal", + channel: "", + managed_pin: "metal", + build_platform: "macos", + runner: ["macos-latest"], + models: ["K2-Horizon-0.9B-GGUF"], + capability_profile: "k2-horizon-v1", + capability_models: ["K2-Horizon-0.9B-GGUF"], + lite: false + }, + { + target: "windows-cuda", + backend: "cuda", + channel: "", + managed_pin: "cuda", + build_platform: "windows", + runner: ["self-hosted", "Windows", "cuda", "lemon-prod"], + models: ["K2-Horizon-0.9B-GGUF"], + capability_profile: "k2-horizon-v1", + capability_models: ["K2-Horizon-0.9B-GGUF"], + lite: false + }, + { + target: "windows-rocm-stable", backend: "rocm", channel: "stable", + managed_pin: "rocm-stable", + build_platform: "windows", runner: ["self-hosted", "Windows", "stx-halo", "rocm", "lemon-prod"], models: ["K2-Horizon-0.9B-GGUF"], capability_profile: "k2-horizon-v1", @@ -128,9 +352,11 @@ jobs: lite: false }, { - target: "rocm-nightly", + target: "windows-rocm-nightly", backend: "rocm", channel: "nightly", + managed_pin: "rocm-nightly", + build_platform: "windows", runner: ["self-hosted", "Windows", "stx-halo", "rocm", "lemon-prod"], models: ["K2-Horizon-0.9B-GGUF"], capability_profile: "k2-horizon-v1", @@ -231,7 +457,7 @@ jobs: cuda_available: ${{ steps.assets.outputs.cuda_available }} release_asset_manifest: ${{ steps.manifest.outputs.release_asset_manifest }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ github.repository }} ref: ${{ github.sha }} @@ -270,7 +496,8 @@ jobs: fi upstream_reference="refs/heads/${source_default_branch}" upstream_reference_head=$(gh api \ - "repos/${source_repository}/commits/${source_default_branch}" --jq '.sha') + "repos/${source_repository}/commits/heads/${source_default_branch}" \ + --jq '.sha') if ! [[ "$upstream_reference_head" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Could not resolve ${upstream_reference} to a full commit." exit 1 @@ -283,7 +510,9 @@ jobs: local release_tag_commit publisher_claim_type local publisher_claimed_source_commit comparison_path local source_manifest_asset_id source_manifest_path - release_tag_commit=$(gh api "repos/${repository}/commits/${tag}" --jq '.sha') + local build_target_attestation_path + release_tag_commit=$(gh api \ + "repos/${repository}/commits/tags/${tag}" --jq '.sha') if ! [[ "$release_tag_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Could not resolve ${repository} ${tag} to a release tag commit." exit 1 @@ -291,17 +520,21 @@ jobs: if [ "$repository" = "$source_repository" ]; then publisher_claim_type="source-release-tag" publisher_claimed_source_commit="$release_tag_commit" + build_target_attestation_path="${source_manifest_directory}/source-release-targets.json" + printf '[]\n' > "$build_target_attestation_path" else publisher_claim_type="immutable-source-manifest" source_manifest_asset_id=$(python -m test.utils.llamacpp_release_manifest \ --source-manifest-asset-id "$path" "$repository") source_manifest_path="${source_manifest_directory}/source-${source_manifest_asset_id}.json" + build_target_attestation_path="${source_manifest_path}.build-targets" gh api "repos/${repository}/releases/assets/${source_manifest_asset_id}" \ -H "Accept: application/octet-stream" | head -c "$((source_manifest_max_bytes + 1))" > "$source_manifest_path" publisher_claimed_source_commit=$(python -m test.utils.llamacpp_release_manifest \ --validate-source-manifest "$source_manifest_path" "$path" \ - "$repository" "$tag" "$release_tag_commit") + "$repository" "$tag" "$release_tag_commit" \ + --build-target-attestation-output "$build_target_attestation_path") fi if ! [[ "$publisher_claimed_source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::${repository} ${tag} did not bind one full upstream source commit." @@ -315,13 +548,15 @@ jobs: --require-upstream-ancestry "$comparison_path" "$source_repository" \ "$publisher_claimed_source_commit" "$upstream_reference_head" jq \ + --slurpfile build_target_attestations "$build_target_attestation_path" \ --arg publisher_claim_type "$publisher_claim_type" \ --arg publisher_claimed_source_commit "$publisher_claimed_source_commit" \ --arg release_tag_commit "$release_tag_commit" \ --arg source_repository "$source_repository" \ --arg upstream_reference "$upstream_reference" \ --arg upstream_reference_head "$upstream_reference_head" \ - '.publisher_claim_type = $publisher_claim_type + '.build_target_attestations = $build_target_attestations[0] + | .publisher_claim_type = $publisher_claim_type | .publisher_claimed_source_commit = $publisher_claimed_source_commit | .release_tag_commit = $release_tag_commit | .source_repository = $source_repository @@ -377,11 +612,20 @@ jobs: from test.utils.llamacpp_release_assets import ( build_asset_requirements, + build_rocm_asset_target_requirements, evaluate_asset_group, ) - def read_assets(path): - return set(Path(path).read_text(encoding="utf-8").splitlines()) + def read_asset_metadata(path): + release = json.loads(Path(path).read_text(encoding="utf-8")) + asset_sizes = {} + for asset in release["assets"]: + asset_sizes[asset["name"]] = asset["size"] + attested_targets = { + attestation["name"]: attestation["build_targets"] + for attestation in release.get("build_target_attestations", []) + } + return asset_sizes, attested_targets def update_outputs(values): github_output = os.environ.get("GITHUB_OUTPUT") @@ -391,9 +635,14 @@ jobs: for key, value in values.items(): out.write(f"{key}={value}\n") - def report_group(name, available_assets, backend_requirements): + def report_group( + name, + available_assets, + backend_requirements, + **evaluation_options, + ): update_backends, missing_by_backend = evaluate_asset_group( - available_assets, backend_requirements + available_assets, backend_requirements, **evaluation_options ) for backend, required in backend_requirements.items(): missing = missing_by_backend[backend] @@ -409,9 +658,9 @@ jobs: rocm_release = os.environ["ROCM_RELEASE"] lemonade_release = os.environ["LEMONADE_RELEASE"] - ggml_assets = read_assets("ggml_assets.txt") - rocm_assets = read_assets("rocm_assets.txt") - lemonade_assets = read_assets("lemonade_assets.txt") + ggml_assets, _ = read_asset_metadata("ggml_release.json") + rocm_assets, rocm_attested_targets = read_asset_metadata("rocm_release.json") + lemonade_assets, _ = read_asset_metadata("lemonade_release.json") with open("src/cpp/resources/backend_versions.json", encoding="utf-8") as f: versions = json.load(f) @@ -422,12 +671,20 @@ jobs: lemonade_release=lemonade_release, backend_versions=versions, ) + rocm_target_requirements = build_rocm_asset_target_requirements( + rocm_release=rocm_release, + backend_versions=versions, + ) ggml_updates, ggml_missing = report_group( "ggml-org/llama.cpp", ggml_assets, requirements["ggml"] ) rocm_updates, rocm_missing = report_group( - "lemonade-sdk/llamacpp-rocm", rocm_assets, requirements["rocm"] + "lemonade-sdk/llamacpp-rocm", + rocm_assets, + requirements["rocm"], + required_asset_targets=rocm_target_requirements, + attested_asset_targets=rocm_attested_targets, ) lemonade_updates, lemonade_missing = report_group( "lemonade-sdk/llama.cpp", lemonade_assets, requirements["lemonade"] @@ -464,15 +721,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the protected candidate commit - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ github.repository }} ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} path: candidate persist-credentials: false fetch-depth: 1 - name: Check out trusted validation tools - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ github.repository }} ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} @@ -490,10 +748,24 @@ jobs: "$BASE_SHA" "$CANDIDATE_SHA" build: - name: Build + name: Build ${{ matrix.platform }} needs: [validate-invocation, get-latest-releases, verify-release-assets, verify-protected-release-manifest, plan] if: always() && !failure() && !cancelled() - runs-on: windows-latest + runs-on: ${{ matrix.runner }} + permissions: + contents: read + env: + VALIDATION_CANDIDATE_ROOT: ${{ (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && format('{0}/candidate', github.workspace) || github.workspace }} + strategy: + fail-fast: false + matrix: + include: + - platform: windows + runner: windows-latest + - platform: linux + runner: ubuntu-latest + - platform: macos + runner: macos-latest steps: - name: Prepare Windows runner for long paths and stale workspace caches if: runner.os == 'Windows' @@ -508,43 +780,60 @@ jobs: } - - uses: actions/checkout@v5 + - name: Check out validation control plane + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.sha }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.sha }} persist-credentials: false clean: true - fetch-depth: 0 + fetch-depth: 2 + + - name: Check out the validation candidate + if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} + path: candidate + persist-credentials: false + clean: true + fetch-depth: 2 - name: Verify authorized merge commit if: github.event_name == 'pull_request_target' - shell: PowerShell + shell: bash env: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - $ErrorActionPreference = "Stop" - if ((git rev-parse HEAD^1) -ne $env:BASE_SHA) { - throw "The merge commit does not contain the authorized base commit." + set -euo pipefail + test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^1)" = "$BASE_SHA" || { + echo "::error::The merge commit does not contain the authorized base commit." + exit 1 } - if ((git rev-parse HEAD^2) -ne $env:HEAD_SHA) { - throw "The merge commit does not contain the authorized pull request head." + test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^2)" = "$HEAD_SHA" || { + echo "::error::The merge commit does not contain the authorized pull request head." + exit 1 } - name: Verify merge-group commit if: github.event_name == 'merge_group' - shell: PowerShell + shell: bash env: MERGE_GROUP_SHA: ${{ github.event.merge_group.head_sha }} run: | - $ErrorActionPreference = "Stop" - if ((git rev-parse HEAD) -ne $env:MERGE_GROUP_SHA) { - throw "The checkout does not match the authorized merge-group commit." + set -euo pipefail + test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD)" = "$MERGE_GROUP_SHA" || { + echo "::error::The checkout does not match the authorized merge-group commit." + exit 1 } - name: Update backend_versions.json if: needs.get-latest-releases.result == 'success' - shell: PowerShell + shell: bash + working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} env: LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} @@ -553,9 +842,8 @@ jobs: LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.rocm_update_backends }} LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.lemonade_update_backends }} run: | - $ErrorActionPreference = "Stop" - - $script = @' + set -euo pipefail + python - <<'PY' import json import os import re @@ -601,12 +889,64 @@ jobs: print(f" llamacpp.{key}: {old} -> {new}") else: print(" none") - '@ + PY + + - name: Stage protected validation-only model catalog + if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' + shell: bash + working-directory: ${{ github.workspace }} + env: + PYTHONPATH: ${{ github.workspace }} + run: | + set -euo pipefail + candidate_catalog="$GITHUB_WORKSPACE/candidate/src/cpp/resources/server_models.json" + python - "$GITHUB_WORKSPACE/candidate" "$candidate_catalog" <<'PY' + import os + import stat + import sys + from pathlib import Path - $script | python - + candidate_root = Path(sys.argv[1]).resolve(strict=True) + catalog_path = Path(sys.argv[2]) + resolved_catalog = catalog_path.resolve(strict=True) + try: + resolved_catalog.relative_to(candidate_root) + except ValueError as exc: + raise SystemExit("candidate catalog escapes its checkout") from exc + if not stat.S_ISREG(os.lstat(catalog_path).st_mode): + raise SystemExit("candidate catalog must be a regular file") + PY + python "$GITHUB_WORKSPACE/test/utils/validation_model_catalog.py" \ + --base "$candidate_catalog" \ + --overlay "$GITHUB_WORKSPACE/test/fixtures/llamacpp_validation_models.json" \ + --output "$candidate_catalog" - - name: Build lemond + - name: Stage branch validation-only model catalog + if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' + working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} + run: >- + python -m test.utils.validation_model_catalog + --base src/cpp/resources/server_models.json + --overlay test/fixtures/llamacpp_validation_models.json + --output src/cpp/resources/server_models.json + + - name: Install Linux build dependencies + if: runner.os == 'Linux' + shell: bash + working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} + run: ./setup.sh + + - name: Install macOS build dependencies + if: runner.os == 'macOS' + shell: bash + run: | + set -euo pipefail + command -v ninja >/dev/null 2>&1 || brew install ninja + + - name: Build lemond on Windows + if: runner.os == 'Windows' shell: PowerShell + working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} run: | $ErrorActionPreference = "Stop" Write-Host "Building Lemonade server binaries..." -ForegroundColor Cyan @@ -626,13 +966,35 @@ jobs: } Write-Host "Build successful!" -ForegroundColor Green - - name: Upload build artifacts - uses: actions/upload-artifact@v7 + - name: Build lemond on Linux or macOS + if: runner.os != 'Windows' + shell: bash + working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} + run: | + set -euo pipefail + cmake --preset default -DBUILD_WEB_APP=OFF -DBUILD_TESTING=OFF + cmake --build --preset default --target lemond --parallel 2 + test -x build/lemond + build/lemond --version + + - name: Upload Windows build artifacts + if: runner.os == 'Windows' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: llamacpp-build-${{ matrix.platform }} + path: | + ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/Release/ + ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/resources/ + retention-days: 7 + + - name: Upload Linux or macOS build artifacts + if: runner.os != 'Windows' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: llamacpp-build + name: llamacpp-build-${{ matrix.platform }} path: | - build/Release/ - build/resources/ + ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/lemond + ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/resources/ retention-days: 7 # ======================================================================== @@ -643,12 +1005,16 @@ jobs: needs: [get-latest-releases, build, plan] if: always() && needs.build.result == 'success' runs-on: ${{ matrix.runner }} + permissions: + contents: read + env: + VALIDATION_CANDIDATE_ROOT: ${{ (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && format('{0}/candidate', github.workspace) || github.workspace }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} steps: - name: Capture Vulkan runner configuration - if: matrix.backend == 'vulkan' + if: matrix.backend == 'vulkan' && runner.os == 'Windows' shell: PowerShell run: | $ErrorActionPreference = "Continue" @@ -810,11 +1176,28 @@ jobs: # Environment collection should never make validation fail. exit 0 + - name: Capture Vulkan runner configuration on Linux + if: matrix.backend == 'vulkan' && runner.os == 'Linux' + shell: bash + run: | + set +e + output_directory="$RUNNER_TEMP/vulkan-runner-config" + mkdir -p "$output_directory" + uname -a > "$output_directory/system.txt" + if command -v vulkaninfo >/dev/null 2>&1; then + vulkaninfo --summary > "$output_directory/vulkaninfo-summary.txt" 2>&1 + vulkaninfo > "$output_directory/vulkaninfo.txt" 2>&1 + else + echo "vulkaninfo was not found in PATH." \ + > "$output_directory/vulkaninfo-unavailable.txt" + fi + exit 0 + - name: Upload Vulkan runner configuration if: always() && matrix.backend == 'vulkan' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: vulkan-runner-config + name: vulkan-runner-config-${{ matrix.target }} path: ${{ runner.temp }}/vulkan-runner-config/ retention-days: 30 if-no-files-found: warn @@ -824,7 +1207,9 @@ jobs: shell: PowerShell run: | $ErrorActionPreference = "Stop" - $workspaceRoot = [IO.Path]::GetFullPath($env:GITHUB_WORKSPACE) + $workspaceRoot = [IO.Path]::GetFullPath( + $env:VALIDATION_CANDIDATE_ROOT + ) $lemondPath = [IO.Path]::GetFullPath( (Join-Path $workspaceRoot "build\Release\lemond.exe") ) @@ -862,270 +1247,192 @@ jobs: } } - - uses: actions/checkout@v5 + - name: Check out validation control plane + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.sha }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.sha }} + persist-credentials: false + clean: true + fetch-depth: 2 + + - name: Check out the validation candidate + if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + repository: ${{ github.repository }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} + path: candidate persist-credentials: false clean: true fetch-depth: 2 - name: Verify authorized merge commit if: github.event_name == 'pull_request_target' - shell: PowerShell + shell: bash env: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - $ErrorActionPreference = "Stop" - if ((git rev-parse HEAD^1) -ne $env:BASE_SHA) { - throw "The merge commit does not contain the authorized base commit." + set -euo pipefail + test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^1)" = "$BASE_SHA" || { + echo "::error::The merge commit does not contain the authorized base commit." + exit 1 } - if ((git rev-parse HEAD^2) -ne $env:HEAD_SHA) { - throw "The merge commit does not contain the authorized pull request head." + test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^2)" = "$HEAD_SHA" || { + echo "::error::The merge commit does not contain the authorized pull request head." + exit 1 } - name: Verify merge-group commit if: github.event_name == 'merge_group' - shell: PowerShell + shell: bash env: MERGE_GROUP_SHA: ${{ github.event.merge_group.head_sha }} run: | - $ErrorActionPreference = "Stop" - if ((git rev-parse HEAD) -ne $env:MERGE_GROUP_SHA) { - throw "The checkout does not match the authorized merge-group commit." + set -euo pipefail + test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD)" = "$MERGE_GROUP_SHA" || { + echo "::error::The checkout does not match the authorized merge-group commit." + exit 1 } - name: Cleanup processes + if: runner.os == 'Windows' uses: ./.github/actions/cleanup-processes-windows + - name: Cleanup processes on Linux + if: runner.os == 'Linux' + uses: ./.github/actions/cleanup-processes-linux + - name: Download build artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - name: llamacpp-build - path: build + name: llamacpp-build-${{ matrix.build_platform }} + path: ${{ env.VALIDATION_CANDIDATE_ROOT }}/build - - name: Verify binaries + - name: Verify Windows binary artifact + if: runner.os == 'Windows' shell: PowerShell run: | - $lemondExe = "build\Release\lemond.exe" + $lemondExe = Join-Path ` + $env:VALIDATION_CANDIDATE_ROOT "build\Release\lemond.exe" if (-not (Test-Path $lemondExe)) { Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red - Get-ChildItem -Recurse build | Select-Object FullName + Get-ChildItem -Recurse ` + (Join-Path $env:VALIDATION_CANDIDATE_ROOT "build") | + Select-Object FullName exit 1 } - & $lemondExe --version - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - Write-Host "Binaries verified!" -ForegroundColor Green + if ((Get-Item $lemondExe).Attributes -band ` + [IO.FileAttributes]::ReparsePoint) { + Write-Host "ERROR: lemond.exe must not be a link!" ` + -ForegroundColor Red + exit 1 + } + Write-Host "Binary artifact verified!" -ForegroundColor Green + + - name: Verify Linux or macOS binary artifact + if: runner.os != 'Windows' + shell: bash + run: | + set -euo pipefail + lemond="$VALIDATION_CANDIDATE_ROOT/build/lemond" + test -f "$lemond" + test ! -L "$lemond" + chmod +x "$lemond" - - name: Setup Python and virtual environment + - name: Set up trusted or branch validation environment + id: validation-venv uses: ./.github/actions/setup-venv with: venv-name: '.venv' python-version: '3.10' - requirements-file: 'test/requirements.txt' + requirements-file: '${{ github.workspace }}/test/requirements.txt' - name: Run validation with lemond - shell: PowerShell + id: run-validation env: CAPABILITY_MODELS: ${{ join(matrix.capability_models || fromJSON('[]'), ',') }} CAPABILITY_PROFILE: ${{ matrix.capability_profile || '' }} - HF_TOKEN: ${{ github.event_name != 'pull_request_target' && github.event_name != 'merge_group' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }} + CANDIDATE_LEMOND: ${{ runner.os == 'Windows' && format('{0}/build/Release/lemond.exe', env.VALIDATION_CANDIDATE_ROOT) || format('{0}/build/lemond', env.VALIDATION_CANDIDATE_ROOT) }} + HF_TOKEN: ${{ github.event_name == 'schedule' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }} + HUGGINGFACE_TOKEN: '' + HUGGING_FACE_HUB_TOKEN: '' + LLAMACPP_BACKEND: ${{ matrix.backend }} + LLAMACPP_CHANNEL: ${{ matrix.channel }} LLAMACPP_TEST_MODELS: ${{ join(matrix.models, ',') }} LITE_MODE: ${{ matrix.lite }} - run: | - $ErrorActionPreference = "Stop" - $port = 13305 - $lemondExe = (Resolve-Path "build\Release\lemond.exe").Path - $backend = "${{ matrix.backend }}" - $channel = "${{ matrix.channel }}" - $label = "${{ matrix.target }}" - $logsDir = "server-logs-$label" - $cacheDir = Join-Path $PWD "ci-cache-$label" - $cachePrefix = [IO.Path]::GetFullPath($cacheDir) - $separator = [string][IO.Path]::DirectorySeparatorChar - if (-not $cachePrefix.EndsWith($separator)) { $cachePrefix += $separator } - $venvPython = ".\.venv\Scripts\python.exe" - $tempRoot = if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { $PWD } - $pidFile = Join-Path $tempRoot "lemond-validate-llamacpp.pid" - - function Get-ValidationLlamaServers { - Get-CimInstance Win32_Process ` - -Filter "Name = 'llama-server.exe'" ` - -ErrorAction SilentlyContinue | Where-Object { - $_.ExecutablePath -and $_.ExecutablePath.StartsWith( - $cachePrefix, - [StringComparison]::OrdinalIgnoreCase - ) - } - } - - New-Item -ItemType Directory -Force -Path $logsDir | Out-Null - New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null - - $stdoutLog = Join-Path $PWD "$logsDir\lemond.stdout.log" - $stderrLog = Join-Path $PWD "$logsDir\lemond.stderr.log" - - # Do not benchmark while another known inference process is active. - $conflicts = @( - Get-Process -Name @( - "lemond", "llama-server", "lemonade", "LemonadeServer", - "flm", "ort-server", "moonshine-server" - ) -ErrorAction SilentlyContinue - ) - if ($conflicts.Count -gt 0) { - $details = ($conflicts | ForEach-Object { - "$($_.ProcessName):$($_.Id)" - }) -join ", " - throw "Runner is not idle: $details" - } - - if (Get-NetTCPConnection -State Listen -LocalPort $port -ErrorAction SilentlyContinue) { - throw "Port $port is already in use." - } - - $proc = $null - $validationExitCode = 0 - $cleanupExitCode = 0 - - try { - $proc = Start-Process ` - -FilePath $lemondExe ` - -ArgumentList @($cacheDir, "--port", $port, "--host", "127.0.0.1") ` - -RedirectStandardOutput $stdoutLog ` - -RedirectStandardError $stderrLog ` - -PassThru - - # Same-job fallback for the existing final cleanup action. - Set-Content -Path $pidFile -Value $proc.Id -Encoding ASCII - Write-Host "Started lemond PID $($proc.Id)" -ForegroundColor Cyan - - if ($proc.WaitForExit(500)) { - throw "lemond exited before validation started. See $stderrLog." - } - - $validationArgs = @( - "test/validate_llamacpp.py", - "--backend", $backend, - "--port", $port, - "--output", "llamacpp_validation_$label.json", - "--logs-dir", $logsDir - ) - - if ($channel) { - $validationArgs += "--channel" - $validationArgs += $channel - } - - . ".github/scripts/llamacpp_validation_args.ps1" - $validationArgs += @(Get-LlamaCppValidationModelArguments ` - -Models $env:LLAMACPP_TEST_MODELS ` - -Lite ($env:LITE_MODE -eq "true") ` - -CapabilityProfile $env:CAPABILITY_PROFILE ` - -CapabilityModels $env:CAPABILITY_MODELS) - - & $venvPython @validationArgs - $validationExitCode = $LASTEXITCODE - } finally { - if ($null -ne $proc) { - if ($proc.HasExited) { - $cleanupExitCode = 1 - Write-Warning "lemond exited before graceful shutdown." - } - try { - if (-not $proc.HasExited) { - Invoke-WebRequest -Uri "http://127.0.0.1:$port/internal/shutdown" ` - -Method POST -TimeoutSec 10 -UseBasicParsing | Out-Null - $null = $proc.WaitForExit(10000) - } - } catch { - $cleanupExitCode = 1 - Write-Host "Graceful shutdown failed; forcing targeted cleanup." ` - -ForegroundColor Yellow - } - - if (-not $proc.HasExited) { - $cleanupExitCode = 1 - Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue - $null = $proc.WaitForExit(5000) - } - - # Catch a llama-server orphan even if its lemond parent exited first. - $orphans = @(Get-ValidationLlamaServers) - if ($orphans.Count -gt 0) { - $cleanupExitCode = 1 - } - foreach ($orphan in $orphans) { - Write-Host "Stopping validation llama-server PID $($orphan.ProcessId)." ` - -ForegroundColor Yellow - Stop-Process -Id $orphan.ProcessId -Force -ErrorAction SilentlyContinue - } - - # Process teardown can lag slightly on Windows, especially after - # releasing Vulkan resources. Give it a short bounded grace period. - $remainingOrphans = @(Get-ValidationLlamaServers) - $cleanupDeadline = (Get-Date).AddSeconds(5) - - while ( - $remainingOrphans.Count -gt 0 -and - (Get-Date) -lt $cleanupDeadline - ) { - Start-Sleep -Milliseconds 500 - $remainingOrphans = @(Get-ValidationLlamaServers) - } - - if ($proc.HasExited -and $remainingOrphans.Count -eq 0) { - Remove-Item $pidFile -Force -ErrorAction SilentlyContinue - } else { - $cleanupExitCode = 1 - if (-not $proc.HasExited) { - Write-Warning "lemond PID $($proc.Id) is still running after cleanup." - } - - foreach ($orphan in $remainingOrphans) { - Write-Warning ( - "llama-server still present after cleanup: " + - "PID=$($orphan.ProcessId), Path=$($orphan.ExecutablePath)" - ) - } - - Write-Warning ( - "Validation cleanup did not fully complete. " + - "The validation result will be preserved." - ) - } - } - } - - if ($validationExitCode -ne 0) { - exit $validationExitCode - } - if ($cleanupExitCode -ne 0) { - exit $cleanupExitCode - } + LEMONADE_PROTECTED_RUNNER_CONTEXT: ${{ matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && join(matrix.runner, ',') == 'macos-latest' && 'github-hosted:macos-latest:macos-metal' || '' }} + PYTHONPATH: ${{ github.workspace }} + TARGET: "${{ matrix.target }}" + VALIDATION_ROOT: ${{ github.workspace }} + shell: bash + working-directory: ${{ github.workspace }} + run: >- + python "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py" + --lemond "$CANDIDATE_LEMOND" + --python "${{ steps.validation-venv.outputs.python-path }}" + ${{ github.event_name == 'schedule' && '--allow-huggingface-download-credentials' || '' }} + ${{ matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && join(matrix.runner, ',') == 'macos-latest' && '--allow-darwin-github-hosted-ephemeral-runner' || '' }} - name: Upload results - if: always() - uses: actions/upload-artifact@v7 + if: >- + steps.run-validation.outcome == 'success' && + !((github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && + matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && + join(matrix.runner, ',') == 'macos-latest') + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: validation-results-${{ matrix.target }} - path: llamacpp_validation_${{ matrix.target }}.json + path: ${{ steps.run-validation.outputs.validation_result }} + retention-days: 30 + if-no-files-found: error + + - name: Upload restart results + if: >- + steps.run-validation.outcome == 'success' && + !((github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && + matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && + join(matrix.runner, ',') == 'macos-latest') + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: restart-validation-results-${{ matrix.target }} + path: ${{ steps.run-validation.outputs.restart_result }} retention-days: 30 if-no-files-found: error - name: Upload server logs - if: always() - uses: actions/upload-artifact@v7 + if: >- + steps.run-validation.outcome == 'success' && + !((github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && + matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && + join(matrix.runner, ',') == 'macos-latest') + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: server-logs-${{ matrix.target }} - path: server-logs-${{ matrix.target }}/ + path: | + ${{ steps.run-validation.outputs.prepare_stdout_log }} + ${{ steps.run-validation.outputs.prepare_stderr_log }} + ${{ steps.run-validation.outputs.validation_stdout_log }} + ${{ steps.run-validation.outputs.validation_stderr_log }} + ${{ steps.run-validation.outputs.restart_stdout_log }} + ${{ steps.run-validation.outputs.restart_stderr_log }} retention-days: 30 - if-no-files-found: ignore + if-no-files-found: error - name: Cleanup - if: always() + if: >- + always() && runner.os == 'Windows' && + github.event_name != 'pull_request_target' && github.event_name != 'merge_group' uses: ./.github/actions/cleanup-processes-windows + - name: Cleanup on Linux + if: >- + always() && runner.os == 'Linux' && + github.event_name != 'pull_request_target' && github.event_name != 'merge_group' + uses: ./.github/actions/cleanup-processes-linux + reverify-protected-release-manifest: name: Reverify protected release manifest needs: [validate, verify-protected-release-manifest] @@ -1140,16 +1447,17 @@ jobs: run: echo "This invocation does not review a protected pin change." - name: Check out the protected candidate commit if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ github.repository }} ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} + allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} path: candidate persist-credentials: false fetch-depth: 1 - name: Check out trusted validation tools if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ github.repository }} ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} diff --git a/.github/workflows/validate_llamacpp_manual.yml b/.github/workflows/validate_llamacpp_manual.yml index 53c5f87013..d7be8812e7 100644 --- a/.github/workflows/validate_llamacpp_manual.yml +++ b/.github/workflows/validate_llamacpp_manual.yml @@ -4,11 +4,11 @@ on: workflow_dispatch: inputs: lite: - description: 'Run in lite mode (smallest model only)' + description: 'Run only the K2-Horizon 0.9B smoke model' type: boolean default: false models: - description: 'Optional comma-separated model IDs (empty uses hot models)' + description: 'Additional comma-separated IDs; empty uses hot models + K2 0.9B' type: string required: false default: '' @@ -20,10 +20,9 @@ jobs: validate: name: Run manual validation permissions: + checks: write contents: read uses: ./.github/workflows/validate_llamacpp_core.yml with: lite: ${{ inputs.lite }} models: ${{ inputs.models }} - secrets: - HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} diff --git a/.github/workflows/validate_llamacpp_pr.yml b/.github/workflows/validate_llamacpp_pr.yml index c99895b823..c872c3f0a8 100644 --- a/.github/workflows/validate_llamacpp_pr.yml +++ b/.github/workflows/validate_llamacpp_pr.yml @@ -2,381 +2,1302 @@ name: Validate llama.cpp protected change on: pull_request_target: - types: [synchronize, edited, labeled, unlabeled, closed] + types: [opened, reopened, synchronize, edited, labeled, unlabeled, closed] merge_group: types: [checks_requested] permissions: contents: read +env: + DEPENDABOT_PR: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.user.login == 'dependabot[bot]' && github.event.pull_request.user.id == 49699333 && github.event.pull_request.user.type == 'Bot' }} + concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}-{1}', github.event.pull_request.number, (github.event.action == 'synchronize' || github.event.action == 'closed' || (github.event.action == 'edited' && github.event.changes.base.ref.from) || ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name == 'ci:upgrades')) && 'validation' || github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'merge_group' || github.event.action == 'synchronize' || github.event.action == 'closed' || (github.event.action == 'edited' && github.event.changes.base.ref.from) || (github.event.action == 'unlabeled' && github.event.label.name == 'ci:upgrades') }} + group: ${{ github.event_name == 'merge_group' && format('{0}-merge-group-{1}-{2}-{3}', github.workflow, github.event.merge_group.head_sha, github.run_number, github.run_attempt) || ((github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || github.event.action == 'edited' || github.event.action == 'labeled' || github.event.action == 'unlabeled' || github.event.action == 'closed') && format('{0}-pr-{1}-event-{2}-{3}', github.workflow, github.event.pull_request.number, github.run_number, github.run_attempt) || format('{0}-unsupported-{1}-{2}', github.workflow, github.run_number, github.run_attempt)) }} + cancel-in-progress: false jobs: - authorize-invocation: - name: Authorize invocation + policy: + name: Resolve protected validation policy runs-on: ubuntu-latest permissions: + actions: read + checks: read contents: read + issues: read pull-requests: read outputs: - run_validation: ${{ steps.authorize.outputs.run_validation }} + authorization_actor: ${{ steps.decision.outputs.authorization_actor }} + authorization_actor_id: ${{ steps.decision.outputs.authorization_actor_id }} + authorization_boundary: ${{ steps.decision.outputs.authorization_boundary }} + authorization_created_at: ${{ steps.decision.outputs.authorization_created_at }} + authorization_event_id: ${{ steps.decision.outputs.authorization_event_id }} + base_ref: ${{ steps.identity.outputs.base_ref }} + base_sha: ${{ steps.identity.outputs.base_sha }} + conclusion: ${{ steps.decision.outputs.conclusion }} + head_sha: ${{ steps.identity.outputs.head_sha }} + merge_sha: ${{ steps.identity.outputs.merge_sha }} + observed_merge_sha: ${{ steps.identity.outputs.observed_merge_sha }} + secondary_observed_merge_sha: ${{ steps.identity.outputs.secondary_observed_merge_sha }} + mode: ${{ steps.decision.outputs.mode }} + run_validation: ${{ steps.decision.outputs.run_validation }} + summary: ${{ steps.decision.outputs.summary }} steps: - - name: Verify trusted invocation - id: authorize + - name: Resolve the live candidate identity + id: identity env: - ACTOR: ${{ github.actor }} - EVENT_ACTION: ${{ github.event.action || '' }} - EVENT_LABEL: ${{ github.event.label.name || '' }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} + EVENT_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha || '' }} EVENT_NAME: ${{ github.event_name }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha || '' }} + MERGE_GROUP_HEAD_SHA: ${{ github.event.merge_group.head_sha || '' }} PR_NUMBER: ${{ github.event.pull_request.number || '' }} run: | set -euo pipefail + oid_pattern='^[0-9a-fA-F]{40}$' if [ "$EVENT_NAME" = "merge_group" ]; then - echo "run_validation=true" >> "$GITHUB_OUTPUT" + if ! [[ "$MERGE_GROUP_BASE_SHA" =~ $oid_pattern && "$MERGE_GROUP_HEAD_SHA" =~ $oid_pattern ]]; then + echo "::error::Merge-group commits must be full Git OIDs." + exit 1 + fi + { + echo "base_sha=$MERGE_GROUP_BASE_SHA" + echo "base_ref=" + echo "head_sha=$MERGE_GROUP_HEAD_SHA" + echo "merge_sha=$MERGE_GROUP_HEAD_SHA" + echo "observed_merge_sha=" + echo "secondary_observed_merge_sha=" + echo "merge_candidate_available=true" + echo "state=open" + echo "merged=false" + echo "changed_files=0" + echo "authorization_label_present=false" + echo "identity_matches=true" + } >> "$GITHUB_OUTPUT" exit 0 fi - if [ "$EVENT_NAME" != "pull_request_target" ]; then - echo "::error::Unsupported invocation event: $EVENT_NAME" + if [ "$EVENT_NAME" != "pull_request_target" ] || ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + echo "::error::Unsupported protected-validation invocation." exit 1 fi - if [ "$EVENT_ACTION" != "labeled" ] || [ "$EVENT_LABEL" != "ci:upgrades" ]; then - echo "run_validation=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") - current_state=$(jq -r '.state' <<<"$pull_request") + pull_request="" + for attempt in 1 2 3; do + pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") + snapshot_state=$(jq -r '.state // empty' <<<"$pull_request") + snapshot_merged=$(jq -r '.merged | tostring' <<<"$pull_request") + snapshot_mergeable=$(jq -r \ + '.mergeable | if . == null then "null" else tostring end' \ + <<<"$pull_request") + if [ "$snapshot_state" != "open" ] || [ "$snapshot_merged" = "true" ] || \ + [ "$snapshot_mergeable" = "true" ] || [ "$snapshot_mergeable" = "false" ] || \ + [ "$attempt" -eq 3 ]; then + break + fi + sleep 2 + done + current_number=$(jq -er '.number' <<<"$pull_request") + current_state=$(jq -er '.state' <<<"$pull_request") + current_merged=$(jq -er \ + '.merged | if type == "boolean" then tostring else error("invalid merged state") end' \ + <<<"$pull_request") + current_base=$(jq -er '.base.sha' <<<"$pull_request") + current_base_ref=$(jq -er \ + '.base.ref | select(type == "string" and test("^[0-9A-Za-z._/-]+$"))' \ + <<<"$pull_request") + current_head=$(jq -er '.head.sha' <<<"$pull_request") + current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") + current_mergeable=$(jq -r \ + '.mergeable | if . == null then "null" else tostring end' \ + <<<"$pull_request") + current_changed_files=$(jq -er '.changed_files' <<<"$pull_request") current_authorized=$(jq -r \ 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") - if [ "$current_state" != "open" ] || [ "$current_authorized" != "true" ]; then - echo "run_validation=false" >> "$GITHUB_OUTPUT" - exit 0 + + if [ "$current_number" != "$PR_NUMBER" ] || \ + ! [[ "$current_state" =~ ^(open|closed)$ ]] || \ + ! [[ "$current_merged" =~ ^(true|false)$ ]] || \ + ! [[ "$current_mergeable" =~ ^(true|false|null)$ ]] || \ + ! [[ "$current_changed_files" =~ ^[0-9]+$ ]] || \ + ! [[ "$current_base" =~ $oid_pattern && "$current_head" =~ $oid_pattern ]] || \ + ! git check-ref-format "refs/heads/${current_base_ref}" >/dev/null || \ + { [ -n "$current_merge" ] && ! [[ "$current_merge" =~ $oid_pattern ]]; }; then + echo "::error::The live pull request returned an invalid identity." + exit 1 + fi + if [ "$current_merged" = "true" ] && [ "$current_state" != "closed" ]; then + echo "::error::The live pull request returned an inconsistent lifecycle state." + exit 1 fi - permission=$(gh api \ - "repos/${GITHUB_REPOSITORY}/collaborators/${ACTOR}/permission" \ - --jq '.permission') - case "$permission" in - admin|maintain|write|push) - echo "run_validation=true" >> "$GITHUB_OUTPUT" - ;; - *) - echo "::error::$ACTOR cannot authorize self-hosted validation." - exit 1 - ;; - esac - - revoke-check: - name: Revoke removed authorization - if: >- - github.event_name == 'pull_request_target' && - github.event.action == 'unlabeled' && - github.event.label.name == 'ci:upgrades' && - github.event.pull_request.state == 'open' - runs-on: ubuntu-latest - permissions: - checks: write - contents: read - pull-requests: read - steps: - - name: Revoke the authorized merge check + candidate_sha="$current_head" + merge_candidate_available=false + observed_merge_sha="" + secondary_observed_merge_sha="" + if [ "$current_mergeable" = "true" ] && [[ "$current_merge" =~ $oid_pattern ]]; then + candidate_sha="$current_merge" + merge_candidate_available=true + elif [[ "$current_merge" =~ $oid_pattern ]]; then + observed_merge_sha="$current_merge" + fi + if [ "$merge_candidate_available" = "false" ] && \ + [[ "$EVENT_MERGE_SHA" =~ $oid_pattern ]] && \ + [ "$EVENT_MERGE_SHA" != "$observed_merge_sha" ]; then + if [ -z "$observed_merge_sha" ]; then + observed_merge_sha="$EVENT_MERGE_SHA" + else + secondary_observed_merge_sha="$EVENT_MERGE_SHA" + fi + fi + + merge_identity_matches=true + if [ "$merge_candidate_available" = "true" ] && \ + [ "$current_merge" != "$EVENT_MERGE_SHA" ]; then + merge_identity_matches=false + fi + identity_matches=false + if [ "$current_base_ref" = "$EVENT_BASE_REF" ] && \ + [ "$current_base" = "$EVENT_BASE_SHA" ] && \ + [ "$current_head" = "$EVENT_HEAD_SHA" ] && \ + [ "$merge_identity_matches" = "true" ]; then + identity_matches=true + fi + { + echo "base_ref=$current_base_ref" + echo "base_sha=$current_base" + echo "head_sha=$current_head" + echo "merge_sha=$candidate_sha" + echo "observed_merge_sha=$observed_merge_sha" + echo "secondary_observed_merge_sha=$secondary_observed_merge_sha" + echo "merge_candidate_available=$merge_candidate_available" + echo "state=$current_state" + echo "merged=$current_merged" + echo "changed_files=$current_changed_files" + echo "authorization_label_present=$current_authorized" + echo "identity_matches=$identity_matches" + } >> "$GITHUB_OUTPUT" + + - name: Check out trusted policy code + if: github.event_name == 'pull_request_target' + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + repository: ${{ github.repository }} + ref: ${{ steps.identity.outputs.base_sha }} + path: trusted + persist-credentials: false + fetch-depth: 1 + + - name: Classify the complete pull request file set + id: classify + if: github.event_name == 'pull_request_target' && steps.identity.outputs.state == 'open' env: - CHECK_NAME: llama.cpp authorized validation v2 + BASE_SHA: ${{ steps.identity.outputs.base_sha }} + EXPECTED_COUNT: ${{ steps.identity.outputs.changed_files }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + MERGE_SHA: ${{ steps.identity.outputs.merge_sha }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -euo pipefail - pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") - current_state=$(jq -r '.state' <<<"$pull_request") - current_authorized=$(jq -r \ - 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") - if [ "$current_state" != "open" ] || [ "$current_authorized" = "true" ]; then - echo "Pull request is closed or authorization was restored; skipping revocation." + test "$(git -C trusted rev-parse HEAD)" = "$BASE_SHA" + if ! [[ "$EXPECTED_COUNT" =~ ^[0-9]+$ ]]; then + echo "::error::The pull-request file count is invalid." + exit 1 + fi + if [ "$EXPECTED_COUNT" -gt 300 ]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" exit 0 fi - - external_prefix="lemonade-llamacpp-pr-${PR_NUMBER}-" - external_id="${external_prefix}revoked-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - check_runs=$(gh api --method GET \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + comparison_path="$RUNNER_TEMP/llamacpp-pr-comparison-${PR_NUMBER}.json" + gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${MERGE_SHA}" \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" \ - -f filter=all \ - -F per_page=100) - revoked_id=$(jq -r --arg external_id "$external_id" \ - '[.check_runs[] | select(.external_id == $external_id) | .id] | last // empty' \ - <<<"$check_runs") - - if [ -z "$revoked_id" ]; then - response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f head_sha="$MERGE_SHA" \ - -f external_id="$external_id" \ - -f status=completed \ - -f conclusion=cancelled \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=The ci:upgrades authorization label was removed.") - revoked_id=$(jq -r '.id // empty' <<<"$response") - fi - if ! [[ "$revoked_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid revocation check-run ID." - exit 1 - fi + > "$comparison_path" + python trusted/.github/scripts/llamacpp_required_check_policy.py \ + classify-compare \ + --response "$comparison_path" \ + --expected-base-sha "$BASE_SHA" \ + --expected-target-sha "$MERGE_SHA" \ + --expected-count "$EXPECTED_COUNT" \ + --github-output "$GITHUB_OUTPUT" - prior_check_ids=$(jq -r \ - --arg external_id "$external_id" \ - --arg external_prefix "$external_prefix" \ - '.check_runs[] - | select((.external_id // "") | startswith($external_prefix)) - | select(.external_id != $external_id) - | .id' <<<"$check_runs") - while IFS= read -r prior_check_id; do - [ -n "$prior_check_id" ] || continue - if ! [[ "$prior_check_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid prior check-run ID." - exit 1 - fi - gh api --method PATCH \ - "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f status=completed \ - -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=The ci:upgrades authorization label was removed." >/dev/null - done <<<"$prior_check_ids" - - start-check: - name: Start authorized merge check - needs: authorize-invocation - if: needs.authorize-invocation.outputs.run_validation == 'true' - runs-on: ubuntu-latest - permissions: - checks: write - contents: read - outputs: - check_run_id: ${{ steps.create.outputs.check_run_id }} - steps: - - name: Mark validation in progress - id: create + - name: Decide whether protected validation may run + id: decision env: + AUTHORIZATION_LABEL_PRESENT: ${{ steps.identity.outputs.authorization_label_present }} + BASE_REF_CHANGED: ${{ github.event.action == 'edited' && github.event.changes.base.ref.from != '' }} CHECK_NAME: llama.cpp authorized validation v2 + CURRENT_RUN_ATTEMPT: ${{ github.run_attempt }} + CURRENT_RUN_ID: ${{ github.run_id }} + CURRENT_RUN_NUMBER: ${{ github.run_number }} + DEPENDABOT_PR: ${{ env.DEPENDABOT_PR }} + EVENT_ACTION: ${{ github.event.action || '' }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} + EVENT_LABEL: ${{ github.event.label.name || '' }} + EVENT_NAME: ${{ github.event_name }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MERGE_SHA: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.event.pull_request.merge_commit_sha }} - RUN_ATTEMPT: ${{ github.run_attempt }} - RUN_ID: ${{ github.run_id }} - TARGET_KEY: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}', github.event.pull_request.number) }} + IDENTITY_MATCHES: ${{ steps.identity.outputs.identity_matches }} + MERGED: ${{ steps.identity.outputs.merged }} + MERGE_CANDIDATE_AVAILABLE: ${{ steps.identity.outputs.merge_candidate_available }} + MERGE_SHA: ${{ steps.identity.outputs.merge_sha }} + PR_NUMBER: ${{ github.event.pull_request.number || '' }} + RELEVANT: ${{ steps.classify.outputs.relevant || 'true' }} + STATE: ${{ steps.identity.outputs.state }} + TRIGGERING_ACTOR: ${{ github.triggering_actor || '' }} run: | set -euo pipefail - EXTERNAL_PREFIX="lemonade-llamacpp-${TARGET_KEY}-" - EXTERNAL_ID="${EXTERNAL_PREFIX}${RUN_ID}-${RUN_ATTEMPT}" - check_runs=$(gh api --method GET \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" \ - -f filter=all \ - -F per_page=100) - check_run_id=$(jq -r --arg external_id "$EXTERNAL_ID" \ - '[.check_runs[] | select(.external_id == $external_id) | .id] | last // empty' \ - <<<"$check_runs") - - if [ -z "$check_run_id" ]; then - response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f head_sha="$MERGE_SHA" \ - -f external_id="$EXTERNAL_ID" \ - -f status=in_progress \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}") - check_run_id=$(jq -r '.id // empty' <<<"$response") - fi - if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid check-run ID." - exit 1 + if [ "$EVENT_NAME" = "merge_group" ]; then + { + echo "authorization_actor=" + echo "authorization_actor_id=0" + echo "authorization_created_at=" + echo "authorization_event_id=0" + echo "mode=validate" + echo "run_validation=true" + echo "conclusion=" + echo "summary=Merge-queue validation is mandatory." + } >> "$GITHUB_OUTPUT" + exit 0 fi - prior_check_ids=$(jq -r \ - --arg external_id "$EXTERNAL_ID" \ - --arg external_prefix "$EXTERNAL_PREFIX" \ - '.check_runs[] - | select((.external_id // "") | startswith($external_prefix)) - | select(.external_id != $external_id) - | .id' <<<"$check_runs") - while IFS= read -r prior_check_id; do - [ -n "$prior_check_id" ] || continue - if ! [[ "$prior_check_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid prior check-run ID." - exit 1 + authorization_actor="" + authorization_actor_id=0 + authorization_boundary="" + authorization_created_at="" + authorization_epoch_valid=false + authorization_event_id=0 + authorization_rerun_matches=false + actor_permission="" + has_existing_check=false + has_newer_check=false + if [ "$MERGED" != "true" ] && [ "$STATE" = "open" ]; then + if [[ "$CURRENT_RUN_ATTEMPT" =~ ^([2-9]|[1-9][0-9]+)$ ]] && \ + [[ "$CURRENT_RUN_ID" =~ ^[1-9][0-9]*$ ]] && \ + [[ "$CURRENT_RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] && \ + { [ "$EVENT_ACTION" = "opened" ] || \ + [ "$EVENT_ACTION" = "reopened" ] || \ + [ "$EVENT_ACTION" = "synchronize" ]; }; then + first_attempt=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${CURRENT_RUN_ID}/attempts/1" \ + 2>/dev/null || true) + if jq -e --arg base_ref "$EVENT_BASE_REF" \ + --arg head_sha "$EVENT_HEAD_SHA" \ + --arg path ".github/workflows/validate_llamacpp_pr.yml" \ + --arg repository "$GITHUB_REPOSITORY" \ + --argjson run_id "$CURRENT_RUN_ID" \ + --argjson run_number "$CURRENT_RUN_NUMBER" \ + '.id == $run_id and .run_number == $run_number and + .run_attempt == 1 and .event == "pull_request_target" and + (.path == $path or .path == ($path + "@" + $base_ref) or + .path == ($path + "@refs/heads/" + $base_ref)) and + .head_sha == $head_sha and + .repository.full_name == $repository and + ((.pull_requests | type) == "array") and + .status == "completed" and + .conclusion == "failure" and + (.created_at | type == "string" and length > 0) and + (.run_started_at | type == "string" and length > 0)' \ + <<<"$first_attempt" >/dev/null 2>&1; then + authorization_boundary=$(jq -r '.run_started_at' <<<"$first_attempt") + fi fi - gh api --method PATCH \ - "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ + events_path="$RUNNER_TEMP/llamacpp-pr-events-${PR_NUMBER}.json" + gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f status=completed \ - -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=Superseded by a newer authorized validation run." >/dev/null - done <<<"$prior_check_ids" + -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path" + provenance_arguments=(provenance --pages "$events_path") + if [ -n "$authorization_boundary" ]; then + provenance_arguments+=(--authorization-after "$authorization_boundary") + fi + provenance=$(python \ + trusted/.github/scripts/llamacpp_required_check_policy.py \ + "${provenance_arguments[@]}") + authorization_actor=$(jq -er '.authorization_actor' <<<"$provenance") + authorization_actor_id=$(jq -er '.authorization_actor_id' <<<"$provenance") + authorization_created_at=$(jq -er '.authorization_created_at' <<<"$provenance") + authorization_epoch_valid=$(jq -er \ + '.epoch_valid | if type == "boolean" then tostring else error("invalid epoch") end' \ + <<<"$provenance") + authorization_event_id=$(jq -er '.authorization_event_id' <<<"$provenance") + if [ -n "$authorization_actor" ]; then + permission_response=$(gh api \ + "repos/${GITHUB_REPOSITORY}/collaborators/${authorization_actor}/permission" \ + 2>/dev/null || true) + if jq -e --argjson actor_id "$authorization_actor_id" \ + '.user.id == $actor_id and (.permission | type == "string")' \ + <<<"$permission_response" >/dev/null 2>&1; then + actor_permission=$(jq -r '.permission' <<<"$permission_response") + fi + fi + if [ -n "$authorization_boundary" ] && \ + [ "$TRIGGERING_ACTOR" = "$authorization_actor" ]; then + authorization_rerun_matches=true + fi - echo "check_run_id=$check_run_id" >> "$GITHUB_OUTPUT" + if [ "$DEPENDABOT_PR" != "true" ]; then + external_prefix="lemonade-llamacpp-required-pr-${PR_NUMBER}-" + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) + IFS=$'\t' read -r newest_check_epoch newest_check_number \ + newest_check_attempt newest_check_status newest_check_conclusion < <( + jq -r --arg check_name "$CHECK_NAME" \ + --arg external_prefix "$external_prefix" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | . as $check + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {epoch: (.epoch | tonumber), number: (.number | tonumber), + attempt: (.attempt | tonumber), + status: ($check.status // ""), + conclusion: ($check.conclusion // "")}) catch empty] + | sort_by(.number, .attempt) + | (last // {epoch: 0, number: 0, attempt: 0, + status: "", conclusion: ""}) + | [.epoch, .number, .attempt, .status, .conclusion] + | @tsv' <<<"$check_runs" + ) + if [ "$newest_check_epoch" = "$authorization_event_id" ] && \ + [ "$newest_check_status" = "completed" ] && \ + [ "$newest_check_conclusion" = "success" ]; then + has_existing_check=true + fi + if [ "$newest_check_number" -gt "$CURRENT_RUN_NUMBER" ] || \ + { [ "$newest_check_number" -eq "$CURRENT_RUN_NUMBER" ] && \ + [ "$newest_check_attempt" -gt "$CURRENT_RUN_ATTEMPT" ]; }; then + has_newer_check=true + fi + fi + fi + + python trusted/.github/scripts/llamacpp_required_check_policy.py decide \ + --action "$EVENT_ACTION" --event-label "$EVENT_LABEL" \ + --base-ref-changed "$BASE_REF_CHANGED" --state "$STATE" \ + --merged "$MERGED" \ + --merge-candidate-available "$MERGE_CANDIDATE_AVAILABLE" \ + --relevant "$RELEVANT" \ + --identity-matches "$IDENTITY_MATCHES" \ + --authorization-label-present "$AUTHORIZATION_LABEL_PRESENT" \ + --authorization-epoch-valid "$authorization_epoch_valid" \ + --authorization-rerun-matches "$authorization_rerun_matches" \ + --actor-permission "$actor_permission" \ + --has-existing-check "$has_existing_check" \ + --has-newer-check "$has_newer_check" \ + --required-workflow-only "$DEPENDABOT_PR" \ + --github-output "$GITHUB_OUTPUT" + { + echo "authorization_actor=$authorization_actor" + echo "authorization_actor_id=$authorization_actor_id" + echo "authorization_boundary=$authorization_boundary" + echo "authorization_created_at=$authorization_created_at" + echo "authorization_event_id=$authorization_event_id" + } >> "$GITHUB_OUTPUT" validate: name: Run authorized validation - needs: [authorize-invocation, start-check] - if: >- - needs.authorize-invocation.outputs.run_validation == 'true' && - needs.start-check.result == 'success' + needs: policy + if: needs.policy.result == 'success' && needs.policy.outputs.run_validation == 'true' permissions: + checks: write contents: read uses: ./.github/workflows/validate_llamacpp_core.yml + with: + authorization_epoch: ${{ needs.policy.outputs.authorization_event_id }} - report: - name: Report authorized validation check - needs: [authorize-invocation, start-check, validate] - if: >- - always() && - (github.event_name == 'merge_group' || - (github.event.action == 'labeled' && - github.event.label.name == 'ci:upgrades')) + reconcile: + name: Reconcile required validation check + needs: [policy, validate] + if: always() runs-on: ubuntu-latest permissions: checks: write contents: read + issues: read pull-requests: read steps: - - name: Assess authorized validation - id: assess + - name: Check out trusted reconciliation policy code + if: >- + needs.policy.result == 'success' && + (needs.policy.outputs.mode == 'validate' || + needs.policy.outputs.mode == 'preserve') + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + repository: ${{ github.repository }} + ref: ${{ needs.policy.outputs.base_sha }} + path: trusted-reconcile + persist-credentials: false + fetch-depth: 1 + + - name: Assess policy and validation + id: assess-policy + if: always() env: - AUTHORIZATION_RESULT: ${{ needs.authorize-invocation.result }} - RUN_VALIDATION: ${{ needs.authorize-invocation.outputs.run_validation }} - START_CHECK_RESULT: ${{ needs.start-check.result }} + EVENT_ACTION: ${{ github.event.action || '' }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} + EVENT_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha || '' }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MODE: ${{ needs.policy.outputs.mode }} + POLICY_CONCLUSION: ${{ needs.policy.outputs.conclusion }} + POLICY_RESULT: ${{ needs.policy.result }} + POLICY_SUMMARY: ${{ needs.policy.outputs.summary }} + PR_NUMBER: ${{ github.event.pull_request.number || '' }} VALIDATION_RESULT: ${{ needs.validate.result }} run: | set -euo pipefail - - conclusion=failure + mode="${MODE:-policy_error}" + conclusion="${POLICY_CONCLUSION:-failure}" + summary="${POLICY_SUMMARY:-Protected validation policy resolution failed.}" passed=false - summary="Invocation authorization failed." - if [ "$AUTHORIZATION_RESULT" = "success" ] && [ "$RUN_VALIDATION" = "true" ]; then - if [ "$START_CHECK_RESULT" != "success" ]; then - summary="Could not start the authorized merge validation check." - elif [ "$VALIDATION_RESULT" = "success" ]; then - conclusion=success + fallback_base_ref="" + fallback_base_sha="" + fallback_head_sha="" + fallback_merge_sha="" + fallback_observed_merge_sha="" + fallback_secondary_observed_merge_sha="" + + if [ "$POLICY_RESULT" != "success" ]; then + live_closed=false + live_identity_valid=false + if [[ "${PR_NUMBER:-}" =~ ^[0-9]+$ ]]; then + pull_request=$(gh api \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" 2>/dev/null || true) + if jq -e --argjson number "$PR_NUMBER" \ + '.number == $number and (.state == "open" or .state == "closed") and + (.merged | type == "boolean") and (.base.ref | type == "string") and + (.base.sha | type == "string") and (.head.sha | type == "string") and + ((.mergeable == null) or (.mergeable | type == "boolean")) and + ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string"))' \ + <<<"$pull_request" >/dev/null 2>&1; then + current_state=$(jq -r '.state' <<<"$pull_request") + current_merged=$(jq -r '.merged' <<<"$pull_request") + current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") + current_base=$(jq -r '.base.sha' <<<"$pull_request") + current_head=$(jq -r '.head.sha' <<<"$pull_request") + current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") + current_mergeable=$(jq -r \ + '.mergeable | if . == null then "null" else tostring end' \ + <<<"$pull_request") + if git check-ref-format "refs/heads/${current_base_ref}" >/dev/null && \ + [[ "$current_base" =~ ^[0-9a-fA-F]{40}$ ]] && \ + [[ "$current_head" =~ ^[0-9a-fA-F]{40}$ ]] && \ + { [ -z "$current_merge" ] || \ + [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; } && \ + { [ "$current_merged" = "false" ] || \ + [ "$current_state" = "closed" ]; }; then + live_identity_valid=true + if [ "$current_state" = "closed" ]; then + live_closed=true + else + fallback_base_ref="$current_base_ref" + fallback_base_sha="$current_base" + fallback_head_sha="$current_head" + fallback_merge_sha="$current_head" + if [ "$current_mergeable" = "true" ] && \ + [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; then + fallback_merge_sha="$current_merge" + elif [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; then + fallback_observed_merge_sha="$current_merge" + fi + if [ "$current_mergeable" != "true" ] && \ + [[ "$EVENT_MERGE_SHA" =~ ^[0-9a-fA-F]{40}$ ]] && \ + [ "$EVENT_MERGE_SHA" != "$fallback_observed_merge_sha" ]; then + if [ -z "$fallback_observed_merge_sha" ]; then + fallback_observed_merge_sha="$EVENT_MERGE_SHA" + else + fallback_secondary_observed_merge_sha="$EVENT_MERGE_SHA" + fi + fi + fi + fi + fi + fi + if [ "$live_identity_valid" = "false" ] && \ + [[ "${EVENT_BASE_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] && \ + [[ "${EVENT_HEAD_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] && \ + git check-ref-format "refs/heads/${EVENT_BASE_REF:-}" >/dev/null; then + fallback_base_ref="$EVENT_BASE_REF" + fallback_base_sha="$EVENT_BASE_SHA" + fallback_head_sha="$EVENT_HEAD_SHA" + fallback_merge_sha="$EVENT_HEAD_SHA" + if [[ "${EVENT_MERGE_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]]; then + fallback_observed_merge_sha="$EVENT_MERGE_SHA" + fi + fi + if [ "$live_closed" = "true" ]; then + mode=skip_closed + conclusion="" + summary="The pull request is still closed; no commit check was changed." passed=true - summary="All authorized llama.cpp validation jobs passed." else - summary="Authorized validation result: $VALIDATION_RESULT." + mode=policy_error + conclusion=failure + summary="Protected validation policy resolution failed closed." fi + else + case "$mode" in + validate) + if [ "$VALIDATION_RESULT" = "success" ]; then + conclusion=success + summary="All authorized llama.cpp validation jobs passed." + passed=true + else + conclusion=failure + summary="Authorized validation result: $VALIDATION_RESULT." + fi + ;; + neutral) + [ "$conclusion" = "success" ] && passed=true + ;; + awaiting_rerun) + conclusion=failure + ;; + superseded) + conclusion=failure + ;; + stale_required) + conclusion=failure + ;; + preserve|stale|merged|closed) + passed=true + ;; + *) + conclusion=failure + ;; + esac fi - { + echo "mode=$mode" echo "conclusion=$conclusion" - echo "passed=$passed" echo "summary=$summary" + echo "passed=$passed" + echo "fallback_base_ref=$fallback_base_ref" + echo "fallback_base_sha=$fallback_base_sha" + echo "fallback_head_sha=$fallback_head_sha" + echo "fallback_merge_sha=$fallback_merge_sha" + echo "fallback_observed_merge_sha=$fallback_observed_merge_sha" + echo "fallback_secondary_observed_merge_sha=$fallback_secondary_observed_merge_sha" } >> "$GITHUB_OUTPUT" - - name: Report result on the authorized merge commit + - name: Preserve the exact-commit gate for unrelated metadata + id: preserve-gate + if: needs.policy.outputs.mode == 'preserve' && env.DEPENDABOT_PR != 'true' + env: + AUTHORIZATION_ACTOR: ${{ needs.policy.outputs.authorization_actor }} + AUTHORIZATION_ACTOR_ID: ${{ needs.policy.outputs.authorization_actor_id }} + AUTHORIZATION_CREATED_AT: ${{ needs.policy.outputs.authorization_created_at }} + AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id }} + BASE_REF: ${{ needs.policy.outputs.base_ref }} + BASE_SHA: ${{ needs.policy.outputs.base_sha }} + CHECK_NAME: llama.cpp authorized validation v2 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ needs.policy.outputs.head_sha }} + MERGE_SHA: ${{ needs.policy.outputs.merge_sha }} + POLICY_HELPER: trusted-reconcile/.github/scripts/llamacpp_required_check_policy.py + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_ATTEMPT: ${{ github.run_attempt }} + RUN_NUMBER: ${{ github.run_number }} + run: | + set -euo pipefail + preserve_valid=false + external_prefix="lemonade-llamacpp-required-pr-${PR_NUMBER}-" + pull_request=$(gh api \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" 2>/dev/null || true) + if jq -e --argjson number "$PR_NUMBER" \ + '.number == $number and .state == "open" and .merged == false and + (.base.ref | type == "string") and (.base.sha | type == "string") and + (.head.sha | type == "string") and + ((.mergeable == null) or (.mergeable | type == "boolean")) and + ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string")) and + any(.labels[]?; .name == "ci:upgrades")' \ + <<<"$pull_request" >/dev/null 2>&1; then + current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") + current_base=$(jq -r '.base.sha' <<<"$pull_request") + current_head=$(jq -r '.head.sha' <<<"$pull_request") + current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") + current_mergeable=$(jq -r \ + '.mergeable | if . == null then "null" else tostring end' \ + <<<"$pull_request") + current_candidate="$current_head" + if [ "$current_mergeable" = "true" ] && \ + [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; then + current_candidate="$current_merge" + fi + if git check-ref-format "refs/heads/${current_base_ref}" >/dev/null && \ + [[ "$current_base" =~ ^[0-9a-fA-F]{40}$ ]] && \ + [[ "$current_head" =~ ^[0-9a-fA-F]{40}$ ]] && \ + { [ -z "$current_merge" ] || \ + [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; } && \ + [ "$current_base_ref" = "$BASE_REF" ] && \ + [ "$current_base" = "$BASE_SHA" ] && \ + [ "$current_head" = "$HEAD_SHA" ] && \ + [ "$current_candidate" = "$MERGE_SHA" ] && \ + [[ "$AUTHORIZATION_EPOCH" =~ ^[1-9][0-9]*$ ]] && \ + [[ "$AUTHORIZATION_ACTOR_ID" =~ ^[1-9][0-9]*$ ]] && \ + [[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] && \ + [[ "$RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] && \ + [ -f "$POLICY_HELPER" ]; then + events_path=$(mktemp "${RUNNER_TEMP}/llamacpp-preserve-events.XXXXXX") + if gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path"; then + provenance=$(python "$POLICY_HELPER" provenance \ + --pages "$events_path" 2>/dev/null || true) + if jq -e --arg actor "$AUTHORIZATION_ACTOR" \ + --arg created_at "$AUTHORIZATION_CREATED_AT" \ + --argjson actor_id "$AUTHORIZATION_ACTOR_ID" \ + --argjson epoch "$AUTHORIZATION_EPOCH" \ + '.authorization_actor == $actor and + .authorization_actor_id == $actor_id and + .authorization_created_at == $created_at and + .authorization_event_id == $epoch and + .authorization_event == "labeled" and .epoch_valid == true' \ + <<<"$provenance" >/dev/null 2>&1; then + permission_response=$(gh api \ + "repos/${GITHUB_REPOSITORY}/collaborators/${AUTHORIZATION_ACTOR}/permission" \ + 2>/dev/null || true) + if jq -e --argjson actor_id "$AUTHORIZATION_ACTOR_ID" \ + '.user.id == $actor_id and + (.permission == "admin" or .permission == "maintain" or + .permission == "push" or .permission == "write")' \ + <<<"$permission_response" >/dev/null 2>&1; then + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" -f filter=all -F per_page=100 \ + 2>/dev/null || true) + IFS=$'\t' read -r newest_check_epoch newest_check_number \ + newest_check_attempt newest_check_status \ + newest_check_conclusion < <( + jq -r --arg check_name "$CHECK_NAME" \ + --arg external_prefix "$external_prefix" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and + .app.slug == "github-actions") + | select((.external_id // "") | + startswith($external_prefix)) + | . as $check + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {epoch: (.epoch | tonumber), + number: (.number | tonumber), + attempt: (.attempt | tonumber), + status: ($check.status // ""), + conclusion: ($check.conclusion // "")}) catch empty] + | sort_by(.number, .attempt) + | (last // {epoch: 0, number: 0, attempt: 0, + status: "", conclusion: ""}) + | [.epoch, .number, .attempt, .status, .conclusion] + | @tsv' <<<"$check_runs" + ) + if [ "$newest_check_epoch" = "$AUTHORIZATION_EPOCH" ] && \ + [ "$newest_check_status" = "completed" ] && \ + [ "$newest_check_conclusion" = "success" ] && \ + { [ "$newest_check_number" -lt "$RUN_NUMBER" ] || \ + { [ "$newest_check_number" -eq "$RUN_NUMBER" ] && \ + [ "$newest_check_attempt" -le "$RUN_ATTEMPT" ]; }; }; then + preserve_valid=true + fi + fi + fi + fi + rm -f "$events_path" + fi + fi + echo "valid=$preserve_valid" >> "$GITHUB_OUTPUT" + if [ "$preserve_valid" != "true" ]; then + echo "::error::The live authorization epoch cannot preserve this exact-commit gate." + fi + + - name: Revalidate an authorized Dependabot result + if: >- + always() && + env.DEPENDABOT_PR == 'true' && + steps.assess-policy.outputs.mode == 'validate' && + steps.assess-policy.outputs.conclusion == 'success' + env: + AUTHORIZATION_ACTOR: ${{ needs.policy.outputs.authorization_actor }} + AUTHORIZATION_ACTOR_ID: ${{ needs.policy.outputs.authorization_actor_id }} + AUTHORIZATION_BOUNDARY: ${{ needs.policy.outputs.authorization_boundary }} + AUTHORIZATION_CREATED_AT: ${{ needs.policy.outputs.authorization_created_at }} + AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id }} + BASE_REF: ${{ needs.policy.outputs.base_ref }} + BASE_SHA: ${{ needs.policy.outputs.base_sha }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ needs.policy.outputs.head_sha }} + MERGE_SHA: ${{ needs.policy.outputs.merge_sha }} + POLICY_HELPER: trusted-reconcile/.github/scripts/llamacpp_required_check_policy.py + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + oid_pattern='^[0-9a-fA-F]{40}$' + pull_request=$(gh api \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") + if ! jq -e --argjson number "$PR_NUMBER" \ + '.number == $number and .state == "open" and .merged == false and + (.base.ref | type == "string") and (.base.sha | type == "string") and + (.head.sha | type == "string") and + ((.mergeable == null) or (.mergeable | type == "boolean")) and + ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string"))' \ + <<<"$pull_request" >/dev/null; then + echo "::error::The live Dependabot pull request identity is invalid." + exit 1 + fi + current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") + current_base=$(jq -r '.base.sha' <<<"$pull_request") + current_head=$(jq -r '.head.sha' <<<"$pull_request") + current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") + current_mergeable=$(jq -r \ + '.mergeable | if . == null then "null" else tostring end' \ + <<<"$pull_request") + if ! git check-ref-format "refs/heads/${current_base_ref}" >/dev/null || \ + ! [[ "$current_base" =~ $oid_pattern && "$current_head" =~ $oid_pattern ]] || \ + { [ -n "$current_merge" ] && ! [[ "$current_merge" =~ $oid_pattern ]]; }; then + echo "::error::The live Dependabot pull request identity is invalid." + exit 1 + fi + current_candidate="$current_head" + if [ "$current_mergeable" = "true" ] && \ + [[ "$current_merge" =~ $oid_pattern ]]; then + current_candidate="$current_merge" + fi + if [ "$current_base_ref" != "$BASE_REF" ] || \ + [ "$current_base" != "$BASE_SHA" ] || \ + [ "$current_head" != "$HEAD_SHA" ] || \ + [ "$current_candidate" != "$MERGE_SHA" ]; then + echo "::error::The Dependabot candidate changed during validation." + exit 1 + fi + current_authorized=$(jq -r \ + 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") + if [ "$current_authorized" != "true" ] || \ + ! [[ "$AUTHORIZATION_EPOCH" =~ ^[1-9][0-9]*$ ]] || \ + ! [[ "$AUTHORIZATION_ACTOR_ID" =~ ^[1-9][0-9]*$ ]] || \ + [ -z "$AUTHORIZATION_BOUNDARY" ] || [ ! -f "$POLICY_HELPER" ]; then + echo "::error::The Dependabot authorization is no longer live." + exit 1 + fi + + events_path=$(mktemp "${RUNNER_TEMP:-/tmp}/llamacpp-dependabot-events.XXXXXX") + trap 'rm -f "$events_path"' EXIT + gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path" + provenance=$(python "$POLICY_HELPER" provenance --pages "$events_path" \ + --authorization-after "$AUTHORIZATION_BOUNDARY") + authorization_actor=$(jq -er '.authorization_actor' <<<"$provenance") + authorization_actor_id=$(jq -er '.authorization_actor_id' <<<"$provenance") + authorization_created_at=$(jq -er '.authorization_created_at' <<<"$provenance") + authorization_event_id=$(jq -er '.authorization_event_id' <<<"$provenance") + authorization_event=$(jq -er '.authorization_event' <<<"$provenance") + authorization_epoch_valid=$(jq -er \ + '.epoch_valid | if type == "boolean" then tostring else error("invalid epoch") end' \ + <<<"$provenance") + if [ "$authorization_actor" != "$AUTHORIZATION_ACTOR" ] || \ + [ "$authorization_actor_id" != "$AUTHORIZATION_ACTOR_ID" ] || \ + [ "$authorization_created_at" != "$AUTHORIZATION_CREATED_AT" ] || \ + [ "$authorization_event_id" != "$AUTHORIZATION_EPOCH" ] || \ + [ "$authorization_event" != "labeled" ] || \ + [ "$authorization_epoch_valid" != "true" ]; then + echo "::error::The Dependabot authorization epoch changed during validation." + exit 1 + fi + permission_response=$(gh api \ + "repos/${GITHUB_REPOSITORY}/collaborators/${authorization_actor}/permission") + if ! jq -e --argjson actor_id "$authorization_actor_id" \ + '.user.id == $actor_id and + (.permission == "admin" or .permission == "maintain" or + .permission == "push" or .permission == "write")' \ + <<<"$permission_response" >/dev/null; then + echo "::error::The Dependabot authorization actor lost write permission." + exit 1 + fi + + - name: Invalidate an unconfirmed observed merge commit + if: >- + always() && + env.DEPENDABOT_PR != 'true' && + (needs.policy.outputs.observed_merge_sha != '' || + needs.policy.outputs.secondary_observed_merge_sha != '' || + steps.assess-policy.outputs.fallback_observed_merge_sha != '' || + steps.assess-policy.outputs.fallback_secondary_observed_merge_sha != '') && + steps.assess-policy.outputs.mode != 'preserve' && + steps.assess-policy.outputs.mode != 'awaiting_rerun' && + steps.assess-policy.outputs.mode != 'superseded' && + steps.assess-policy.outputs.mode != 'stale_required' && + steps.assess-policy.outputs.mode != 'stale' && + steps.assess-policy.outputs.mode != 'merged' && + steps.assess-policy.outputs.mode != 'skip_closed' + env: + AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id || 0 }} + CHECK_NAME: llama.cpp authorization diagnostics v2 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OBSERVED_MERGE_SHA: ${{ steps.assess-policy.outputs.fallback_observed_merge_sha || needs.policy.outputs.observed_merge_sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_ATTEMPT: ${{ github.run_attempt }} + RUN_ID: ${{ github.run_id }} + RUN_NUMBER: ${{ github.run_number }} + SECONDARY_OBSERVED_MERGE_SHA: ${{ steps.assess-policy.outputs.fallback_secondary_observed_merge_sha || needs.policy.outputs.secondary_observed_merge_sha }} + run: | + set -euo pipefail + external_prefix="lemonade-llamacpp-metadata-pr-${PR_NUMBER}-" + if ! [[ "$AUTHORIZATION_EPOCH" =~ ^[0-9]+$ ]]; then + AUTHORIZATION_EPOCH=0 + fi + + invalidate_observed_merge() { + local observed_merge_sha="$1" + local check_runs check_run_id newest_number newest_attempt prior_check_ids + local external_id summary + if ! [[ "$observed_merge_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::The observed merge commit must be a full Git OID." + return 1 + fi + + external_id="${external_prefix}${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}" + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${observed_merge_sha}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) + IFS=$'\t' read -r newest_number newest_attempt < <( + jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] + | sort_by(.number, .attempt) + | (last // {number: 0, attempt: 0}) + | [.number, .attempt] | @tsv' <<<"$check_runs" + ) + if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ + { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ + [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then + echo "::notice::A newer policy run already invalidated ${observed_merge_sha}." + return 0 + fi + + check_run_id=$(jq -r --arg check_name "$CHECK_NAME" \ + --arg external_id "$external_id" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions" and + .external_id == $external_id) + | .id] | last // empty' <<<"$check_runs") + summary="GitHub did not confirm this observed synthetic merge commit as executable." + if [[ "$check_run_id" =~ ^[0-9]+$ ]]; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" -f status=completed -f conclusion=failure \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=$CHECK_NAME" -f "output[summary]=$summary" >/dev/null + else + response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" -f head_sha="$observed_merge_sha" \ + -f external_id="$external_id" -f status=completed -f conclusion=failure \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=$CHECK_NAME" -f "output[summary]=$summary") + check_run_id=$(jq -r '.id // empty' <<<"$response") + fi + if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid observed-merge check-run ID." + return 1 + fi + + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${observed_merge_sha}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) + IFS=$'\t' read -r newest_number newest_attempt < <( + jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] + | sort_by(.number, .attempt) + | (last // {number: 0, attempt: 0}) + | [.number, .attempt] | @tsv' <<<"$check_runs" + ) + if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ + { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ + [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=Superseded by a newer policy decision." >/dev/null + return 0 + fi + + prior_check_ids=$(jq -r --arg check_name "$CHECK_NAME" \ + --arg external_prefix "$external_prefix" \ + --argjson current_number "$RUN_NUMBER" --argjson current_attempt "$RUN_ATTEMPT" \ + '.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | . as $check + | try (($check.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty + | select(.number < $current_number or + (.number == $current_number and .attempt < $current_attempt)) + | $check.id' <<<"$check_runs") + while IFS= read -r prior_check_id; do + [ -n "$prior_check_id" ] || continue + [[ "$prior_check_id" =~ ^[0-9]+$ ]] || return 1 + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=Superseded by a newer policy decision." >/dev/null + done <<<"$prior_check_ids" + } + + invalidate_observed_merge "$OBSERVED_MERGE_SHA" + if [ -n "$SECONDARY_OBSERVED_MERGE_SHA" ] && \ + [ "$SECONDARY_OBSERVED_MERGE_SHA" != "$OBSERVED_MERGE_SHA" ]; then + invalidate_observed_merge "$SECONDARY_OBSERVED_MERGE_SHA" + fi + + - name: Report result on the exact candidate commit if: >- always() && - (github.event_name == 'merge_group' || - (github.event.action == 'labeled' && - github.event.label.name == 'ci:upgrades')) + env.DEPENDABOT_PR != 'true' && + (steps.assess-policy.outputs.mode != 'preserve' || + steps.preserve-gate.outputs.valid != 'true') && + steps.assess-policy.outputs.mode != 'awaiting_rerun' && + steps.assess-policy.outputs.mode != 'superseded' && + steps.assess-policy.outputs.mode != 'stale_required' && + steps.assess-policy.outputs.mode != 'stale' && + steps.assess-policy.outputs.mode != 'merged' && + steps.assess-policy.outputs.mode != 'skip_closed' env: - BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} + AUTHORIZATION_ACTOR: ${{ needs.policy.outputs.authorization_actor || '' }} + AUTHORIZATION_ACTOR_ID: ${{ needs.policy.outputs.authorization_actor_id || 0 }} + AUTHORIZATION_BOUNDARY: ${{ needs.policy.outputs.authorization_boundary || '' }} + AUTHORIZATION_CREATED_AT: ${{ needs.policy.outputs.authorization_created_at || '' }} + AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id || 0 }} + BASE_REF: ${{ steps.assess-policy.outputs.fallback_base_ref || needs.policy.outputs.base_ref || github.event.pull_request.base.ref || '' }} + BASE_SHA: ${{ steps.assess-policy.outputs.fallback_base_sha || needs.policy.outputs.base_sha || github.event.pull_request.base.sha || '' }} CHECK_NAME: llama.cpp authorized validation v2 - CHECK_RUN_ID: ${{ needs.start-check.outputs.check_run_id }} - CONCLUSION: ${{ steps.assess.outputs.conclusion || 'failure' }} + CHECK_RUN_ID: ${{ needs.validate.outputs.check_run_id }} + CONCLUSION: ${{ steps.assess-policy.outputs.conclusion || 'failure' }} + EVENT_ACTION: ${{ github.event.action || '' }} EVENT_NAME: ${{ github.event_name }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} - MERGE_SHA: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.event.pull_request.merge_commit_sha }} + HEAD_SHA: ${{ steps.assess-policy.outputs.fallback_head_sha || needs.policy.outputs.head_sha || github.event.pull_request.head.sha || '' }} + MERGE_SHA: ${{ steps.assess-policy.outputs.fallback_merge_sha || needs.policy.outputs.merge_sha || github.event.merge_group.head_sha || github.event.pull_request.head.sha || '' }} + METADATA_CHECK_NAME: llama.cpp authorization diagnostics v2 + MODE: ${{ steps.assess-policy.outputs.mode }} + POLICY_HELPER: trusted-reconcile/.github/scripts/llamacpp_required_check_policy.py + PRESERVE_VALID: ${{ steps.preserve-gate.outputs.valid || 'false' }} PR_NUMBER: ${{ github.event.pull_request.number || '' }} RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} - SUMMARY: ${{ steps.assess.outputs.summary || 'Validation assessment failed.' }} + RUN_NUMBER: ${{ github.run_number }} + SUMMARY: ${{ steps.assess-policy.outputs.summary || 'Validation assessment failed.' }} TARGET_KEY: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}', github.event.pull_request.number) }} run: | set -euo pipefail - EXTERNAL_PREFIX="lemonade-llamacpp-${TARGET_KEY}-" - EXTERNAL_ID="${EXTERNAL_PREFIX}${RUN_ID}-${RUN_ATTEMPT}" - stale=false - if [ "$CONCLUSION" = "success" ] && [ "$EVENT_NAME" = "pull_request_target" ]; then - pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") + oid_pattern='^[0-9a-fA-F]{40}$' + if ! [[ "$MERGE_SHA" =~ $oid_pattern ]]; then + echo "::error::Cannot report a required check without an exact candidate commit." + exit 1 + fi + + if ! [[ "$AUTHORIZATION_EPOCH" =~ ^[0-9]+$ ]]; then + AUTHORIZATION_EPOCH=0 + fi + if [ "$MODE" = "preserve" ] && [ "$PRESERVE_VALID" != "true" ]; then + CONCLUSION=failure + SUMMARY="The live authorization epoch could not preserve this exact-commit gate." + fi + + verify_live_candidate() { + local pull_request current_number current_state current_merged + local current_base_ref current_base current_head current_merge + local current_mergeable current_candidate current_authorized + local events_path provenance permission_response + local authorization_actor authorization_actor_id + local authorization_created_at authorization_event_id + local authorization_event authorization_epoch_valid + pull_request=$(gh api \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") || return 1 + if ! jq -e --argjson number "$PR_NUMBER" \ + '.number == $number and .state == "open" and .merged == false and + (.base.ref | type == "string") and (.base.sha | type == "string") and + (.head.sha | type == "string") and + ((.mergeable == null) or (.mergeable | type == "boolean")) and + ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string"))' \ + <<<"$pull_request" >/dev/null; then + return 1 + fi + current_number=$(jq -r '.number' <<<"$pull_request") current_state=$(jq -r '.state' <<<"$pull_request") + current_merged=$(jq -r '.merged' <<<"$pull_request") + current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") current_base=$(jq -r '.base.sha' <<<"$pull_request") current_head=$(jq -r '.head.sha' <<<"$pull_request") current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") - current_authorized=$(jq -r \ - 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") - if [ "$current_state" != "open" ] || \ - [ "$current_authorized" != "true" ] || \ + current_mergeable=$(jq -r \ + '.mergeable | if . == null then "null" else tostring end' \ + <<<"$pull_request") + if [ "$current_number" != "$PR_NUMBER" ] || \ + [ "$current_state" != "open" ] || [ "$current_merged" != "false" ] || \ + ! git check-ref-format "refs/heads/${current_base_ref}" >/dev/null || \ + ! [[ "$current_base" =~ $oid_pattern && "$current_head" =~ $oid_pattern ]] || \ + { [ -n "$current_merge" ] && ! [[ "$current_merge" =~ $oid_pattern ]]; }; then + return 1 + fi + current_candidate="$current_head" + if [ "$current_mergeable" = "true" ] && [[ "$current_merge" =~ $oid_pattern ]]; then + current_candidate="$current_merge" + fi + if [ "$current_base_ref" != "$BASE_REF" ] || \ [ "$current_base" != "$BASE_SHA" ] || \ [ "$current_head" != "$HEAD_SHA" ] || \ - [ "$current_merge" != "$MERGE_SHA" ]; then - stale=true - CONCLUSION=failure - SUMMARY="The pull request authorization or merge identity changed during validation." + [ "$current_candidate" != "$MERGE_SHA" ]; then + return 1 + fi + + if [ "$MODE" != "validate" ]; then + return 0 + fi + current_authorized=$(jq -r \ + 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") + [ "$current_authorized" = "true" ] || return 1 + [ -n "$AUTHORIZATION_BOUNDARY" ] || return 1 + [ "$AUTHORIZATION_EPOCH" -gt 0 ] || return 1 + [ -f "$POLICY_HELPER" ] || return 1 + events_path=$(mktemp "${RUNNER_TEMP:-/tmp}/llamacpp-reconcile-events.XXXXXX") + if ! gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path"; then + rm -f "$events_path" + return 1 + fi + provenance=$(python "$POLICY_HELPER" provenance --pages "$events_path" \ + --authorization-after "$AUTHORIZATION_BOUNDARY") || { + rm -f "$events_path" + return 1 + } + rm -f "$events_path" + authorization_actor=$(jq -er '.authorization_actor' <<<"$provenance") || return 1 + authorization_actor_id=$(jq -er '.authorization_actor_id' <<<"$provenance") || return 1 + authorization_created_at=$(jq -er '.authorization_created_at' <<<"$provenance") || return 1 + authorization_event_id=$(jq -er '.authorization_event_id' <<<"$provenance") || return 1 + authorization_event=$(jq -er '.authorization_event' <<<"$provenance") || return 1 + authorization_epoch_valid=$(jq -er \ + '.epoch_valid | if type == "boolean" then tostring else error("invalid epoch") end' \ + <<<"$provenance") || return 1 + if [ "$authorization_actor" != "$AUTHORIZATION_ACTOR" ] || \ + [ "$authorization_actor_id" != "$AUTHORIZATION_ACTOR_ID" ] || \ + [ "$authorization_created_at" != "$AUTHORIZATION_CREATED_AT" ] || \ + [ "$authorization_event_id" != "$AUTHORIZATION_EPOCH" ] || \ + [ "$authorization_event" != "labeled" ] || \ + [ "$authorization_epoch_valid" != "true" ]; then + return 1 + fi + permission_response=$(gh api \ + "repos/${GITHUB_REPOSITORY}/collaborators/${authorization_actor}/permission") || \ + return 1 + if ! jq -e --argjson actor_id "$authorization_actor_id" \ + '.user.id == $actor_id and + (.permission == "admin" or .permission == "maintain" or + .permission == "push" or .permission == "write")' \ + <<<"$permission_response" >/dev/null; then + return 1 fi + } + + stale=false + if [ "$CONCLUSION" = "success" ] && \ + [ "$EVENT_NAME" = "pull_request_target" ] && \ + ! verify_live_candidate; then + stale=true + CONCLUSION=failure + SUMMARY="The pull request identity or authorization changed before reporting." + fi + + decision_scope=metadata + if [ "$EVENT_NAME" = "merge_group" ] || \ + [ "$EVENT_ACTION" = "opened" ] || \ + [ "$EVENT_ACTION" = "reopened" ] || \ + [ "$EVENT_ACTION" = "synchronize" ]; then + decision_scope=required fi + if [ "$decision_scope" = "metadata" ]; then + CHECK_NAME="$METADATA_CHECK_NAME" + fi + external_prefix="lemonade-llamacpp-${decision_scope}-${TARGET_KEY}-" + external_id="${external_prefix}${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}" + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) + IFS=$'\t' read -r newest_number newest_attempt < <( + jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] + | sort_by(.number, .attempt) + | (last // {number: 0, attempt: 0}) + | [.number, .attempt] | @tsv' <<<"$check_runs" + ) + if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ + { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ + [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then + echo "::error::A newer policy run superseded this required-workflow run." + exit 1 + fi + + if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then + CHECK_RUN_ID=$(jq -r --argjson check_run_id "$CHECK_RUN_ID" \ + --arg check_name "$CHECK_NAME" --arg external_id "$external_id" \ + '[.[] | .check_runs[]? + | select(.id == $check_run_id and .name == $check_name and + .app.slug == "github-actions" and .external_id == $external_id) + | .id] | last // empty' <<<"$check_runs") + fi if ! [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then - check_runs=$(gh api --method GET \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" \ - -f filter=all \ - -F per_page=100) - CHECK_RUN_ID=$(jq -r --arg external_id "$EXTERNAL_ID" \ - '[.check_runs[] | select(.external_id == $external_id) | .id] | last // empty' \ - <<<"$check_runs") + CHECK_RUN_ID=$(jq -r --arg check_name "$CHECK_NAME" \ + --arg external_id "$external_id" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions" and + .external_id == $external_id) + | .id] | last // empty' <<<"$check_runs") fi if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then - gh api --method PATCH \ - "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f status=completed \ - -f conclusion="$CONCLUSION" \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=$SUMMARY" >/dev/null + -f name="$CHECK_NAME" -f status=completed -f conclusion="$CONCLUSION" \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=$CHECK_NAME" -f "output[summary]=$SUMMARY" >/dev/null else - gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ + response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f head_sha="$MERGE_SHA" \ - -f external_id="$EXTERNAL_ID" \ - -f status=completed \ - -f conclusion="$CONCLUSION" \ + -f name="$CHECK_NAME" -f head_sha="$MERGE_SHA" -f external_id="$external_id" \ + -f status=completed -f conclusion="$CONCLUSION" \ -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=$SUMMARY" >/dev/null + -f "output[title]=$CHECK_NAME" -f "output[summary]=$SUMMARY") + CHECK_RUN_ID=$(jq -r '.id // empty' <<<"$response") + fi + if ! [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then + echo "::error::Checks API returned an invalid check-run ID." + exit 1 fi - if [ "$stale" = "true" ]; then + check_runs=$(gh api --method GET --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) + IFS=$'\t' read -r newest_number newest_attempt < <( + jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ + '[.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | try ((.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] + | sort_by(.number, .attempt) + | (last // {number: 0, attempt: 0}) + | [.number, .attempt] | @tsv' <<<"$check_runs" + ) + if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ + { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ + [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=Superseded by a newer policy decision." >/dev/null + echo "::error::A newer policy run superseded this required-workflow run." exit 1 fi - - name: Enforce result - if: steps.assess.outputs.passed != 'true' + if [ "$CONCLUSION" = "success" ] && \ + [ "$EVENT_NAME" = "pull_request_target" ] && \ + ! verify_live_candidate; then + stale=true + CONCLUSION=failure + SUMMARY="The pull request identity or authorization changed after reporting." + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" -f status=completed -f conclusion=failure \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=$CHECK_NAME" -f "output[summary]=$SUMMARY" >/dev/null + fi + + prior_check_ids=$(jq -r \ + --arg check_name "$CHECK_NAME" \ + --arg external_prefix "$external_prefix" \ + --argjson current_number "$RUN_NUMBER" --argjson current_attempt "$RUN_ATTEMPT" \ + '.[] | .check_runs[]? + | select(.name == $check_name and .app.slug == "github-actions") + | select((.external_id // "") | startswith($external_prefix)) + | . as $check + | try (($check.external_id | ltrimstr($external_prefix) + | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) + | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty + | select(.number < $current_number or + (.number == $current_number and .attempt < $current_attempt)) + | $check.id' <<<"$check_runs") + while IFS= read -r prior_check_id; do + [ -n "$prior_check_id" ] || continue + [[ "$prior_check_id" =~ ^[0-9]+$ ]] || exit 1 + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ + -f "output[title]=$CHECK_NAME" \ + -f "output[summary]=Superseded by a newer policy decision." >/dev/null + done <<<"$prior_check_ids" + [ "$stale" = "false" ] + + - name: Enforce policy result + if: >- + always() && + (steps.assess-policy.outputs.passed != 'true' || + (steps.assess-policy.outputs.mode == 'preserve' && + env.DEPENDABOT_PR != 'true' && + steps.preserve-gate.outputs.valid != 'true')) run: | - echo "${{ steps.assess.outputs.summary || 'Validation assessment failed.' }}" + echo "${{ steps.assess-policy.outputs.summary || 'Validation assessment failed.' }}" exit 1 diff --git a/.github/workflows/validate_llamacpp_schedule.yml b/.github/workflows/validate_llamacpp_schedule.yml index ad5b9261f1..c939a5b8da 100644 --- a/.github/workflows/validate_llamacpp_schedule.yml +++ b/.github/workflows/validate_llamacpp_schedule.yml @@ -15,6 +15,7 @@ jobs: validate: name: Validate release candidates permissions: + checks: write contents: read uses: ./.github/workflows/validate_llamacpp_core.yml secrets: @@ -31,19 +32,48 @@ jobs: group: llamacpp-auto-update-publication cancel-in-progress: false permissions: - contents: write - pull-requests: write + contents: read steps: - - uses: actions/checkout@v5 + - name: Require dedicated publication identity + shell: bash + env: + GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} + EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }} + run: | + set -euo pipefail + + : "${GH_TOKEN:?LLAMACPP_UPDATE_TOKEN repository secret is required}" + : "${EXPECTED_PUBLICATION_ACTOR:?LLAMACPP_UPDATE_ACTOR repository variable is required}" + if ! authenticated_actor=$(gh api graphql \ + -f 'query=query { viewer { login } }' \ + --jq '.data.viewer.login'); then + echo "Could not authenticate the dedicated publication token." >&2 + exit 1 + fi + if [[ -z "$authenticated_actor" || + "$authenticated_actor" != "$EXPECTED_PUBLICATION_ACTOR" ]]; then + echo "The authenticated publication actor does not match LLAMACPP_UPDATE_ACTOR." >&2 + exit 1 + fi + + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: fetch-depth: 0 + persist-credentials: false + token: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} - - name: Download validation results - uses: actions/download-artifact@v7 + - name: Download primary validation results + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: pattern: validation-results-* merge-multiple: true + - name: Download restart validation results + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + pattern: restart-validation-results-* + merge-multiple: true + - name: Validate complete validation evidence env: EXPECTED_VALIDATION_MATRIX: ${{ needs.validate.outputs.validation_matrix }} @@ -224,7 +254,7 @@ jobs: - name: Verify release asset manifest env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} EXPECTED_RELEASE_ASSET_MANIFEST: ${{ needs.validate.outputs.release_asset_manifest }} run: | set -euo pipefail @@ -240,7 +270,8 @@ jobs: - name: Publish update pull request env: BASE_BRANCH: ${{ github.event.repository.default_branch }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} + EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }} EXPECTED_RELEASE_ASSET_MANIFEST: ${{ needs.validate.outputs.release_asset_manifest }} LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 92a6893b11..3858d35522 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1192,6 +1192,16 @@ if(BUILD_TESTING AND EXISTS "${MCP_CLIENT_CONFIG_TEST_SOURCE}") ) target_link_libraries(test_mcp_client_config PRIVATE lemonade-server-core) + if(NOT WIN32) + set_property(SOURCE src/cpp/server/mcp_client.cpp APPEND PROPERTY + COMPILE_DEFINITIONS + LEMONADE_MCP_PROCESS_TEST_HOOK + ) + target_compile_definitions(test_mcp_client_config PRIVATE + LEMONADE_MCP_PROCESS_TEST_HOOK + ) + endif() + if(Python3_Interpreter_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/fixtures/mock_mcp_stdio_server.py") file(TO_CMAKE_PATH "${Python3_EXECUTABLE}" LEMONADE_TEST_PYTHON_PATH) file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/fixtures/mock_mcp_stdio_server.py" LEMONADE_TEST_MCP_STDIO_SERVER_PATH) @@ -1203,6 +1213,20 @@ if(BUILD_TESTING AND EXISTS "${MCP_CLIENT_CONFIG_TEST_SOURCE}") add_cpp_ci_test(mcp_client_config CI ON COMMAND test_mcp_client_config) endif() +set(MCP_CLIENT_PROCESS_CONTRACT_TEST_SOURCE + "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_mcp_client_process_contract.cpp" +) +if(BUILD_TESTING AND EXISTS "${MCP_CLIENT_PROCESS_CONTRACT_TEST_SOURCE}") + add_executable(test_mcp_client_process_contract + ${MCP_CLIENT_PROCESS_CONTRACT_TEST_SOURCE} + ) + target_compile_definitions(test_mcp_client_process_contract PRIVATE + "LEMONADE_MCP_CLIENT_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/server/mcp_client.cpp\"" + ) + add_cpp_ci_test(mcp_client_process_contract CI ON + COMMAND test_mcp_client_process_contract) +endif() + # libwebsockets for the realtime and log-stream WebSocket server if(USE_SYSTEM_LWS) target_link_libraries(lemonade-server-core PUBLIC ${LWS_LIBRARIES}) @@ -2230,27 +2254,57 @@ if(BUILD_TESTING AND EXISTS "${_ARCHIVE_PLATFORM_TEST_SRC}") add_cpp_ci_test(ArchivePlatformTest CI ON COMMAND test_archive_platform) endif() -# ProcessManager zombie detection and non-mutating reap contract. -if(BUILD_TESTING AND CMAKE_SYSTEM_NAME STREQUAL "Linux") +# ProcessManager lifecycle and output-capture contracts. +if(BUILD_TESTING AND (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR + CMAKE_SYSTEM_NAME STREQUAL "Darwin")) set(_PROCESS_MANAGER_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_process_manager.cpp" ) if(EXISTS "${_PROCESS_MANAGER_TEST_SRC}") + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(_PROCESS_MANAGER_PLATFORM_SRC + src/cpp/server/utils/platform/process_linux.cpp + ) + else() + set(_PROCESS_MANAGER_PLATFORM_SRC + src/cpp/server/utils/platform/process_macos.cpp + ) + endif() add_executable(test_process_manager test/cpp/test_process_manager.cpp src/cpp/server/utils/process_manager.cpp - src/cpp/server/utils/platform/process_linux.cpp + ${_PROCESS_MANAGER_PLATFORM_SRC} ) target_include_directories(test_process_manager PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/include ${CMAKE_CURRENT_BINARY_DIR}/include ) + target_compile_definitions(test_process_manager PRIVATE + LEMONADE_PROCESS_TEST_HOOK + ) find_package(Threads REQUIRED) target_link_libraries(test_process_manager PRIVATE Threads::Threads) add_cpp_ci_test(ProcessManagerTest CI ON COMMAND test_process_manager) endif() endif() +set(_PROCESS_MANAGER_WINDOWS_TEST_SRC + "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_process_manager_windows.cpp" +) +if(BUILD_TESTING AND WIN32 AND EXISTS "${_PROCESS_MANAGER_WINDOWS_TEST_SRC}") + add_executable(test_process_manager_windows + test/cpp/test_process_manager_windows.cpp + src/cpp/server/utils/process_manager.cpp + src/cpp/server/utils/platform/process_windows.cpp + ) + target_include_directories(test_process_manager_windows PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/include + ${CMAKE_CURRENT_BINARY_DIR}/include + ) + add_cpp_ci_test(ProcessManagerWindowsTest CI ON + COMMAND test_process_manager_windows) +endif() + # FastFlowLM argument validation. Keep this test independent from the server set(_FLM_ARG_RESOLVER_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_flm_arg_resolver.cpp" @@ -3222,6 +3276,13 @@ if(BUILD_TESTING AND EXISTS "${_STREAMING_PROXY_CANCEL_TEST_SRC}") add_cpp_ci_test(StreamingProxyCancelTest CI OFF COMMAND test_streaming_proxy_cancel) endif() +set(_LLAMACPP_STREAM_MODEL_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_llamacpp_stream_model.cpp") +if(BUILD_TESTING AND EXISTS "${_LLAMACPP_STREAM_MODEL_TEST_SRC}") + add_executable(test_llamacpp_stream_model test/cpp/test_llamacpp_stream_model.cpp) + target_link_libraries(test_llamacpp_stream_model PRIVATE lemonade-server-core) + add_cpp_ci_test(LlamaCppStreamModelTest CI ON COMMAND test_llamacpp_stream_model) +endif() + set(_STREAMING_HEARTBEAT_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_streaming_heartbeat.cpp") if(BUILD_TESTING AND EXISTS "${_STREAMING_HEARTBEAT_TEST_SRC}") add_executable(test_streaming_heartbeat test/cpp/test_streaming_heartbeat.cpp) diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 7334ddea88..315c4ab14a 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -145,16 +145,55 @@ Every PR runs the C++ `cpp-ci` tests, the endpoint/CLI suites on Windows and Lin ### What defers to the merge queue -Packaging, distro, PPA, backend-validation, self-hosted inference and most macOS jobs do **not** run on PR pushes. They run in the merge queue, so a break there blocks the merge rather than every push. Each group reports through one aggregate check whose name is stable enough to be required: +Packaging, distro, PPA, backend-validation, self-hosted inference and most macOS jobs do **not** run on PR pushes. They run in the merge queue, so a break there blocks the merge rather than every push. Each group reports an aggregate gate signal. The llama.cpp custom checks are diagnostic only; the trusted wrapper workflow is their enforcement boundary. -| Group | Gate check | Opt in on a PR with | +| Group | Gate signal | Opt in on a PR with | |---|---|---| | Fedora RPM, Debian 13, Arch, openSUSE, Launchpad PPA, `Build Lemonade Desktop Installer` | `Packaging builds`, `Linux distro builds`, `Launchpad PPA builds` | `ci:distros` | | macOS `.dmg`, `Test CLI/Endpoints (macos-latest)`, `Test Embeddable (macOS)`, `Test .dmg - macOS inference` | `macOS builds` | `ci:macos` | -| llama.cpp, vLLM, stable-diffusion.cpp validation | `llama.cpp validation`, `vLLM validation`, `stable-diffusion.cpp validation` | `ci:upgrades` | +| llama.cpp, vLLM, stable-diffusion.cpp validation | Required workflow `Validate llama.cpp protected change`; `vLLM validation`; `stable-diffusion.cpp validation` | `ci:upgrades` | | `Test .exe - *` and `Test .deb - *` inference suites on the self-hosted rigs | `Inference backend tests` | `ci:backends` | -Every gated job is reachable from a PR by label — nothing is merge-queue-only. Apply the label when your change plausibly affects that surface (a backend version pin wants `ci:upgrades`, packaging or install-path changes want `ci:distros`, `#ifdef __APPLE__` or CMake changes want `ci:macos`, a wrapped-server or inference-path change wants `ci:backends`); the label takes effect immediately, without a push. Fork contributors can't apply labels themselves — ask a maintainer (or the [Discord](https://discord.gg/5xXzkMu8Zk)) to add one. Note that `Test .dmg - macOS inference` exercises several of the same wrapped servers (llama.cpp, whisper.cpp, moonshine, kokoro) on Metal but lives in the macOS group — a change to one of those that could break on Metal wants `ci:macos` too. `Build Embeddable Lemonade (macOS)` still runs on every PR as the AppleClang compile check. +The base-branch llama.cpp wrapper reports `llama.cpp authorized validation v2` for required default-event and merge-queue decisions, and `llama.cpp authorization diagnostics v2` for label and lifecycle metadata. Each check targets GitHub's synthetic test-merge commit when one is available and falls back to the immutable current head while GitHub is still computing mergeability or the PR conflicts; protected validation never executes without a test-merge commit. Each provides exact-candidate diagnostics and stale-run reconciliation, while the wrapper's own conclusion fails whenever policy resolution, authorization, validation, or reconciliation fails. Changes confined to `docs/`, `src/app/`, or `src/web-app/` receive a neutral success; `src/app/src/renderer/utils/toolDefinitions.json` is an exception because it is copied into the server resources. Every other path is treated as protected, including unknown future paths, because the validation builds or executes candidate-controlled inputs. Protected changes use the authorization and rerun procedure below. Shared CMake or runtime changes may need both `ci:upgrades` and the other matching surface label. + +Dependabot-authored `pull_request_target` runs receive a read-only `GITHUB_TOKEN`, including when a maintainer re-runs them. These runs rely solely on the organization required-workflow conclusion and intentionally omit custom diagnostic Checks API runs. Authorization-required and awaiting-rerun states still fail the required workflow, while an authorized same-run rerun can pass. + +Do not require `llama.cpp validation`; that is the reusable workflow's internal aggregate job and does not prove that a PR was authorized. Do **not** require `llama.cpp authorized validation v2` during bootstrap, and do not require `llama.cpp authorization diagnostics v2` either. Never configure `llama.cpp authorized validation v2` as a required status check, including with **GitHub Actions** selected as its expected source: the shared `github-actions` App can also report a same-name check from candidate-controlled workflow code, so its App identity does not identify the workflow or event that produced the result. Both custom checks are diagnostic only. + +The validation-infrastructure change that introduces the trusted wrapper and policy helper must land on the default branch as a separate bootstrap merge before an organization required-workflow ruleset is created or activated; a rule cannot enforce a workflow that is not yet on the default branch. Enforcement therefore starts with later protected pull requests, not with the bootstrap pull request itself. Before relying on that enforcement, create the `ci:upgrades` label and provision ephemeral protected self-hosted runners that are discarded after every job. + +The current `.github/CODEOWNERS` covers only agent configuration files. It does not cover the trusted llama.cpp workflows, policy and publication scripts, validation runner, validator, or trusted test helpers and fixtures. A maintainer must add explicit entries for every trusted validation and publication surface, using real eligible owners, and a repository administrator must enforce code-owner review on the protected branch. Do not invent an owner in repository code. Missing exact CODEOWNERS coverage or missing enforced code-owner review is an external activation **NO-GO**. + +Scheduled publication has two additional external activation blockers. Add the repository secret `LLAMACPP_UPDATE_TOKEN` containing a separately provisioned, repository-scoped fine-grained PAT with **Contents: read and write** and **Pull requests: read and write**. Add the repository variable `LLAMACPP_UPDATE_ACTOR` containing that PAT's exact `viewer.login`. The current workflow does not support storing a GitHub App installation token in this secret: installation tokens expire after one hour, so App support would have to mint a fresh token inside each publication job. The workflow and publisher both fail before mutation when either value is absent or the authenticated actor differs. They use only this credential for checkout, release and pull-request APIs, push, pull-request creation, and draft promotion. Never substitute the repository's `GITHUB_TOKEN` or enable **Allow GitHub Actions to create and approve pull requests** as a workaround: that token does not trigger the required workflow for the pull request it creates. The publisher deliberately creates a draft first so its `opened` event records the expected initial failed required-workflow run, then marks it ready for a maintainer to authorize with the procedure below. Provisioning and rotation of the dedicated credential cannot be completed by repository code, so missing secret or variable configuration is an activation **NO-GO**. + +The validation runner uses Linux child-subreaper adoption plus process groups. On Windows it starts a trusted gate wrapper, assigns the wrapper to a kill-on-close Job Object before releasing candidate execution, and relies on Job inheritance for candidate descendants. macOS has no equivalent kernel API that can retain a fast double-fork after it creates a new session, so Darwin validation fails closed by default. The trusted core workflow may pass `--allow-darwin-github-hosted-ephemeral-runner` only for its static `macos-latest` Metal lane when GitHub reports `runner.environment == 'github-hosted'`; for that lane, disposal of the GitHub-hosted VM is the final containment boundary. Ephemeral workers remain part of the Windows trust boundary because process containment is not an adversarial sandbox. + +For protected validation, the base-branch checkout supplies the setup and cleanup actions, Python runner, validator, fixtures, and every imported `test.utils` helper. The candidate supplies only its build inputs, built `lemond` and adjacent build resources (including `defaults.json`), plus its base model catalog before the trusted overlay is applied. This prevents a pull request from substituting validation code, but it does **not** make candidate native code an adversarial sandbox: candidate build scripts and `lemond` execute as the runner's OS user and can access anything that user can. Jobs that execute protected candidate code must therefore use disposable GitHub-hosted workers or single-job ephemeral protected self-hosted workers, expose no repository secrets, and grant only a read-only repository token. The protected GitHub-hosted `macos-latest` lane receives no Hugging Face credential and runs no artifact upload, cache upload, or other token-bearing step after candidate runtime begins. + +Before K2 promotion, all three pinned release repositories must provide immutable K2-capable backend releases. The Lemonade-controlled publishers must enable release immutability. Because the current `ggml-org/llama.cpp` b-release history is not immutable, upstream must enable immutable releases or Lemonade must use a controlled immutable mirror and revise the trust contract accordingly. The `lemonade-sdk/llama.cpp` publisher must verify its complete CUDA matrix before making a release immutable, including all three `sm_121` artifacts: Windows x64 plus Ubuntu x64 and arm64. + +The `lemonade-sdk/llamacpp-rocm` publisher must emit the full advertised nightly matrix and immutable source sidecars that bind each exact asset to its concrete ROCm build targets; the `gfx103X` assets must cover `gfx1033`, `gfx1035`, and `gfx1036`. In particular, it currently needs Windows `gfx1152` plus Ubuntu `gfx1152` and `gfx942` artifacts. The scheduled updater intentionally keeps ROCm nightly ineligible while an asset or required build-target attestation is absent; do not weaken the matrix to make an incomplete release pass. Land and validate the matching managed pins plus validation-only model catalog (or temporarily phase K2 out of the gate). + +Only after those prerequisites pass may an LLM360 organization owner create an organization-level branch ruleset for `LLM360/lemonade`. Target the protected `main` branch and add **Require workflows to pass before merging** with source repository `LLM360/lemonade`, source branch `main`, and workflow `.github/workflows/validate_llamacpp_pr.yml`. That workflow declares the supported `pull_request_target` and `merge_group` events, and its failing reconciliation job makes a policy failure fail the required workflow itself. Do not substitute a required status-check name for this workflow-bound rule. See GitHub's documentation for [organization ruleset setup](https://docs.github.com/en/enterprise-cloud@latest/organizations/managing-organization-settings/creating-rulesets-for-repositories-in-your-organization) and [workflow-rule event semantics](https://docs.github.com/en/enterprise-cloud@latest/repositories/configuring-branches-and-merges-in-your-repository/managing-rulesets/available-rules-for-rulesets#require-workflows-to-pass-before-merging). + +GitHub ignores configured activity filters for a ruleset workflow. It starts required `pull_request_target` runs only for `opened`, `synchronize`, and `reopened`. Thus, an authorization-label run cannot satisfy the workflow rule. Use this procedure for a protected PR: + +1. Find the newest failed **Validate llama.cpp protected change** run that GitHub created for `opened`, `synchronize`, or `reopened`. Do not use a run that a label or metadata change created. +2. If `ci:upgrades` is present, remove it. Apply `ci:upgrades` strictly after the original attempt's recorded run start time, not its queued creation time. A label applied while the run was queued is stale. The maintainer who applies the label must have current write permission. +3. In the selected run, select **Re-run all jobs**. The same maintainer must start the rerun. Do not select **Re-run failed jobs**, because that option does not rerun the successful policy job. +4. If the head, base branch, base commit, merge commit, or authorization actor changes, or a newer `opened`, `synchronize`, or `reopened` run starts, stop. Use the newest failed default-event run and repeat the procedure. + +Metadata diagnostic runs do not supersede the selected required default-event run. `llama.cpp authorization diagnostics v2` uses a distinct visible name, so a red metadata result does not conflict with a recovered required result after the label is reapplied and the selected run is rerun. The `unlabeled` and `labeled` runs produced by step 2 are expected. If authorization is removed later, apply a fresh label and rerun the same selected default-event run, provided no newer default-event run exists. A removal during validation makes that attempt fail its live authorization recheck. + +Removal after a completed green rerun is diagnostic only: because the workflow rule ignores `unlabeled`, that metadata run may not revoke the already-green organization workflow result. The authorization epoch applies only to reruns of one exact default-event run and candidate while the live identity remains current; do not describe it as durable merge state. Activation is **NO-GO** unless Evaluate mode proves that post-success removal revokes the required result, or an exclusive GitHub App or equivalent separately trusted status primitive is provisioned to enforce revocation. + +A base-branch retarget starts an `edited` run, which the workflow rule cannot require. After a retarget, push a new commit or close and reopen the PR to create a supported default-event run. Then, apply a new label and rerun all jobs on that run. The policy rejects attempt 1, a rerun of a label event, a label at or before the original run start time, a rerun by a different actor, and an older run after an identity or lifecycle change. + +The synthetic merge SHA proves one exact base-plus-head candidate. Before activation, require a merge queue, strict up-to-date enforcement, or a live-proven equivalent so the exact validated base-plus-head candidate is the candidate that will eventually merge. In Evaluate mode, test what happens when the default branch advances after a green rerun: the PR must become unmergeable until the new candidate passes a fresh required workflow, or a mandatory merge queue must validate the eventual merge-group candidate. Activation is **NO-GO** if a prior green result can carry over to a different base-plus-head candidate on the ordinary merge path. + +After the bootstrap merge is on the default branch, an organization owner can create the rule in **Evaluate** mode. Exercise it with separate, unmerged live fork PRs before making it **Active** with no bypass for ordinary merges. The live checks must cover a neutral docs change, an unauthorized protected change, the full authorization and rerun procedure, label removal both during and after a successful run, a subsequent head update, a default-branch advance after a green rerun, a base retarget followed by a supported retrigger, and a merge-queue entry. Do not combine validation-infrastructure bootstrap and the protected feature change that depends on it into one activation step. Ruleset activation is an external deployment blocker; repository code and a repository administrator cannot complete it. If the organization-level workflow rule cannot enforce this lifecycle, the alternative is a separately provisioned GitHub App whose identity is exclusive to this gate, not the repository's ordinary `GITHUB_TOKEN` or the shared GitHub Actions expected source. + +Every gated job is reachable from a PR by label — nothing is merge-queue-only. Apply the label when your change plausibly affects that surface (a backend version pin wants `ci:upgrades`, packaging or install-path changes want `ci:distros`, `#ifdef __APPLE__` or CMake changes want `ci:macos`, a wrapped-server or inference-path change wants `ci:backends`). For llama.cpp, complete the required-workflow rerun procedure after you apply `ci:upgrades`. Other surface labels take effect immediately without a push. Fork contributors cannot apply labels themselves — ask a maintainer (or the [Discord](https://discord.gg/5xXzkMu8Zk)) to add one. Note that `Test .dmg - macOS inference` exercises several of the same wrapped servers (llama.cpp, whisper.cpp, moonshine, kokoro) on Metal but lives in the macOS group — a change to one of those that could break on Metal wants `ci:macos` too. `Build Embeddable Lemonade (macOS)` still runs on every PR as the AppleClang compile check. Two consequences worth knowing: diff --git a/docs/guide/configuration/llamacpp.md b/docs/guide/configuration/llamacpp.md index 4857cb54b9..416f1047c2 100644 --- a/docs/guide/configuration/llamacpp.md +++ b/docs/guide/configuration/llamacpp.md @@ -2,78 +2,6 @@ Lemonade uses [llama.cpp](https://github.com/ggerganov/llama.cpp) as its primary LLM inference backend, supporting multiple hardware acceleration options. This document explains the available backends and how to choose between them. -- [K2-Horizon](#k2-horizon) -- [Available Backends](#available-backends) -- [ROCm Channel Configuration](#rocm-channel-configuration) -- [Choosing the Right Backend](#choosing-the-right-backend) -- [Platform Specifics](#platform-specifics) - -## K2-Horizon - -Lemonade includes these official IFM BF16 GGUF models: - -| Model ID | Weight Size | Maximum Context | Suggested Starting Values | -| --- | ---: | ---: | --- | -| `K2-Horizon-0.9B-GGUF` | 2.16 GB | 131,072 tokens | high reasoning effort, temperature 0.6, top-p 0.95 | -| `K2-Horizon-3.7B-GGUF` | 10.13 GB | 524,288 tokens | high reasoning effort, temperature 1.0, top-p 0.95 | -| `K2-Horizon-7B-GGUF` | 18.01 GB | 524,288 tokens | high reasoning effort, temperature 1.0, top-p 0.95 | - -These are caller-supplied starting points. Lemonade does not install per-model sampling defaults. - -These model IDs must ship only with managed llama.cpp binaries built from accepted K2-Horizon -support on current upstream llama.cpp. The catalog entries are staged while that upstream -port, the coordinated backend pins, and packaged runtime validation are completed. Do not -publish them with the current managed pins or substitute the IFM development branch for a -managed release. - -After compatible managed binaries are pinned, start a model with a bounded context: - -```bash -lemonade run K2-Horizon-0.9B-GGUF --ctx-size 8192 -``` - -You can also call the OpenAI-compatible API: - -```python -from openai import OpenAI - -client = OpenAI(base_url="http://localhost:13305/api/v1", api_key="not-needed") -response = client.chat.completions.create( - model="K2-Horizon-0.9B-GGUF", - messages=[ - {"role": "user", "content": "What is 17 plus 25? Return the number only."} - ], - temperature=0.6, - top_p=0.95, - max_tokens=256, - extra_body={ - "chat_template_kwargs": { - "reasoning_effort": "high", - "tool_call_format": "xml", - } - }, -) -message = response.choices[0].message -reasoning = getattr(message, "reasoning_content", None) -if reasoning: - print(reasoning) -print(message.content) -``` - -The upstream parser and packaged-runtime gate must prove high, medium, low, and disabled -reasoning plus the `json`, `xml`, and `xml_typed` tool-call formats before publication. The -IFM development branch does not pass that complete matrix: medium and low reasoning -can expose IFM control tags, and the non-default tool formats can cause parser errors. To -disable thinking after the compatible managed release ships, set -`chat_template_kwargs.enable_thinking` to `false`. Use a large output allowance for long -reasoning tasks. Do not use a 32K output allowance as a server-wide default. - -The table gives only the weight size. The context cache and backend buffers need more memory. -The maximum 128K and 512K contexts need substantially more memory than the 8K example. - -An older custom llama.cpp executable cannot load these models. This integration does not -support K2-Horizon Uno. - ## Available Backends ### CPU @@ -111,7 +39,7 @@ support K2-Horizon Uno. - **Use Case**: NVIDIA GPU-optimized inference - **Performance**: Optimized for NVIDIA hardware, typically outperforms Vulkan on supported GPUs - **Source**: Per-architecture builds from [lemonade-sdk/llama.cpp](https://github.com/lemonade-sdk/llama.cpp) -- **Binaries**: Compute-capability-specific builds (sm_75, sm_80, sm_86, sm_89, sm_90, sm_100, sm_120, sm_121) +- **Binaries**: Compute-capability-specific builds (sm_75, sm_80, sm_86, sm_89, sm_90, sm_100, sm_120) - **Runtime**: Bundled CUDA runtime libraries (no system-wide CUDA toolkit installation required) - **Notes**: On Windows, .7z extraction requires the bsdtar bundled with Windows 11 22H2+. On Linux, the build is shipped as .tar.xz and extracts with the system `tar`. diff --git a/src/cpp/include/lemon/streaming_proxy.h b/src/cpp/include/lemon/streaming_proxy.h index 851ea47533..23605a9bd8 100644 --- a/src/cpp/include/lemon/streaming_proxy.h +++ b/src/cpp/include/lemon/streaming_proxy.h @@ -1,7 +1,8 @@ #pragma once -#include +#include #include +#include #include #include #include "utils/http_client.h" @@ -14,6 +15,11 @@ using json = nlohmann::json; class StreamingProxy { public: + static constexpr std::size_t kMaxSseFrameBytes = 1024 * 1024; + + using SseFrameTransform = + std::function; + struct TelemetryData { int input_tokens = 0; int output_tokens = 0; @@ -54,8 +60,21 @@ class StreamingProxy { httplib::DataSink& sink, std::function on_complete = nullptr, long timeout_seconds = 300, - std::function on_chunk = nullptr, - long heartbeat_interval_ms = 1000 + std::function on_backend_progress = nullptr, + long heartbeat_interval_ms = 1000, + std::function on_frame = nullptr + ); + + static void forward_transformed_sse_stream( + const std::string& backend_url, + const std::string& request_body, + httplib::DataSink& sink, + SseFrameTransform frame_transform, + std::function on_complete = nullptr, + long timeout_seconds = 300, + std::function on_backend_progress = nullptr, + long heartbeat_interval_ms = 1000, + std::function on_frame = nullptr ); static void forward_byte_stream( @@ -78,6 +97,17 @@ class StreamingProxy { static void accumulate_responses_delta(const nlohmann::json& parsed, std::string& accumulated_text); private: + static void forward_sse_stream_impl( + const std::string& backend_url, + const std::string& request_body, + httplib::DataSink& sink, + SseFrameTransform frame_transform, + std::function on_complete, + long timeout_seconds, + std::function on_backend_progress, + long heartbeat_interval_ms, + std::function on_frame + ); }; } // namespace lemon diff --git a/src/cpp/include/lemon/utils/process_manager.h b/src/cpp/include/lemon/utils/process_manager.h index 4cd9553124..02eb392b4a 100644 --- a/src/cpp/include/lemon/utils/process_manager.h +++ b/src/cpp/include/lemon/utils/process_manager.h @@ -1,15 +1,36 @@ #pragma once +#include +#include +#include +#include +#include #include #include -#include namespace lemon { namespace utils { +class ProcessOutputCapture { +public: + explicit ProcessOutputCapture(std::size_t max_bytes); + + void append(const char* data, std::size_t size); + void finish_reader(); + std::string read_tail(std::size_t max_bytes, int wait_timeout_ms); + +private: + const std::size_t max_bytes_; + std::mutex mutex_; + std::condition_variable completion_cv_; + std::string tail_; + int active_readers_ = 2; +}; + struct ProcessHandle { void* handle; int pid; + std::shared_ptr output_capture; }; // Returns true to continue, false to kill the process @@ -23,7 +44,8 @@ class ProcessManager { const std::string& working_dir = "", bool inherit_output = false, bool filter_health_logs = false, - const std::vector>& env_vars = {}); + const std::vector>& env_vars = {}, + std::size_t capture_output_bytes = 0); // Blocks until process exits or callback returns false (which kills the process) // Returns exit code, or -1 if killed by callback @@ -52,7 +74,9 @@ class ProcessManager { // read-only while lifecycle cleanup can reliably remove zombies. static int reap_process(ProcessHandle handle); - static std::string read_output(ProcessHandle handle, int max_bytes = 4096); + static std::string read_output(ProcessHandle handle, + int max_bytes = 4096, + int wait_timeout_ms = 1000); // Kill process forcefully and wait/close owned handles. static void kill_process(ProcessHandle handle); diff --git a/src/cpp/include/lemon/utils/process_platform.h b/src/cpp/include/lemon/utils/process_platform.h index 406a75d865..d137804c62 100644 --- a/src/cpp/include/lemon/utils/process_platform.h +++ b/src/cpp/include/lemon/utils/process_platform.h @@ -1,9 +1,10 @@ #pragma once #include + +#include #include #include -#include namespace lemon::utils { @@ -19,7 +20,8 @@ class ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars) = 0; + const std::vector>& env_vars, + std::shared_ptr output_capture) = 0; virtual void terminate(ProcessHandle handle) = 0; virtual bool is_running(ProcessHandle handle) = 0; diff --git a/src/cpp/include/lemon/wrapped_server.h b/src/cpp/include/lemon/wrapped_server.h index 56b07da4ba..2d11b7a161 100644 --- a/src/cpp/include/lemon/wrapped_server.h +++ b/src/cpp/include/lemon/wrapped_server.h @@ -531,8 +531,8 @@ class WrappedServer : public ICompletionServer { } // Mark observable backend progress. Streaming proxies call this for every - // delivered chunk; non-streaming requests call it on start/finish and when - // the watchdog observes a healthy out-of-band probe. + // nonempty successful backend body chunk; non-streaming requests call it + // on start/finish and when the watchdog observes a healthy out-of-band probe. void note_backend_activity(); void set_telemetry(int input_tokens, int output_tokens, @@ -612,6 +612,15 @@ class WrappedServer : public ICompletionServer { // Common method to forward requests to the wrapped server (non-streaming) json forward_request(const std::string& endpoint, const json& request, long timeout_seconds = 0); + void forward_streaming_request_impl( + const std::string& endpoint, + const std::string& request_body, + httplib::DataSink& sink, + bool sse, + long timeout_seconds, + TelemetryCallback telemetry_callback, + StreamingProxy::SseFrameTransform frame_transform); + json forward_get_request(const std::string& endpoint, long timeout_seconds = 0); // Forward multipart form data to the wrapped server diff --git a/src/cpp/resources/server_models.json b/src/cpp/resources/server_models.json index cd49fafc14..9f12321427 100644 --- a/src/cpp/resources/server_models.json +++ b/src/cpp/resources/server_models.json @@ -731,40 +731,6 @@ ], "size": 13.4 }, - "K2-Horizon-0.9B-GGUF": { - "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf", - "recipe": "llamacpp", - "suggested": true, - "labels": [ - "chat", - "reasoning", - "tool-calling", - "hot" - ], - "size": 2.16 - }, - "K2-Horizon-3.7B-GGUF": { - "checkpoint": "IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf", - "recipe": "llamacpp", - "suggested": true, - "labels": [ - "chat", - "reasoning", - "tool-calling" - ], - "size": 10.13 - }, - "K2-Horizon-7B-GGUF": { - "checkpoint": "IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf", - "recipe": "llamacpp", - "suggested": true, - "labels": [ - "chat", - "reasoning", - "tool-calling" - ], - "size": 18.01 - }, "MiniCPM5-1B-GGUF": { "source": "modelscope", "checkpoint": "OpenBMB/MiniCPM5-1B-GGUF:MiniCPM5-1B-Q4_K_M.gguf", diff --git a/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp b/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp index 14cf086515..8457745082 100644 --- a/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp +++ b/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp @@ -160,17 +160,7 @@ std::vector flm_installed_checkpoints() { int rc = lemon::utils::ProcessManager::run_command(command, output); #else std::string command = "\"" + flm_path + "\" list --filter installed --quiet --json 2>/dev/null"; - FILE* pipe = popen(command.c_str(), "r"); - if (!pipe) { - return installed_models; - } - - char buffer[256]; - while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - output += buffer; - } - - pclose(pipe); + (void)lemon::utils::ProcessManager::run_command(command, output); #endif // Parse output: { "models": [ { "name": "modelname:tag", ... }, ... ] } @@ -240,17 +230,7 @@ std::vector flm_discover_models() { } #else std::string command = "\"" + flm_path + "\" list --json 2>/dev/null"; - FILE* pipe = popen(command.c_str(), "r"); - if (!pipe) { - return flm_models; - } - - char buffer[256]; - while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - output += buffer; - } - - pclose(pipe); + (void)lemon::utils::ProcessManager::run_command(command, output); #endif // Parse output: { "models": [ { "name": "modelname:tag", "footprint": 1.23, ... }, ... ] } @@ -565,17 +545,7 @@ std::string flm_version() { int rc = lemon::utils::ProcessManager::run_command(command, output); #else std::string command = "\"" + flm_path + "\" version --json 2>/dev/null"; - FILE* pipe = popen(command.c_str(), "r"); - if (!pipe) { - return "unknown"; - } - - char buffer[256]; - while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - output += buffer; - } - - pclose(pipe); + (void)lemon::utils::ProcessManager::run_command(command, output); #endif // Parse JSON output: { "version": "0.9.34" } @@ -680,18 +650,7 @@ bool run_flm_validate(const std::string& flm_path, std::string& error_message) { #ifdef _WIN32 exit_code = lemon::utils::ProcessManager::run_command(command, output); #else - FILE* pipe = popen(command.c_str(), "r"); - if (!pipe) { - error_message = "Failed to execute " + flm_exe; - return false; - } - - char buffer[1024]; - while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - output += buffer; - } - - exit_code = pclose(pipe); + exit_code = lemon::utils::ProcessManager::run_command(command, output); if (exit_code != -1) { exit_code = WEXITSTATUS(exit_code); } diff --git a/src/cpp/server/backends/llamacpp/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp/llamacpp_server.cpp index 0f8e021f3a..4bb03e6ced 100644 --- a/src/cpp/server/backends/llamacpp/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp/llamacpp_server.cpp @@ -11,8 +11,8 @@ #include "lemon/model_manager.h" #include #include +#include #include -#include #include #include #include "lemon/auto_tune.h" @@ -81,6 +81,108 @@ static void push_arg(std::vector& args, push_reserved(reserved, key, aliases); } +static std::string normalize_sse_response_model_frame( + const std::string& frame, + const std::string& model_name) { + struct Line { + std::string text; + std::size_t content_end = 0; + std::size_t payload_start = 0; + bool is_data = false; + }; + + std::vector lines; + std::size_t line_start = 0; + while (line_start < frame.size()) { + const std::size_t terminator = frame.find_first_of("\r\n", line_start); + std::size_t line_end = frame.size(); + if (terminator != std::string::npos) { + line_end = terminator + 1; + if (frame[terminator] == '\r' && line_end < frame.size() && + frame[line_end] == '\n') { + ++line_end; + } + } + + Line line; + line.text = frame.substr(line_start, line_end - line_start); + line.content_end = line.text.size(); + if (line.content_end > 0 && line.text[line.content_end - 1] == '\n') { + --line.content_end; + } + if (line.content_end > 0 && line.text[line.content_end - 1] == '\r') { + --line.content_end; + } + if (line.text.rfind("data:", 0) == 0) { + line.is_data = true; + line.payload_start = std::strlen("data:"); + if (line.payload_start < line.content_end && + line.text[line.payload_start] == ' ') { + ++line.payload_start; + } + } + lines.push_back(std::move(line)); + line_start = line_end; + } + + std::string event_data; + std::size_t first_data_line = std::string::npos; + for (std::size_t index = 0; index < lines.size(); ++index) { + const Line& line = lines[index]; + if (!line.is_data) { + continue; + } + if (first_data_line == std::string::npos) { + first_data_line = index; + } else { + event_data.push_back('\n'); + } + event_data.append( + line.text, line.payload_start, + line.content_end - line.payload_start); + } + + if (first_data_line == std::string::npos || event_data.empty() || + event_data == "[DONE]") { + return frame; + } + + json payload = json::parse(event_data, nullptr, false); + if (!payload.is_object()) { + throw std::runtime_error( + "backend connection failed during SSE stream before DONE: " + "invalid OpenAI SSE data event"); + } + + bool normalized = false; + if (payload.contains("model")) { + payload["model"] = model_name; + normalized = true; + } + if (payload.contains("response") && payload["response"].is_object() && + payload["response"].contains("model")) { + payload["response"]["model"] = model_name; + normalized = true; + } + if (!normalized) { + return frame; + } + + std::string output; + for (std::size_t index = 0; index < lines.size(); ++index) { + const Line& line = lines[index]; + if (!line.is_data) { + output += line.text; + } else if (index == first_data_line) { + output.append(line.text, 0, line.payload_start); + output += payload.dump(); + output.append(line.text, line.content_end, + line.text.size() - line.content_end); + } + } + return output; +} + static std::string resolve_llamacpp_backend(const std::string& backend) { if (backend == "rocm") { // Map "rocm" to the appropriate channel based on config @@ -93,6 +195,93 @@ static std::string resolve_llamacpp_backend(const std::string& backend) { return backend; } +static std::string resolve_system_llamacpp_executable_path( + const std::string& executable) { + std::error_code ec; + fs::path executable_path = utils::path_from_utf8(executable); + if (executable_path.has_parent_path()) { + fs::path absolute_path = fs::absolute(executable_path, ec); + return ec ? executable : path_to_utf8(absolute_path); + } + +#ifdef _WIN32 + const std::string resolved = utils::find_executable_in_path(executable); + return resolved.empty() ? executable : resolved; +#else + const char* path_env = std::getenv("PATH"); + if (!path_env || path_env[0] == '\0') { + return executable; + } + + std::string path_value(path_env); + size_t start = 0; + while (start <= path_value.size()) { + const size_t end = path_value.find(':', start); + const std::string directory = path_value.substr( + start, end == std::string::npos ? std::string::npos : end - start); + if (!directory.empty()) { + fs::path candidate = fs::path(directory) / executable; + ec.clear(); + if (fs::is_regular_file(candidate, ec) && + access(candidate.c_str(), X_OK) == 0) { + fs::path absolute_path = fs::absolute(candidate, ec); + return ec ? path_to_utf8(candidate) : path_to_utf8(absolute_path); + } + } + if (end == std::string::npos) { + break; + } + start = end + 1; + } + return executable; +#endif +} + +static std::string detect_system_llamacpp_version( + const std::string& executable_path) { + constexpr std::size_t max_version_output_bytes = 4096; + ProcessHandle handle{}; + try { + handle = ProcessManager::start_process( + executable_path, {"--version"}, "", false, false, {}, + max_version_output_bytes); + const int exit_code = ProcessManager::wait_for_exit(handle, 10); + if (exit_code < 0) { + ProcessManager::kill_process(handle); + return "unknown"; + } + +#ifndef _WIN32 + handle.pid = 0; +#endif + const std::string output = ProcessManager::read_output( + handle, static_cast(max_version_output_bytes)); +#ifdef _WIN32 + ProcessManager::reap_process(handle); +#endif + handle = {}; + return llamacpp::detail::parse_system_llamacpp_version(output, + exit_code); + } catch (const std::exception&) { + if (handle.pid > 0 || handle.handle != nullptr) { + if (ProcessManager::is_running(handle)) { + ProcessManager::kill_process(handle); + } else { + ProcessManager::reap_process(handle); + } + } + return "unknown"; + } +} + +static bool custom_args_control_llamacpp_logging( + const std::vector& args) { + return std::any_of(args.begin(), args.end(), [](const std::string& arg) { + return arg == "--log-file" || arg.rfind("--log-file=", 0) == 0 || + arg == "--log-disable"; + }); +} + static bool is_llamacpp_rocm_backend(const std::string& backend) { return backend == "rocm-stable" || backend == "rocm-nightly"; } @@ -393,6 +582,9 @@ void LlamaCppServer::load(const std::string& model_name, } push_reserved(reserved_flags, "--reranking", std::vector{"--rerank"}); + bool user_controls_logging = + !utils::get_environment_variable_utf8("LLAMA_ARG_LOG_FILE").empty(); + // Validate and append custom arguments if (!llamacpp_args.empty()) { std::string validation_error = validate_custom_args(llamacpp_args, reserved_flags); @@ -404,6 +596,8 @@ void LlamaCppServer::load(const std::string& model_name, LOG(DEBUG, "LlamaCpp") << "Adding custom arguments: " << llamacpp_args << std::endl; std::vector custom_args_vec = parse_custom_args(llamacpp_args); + user_controls_logging = user_controls_logging || + custom_args_control_llamacpp_logging(custom_args_vec); args.insert(args.end(), custom_args_vec.begin(), custom_args_vec.end()); } @@ -573,16 +767,51 @@ void LlamaCppServer::load(const std::string& model_name, #endif bool inherit_llama_output = (log_level_ == "info") || is_debug(); - set_process_handle(ProcessManager::start_process( - process_executable, args, working_dir, inherit_llama_output, true, env_vars), - process_executable, args); + constexpr std::size_t max_startup_output_bytes = 64 * 1024; + const std::size_t startup_output_bytes = + llamacpp_backend == "system" && !user_controls_logging && + llamacpp::detail::identifies_k2_horizon_model( + model_name, model_info.checkpoint(), + model_info.gguf.architecture) + ? max_startup_output_bytes + : 0; + const ProcessHandle started_handle = ProcessManager::start_process( + process_executable, args, working_dir, inherit_llama_output, true, + env_vars, startup_output_bytes); + set_process_handle(started_handle, process_executable, args); if (!wait_for_ready("/health")) { const ProcessHandle handle = consume_process_handle_for_cleanup(); + const bool process_still_running = + has_process_handle(handle) && ProcessManager::is_running(handle); if (has_process_handle(handle)) { ProcessManager::stop_process(handle); } - throw std::runtime_error("llama-server failed to start"); + const std::string startup_error = "llama-server failed to start"; + const bool load_cancelled = load_cancel_ && load_cancel_->load(); + const bool k2_system_failure = + startup_output_bytes > 0 && !load_cancelled && !process_still_running; + const std::string startup_output = + k2_system_failure + ? ProcessManager::read_output(started_handle, 64 * 1024) + : ""; + if (llamacpp::detail::should_report_k2_system_startup_diagnostic( + llamacpp_backend, load_cancelled, model_name, + model_info.checkpoint(), model_info.gguf.architecture, + startup_output)) { + const std::string system_executable = + resolve_system_llamacpp_executable_path(executable); + const std::string original_error = + llamacpp::detail::k2_horizon_unsupported_architecture_error( + startup_output); + throw std::runtime_error( + llamacpp::detail::k2_horizon_system_startup_error( + model_name, + system_executable, + detect_system_llamacpp_version(system_executable), + original_error)); + } + throw std::runtime_error(startup_error); } LOG(DEBUG, "LlamaCpp") << "Model loaded on port " << get_backend_port() << std::endl; @@ -675,16 +904,37 @@ void LlamaCppServer::forward_streaming_request(const std::string& endpoint, long timeout_seconds, TelemetryCallback telemetry_callback) { std::string body = request_body; - if (endpoint == "/v1/chat/completions" || endpoint == "/v1/responses") { + std::string response_model; + const bool is_openai_text_endpoint = + endpoint == "/v1/chat/completions" || + endpoint == "/v1/completions" || endpoint == "/v1/responses"; + if (is_openai_text_endpoint) { json request = json::parse(request_body, nullptr, false); if (!request.is_discarded()) { + if (request.contains("model") && request["model"].is_string()) { + response_model = request["model"].get(); + } if (endpoint == "/v1/chat/completions") { JsonUtils::add_legacy_max_tokens_alias(request); + request = llamacpp::sanitize_tool_schema_limits(std::move(request)); + body = request.dump(); + } else if (endpoint == "/v1/responses") { + request = llamacpp::sanitize_tool_schema_limits(std::move(request)); + body = request.dump(); } - body = llamacpp::sanitize_tool_schema_limits(std::move(request)).dump(); } } + if (sse && !response_model.empty()) { + WrappedServer::forward_streaming_request_impl( + endpoint, body, sink, sse, timeout_seconds, telemetry_callback, + [response_model](const std::string& frame) { + return normalize_sse_response_model_frame( + frame, response_model); + }); + return; + } + WrappedServer::forward_streaming_request( endpoint, body, sink, sse, timeout_seconds, telemetry_callback); } @@ -701,44 +951,6 @@ std::unique_ptr create(const BackendContext& ctx) { } namespace { -std::string system_llamacpp_version() { - std::string output; - #ifdef _WIN32 - std::string command = "llama-server --version 2>NUL"; - int rc = lemon::utils::ProcessManager::run_command(command, output); - #else - FILE* pipe = popen("llama-server --version 2>/dev/null", "r"); - if (!pipe) { - return "unknown"; - } - - char buffer[256]; - if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - output = buffer; - } - - pclose(pipe); - #endif - - // Parse version from output like "version: 3432 (e2b2a632)" or "llama.cpp version b3432" - if (!output.empty()) { - // Try to find a version number - std::regex version_regex(R"(version:\s*(\d+)|version\s+b?(\d+))"); - std::smatch match; - if (std::regex_search(output, match, version_regex)) { - for (size_t i = 1; i < match.size(); ++i) { - if (match[i].matched) { - return "b" + match[i].str(); - } - } - } - return "detected"; - } - - return "unknown"; -} - - bool is_ggml_hip_plugin_available() { #ifdef __linux__ // Allow distros/packagers that install outside the FHS paths below @@ -878,7 +1090,8 @@ class LlamaCppOps : public BackendOps { const std::string& file_version) const override { // The PATH-installed "system" llama-server has no version.txt; query it. if (backend == "system") { - return system_llamacpp_version(); + return detect_system_llamacpp_version( + resolve_system_llamacpp_executable_path(descriptor.binary)); } return file_version; } diff --git a/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h b/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h index 53a673e42d..268edbb484 100644 --- a/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h +++ b/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -16,6 +17,115 @@ namespace backends { namespace llamacpp { namespace detail { +inline std::string lowercase_ascii(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return value; +} + +inline bool identifies_k2_horizon_model(const std::string& model_name, + const std::string& checkpoint, + const std::string& architecture) { + const std::string normalized_architecture = lowercase_ascii(architecture); + if (normalized_architecture == "k2-horizon" || + normalized_architecture == "k2_horizon") { + return true; + } + if (!normalized_architecture.empty()) { + return false; + } + + const auto contains_k2_horizon = [](const std::string& value) { + const std::string normalized = lowercase_ascii(value); + return normalized.find("k2-horizon") != std::string::npos || + normalized.find("k2_horizon") != std::string::npos; + }; + return contains_k2_horizon(model_name) || contains_k2_horizon(checkpoint); +} + +inline std::string k2_horizon_unsupported_architecture_error( + const std::string& startup_output) { + size_t start = 0; + while (start <= startup_output.size()) { + const size_t end = startup_output.find_first_of("\r\n", start); + std::string line = startup_output.substr( + start, end == std::string::npos ? std::string::npos : end - start); + const std::string normalized = lowercase_ascii(line); + const bool names_k2 = + normalized.find("k2-horizon") != std::string::npos || + normalized.find("k2_horizon") != std::string::npos; + if (names_k2 && + normalized.find("unknown model architecture") != std::string::npos) { + const size_t first = line.find_first_not_of(" \t"); + const size_t last = line.find_last_not_of(" \t"); + return first == std::string::npos + ? std::string() + : line.substr(first, last - first + 1); + } + if (end == std::string::npos) { + break; + } + start = end + 1; + if (startup_output[end] == '\r' && start < startup_output.size() && + startup_output[start] == '\n') { + ++start; + } + } + return ""; +} + +inline bool should_report_k2_system_startup_diagnostic( + const std::string& backend, + bool load_cancelled, + const std::string& model_name, + const std::string& checkpoint, + const std::string& architecture, + const std::string& startup_output = "") { + return backend == "system" && !load_cancelled && + identifies_k2_horizon_model(model_name, checkpoint, architecture) && + !k2_horizon_unsupported_architecture_error(startup_output).empty(); +} + +inline std::string parse_system_llamacpp_version(const std::string& output, + int exit_code = 0) { + if (exit_code != 0) { + return "unknown"; + } + + std::smatch match; + if (std::regex_search(output, match, std::regex(R"(build\s+(\d+))"))) { + return "b" + match[1].str(); + } + + if (std::regex_search( + output, + match, + std::regex(R"(version:\s*(\d+)|version\s+b?(\d+))"))) { + for (size_t i = 1; i < match.size(); ++i) { + if (match[i].matched) { + return "b" + match[i].str(); + } + } + } + return output.empty() ? "unknown" : "detected"; +} + +inline std::string k2_horizon_system_startup_error( + const std::string& model_name, + const std::string& executable_path, + const std::string& version, + const std::string& original_error) { + const std::string displayed_path = executable_path.empty() ? "unknown" : executable_path; + const std::string displayed_version = + version.empty() || version == "detected" ? "unknown" : version; + return "System llama-server failed to start while loading K2-Horizon model '" + + model_name + "' using executable '" + displayed_path + "' (version " + + displayed_version + "). Original error: " + original_error + + ". Install a K2-Horizon-capable llama.cpp build from " + "https://github.com/MBZUAI-IFM/llama.cpp/tree/model/K2Horizon and " + "ensure that executable is first on PATH, then retry."; +} + // Private, deterministic pieces of the system llama.cpp HIP lookup. The full // production availability check stays in llamacpp_server.cpp so this header is // only a narrow test seam, not a second implementation of the policy. diff --git a/src/cpp/server/mcp_client.cpp b/src/cpp/server/mcp_client.cpp index 5f2ca7a46a..a34b0436e0 100644 --- a/src/cpp/server/mcp_client.cpp +++ b/src/cpp/server/mcp_client.cpp @@ -36,6 +36,9 @@ #include #include #include + #ifdef __APPLE__ + #include + #endif #include #include #include @@ -46,6 +49,26 @@ extern char** environ; namespace fs = std::filesystem; +#if defined(LEMONADE_MCP_PROCESS_TEST_HOOK) && !defined(_WIN32) +using McpProcessTestHook = void (*)(pid_t); + +static std::atomic mcp_exit_observed_hook{nullptr}; +static std::atomic mcp_group_cleanup_hook{nullptr}; +static std::atomic mcp_final_reap_hook{nullptr}; + +extern "C" void lemonade_test_set_mcp_process_hooks( + McpProcessTestHook exit_observed, McpProcessTestHook group_cleanup, + McpProcessTestHook final_reap); + +extern "C" void lemonade_test_set_mcp_process_hooks( + McpProcessTestHook exit_observed, McpProcessTestHook group_cleanup, + McpProcessTestHook final_reap) { + mcp_exit_observed_hook.store(exit_observed, std::memory_order_release); + mcp_group_cleanup_hook.store(group_cleanup, std::memory_order_release); + mcp_final_reap_hook.store(final_reap, std::memory_order_release); +} +#endif + namespace lemon { namespace { @@ -610,42 +633,76 @@ class StdioProcess { #else if (pid_ > 0) { int status = 0; - bool reaped = false; - auto deadline = std::chrono::steady_clock::now() + - std::chrono::seconds(1); - while (std::chrono::steady_clock::now() < deadline) { - const pid_t result = ::waitpid(pid_, &status, WNOHANG); - if (result == pid_ || (result < 0 && errno == ECHILD)) { - reaped = true; - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(25)); - } - if (!reaped) { - ::kill(-pid_, SIGTERM); - deadline = std::chrono::steady_clock::now() + - std::chrono::seconds(1); + bool child_identity_retained = true; + auto wait_for_exit_without_reaping = [&](auto deadline) { while (std::chrono::steady_clock::now() < deadline) { - const pid_t result = ::waitpid(pid_, &status, WNOHANG); - if (result == pid_ || - (result < 0 && errno == ECHILD)) { - reaped = true; - break; + siginfo_t child_info{}; + int result; + do { + result = ::waitid( + P_PID, static_cast(pid_), &child_info, + WEXITED | WNOHANG | WNOWAIT); + } while (result < 0 && errno == EINTR); + if (result < 0) { + child_identity_retained = false; + return false; } + if (child_info.si_pid == pid_) return true; std::this_thread::sleep_for( std::chrono::milliseconds(25)); } + return false; + }; + + const bool exit_observed = wait_for_exit_without_reaping( + std::chrono::steady_clock::now() + + std::chrono::seconds(1)); +#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK + if (exit_observed) { + if (const auto hook = mcp_exit_observed_hook.load( + std::memory_order_acquire)) { + hook(pid_); + } + } +#endif + if (child_identity_retained) { + ::kill(-pid_, SIGTERM); + if (exit_observed) { + std::this_thread::sleep_for( + std::chrono::milliseconds(50)); + } else { + const bool observed_after_terminate = + wait_for_exit_without_reaping( + std::chrono::steady_clock::now() + + std::chrono::seconds(1)); +#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK + if (observed_after_terminate) { + if (const auto hook = mcp_exit_observed_hook.load( + std::memory_order_acquire)) { + hook(pid_); + } + } +#else + (void)observed_after_terminate; +#endif + } } - if (!reaped) { + if (child_identity_retained) { ::kill(-pid_, SIGKILL); +#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK + if (const auto hook = mcp_group_cleanup_hook.load( + std::memory_order_acquire)) { + hook(pid_); + } +#endif while (::waitpid(pid_, &status, 0) < 0 && errno == EINTR) { } - } else { - // Clean up descendants in the process group even when the direct - // child exited after stdin was closed. - ::kill(-pid_, SIGTERM); - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - ::kill(-pid_, SIGKILL); +#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK + if (const auto hook = mcp_final_reap_hook.load( + std::memory_order_acquire)) { + hook(pid_); + } +#endif } } #endif @@ -944,19 +1001,40 @@ class StdioProcess { static int create_cloexec_pipe(int fds[2]) { #ifdef __linux__ - return ::pipe2(fds, O_CLOEXEC); + if (::pipe2(fds, O_CLOEXEC) != 0) return -1; #else if (::pipe(fds) != 0) return -1; - if (set_close_on_exec(fds[0]) && set_close_on_exec(fds[1])) { - return 0; + if (!set_close_on_exec(fds[0]) || !set_close_on_exec(fds[1])) { + const int saved_errno = errno; + ::close(fds[0]); + ::close(fds[1]); + fds[0] = fds[1] = -1; + errno = saved_errno; + return -1; } - const int saved_errno = errno; - ::close(fds[0]); - ::close(fds[1]); - fds[0] = fds[1] = -1; - errno = saved_errno; - return -1; #endif + + for (int index = 0; index < 2; ++index) { + int& fd = fds[index]; + if (fd > STDERR_FILENO) continue; + + int replacement; + do { + replacement = + ::fcntl(fd, F_DUPFD_CLOEXEC, STDERR_FILENO + 1); + } while (replacement < 0 && errno == EINTR); + if (replacement < 0) { + const int saved_errno = errno; + ::close(fds[0]); + ::close(fds[1]); + fds[0] = fds[1] = -1; + errno = saved_errno; + return -1; + } + ::close(fd); + fd = replacement; + } + return 0; } void start_posix(const McpServerConfig& config) { @@ -987,27 +1065,108 @@ class StdioProcess { int stdin_pipe[2] = {-1, -1}; int stdout_pipe[2] = {-1, -1}; int stderr_pipe[2] = {-1, -1}; +#ifndef __APPLE__ int exec_error_pipe[2] = {-1, -1}; +#endif auto close_pair = [](int pair[2]) { if (pair[0] >= 0) ::close(pair[0]); if (pair[1] >= 0) ::close(pair[1]); pair[0] = pair[1] = -1; }; - if (create_cloexec_pipe(stdin_pipe) != 0 || - create_cloexec_pipe(stdout_pipe) != 0 || - create_cloexec_pipe(stderr_pipe) != 0 || - create_cloexec_pipe(exec_error_pipe) != 0) { + bool pipe_creation_failed = create_cloexec_pipe(stdin_pipe) != 0 || + create_cloexec_pipe(stdout_pipe) != 0 || + create_cloexec_pipe(stderr_pipe) != 0; +#ifndef __APPLE__ + pipe_creation_failed = pipe_creation_failed || + create_cloexec_pipe(exec_error_pipe) != 0; +#endif + if (pipe_creation_failed) { const int error = errno; close_pair(stdin_pipe); close_pair(stdout_pipe); close_pair(stderr_pipe); +#ifndef __APPLE__ close_pair(exec_error_pipe); +#endif throw std::runtime_error( std::string("pipe creation failed: ") + std::strerror(error)); } +#ifdef __APPLE__ + posix_spawn_file_actions_t file_actions; + bool file_actions_initialized = false; + int launch_error = posix_spawn_file_actions_init(&file_actions); + if (launch_error == 0) { + file_actions_initialized = true; + launch_error = posix_spawn_file_actions_adddup2( + &file_actions, stdin_pipe[0], STDIN_FILENO); + } + if (launch_error == 0) { + launch_error = posix_spawn_file_actions_adddup2( + &file_actions, stdout_pipe[1], STDOUT_FILENO); + } + if (launch_error == 0) { + launch_error = posix_spawn_file_actions_adddup2( + &file_actions, stderr_pipe[1], STDERR_FILENO); + } + for (int fd : {stdin_pipe[0], stdin_pipe[1], stdout_pipe[0], + stdout_pipe[1], stderr_pipe[0], stderr_pipe[1]}) { + if (launch_error == 0) { + launch_error = + posix_spawn_file_actions_addclose(&file_actions, fd); + } + } + if (launch_error == 0 && !config.working_dir.empty()) { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + launch_error = posix_spawn_file_actions_addchdir_np( + &file_actions, config.working_dir.c_str()); +#pragma clang diagnostic pop + } + + posix_spawnattr_t attributes; + bool attributes_initialized = false; + if (launch_error == 0) { + launch_error = posix_spawnattr_init(&attributes); + attributes_initialized = launch_error == 0; + } + if (launch_error == 0) { + launch_error = posix_spawnattr_setpgroup(&attributes, 0); + } + if (launch_error == 0) { + launch_error = posix_spawnattr_setflags( + &attributes, POSIX_SPAWN_CLOEXEC_DEFAULT | + POSIX_SPAWN_SETPGROUP); + } + + pid_t child = 0; + if (launch_error == 0) { + launch_error = posix_spawn( + &child, executable.c_str(), &file_actions, &attributes, + argv.data(), envp.data()); + } + if (attributes_initialized) { + posix_spawnattr_destroy(&attributes); + } + if (file_actions_initialized) { + posix_spawn_file_actions_destroy(&file_actions); + } + + if (launch_error != 0) { + close_pair(stdin_pipe); + close_pair(stdout_pipe); + close_pair(stderr_pipe); + throw std::runtime_error( + "Failed to launch MCP server '" + config.command + "': " + + std::strerror(launch_error)); + } + + ::close(stdin_pipe[0]); + ::close(stdout_pipe[1]); + ::close(stderr_pipe[1]); +#else const pid_t child = ::fork(); if (child < 0) { const int error = errno; @@ -1089,6 +1248,7 @@ class StdioProcess { std::string("Failed to verify MCP exec: ") + std::strerror(error)); } +#endif pid_ = child; stdin_write_ = stdin_pipe[1]; diff --git a/src/cpp/server/streaming_proxy.cpp b/src/cpp/server/streaming_proxy.cpp index 17c0e432bd..1b492fd334 100644 --- a/src/cpp/server/streaming_proxy.cpp +++ b/src/cpp/server/streaming_proxy.cpp @@ -1,9 +1,13 @@ #include "lemon/streaming_proxy.h" -#include -#include +#include #include #include +#include +#include +#include +#include #include +#include #include #include @@ -89,6 +93,209 @@ bool is_gpu_hang_or_compute_error(const std::string& message) { lowered.find("gpu hang") != std::string::npos; } +std::optional sse_event_data(const std::string& frame) { + std::string event_data; + bool has_data_field = false; + std::size_t line_start = 0; + while (line_start < frame.size()) { + const std::size_t terminator = + frame.find_first_of("\r\n", line_start); + const std::size_t content_end = + terminator == std::string::npos ? frame.size() : terminator; + const std::size_t content_length = content_end - line_start; + const bool empty_data_field = + content_length == 4 && + frame.compare(line_start, content_length, "data") == 0; + const bool data_field = + content_length >= 5 && frame.compare(line_start, 5, "data:") == 0; + if (empty_data_field || data_field) { + if (has_data_field) { + event_data.push_back('\n'); + } + if (data_field) { + std::size_t payload_start = line_start + 5; + if (payload_start < content_end && + frame[payload_start] == ' ') { + ++payload_start; + } + event_data.append( + frame, payload_start, content_end - payload_start); + } + has_data_field = true; + } + + if (terminator == std::string::npos) { + break; + } + line_start = terminator + 1; + if (frame[terminator] == '\r' && line_start < frame.size() && + frame[line_start] == '\n') { + ++line_start; + } + } + + if (!has_data_field) { + return std::nullopt; + } + return event_data; +} + +bool is_sse_done_frame(const std::string& frame) { + const std::optional event_data = sse_event_data(frame); + return event_data.has_value() && *event_data == "[DONE]"; +} + +class SseFrameBuffer { +public: + explicit SseFrameBuffer(std::size_t max_frame_bytes) + : max_frame_bytes_(max_frame_bytes) {} + + template + bool append(const char* data, + std::size_t length, + FrameCallback&& frame_callback, + LineCallback&& line_callback) { + std::size_t offset = 0; + while (offset < length) { + const std::size_t buffer_limit = max_frame_bytes_ + 1; + const std::size_t available = buffer_limit - buffer_.size(); + const std::size_t append_size = + std::min(length - offset, available); + buffer_.append(data + offset, append_size); + offset += append_size; + if (!drain(false, frame_callback, line_callback)) { + return false; + } + } + return true; + } + + template + bool finish(FrameCallback&& frame_callback, + LineCallback&& line_callback) { + return drain(true, frame_callback, line_callback); + } + + bool empty() const { + return buffer_.empty(); + } + + void clear() { + buffer_.clear(); + scan_position_ = 0; + line_start_ = 0; + } + +private: + bool prepare_stream_start(bool end_of_stream) { + if (!stream_start_) { + return true; + } + + static constexpr char utf8_bom[] = "\xEF\xBB\xBF"; + constexpr std::size_t utf8_bom_size = sizeof(utf8_bom) - 1; + const std::size_t comparison_size = + std::min(buffer_.size(), utf8_bom_size); + if (buffer_.compare( + 0, comparison_size, utf8_bom, comparison_size) != 0) { + stream_start_ = false; + return true; + } + if (buffer_.size() < utf8_bom_size && !end_of_stream) { + return false; + } + if (buffer_.size() >= utf8_bom_size) { + if (utf8_bom_removed_) { + throw std::runtime_error( + "backend connection failed during SSE stream before DONE: " + "multiple leading UTF-8 BOMs"); + } + buffer_.erase(0, utf8_bom_size); + utf8_bom_removed_ = true; + return prepare_stream_start(end_of_stream); + } + stream_start_ = false; + return true; + } + + template + bool drain(bool end_of_stream, + FrameCallback&& frame_callback, + LineCallback&& line_callback) { + if (!prepare_stream_start(end_of_stream)) { + return true; + } + + std::size_t consumed = 0; + while (scan_position_ < buffer_.size()) { + const std::size_t terminator = + buffer_.find_first_of("\r\n", scan_position_); + if (terminator == std::string::npos) { + scan_position_ = buffer_.size(); + break; + } + + std::size_t terminator_length = 1; + if (buffer_[terminator] == '\r') { + if (terminator + 1 == buffer_.size() && !end_of_stream) { + scan_position_ = terminator; + break; + } + if (terminator + 1 < buffer_.size() && + buffer_[terminator + 1] == '\n') { + terminator_length = 2; + } + } + + const std::size_t terminated_end = + terminator + terminator_length; + if (terminated_end - consumed > max_frame_bytes_) { + throw_frame_too_large(); + } + + line_callback( + buffer_.substr(line_start_, terminator - line_start_)); + const bool frame_complete = terminator == line_start_; + scan_position_ = terminated_end; + line_start_ = terminated_end; + if (!frame_complete) { + continue; + } + + if (!frame_callback( + buffer_.substr(consumed, terminated_end - consumed))) { + clear(); + return false; + } + consumed = terminated_end; + } + + if (buffer_.size() - consumed > max_frame_bytes_) { + throw_frame_too_large(); + } + if (consumed > 0) { + buffer_.erase(0, consumed); + scan_position_ -= consumed; + line_start_ -= consumed; + } + return true; + } + + [[noreturn]] void throw_frame_too_large() const { + throw std::runtime_error( + "backend connection failed during SSE stream before DONE: " + "SSE frame exceeds the maximum size of " + + std::to_string(max_frame_bytes_) + " bytes"); + } + + const std::size_t max_frame_bytes_; + std::string buffer_; + std::size_t scan_position_ = 0; + std::size_t line_start_ = 0; + bool stream_start_ = true; + bool utf8_bom_removed_ = false; +}; + } // namespace @@ -98,8 +305,42 @@ void StreamingProxy::forward_sse_stream( httplib::DataSink& sink, std::function on_complete, long timeout_seconds, - std::function on_chunk, - long heartbeat_interval_ms) { + std::function on_backend_progress, + long heartbeat_interval_ms, + std::function on_frame) { + forward_sse_stream_impl( + backend_url, request_body, sink, nullptr, std::move(on_complete), + timeout_seconds, std::move(on_backend_progress), heartbeat_interval_ms, + std::move(on_frame)); +} + +void StreamingProxy::forward_transformed_sse_stream( + const std::string& backend_url, + const std::string& request_body, + httplib::DataSink& sink, + SseFrameTransform frame_transform, + std::function on_complete, + long timeout_seconds, + std::function on_backend_progress, + long heartbeat_interval_ms, + std::function on_frame) { + forward_sse_stream_impl( + backend_url, request_body, sink, std::move(frame_transform), + std::move(on_complete), timeout_seconds, + std::move(on_backend_progress), heartbeat_interval_ms, + std::move(on_frame)); +} + +void StreamingProxy::forward_sse_stream_impl( + const std::string& backend_url, + const std::string& request_body, + httplib::DataSink& sink, + SseFrameTransform frame_transform, + std::function on_complete, + long timeout_seconds, + std::function on_backend_progress, + long heartbeat_interval_ms, + std::function on_frame) { TelemetryData telemetry; try { @@ -108,13 +349,14 @@ void StreamingProxy::forward_sse_stream( telemetry.model_name = req_json["model"].get(); } } catch (...) {} - std::string line_buffer; + SseFrameBuffer frame_buffer(kMaxSseFrameBytes); + std::exception_ptr frame_processing_error; bool stream_error = false; bool has_done_marker = false; bool has_first_token = false; double time_to_first_token = 0.0; const auto start_time = std::chrono::steady_clock::now(); - auto last_activity_time = start_time; + auto last_downstream_write_time = start_time; int backend_status = 200; std::string error_body; @@ -135,13 +377,54 @@ void StreamingProxy::forward_sse_stream( } }; + auto process_frame = [&](const std::string& source_frame) { + const bool is_done_frame = is_sse_done_frame(source_frame); + has_done_marker = has_done_marker || is_done_frame; + std::string frame = frame_transform + ? frame_transform(source_frame) + : source_frame; + if (frame.empty()) { + return true; + } + if (!sink.write(frame.data(), frame.size())) { + return false; + } + last_downstream_write_time = std::chrono::steady_clock::now(); + + if (on_frame) { + on_frame(); + } + if (!has_first_token && + frame.find("data: ") != std::string::npos) { + has_first_token = true; + time_to_first_token = std::chrono::duration( + std::chrono::steady_clock::now() - start_time) + .count(); + } + return true; + }; + + auto process_frame_bytes = [&](const char* data, + std::size_t length, + bool end_of_stream) { + try { + if (end_of_stream) { + return frame_buffer.finish(process_frame, process_line); + } + return frame_buffer.append( + data, length, process_frame, process_line); + } catch (...) { + frame_processing_error = std::current_exception(); + frame_buffer.clear(); + return false; + } + }; + utils::HttpResponse result = utils::HttpClient::post_stream( backend_url, request_body, - [&sink, &line_buffer, &has_done_marker, &has_first_token, &time_to_first_token, - &start_time, &last_activity_time, &on_chunk, &process_line, &backend_status, &error_body](const char* data, size_t length) { - last_activity_time = std::chrono::steady_clock::now(); - + [&backend_status, &error_body, &on_backend_progress, + &process_frame_bytes](const char* data, size_t length) { if (backend_status != 200) { if (error_body.size() < max_error_body) { error_body.append(data, std::min(length, max_error_body - error_body.size())); @@ -149,35 +432,18 @@ void StreamingProxy::forward_sse_stream( return true; } - if (on_chunk) { - on_chunk(); + if (length > 0 && on_backend_progress) { + on_backend_progress(); } - line_buffer.append(data, length); - process_sse_lines(line_buffer, process_line); - - std::string chunk(data, length); - if (!has_first_token && chunk.find("data: ") != std::string::npos) { - has_first_token = true; - time_to_first_token = std::chrono::duration( - std::chrono::steady_clock::now() - start_time).count(); - } - - if (chunk.find("data: [DONE]") != std::string::npos) { - has_done_marker = true; - } - - if (!sink.write(data, length)) { - return false; - } - - return true; + return process_frame_bytes(data, length, false); }, {}, timeout_seconds, [&backend_status](int status) { backend_status = status; }, utils::HttpSecurityPolicy::TrustedLoopback, - [&sink, &last_activity_time, &backend_status, heartbeat_interval_ms]() { + [&sink, &last_downstream_write_time, &backend_status, + heartbeat_interval_ms]() { if (sink.is_writable && !sink.is_writable()) { return true; } @@ -185,13 +451,13 @@ void StreamingProxy::forward_sse_stream( if (heartbeat_interval_ms > 0 && backend_status == 200) { const auto now = std::chrono::steady_clock::now(); const auto elapsed = std::chrono::duration_cast( - now - last_activity_time).count(); + now - last_downstream_write_time).count(); if (elapsed >= heartbeat_interval_ms) { static constexpr const char* heartbeat = ": ping\n\n"; if (!sink.write(heartbeat, std::strlen(heartbeat))) { return true; } - last_activity_time = now; + last_downstream_write_time = now; } } @@ -199,13 +465,35 @@ void StreamingProxy::forward_sse_stream( } ); + const bool may_have_final_frame = + result.curl_code == CURLE_OK || + result.curl_code == CURLE_PARTIAL_FILE || + result.curl_code == CURLE_RECV_ERROR; + const bool eof_sink_rejected = + backend_status == 200 && may_have_final_frame && + !process_frame_bytes(nullptr, 0, true); + const bool has_incomplete_frame = !frame_buffer.empty(); + frame_buffer.clear(); + if (frame_processing_error) { + std::rethrow_exception(frame_processing_error); + } + if (result.curl_code == CURLE_OK && backend_status == 200 && + has_incomplete_frame) { + throw std::runtime_error( + "backend connection failed during SSE stream before DONE: " + "incomplete final event"); + } + const bool client_disconnected = result.curl_code == CURLE_WRITE_ERROR || - result.curl_code == CURLE_ABORTED_BY_CALLBACK; + result.curl_code == CURLE_ABORTED_BY_CALLBACK || eof_sink_rejected; const bool transport_interrupted = result.curl_code == CURLE_PARTIAL_FILE || result.curl_code == CURLE_RECV_ERROR; - if (result.curl_code != CURLE_OK) { + if (eof_sink_rejected && result.curl_code == CURLE_OK) { + stream_error = true; + telemetry.error_message = "Client disconnected during stream"; + } else if (result.curl_code != CURLE_OK) { if (client_disconnected) { stream_error = true; LOG(WARNING, "StreamingProxy") << "Client disconnected during SSE stream (CURL error: " << result.curl_error << ")" << std::endl; @@ -279,13 +567,6 @@ void StreamingProxy::forward_sse_stream( LOG(INFO, "Server") << "Streaming completed - 200 OK" << std::endl; - if (!line_buffer.empty()) { - if (line_buffer.back() == '\r') { - line_buffer.pop_back(); - } - process_line(line_buffer); - } - if (telemetry.time_to_first_token <= 0.0) { telemetry.time_to_first_token = time_to_first_token; } diff --git a/src/cpp/server/system_info.cpp b/src/cpp/server/system_info.cpp index b14b65f04b..9c4a12042b 100644 --- a/src/cpp/server/system_info.cpp +++ b/src/cpp/server/system_info.cpp @@ -2398,12 +2398,9 @@ static std::vector query_nvidia_smi() { " --format=csv,noheader,nounits 2>/dev/null"; for (const char* smi : {"nvidia-smi", "/usr/bin/nvidia-smi"}) { std::string cmd = std::string(smi) + smi_query; - FILE* pipe = popen(cmd.c_str(), "r"); - if (!pipe) continue; std::string candidate; - char buffer[512]; - while (fgets(buffer, sizeof(buffer), pipe) != nullptr) candidate += buffer; - int rc = pclose(pipe); + const int rc = lemon::utils::ProcessManager::run_command( + cmd, candidate, 10); if (rc == 0 && !candidate.empty()) { output = candidate; break; @@ -3055,7 +3052,7 @@ CPUInfo LinuxSystemInfo::get_cpu_device() { cpu.available = false; // Execute lscpu command - FILE* pipe = popen("lscpu 2>/dev/null", "r"); + FILE* pipe = popen("lscpu 2>/dev/null", "re"); if (!pipe) { cpu.error = "Failed to execute lscpu command"; return cpu; @@ -3268,7 +3265,7 @@ std::vector LinuxSystemInfo::get_nvidia_gpu_devices() { } // Fallback: lspci (for systems where nvidia-smi is unavailable) - FILE* pipe = popen("lspci 2>/dev/null | grep -iE 'vga|3d|display'", "r"); + FILE* pipe = popen("lspci 2>/dev/null | grep -iE 'vga|3d|display'", "re"); if (!pipe) { GPUInfo gpu; gpu.available = false; @@ -3519,7 +3516,7 @@ std::vector LinuxSystemInfo::detect_amd_gpus(const std::string& gpu_typ std::string LinuxSystemInfo::get_nvidia_driver_version() { // Try nvidia-smi first - FILE* pipe = popen("nvidia-smi --query-gpu=driver_version --format=csv,noheader,nounits 2>/dev/null", "r"); + FILE* pipe = popen("nvidia-smi --query-gpu=driver_version --format=csv,noheader,nounits 2>/dev/null", "re"); if (pipe) { char buffer[128]; if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { @@ -3568,7 +3565,7 @@ std::string LinuxSystemInfo::get_nvidia_driver_version() { } double LinuxSystemInfo::get_nvidia_vram() { - FILE* pipe = popen("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null", "r"); + FILE* pipe = popen("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null", "re"); if (!pipe) { return 0.0; } @@ -3704,7 +3701,7 @@ std::string LinuxSystemInfo::get_os_version() { } std::string LinuxSystemInfo::get_processor_name() { - FILE* pipe = popen("lscpu 2>/dev/null", "r"); + FILE* pipe = popen("lscpu 2>/dev/null", "re"); if (!pipe) { return "ERROR - Failed to execute lscpu"; } @@ -3855,20 +3852,17 @@ std::string MacOSSystemInfo::get_os_version() { std::string result = "macOS"; // Get macOS product version (e.g., "14.3.1") - FILE* pipe = popen("sw_vers -productVersion 2>/dev/null", "r"); - if (pipe) { - char buffer[128]; - if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - std::string version = buffer; - // Trim trailing newline - while (!version.empty() && (version.back() == '\n' || version.back() == '\r')) { - version.pop_back(); - } - if (!version.empty()) { - result += " " + version; - } + std::string version; + if (lemon::utils::ProcessManager::run_command( + "sw_vers -productVersion 2>/dev/null", version, 5) == 0) { + // Trim trailing newline + while (!version.empty() && + (version.back() == '\n' || version.back() == '\r')) { + version.pop_back(); + } + if (!version.empty()) { + result += " " + version; } - pclose(pipe); } // Append Darwin kernel version diff --git a/src/cpp/server/utils/platform/process_linux.cpp b/src/cpp/server/utils/platform/process_linux.cpp index 8827cfe62c..b1eb392492 100644 --- a/src/cpp/server/utils/platform/process_linux.cpp +++ b/src/cpp/server/utils/platform/process_linux.cpp @@ -26,6 +26,10 @@ #include #endif +#ifdef LEMONADE_PROCESS_TEST_HOOK +extern "C" void lemonade_test_run_command_pipe_created(int fd); +#endif + namespace lemon::utils { // Helper functions @@ -55,6 +59,95 @@ static void log_process_line(const std::string& line) { } } +static void close_pipe(int pipe_fds[2]) { + for (int i = 0; i < 2; ++i) { + if (pipe_fds[i] >= 0) { + close(pipe_fds[i]); + pipe_fds[i] = -1; + } + } +} + +static bool create_pipe_above_standard_streams(int pipe_fds[2]) { + int pipe_result; + do { + pipe_result = pipe2(pipe_fds, O_CLOEXEC); + } while (pipe_result < 0 && errno == EINTR); + if (pipe_result < 0) { + return false; + } + + for (int i = 0; i < 2; ++i) { + if (pipe_fds[i] > STDERR_FILENO) { + continue; + } + + int replacement; + do { + replacement = + fcntl(pipe_fds[i], F_DUPFD_CLOEXEC, STDERR_FILENO + 1); + } while (replacement < 0 && errno == EINTR); + if (replacement < 0) { + close_pipe(pipe_fds); + return false; + } + close(pipe_fds[i]); + pipe_fds[i] = replacement; + } + return true; +} + +static void read_process_output( + int fd, + bool log_output, + const std::shared_ptr& output_capture) { + char buffer[4096]; + std::string line_buffer; + ssize_t bytes_read; + + while ((bytes_read = read(fd, buffer, sizeof(buffer))) > 0) { + if (output_capture) { + output_capture->append(buffer, static_cast(bytes_read)); + } + if (!log_output) { + continue; + } + + line_buffer.append(buffer, static_cast(bytes_read)); + size_t pos; + while ((pos = line_buffer.find('\n')) != std::string::npos) { + std::string line = line_buffer.substr(0, pos); + line_buffer.erase(0, pos + 1); + log_process_line(line); + } + } + + if (log_output && !line_buffer.empty()) { + log_process_line(line_buffer); + } + close(fd); + if (output_capture) { + output_capture->finish_reader(); + } +} + +static void start_process_output_reader( + int fd, + bool log_output, + const std::shared_ptr& output_capture) { + try { + std::thread(read_process_output, fd, log_output, output_capture).detach(); + } catch (const std::exception& error) { + close(fd); + if (output_capture) { + output_capture->finish_reader(); + } + LOG(ERROR, "ProcessManager") + << "Failed to start process output reader: " << error.what() + << std::endl; + } +} + #ifdef HAVE_LIBCAP static void preserve_capabilities_for_exec() { cap_t caps = cap_get_proc(); @@ -101,7 +194,8 @@ class LinuxProcessPlatform : public ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars) override; + const std::vector>& env_vars, + std::shared_ptr output_capture) override; void terminate(ProcessHandle handle) override; bool is_running(ProcessHandle handle) override; @@ -129,7 +223,7 @@ class LinuxProcessPlatform : public ProcessPlatform { const std::vector& args, const std::string& working_dir, bool inherit_output, - bool filter_health_logs, + bool redirect_output, const std::vector>& env_vars, int stdout_pipe[2], int stderr_pipe[2]); @@ -141,7 +235,7 @@ pid_t LinuxProcessPlatform::spawn_process( const std::vector& args, const std::string& working_dir, bool inherit_output, - bool filter_health_logs, + bool redirect_output, const std::vector>& env_vars, int stdout_pipe[2], int stderr_pipe[2]) { @@ -165,8 +259,7 @@ pid_t LinuxProcessPlatform::spawn_process( setenv(env_pair.first.c_str(), env_pair.second.c_str(), 1); } - // Redirect stdout/stderr to pipes if filtering - if (inherit_output && filter_health_logs) { + if (redirect_output) { close(stdout_pipe[0]); close(stderr_pipe[0]); dup2(stdout_pipe[1], STDOUT_FILENO); @@ -210,18 +303,25 @@ ProcessHandle LinuxProcessPlatform::spawn( const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars) { + const std::vector>& env_vars, + std::shared_ptr output_capture) { ProcessHandle handle; handle.handle = nullptr; handle.pid = 0; + handle.output_capture = output_capture; int stdout_pipe[2] = {-1, -1}; int stderr_pipe[2] = {-1, -1}; + const bool redirect_output = + (inherit_output && filter_health_logs) || output_capture != nullptr; - // Create pipes for filtering if requested - if (inherit_output && filter_health_logs) { - if (pipe(stdout_pipe) < 0 || pipe(stderr_pipe) < 0) { + if (redirect_output) { + if (!create_pipe_above_standard_streams(stdout_pipe)) { + throw std::runtime_error("Failed to create pipes for output filtering"); + } + if (!create_pipe_above_standard_streams(stderr_pipe)) { + close_pipe(stdout_pipe); throw std::runtime_error("Failed to create pipes for output filtering"); } } @@ -238,8 +338,17 @@ ProcessHandle LinuxProcessPlatform::spawn( } } - pid_t pid = spawn_process(executable, args, working_dir, inherit_output, - filter_health_logs, env_vars, stdout_pipe, stderr_pipe); + pid_t pid; + try { + pid = spawn_process(executable, args, working_dir, inherit_output, + redirect_output, env_vars, stdout_pipe, stderr_pipe); + } catch (...) { + if (redirect_output) { + close_pipe(stdout_pipe); + close_pipe(stderr_pipe); + } + throw; + } handle.pid = pid; @@ -247,58 +356,14 @@ ProcessHandle LinuxProcessPlatform::spawn( LOG(INFO, "ProcessManager") << "Process started successfully, PID: " << pid << std::endl; } - // Start filter threads if needed - if (inherit_output && filter_health_logs) { + if (redirect_output) { close(stdout_pipe[1]); close(stderr_pipe[1]); - std::thread([fd = stdout_pipe[0]]() { - char buffer[4096]; - std::string line_buffer; - ssize_t bytes_read; - - while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { - buffer[bytes_read] = '\0'; - line_buffer += buffer; - - size_t pos; - while ((pos = line_buffer.find('\n')) != std::string::npos) { - std::string line = line_buffer.substr(0, pos); - line_buffer = line_buffer.substr(pos + 1); - log_process_line(line); - } - } - - if (!line_buffer.empty()) { - log_process_line(line_buffer); - } - - close(fd); - }).detach(); - - std::thread([fd = stderr_pipe[0]]() { - char buffer[4096]; - std::string line_buffer; - ssize_t bytes_read; - - while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { - buffer[bytes_read] = '\0'; - line_buffer += buffer; - - size_t pos; - while ((pos = line_buffer.find('\n')) != std::string::npos) { - std::string line = line_buffer.substr(0, pos); - line_buffer = line_buffer.substr(pos + 1); - log_process_line(line); - } - } - - if (!line_buffer.empty()) { - log_process_line(line_buffer); - } - - close(fd); - }).detach(); + start_process_output_reader(stdout_pipe[0], inherit_output, + output_capture); + start_process_output_reader(stderr_pipe[0], inherit_output, + output_capture); } return handle; @@ -500,17 +565,16 @@ int LinuxProcessPlatform::run_with_output( int timeout_seconds, bool capture_stderr) { - int stdout_pipe[2]; + int stdout_pipe[2] = {-1, -1}; - if (pipe(stdout_pipe) < 0) { + if (!create_pipe_above_standard_streams(stdout_pipe)) { throw std::runtime_error("Failed to create pipe"); } pid_t pid = fork(); if (pid < 0) { - close(stdout_pipe[0]); - close(stdout_pipe[1]); + close_pipe(stdout_pipe); throw std::runtime_error("Failed to fork process"); } @@ -676,10 +740,13 @@ int LinuxProcessPlatform::find_free_port(int start_port) { int LinuxProcessPlatform::run_command(const std::string& command, std::string& output, int timeout_seconds) { output.clear(); - FILE* pipe = popen(command.c_str(), "r"); + FILE* pipe = popen(command.c_str(), "re"); if (!pipe) { return -1; } +#ifdef LEMONADE_PROCESS_TEST_HOOK + lemonade_test_run_command_pipe_created(fileno(pipe)); +#endif char buf[4096]; while (fgets(buf, sizeof(buf), pipe)) { diff --git a/src/cpp/server/utils/platform/process_macos.cpp b/src/cpp/server/utils/platform/process_macos.cpp index a61e060bf2..186831baaf 100644 --- a/src/cpp/server/utils/platform/process_macos.cpp +++ b/src/cpp/server/utils/platform/process_macos.cpp @@ -20,6 +20,12 @@ #include extern char** environ; +#ifdef LEMONADE_PROCESS_TEST_HOOK +extern "C" void lemonade_test_run_with_output_pipe_created(); +extern "C" void lemonade_test_run_command_exit_observed(pid_t pid); +extern "C" void lemonade_test_run_command_group_cleanup_complete(pid_t pid); +extern "C" void lemonade_test_run_command_final_reap_complete(pid_t pid); +#endif namespace lemon::utils { @@ -50,6 +56,308 @@ static void log_process_line(const std::string& line) { } } +static void close_pipe(int pipe_fds[2]) { + for (int i = 0; i < 2; ++i) { + if (pipe_fds[i] >= 0) { + close(pipe_fds[i]); + pipe_fds[i] = -1; + } + } +} + +static bool set_close_on_exec(int fd) { + int flags; + do { + flags = fcntl(fd, F_GETFD); + } while (flags < 0 && errno == EINTR); + if (flags < 0) { + return false; + } + + int result; + do { + result = fcntl(fd, F_SETFD, flags | FD_CLOEXEC); + } while (result < 0 && errno == EINTR); + return result == 0; +} + +static bool create_pipe_above_standard_streams(int pipe_fds[2]) { + if (pipe(pipe_fds) < 0) { + return false; + } +#ifdef LEMONADE_PROCESS_TEST_HOOK + lemonade_test_run_with_output_pipe_created(); +#endif + + for (int i = 0; i < 2; ++i) { + if (pipe_fds[i] > STDERR_FILENO) { + if (!set_close_on_exec(pipe_fds[i])) { + close_pipe(pipe_fds); + return false; + } + continue; + } + + int replacement; + do { + replacement = + fcntl(pipe_fds[i], F_DUPFD_CLOEXEC, STDERR_FILENO + 1); + } while (replacement < 0 && errno == EINTR); + if (replacement < 0) { + close_pipe(pipe_fds); + return false; + } + close(pipe_fds[i]); + pipe_fds[i] = replacement; + } + return true; +} + +static void kill_process_group(pid_t pid, bool fallback_to_process = true) { + int result; + do { + result = ::kill(-pid, SIGKILL); + } while (result < 0 && errno == EINTR); + if (fallback_to_process && result < 0 && errno == ESRCH) { + do { + result = ::kill(pid, SIGKILL); + } while (result < 0 && errno == EINTR); + } +} + +class SpawnFileActions { +public: + SpawnFileActions() { + error_ = posix_spawn_file_actions_init(&actions_); + initialized_ = error_ == 0; + } + + ~SpawnFileActions() { + if (initialized_) { + posix_spawn_file_actions_destroy(&actions_); + } + } + + SpawnFileActions(const SpawnFileActions&) = delete; + SpawnFileActions& operator=(const SpawnFileActions&) = delete; + + void add_close(int fd) { + if (error_ == 0) { + error_ = posix_spawn_file_actions_addclose(&actions_, fd); + } + } + + void add_dup2(int source, int destination) { + if (error_ == 0) { + error_ = + posix_spawn_file_actions_adddup2(&actions_, source, destination); + } + } + + void add_open(int fd, const char* path, int flags, mode_t mode) { + if (error_ == 0) { + error_ = posix_spawn_file_actions_addopen(&actions_, fd, path, + flags, mode); + } + } + + void add_inherit_if_open(int fd) { + if (error_ != 0) { + return; + } + + int flags; + do { + flags = fcntl(fd, F_GETFD); + } while (flags < 0 && errno == EINTR); + if (flags < 0) { + if (errno != EBADF) { + error_ = errno; + } + return; + } + if ((flags & FD_CLOEXEC) == 0) { + error_ = posix_spawn_file_actions_addinherit_np(&actions_, fd); + } + } + + void add_chdir(const std::string& working_dir) { + if (error_ != 0 || working_dir.empty()) { + return; + } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + error_ = posix_spawn_file_actions_addchdir_np(&actions_, + working_dir.c_str()); +#pragma clang diagnostic pop + } + + int error() const { return error_; } + posix_spawn_file_actions_t* get() { return &actions_; } + +private: + posix_spawn_file_actions_t actions_; + bool initialized_ = false; + int error_ = 0; +}; + +class SpawnAttributes { +public: + SpawnAttributes() { + error_ = posix_spawnattr_init(&attributes_); + initialized_ = error_ == 0; + } + + ~SpawnAttributes() { + if (initialized_) { + posix_spawnattr_destroy(&attributes_); + } + } + + SpawnAttributes(const SpawnAttributes&) = delete; + SpawnAttributes& operator=(const SpawnAttributes&) = delete; + + void set_default_signals() { + if (error_ != 0) { + return; + } + + sigset_t default_signals; + if (sigfillset(&default_signals) != 0) { + error_ = errno; + return; + } + error_ = posix_spawnattr_setsigdefault(&attributes_, &default_signals); + } + + void set_process_group(pid_t process_group) { + if (error_ == 0) { + error_ = posix_spawnattr_setpgroup(&attributes_, process_group); + } + } + + void set_flags(short flags) { + if (error_ == 0) { + error_ = posix_spawnattr_setflags(&attributes_, flags); + } + } + + int error() const { return error_; } + posix_spawnattr_t* get() { return &attributes_; } + +private: + posix_spawnattr_t attributes_; + bool initialized_ = false; + int error_ = 0; +}; + +static pid_t spawn_capturing_output(const std::string& executable, + const std::vector& args, + const std::string& working_dir, + bool capture_stderr, + bool create_process_group, + int output_pipe[2]) { + SpawnFileActions file_actions; + file_actions.add_inherit_if_open(STDIN_FILENO); + file_actions.add_close(output_pipe[0]); + file_actions.add_dup2(output_pipe[1], STDOUT_FILENO); + if (capture_stderr) { + file_actions.add_dup2(output_pipe[1], STDERR_FILENO); + } else { + file_actions.add_inherit_if_open(STDERR_FILENO); + } + file_actions.add_close(output_pipe[1]); + file_actions.add_chdir(working_dir); + + SpawnAttributes attributes; + short spawn_flags = POSIX_SPAWN_CLOEXEC_DEFAULT; + if (create_process_group) { + attributes.set_process_group(0); + spawn_flags |= POSIX_SPAWN_SETPGROUP; + } + attributes.set_flags(spawn_flags); + + std::vector argv_ptrs; + argv_ptrs.reserve(args.size() + 2); + argv_ptrs.push_back(const_cast(executable.c_str())); + for (const auto& arg : args) { + argv_ptrs.push_back(const_cast(arg.c_str())); + } + argv_ptrs.push_back(nullptr); + + int spawn_result = file_actions.error(); + if (spawn_result == 0) { + spawn_result = attributes.error(); + } + + pid_t pid = 0; + if (spawn_result == 0) { + spawn_result = posix_spawnp(&pid, executable.c_str(), + file_actions.get(), attributes.get(), + argv_ptrs.data(), environ); + } + if (spawn_result != 0) { + close_pipe(output_pipe); + throw std::runtime_error(std::string("posix_spawn failed: ") + + strerror(spawn_result)); + } + + close(output_pipe[1]); + output_pipe[1] = -1; + return pid; +} + +static void read_process_output( + int fd, + bool log_output, + const std::shared_ptr& output_capture) { + char buffer[4096]; + std::string line_buffer; + ssize_t bytes_read; + + while ((bytes_read = read(fd, buffer, sizeof(buffer))) > 0) { + if (output_capture) { + output_capture->append(buffer, static_cast(bytes_read)); + } + if (!log_output) { + continue; + } + + line_buffer.append(buffer, static_cast(bytes_read)); + size_t pos; + while ((pos = line_buffer.find('\n')) != std::string::npos) { + std::string line = line_buffer.substr(0, pos); + line_buffer.erase(0, pos + 1); + log_process_line(line); + } + } + + if (log_output && !line_buffer.empty()) { + log_process_line(line_buffer); + } + close(fd); + if (output_capture) { + output_capture->finish_reader(); + } +} + +static void start_process_output_reader( + int fd, + bool log_output, + const std::shared_ptr& output_capture) { + try { + std::thread(read_process_output, fd, log_output, output_capture).detach(); + } catch (const std::exception& error) { + close(fd); + if (output_capture) { + output_capture->finish_reader(); + } + LOG(ERROR, "ProcessManager") + << "Failed to start process output reader: " << error.what() + << std::endl; + } +} + // Forward declare UnixProcessPlatform base class methods class MacOSProcessPlatform : public ProcessPlatform { public: @@ -59,7 +367,8 @@ class MacOSProcessPlatform : public ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars) override; + const std::vector>& env_vars, + std::shared_ptr output_capture) override; void terminate(ProcessHandle handle) override; bool is_running(ProcessHandle handle) override; @@ -87,17 +396,25 @@ ProcessHandle MacOSProcessPlatform::spawn( const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars) { + const std::vector>& env_vars, + std::shared_ptr output_capture) { ProcessHandle handle; handle.handle = nullptr; handle.pid = 0; + handle.output_capture = output_capture; int stdout_pipe[2] = {-1, -1}; int stderr_pipe[2] = {-1, -1}; + const bool redirect_output = + (inherit_output && filter_health_logs) || output_capture != nullptr; - if (inherit_output && filter_health_logs) { - if (pipe(stdout_pipe) < 0 || pipe(stderr_pipe) < 0) { + if (redirect_output) { + if (!create_pipe_above_standard_streams(stdout_pipe)) { + throw std::runtime_error("Failed to create pipes for output filtering"); + } + if (!create_pipe_above_standard_streams(stderr_pipe)) { + close_pipe(stdout_pipe); throw std::runtime_error("Failed to create pipes for output filtering"); } } @@ -128,34 +445,30 @@ ProcessHandle MacOSProcessPlatform::spawn( // pipe/working-dir semantics. Adds POSIX_SPAWN_CLOEXEC_DEFAULT to avoid // leaking lemond FDs into the child, and POSIX_SPAWN_SETSIGDEF to reset // inherited SIG_IGN dispositions. - posix_spawn_file_actions_t file_actions; - posix_spawn_file_actions_init(&file_actions); - - if (inherit_output && filter_health_logs) { - posix_spawn_file_actions_addclose(&file_actions, stdout_pipe[0]); - posix_spawn_file_actions_addclose(&file_actions, stderr_pipe[0]); - posix_spawn_file_actions_adddup2(&file_actions, stdout_pipe[1], STDOUT_FILENO); - posix_spawn_file_actions_adddup2(&file_actions, stderr_pipe[1], STDERR_FILENO); - posix_spawn_file_actions_addclose(&file_actions, stdout_pipe[1]); - posix_spawn_file_actions_addclose(&file_actions, stderr_pipe[1]); + SpawnFileActions file_actions; + file_actions.add_inherit_if_open(STDIN_FILENO); + + if (redirect_output) { + file_actions.add_close(stdout_pipe[0]); + file_actions.add_close(stderr_pipe[0]); + file_actions.add_dup2(stdout_pipe[1], STDOUT_FILENO); + file_actions.add_dup2(stderr_pipe[1], STDERR_FILENO); + file_actions.add_close(stdout_pipe[1]); + file_actions.add_close(stderr_pipe[1]); } else if (!inherit_output) { - posix_spawn_file_actions_addopen(&file_actions, STDOUT_FILENO, "/dev/null", O_WRONLY, 0); - posix_spawn_file_actions_adddup2(&file_actions, STDOUT_FILENO, STDERR_FILENO); + file_actions.add_open(STDOUT_FILENO, "/dev/null", O_WRONLY, 0); + file_actions.add_dup2(STDOUT_FILENO, STDERR_FILENO); + } else { + file_actions.add_inherit_if_open(STDOUT_FILENO); + file_actions.add_inherit_if_open(STDERR_FILENO); } - if (!working_dir.empty()) { -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdeprecated-declarations" - posix_spawn_file_actions_addchdir_np(&file_actions, working_dir.c_str()); -#pragma clang diagnostic pop - } + file_actions.add_chdir(working_dir); - posix_spawnattr_t attr; - posix_spawnattr_init(&attr); - sigset_t default_signals; - sigfillset(&default_signals); - posix_spawnattr_setsigdefault(&attr, &default_signals); - posix_spawnattr_setflags(&attr, POSIX_SPAWN_CLOEXEC_DEFAULT | POSIX_SPAWN_SETSIGDEF); + SpawnAttributes attributes; + attributes.set_default_signals(); + attributes.set_flags(POSIX_SPAWN_CLOEXEC_DEFAULT | + POSIX_SPAWN_SETSIGDEF); // Build envp std::vector env_strings; @@ -190,19 +503,22 @@ ProcessHandle MacOSProcessPlatform::spawn( } argv_ptrs.push_back(nullptr); - pid_t pid = 0; - int spawn_rc = posix_spawnp(&pid, executable.c_str(), &file_actions, &attr, - argv_ptrs.data(), envp.data()); + int spawn_rc = file_actions.error(); + if (spawn_rc == 0) { + spawn_rc = attributes.error(); + } - posix_spawn_file_actions_destroy(&file_actions); - posix_spawnattr_destroy(&attr); + pid_t pid = 0; + if (spawn_rc == 0) { + spawn_rc = posix_spawnp(&pid, executable.c_str(), file_actions.get(), + attributes.get(), argv_ptrs.data(), + envp.data()); + } if (spawn_rc != 0) { - if (inherit_output && filter_health_logs) { - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); + if (redirect_output) { + close_pipe(stdout_pipe); + close_pipe(stderr_pipe); } throw std::runtime_error(std::string("posix_spawn failed: ") + strerror(spawn_rc)); } @@ -213,58 +529,14 @@ ProcessHandle MacOSProcessPlatform::spawn( LOG(INFO, "ProcessManager") << "Process started successfully, PID: " << pid << std::endl; } - // Start filter threads if needed - if (inherit_output && filter_health_logs) { + if (redirect_output) { close(stdout_pipe[1]); close(stderr_pipe[1]); - std::thread([fd = stdout_pipe[0]]() { - char buffer[4096]; - std::string line_buffer; - ssize_t bytes_read; - - while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { - buffer[bytes_read] = '\0'; - line_buffer += buffer; - - size_t pos; - while ((pos = line_buffer.find('\n')) != std::string::npos) { - std::string line = line_buffer.substr(0, pos); - line_buffer = line_buffer.substr(pos + 1); - log_process_line(line); - } - } - - if (!line_buffer.empty()) { - log_process_line(line_buffer); - } - - close(fd); - }).detach(); - - std::thread([fd = stderr_pipe[0]]() { - char buffer[4096]; - std::string line_buffer; - ssize_t bytes_read; - - while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { - buffer[bytes_read] = '\0'; - line_buffer += buffer; - - size_t pos; - while ((pos = line_buffer.find('\n')) != std::string::npos) { - std::string line = line_buffer.substr(0, pos); - line_buffer = line_buffer.substr(pos + 1); - log_process_line(line); - } - } - - if (!line_buffer.empty()) { - log_process_line(line_buffer); - } - - close(fd); - }).detach(); + start_process_output_reader(stdout_pipe[0], inherit_output, + output_capture); + start_process_output_reader(stderr_pipe[0], inherit_output, + output_capture); } return handle; @@ -459,46 +731,13 @@ int MacOSProcessPlatform::run_with_output( int timeout_seconds, bool capture_stderr) { - // For simplicity, reuse fork/exec for run_with_output on macOS - // (This is a less critical path than spawn) - int stdout_pipe[2]; + int stdout_pipe[2] = {-1, -1}; - if (pipe(stdout_pipe) < 0) { + if (!create_pipe_above_standard_streams(stdout_pipe)) { throw std::runtime_error("Failed to create pipe"); } - - pid_t pid = fork(); - - if (pid < 0) { - close(stdout_pipe[0]); - close(stdout_pipe[1]); - throw std::runtime_error("Failed to fork process"); - } - - if (pid == 0) { - close(stdout_pipe[0]); - dup2(stdout_pipe[1], STDOUT_FILENO); - if (capture_stderr) { - dup2(stdout_pipe[1], STDERR_FILENO); - } - close(stdout_pipe[1]); - - if (!working_dir.empty()) { - chdir(working_dir.c_str()); - } - - std::vector argv_ptrs; - argv_ptrs.push_back(const_cast(executable.c_str())); - for (const auto& arg : args) { - argv_ptrs.push_back(const_cast(arg.c_str())); - } - argv_ptrs.push_back(nullptr); - - execvp(executable.c_str(), argv_ptrs.data()); - _exit(127); - } - - close(stdout_pipe[1]); + const pid_t pid = spawn_capturing_output( + executable, args, working_dir, capture_stderr, false, stdout_pipe); std::string line_buffer; char buffer[4096]; @@ -628,18 +867,132 @@ int MacOSProcessPlatform::find_free_port(int start_port) { int MacOSProcessPlatform::run_command(const std::string& command, std::string& output, int timeout_seconds) { output.clear(); + int output_pipe[2] = {-1, -1}; + if (!create_pipe_above_standard_streams(output_pipe)) { + return -1; + } + + pid_t pid; + try { + pid = spawn_capturing_output("/bin/sh", {"-c", command}, "", false, + true, output_pipe); + } catch (...) { + return -1; + } - FILE* pipe = popen(command.c_str(), "r"); - if (!pipe) { + const int flags = fcntl(output_pipe[0], F_GETFL, 0); + if (flags < 0 || + fcntl(output_pipe[0], F_SETFL, flags | O_NONBLOCK) != 0) { + kill_process_group(pid); + close(output_pipe[0]); + int status = 0; + while (waitpid(pid, &status, 0) < 0 && errno == EINTR) { + } return -1; } - char buf[4096]; - while (fgets(buf, sizeof(buf), pipe)) { - output += buf; + const auto start_time = std::chrono::steady_clock::now(); + const auto deadline = + start_time + std::chrono::seconds(timeout_seconds > 0 + ? timeout_seconds + : 0); + bool timed_out = false; + bool read_failed = false; + char buffer[4096]; + while (true) { + if (timeout_seconds > 0) { + if (std::chrono::steady_clock::now() >= deadline) { + kill_process_group(pid); + timed_out = true; + break; + } + } + + const ssize_t bytes_read = read(output_pipe[0], buffer, sizeof(buffer)); + if (bytes_read > 0) { + output.append(buffer, static_cast(bytes_read)); + continue; + } + if (bytes_read == 0) { + break; + } + if (errno == EINTR) { + continue; + } + if (errno != EAGAIN && errno != EWOULDBLOCK) { + kill_process_group(pid); + read_failed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + close(output_pipe[0]); + int status = 0; + pid_t wait_result = -1; + bool child_identity_lost = false; + if (!timed_out && !read_failed) { + while (true) { + siginfo_t child_info{}; + int wait_options = WEXITED | WNOWAIT; + if (timeout_seconds > 0) { + wait_options |= WNOHANG; + } + + int observe_result; + do { + observe_result = + waitid(P_PID, static_cast(pid), &child_info, + wait_options); + } while (observe_result < 0 && errno == EINTR); + + if (observe_result == 0 && child_info.si_pid == pid) { + break; + } + if (observe_result < 0) { + child_identity_lost = errno == ECHILD; + if (!child_identity_lost) { + kill_process_group(pid); + } + read_failed = true; + break; + } + if (timeout_seconds > 0 && + std::chrono::steady_clock::now() >= deadline) { + kill_process_group(pid); + timed_out = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + if (timed_out || read_failed) { + if (child_identity_lost) { + return -1; + } + do { + wait_result = waitpid(pid, &status, 0); + } while (wait_result < 0 && errno == EINTR); + return -1; } - return pclose(pipe); +#ifdef LEMONADE_PROCESS_TEST_HOOK + lemonade_test_run_command_exit_observed(pid); +#endif + kill_process_group(pid, false); +#ifdef LEMONADE_PROCESS_TEST_HOOK + lemonade_test_run_command_group_cleanup_complete(pid); +#endif + do { + wait_result = waitpid(pid, &status, 0); + } while (wait_result < 0 && errno == EINTR); + if (wait_result != pid) { + return -1; + } +#ifdef LEMONADE_PROCESS_TEST_HOOK + lemonade_test_run_command_final_reap_complete(pid); +#endif + return status; } std::unique_ptr create_process_platform() { diff --git a/src/cpp/server/utils/platform/process_windows.cpp b/src/cpp/server/utils/platform/process_windows.cpp index 83b9a3ad33..8e95d7c8e1 100644 --- a/src/cpp/server/utils/platform/process_windows.cpp +++ b/src/cpp/server/utils/platform/process_windows.cpp @@ -22,8 +22,9 @@ #include #include #include -#include +#include #include +#include namespace lemon { namespace utils { @@ -51,6 +52,164 @@ static std::string escape_windows_arg(const std::string& arg) { return result; } +static BOOL create_process_with_restricted_handles( + LPSTR command_line, + DWORD creation_flags, + LPVOID environment, + LPCSTR working_directory, + const STARTUPINFOA& startup_info, + PROCESS_INFORMATION* process_information) { + STARTUPINFOEXA extended_startup_info{}; + extended_startup_info.StartupInfo = startup_info; + + HANDLE source_handles[] = { + startup_info.hStdInput, + startup_info.hStdOutput, + startup_info.hStdError, + }; + HANDLE fallback_handles[3] = {}; + HANDLE unique_source_handles[3] = {}; + HANDLE inherited_handles[3] = {}; + DWORD inherited_handle_count = 0; + SIZE_T attribute_list_size = 0; + void* attribute_storage = nullptr; + bool attribute_list_initialized = false; + auto cleanup = [&]() { + if (attribute_list_initialized) { + DeleteProcThreadAttributeList( + extended_startup_info.lpAttributeList); + } + if (attribute_storage != nullptr) { + HeapFree(GetProcessHeap(), 0, attribute_storage); + } + for (DWORD i = 0; i < inherited_handle_count; ++i) { + CloseHandle(inherited_handles[i]); + } + for (DWORD i = 0; i < 3; ++i) { + if (fallback_handles[i] != nullptr && + fallback_handles[i] != INVALID_HANDLE_VALUE) { + CloseHandle(fallback_handles[i]); + } + } + }; + + if ((startup_info.dwFlags & STARTF_USESTDHANDLES) != 0) { + for (DWORD i = 0; i < 3; ++i) { + bool usable = source_handles[i] != nullptr && + source_handles[i] != INVALID_HANDLE_VALUE; + if (usable) { + SetLastError(ERROR_SUCCESS); + const DWORD type = GetFileType(source_handles[i]); + usable = type != FILE_TYPE_UNKNOWN || + GetLastError() == ERROR_SUCCESS; + } + if (usable) { + continue; + } + + const DWORD access = i == 0 ? GENERIC_READ : GENERIC_WRITE; + const HANDLE fallback = CreateFileA( + "NUL", access, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (fallback == INVALID_HANDLE_VALUE) { + const DWORD error = GetLastError(); + cleanup(); + SetLastError(error); + return FALSE; + } + fallback_handles[i] = fallback; + source_handles[i] = fallback; + } + + for (DWORD i = 0; i < 3; ++i) { + const HANDLE source = source_handles[i]; + DWORD existing = 0; + while (existing < inherited_handle_count && + unique_source_handles[existing] != source) { + ++existing; + } + + HANDLE inherited = nullptr; + if (existing < inherited_handle_count) { + inherited = inherited_handles[existing]; + } else { + if (!DuplicateHandle( + GetCurrentProcess(), source, GetCurrentProcess(), + &inherited, 0, TRUE, DUPLICATE_SAME_ACCESS)) { + const DWORD error = GetLastError(); + cleanup(); + SetLastError(error); + return FALSE; + } + unique_source_handles[inherited_handle_count] = source; + inherited_handles[inherited_handle_count] = inherited; + ++inherited_handle_count; + } + + if (i == 0) { + extended_startup_info.StartupInfo.hStdInput = inherited; + } else if (i == 1) { + extended_startup_info.StartupInfo.hStdOutput = inherited; + } else { + extended_startup_info.StartupInfo.hStdError = inherited; + } + } + } + + BOOL inherit_handles = FALSE; + DWORD effective_creation_flags = creation_flags; + if (inherited_handle_count > 0) { + InitializeProcThreadAttributeList( + nullptr, 1, 0, &attribute_list_size); + if (attribute_list_size == 0) { + const DWORD error = GetLastError(); + cleanup(); + SetLastError(error); + return FALSE; + } + + attribute_storage = + HeapAlloc(GetProcessHeap(), 0, attribute_list_size); + if (attribute_storage == nullptr) { + cleanup(); + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return FALSE; + } + extended_startup_info.lpAttributeList = + static_cast(attribute_storage); + if (!InitializeProcThreadAttributeList( + extended_startup_info.lpAttributeList, 1, 0, + &attribute_list_size)) { + const DWORD error = GetLastError(); + cleanup(); + SetLastError(error); + return FALSE; + } + attribute_list_initialized = true; + if (!UpdateProcThreadAttribute( + extended_startup_info.lpAttributeList, 0, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, inherited_handles, + inherited_handle_count * sizeof(HANDLE), nullptr, nullptr)) { + const DWORD error = GetLastError(); + cleanup(); + SetLastError(error); + return FALSE; + } + extended_startup_info.StartupInfo.cb = sizeof(STARTUPINFOEXA); + effective_creation_flags |= EXTENDED_STARTUPINFO_PRESENT; + inherit_handles = TRUE; + } + + const BOOL success = CreateProcessA( + nullptr, command_line, nullptr, nullptr, inherit_handles, + effective_creation_flags, environment, working_directory, + &extended_startup_info.StartupInfo, process_information); + const DWORD error = success ? ERROR_SUCCESS : GetLastError(); + cleanup(); + SetLastError(error); + return success; +} + // Helper function to check if a line should be filtered static bool should_filter_line(const std::string& line) { // Filter out health check requests (both /health and /v1/health) @@ -82,34 +241,77 @@ static void log_process_line(const std::string& line) { } } -// Thread function to read from pipe and filter output +struct OutputReaderContext { + HANDLE pipe; + bool log_output; + std::shared_ptr output_capture; +}; + static DWORD WINAPI output_filter_thread(LPVOID param) { - HANDLE pipe = static_cast(param); + std::unique_ptr context( + static_cast(param)); char buffer[4096]; DWORD bytes_read; std::string line_buffer; - while (ReadFile(pipe, buffer, sizeof(buffer) - 1, &bytes_read, nullptr) && bytes_read > 0) { - buffer[bytes_read] = '\0'; - line_buffer += buffer; + while (ReadFile(context->pipe, buffer, sizeof(buffer), &bytes_read, nullptr) && + bytes_read > 0) { + if (context->output_capture) { + context->output_capture->append( + buffer, static_cast(bytes_read)); + } + if (!context->log_output) { + continue; + } + + line_buffer.append(buffer, static_cast(bytes_read)); - // Process complete lines size_t pos; while ((pos = line_buffer.find('\n')) != std::string::npos) { std::string line = line_buffer.substr(0, pos); - line_buffer = line_buffer.substr(pos + 1); - + line_buffer.erase(0, pos + 1); log_process_line(line); } } - // Print any remaining partial line - if (!line_buffer.empty()) { + if (context->log_output && !line_buffer.empty()) { log_process_line(line_buffer); } + CloseHandle(context->pipe); + if (context->output_capture) { + context->output_capture->finish_reader(); + } + return 0; +} + +static void start_output_reader( + HANDLE pipe, + bool log_output, + const std::shared_ptr& output_capture) { + auto* context = + new (std::nothrow) OutputReaderContext{pipe, log_output, output_capture}; + if (context == nullptr) { + CloseHandle(pipe); + if (output_capture) { + output_capture->finish_reader(); + } + LOG(ERROR, "ProcessManager") + << "Failed to allocate process output reader context" << std::endl; + return; + } + + HANDLE thread = CreateThread( + nullptr, 0, output_filter_thread, context, 0, nullptr); + if (thread != nullptr) { + CloseHandle(thread); + return; + } CloseHandle(pipe); - return 0; + if (output_capture) { + output_capture->finish_reader(); + } + delete context; } // Helper function: lowercase ASCII string for case-insensitive comparison @@ -180,11 +382,13 @@ class WindowsProcessPlatform : public ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars) override { + const std::vector>& env_vars, + std::shared_ptr output_capture) override { ProcessHandle handle; handle.handle = nullptr; handle.pid = 0; + handle.output_capture = output_capture; std::string cmdline = escape_windows_arg(executable); for (const auto& arg : args) { @@ -203,7 +407,8 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE stderr_write = nullptr; HANDLE nul_input = nullptr; - bool use_filtered_output = (inherit_output && filter_health_logs); + bool use_filtered_output = + (inherit_output && filter_health_logs) || output_capture != nullptr; if (inherit_output && !filter_health_logs) { const HANDLE std_in = GetStdHandle(STD_INPUT_HANDLE); @@ -217,39 +422,32 @@ class WindowsProcessPlatform : public ProcessPlatform { if (invalid_stdio) { use_filtered_output = true; + if (!output_capture) { + output_capture = + std::make_shared(0); + handle.output_capture = output_capture; + } LOG(WARNING, "ProcessManager") << "Parent std handles are unavailable; enabling filtered output capture" << std::endl; } } - // If inherit_output is true, either use pipes with filtering or direct inheritance - if (inherit_output && use_filtered_output) { - // Create pipes for stdout and stderr to filter output - SECURITY_ATTRIBUTES sa; - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.bInheritHandle = TRUE; - sa.lpSecurityDescriptor = nullptr; - - if (!CreatePipe(&stdout_read, &stdout_write, &sa, 0)) { + if (use_filtered_output) { + if (!CreatePipe(&stdout_read, &stdout_write, nullptr, 0)) { throw std::runtime_error("Failed to create stdout pipe"); } - if (!CreatePipe(&stderr_read, &stderr_write, &sa, 0)) { + if (!CreatePipe(&stderr_read, &stderr_write, nullptr, 0)) { CloseHandle(stdout_read); CloseHandle(stdout_write); throw std::runtime_error("Failed to create stderr pipe"); } - // Make sure the read handles are not inherited - SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0); - SetHandleInformation(stderr_read, HANDLE_FLAG_INHERIT, 0); - si.dwFlags = STARTF_USESTDHANDLES; si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); if (si.hStdInput == nullptr || si.hStdInput == INVALID_HANDLE_VALUE) { nul_input = CreateFileA("NUL", GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (nul_input != INVALID_HANDLE_VALUE) { - SetHandleInformation(nul_input, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT); si.hStdInput = nul_input; } else { si.hStdInput = nullptr; @@ -258,7 +456,9 @@ class WindowsProcessPlatform : public ProcessPlatform { si.hStdOutput = stdout_write; si.hStdError = stderr_write; - LOG(DEBUG, "ProcessManager") << "Starting process with filtered output: " << cmdline << std::endl; + if (inherit_output) { + LOG(DEBUG, "ProcessManager") << "Starting process with filtered output: " << cmdline << std::endl; + } } else if (inherit_output) { // Direct inheritance without filtering si.dwFlags |= STARTF_USESTDHANDLES; @@ -273,8 +473,6 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE hNul = CreateFileA("NUL", GENERIC_WRITE, FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (hNul != INVALID_HANDLE_VALUE) { - // Ensure the NUL handle is inheritable - SetHandleInformation(hNul, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT); si.hStdOutput = hNul; si.hStdError = hNul; } @@ -285,21 +483,18 @@ class WindowsProcessPlatform : public ProcessPlatform { environment_block = build_windows_environment_block(env_vars); } - BOOL success = CreateProcessA( - nullptr, + BOOL success = create_process_with_restricted_handles( const_cast(cmdline.c_str()), - nullptr, - nullptr, - TRUE, // Inherit handles (inherit_output && !use_filtered_output) ? 0 : CREATE_NO_WINDOW, environment_block.empty() ? nullptr : environment_block.data(), working_dir.empty() ? nullptr : working_dir.c_str(), - &si, + si, &pi ); // If we opened a NUL handle, we can close it now (the child process has its own inherited handle) - if (!inherit_output && si.hStdOutput != nullptr && si.hStdOutput != INVALID_HANDLE_VALUE) { + if (!inherit_output && !use_filtered_output && + si.hStdOutput != nullptr && si.hStdOutput != INVALID_HANDLE_VALUE) { CloseHandle(si.hStdOutput); } @@ -336,10 +531,9 @@ class WindowsProcessPlatform : public ProcessPlatform { if (stdout_write) CloseHandle(stdout_write); if (stderr_write) CloseHandle(stderr_write); - // Start filter threads if needed - if (inherit_output && use_filtered_output) { - CreateThread(nullptr, 0, output_filter_thread, stdout_read, 0, nullptr); - CreateThread(nullptr, 0, output_filter_thread, stderr_read, 0, nullptr); + if (use_filtered_output) { + start_output_reader(stdout_read, inherit_output, output_capture); + start_output_reader(stderr_read, inherit_output, output_capture); } if (inherit_output) { @@ -466,18 +660,10 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE stdout_read = nullptr; HANDLE stdout_write = nullptr; - SECURITY_ATTRIBUTES sa; - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.bInheritHandle = TRUE; - sa.lpSecurityDescriptor = nullptr; - - if (!CreatePipe(&stdout_read, &stdout_write, &sa, 0)) { + if (!CreatePipe(&stdout_read, &stdout_write, nullptr, 0)) { throw std::runtime_error("Failed to create stdout pipe"); } - // Make sure the read handle is not inherited - SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0); - STARTUPINFOA si; PROCESS_INFORMATION pi; ZeroMemory(&si, sizeof(si)); @@ -488,16 +674,12 @@ class WindowsProcessPlatform : public ProcessPlatform { si.hStdError = capture_stderr ? stdout_write : GetStdHandle(STD_ERROR_HANDLE); ZeroMemory(&pi, sizeof(pi)); - BOOL success = CreateProcessA( - nullptr, + BOOL success = create_process_with_restricted_handles( const_cast(cmdline.c_str()), - nullptr, - nullptr, - TRUE, // Inherit handles CREATE_NO_WINDOW, nullptr, working_dir.empty() ? nullptr : working_dir.c_str(), - &si, + si, &pi ); @@ -665,15 +847,9 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE stdout_read = nullptr; HANDLE stdout_write = nullptr; - SECURITY_ATTRIBUTES sa; - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.bInheritHandle = TRUE; - sa.lpSecurityDescriptor = nullptr; - - if (!CreatePipe(&stdout_read, &stdout_write, &sa, 0)) { + if (!CreatePipe(&stdout_read, &stdout_write, nullptr, 0)) { return -1; } - SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0); STARTUPINFOA si = {}; si.cb = sizeof(si); @@ -685,10 +861,9 @@ class WindowsProcessPlatform : public ProcessPlatform { PROCESS_INFORMATION pi = {}; // Wrap in cmd /c so shell features (redirection, pipes) work std::string cmdline = "cmd /c " + command; - BOOL success = CreateProcessA( - nullptr, const_cast(cmdline.c_str()), - nullptr, nullptr, TRUE, CREATE_NO_WINDOW, - nullptr, nullptr, &si, &pi); + BOOL success = create_process_with_restricted_handles( + const_cast(cmdline.c_str()), CREATE_NO_WINDOW, + nullptr, nullptr, si, &pi); CloseHandle(stdout_write); diff --git a/src/cpp/server/utils/process_manager.cpp b/src/cpp/server/utils/process_manager.cpp index 9a7681e77b..53778ced22 100644 --- a/src/cpp/server/utils/process_manager.cpp +++ b/src/cpp/server/utils/process_manager.cpp @@ -1,24 +1,97 @@ #include #include +#include +#include + namespace lemon { namespace utils { +namespace { + +constexpr int output_reader_drain_timeout_ms = 1000; + +void wait_for_output_readers(const ProcessHandle& handle) { + if (handle.output_capture) { + handle.output_capture->read_tail(0, output_reader_drain_timeout_ms); + } +} + +} // namespace + +ProcessOutputCapture::ProcessOutputCapture(std::size_t max_bytes) + : max_bytes_(max_bytes) { +} + +void ProcessOutputCapture::append(const char* data, std::size_t size) { + if (data == nullptr || size == 0 || max_bytes_ == 0) { + return; + } + + std::lock_guard lock(mutex_); + if (size >= max_bytes_) { + tail_.assign(data + size - max_bytes_, max_bytes_); + return; + } + + const std::size_t overflow = + tail_.size() + size > max_bytes_ ? tail_.size() + size - max_bytes_ : 0; + if (overflow > 0) { + tail_.erase(0, overflow); + } + tail_.append(data, size); +} + +void ProcessOutputCapture::finish_reader() { + std::lock_guard lock(mutex_); + if (active_readers_ > 0) { + --active_readers_; + } + if (active_readers_ == 0) { + completion_cv_.notify_all(); + } +} + +std::string ProcessOutputCapture::read_tail(std::size_t max_bytes, + int wait_timeout_ms) { + std::unique_lock lock(mutex_); + if (wait_timeout_ms > 0 && active_readers_ > 0) { + completion_cv_.wait_for( + lock, + std::chrono::milliseconds(wait_timeout_ms), + [this]() { return active_readers_ == 0; }); + } + + const std::size_t size = std::min(max_bytes, tail_.size()); + return tail_.substr(tail_.size() - size, size); +} + ProcessHandle ProcessManager::start_process( const std::string& executable, const std::vector& args, const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars) { + const std::vector>& env_vars, + std::size_t capture_output_bytes) { + + std::shared_ptr output_capture; + if (capture_output_bytes > 0 || + (inherit_output && filter_health_logs)) { + output_capture = + std::make_shared(capture_output_bytes); + } auto platform = create_process_platform(); - return platform->spawn(executable, args, working_dir, inherit_output, filter_health_logs, env_vars); + return platform->spawn(executable, args, working_dir, inherit_output, + filter_health_logs, env_vars, + std::move(output_capture)); } void ProcessManager::stop_process(ProcessHandle handle) { auto platform = create_process_platform(); platform->terminate(handle); + wait_for_output_readers(handle); } bool ProcessManager::is_running(ProcessHandle handle) { @@ -33,18 +106,30 @@ int ProcessManager::get_exit_code(ProcessHandle handle) { int ProcessManager::wait_for_exit(ProcessHandle handle, int timeout_seconds) { auto platform = create_process_platform(); - return platform->wait_for_exit(handle, timeout_seconds); + const int exit_code = platform->wait_for_exit(handle, timeout_seconds); + if (exit_code >= 0) { + wait_for_output_readers(handle); + } + return exit_code; } int ProcessManager::reap_process(ProcessHandle handle) { auto platform = create_process_platform(); - return platform->reap(handle); + const int exit_code = platform->reap(handle); + if (exit_code >= 0) { + wait_for_output_readers(handle); + } + return exit_code; } -std::string ProcessManager::read_output(ProcessHandle handle, int max_bytes) { - // Note: This is a simplified version. Full implementation would need pipes - // for stdout/stderr capture during process creation. - return ""; +std::string ProcessManager::read_output(ProcessHandle handle, + int max_bytes, + int wait_timeout_ms) { + if (!handle.output_capture || max_bytes <= 0) { + return ""; + } + return handle.output_capture->read_tail( + static_cast(max_bytes), wait_timeout_ms); } int ProcessManager::run_process_with_output( @@ -62,6 +147,7 @@ int ProcessManager::run_process_with_output( void ProcessManager::kill_process(ProcessHandle handle) { auto platform = create_process_platform(); platform->kill(handle); + wait_for_output_readers(handle); } void ProcessManager::terminate_process(ProcessHandle handle) { diff --git a/src/cpp/server/wrapped_server.cpp b/src/cpp/server/wrapped_server.cpp index 38c20527ed..29aa180d5c 100644 --- a/src/cpp/server/wrapped_server.cpp +++ b/src/cpp/server/wrapped_server.cpp @@ -6,10 +6,11 @@ #include #include #include -#include #include -#include #include +#include +#include +#include #include namespace lemon { @@ -792,6 +793,19 @@ void WrappedServer::forward_streaming_request(const std::string& endpoint, bool sse, long timeout_seconds, TelemetryCallback telemetry_callback) { + forward_streaming_request_impl( + endpoint, request_body, sink, sse, timeout_seconds, + std::move(telemetry_callback), nullptr); +} + +void WrappedServer::forward_streaming_request_impl( + const std::string& endpoint, + const std::string& request_body, + httplib::DataSink& sink, + bool sse, + long timeout_seconds, + TelemetryCallback telemetry_callback, + StreamingProxy::SseFrameTransform frame_transform) { if (!is_backend_alive()) { if (was_watchdog_triggered() || has_backend_process_exited()) { if (!was_watchdog_triggered()) { @@ -811,31 +825,46 @@ void WrappedServer::forward_streaming_request(const std::string& endpoint, std::string url = get_base_url() + endpoint; bool streamed_any_bytes = false; - auto mark_stream_progress = [this, &streamed_any_bytes]() { + auto mark_client_delivery = [this, &streamed_any_bytes]() { if (!streamed_any_bytes) { set_streaming(true); } streamed_any_bytes = true; + }; + auto mark_backend_progress = [this]() { note_backend_activity(); }; + auto mark_byte_stream_progress = [&mark_client_delivery, + &mark_backend_progress]() { + mark_client_delivery(); + mark_backend_progress(); + }; try { if (sse) { // Use StreamingProxy to forward the SSE stream with telemetry callback // Use INFERENCE_TIMEOUT_SECONDS (0 = infinite) as chat completions can take a long time - StreamingProxy::forward_sse_stream(url, request_body, sink, - [telemetry_callback](const StreamingProxy::TelemetryData& telemetry) { - if (telemetry_callback) { - telemetry_callback(telemetry); - } - }, - timeout_seconds, - mark_stream_progress - ); + auto report_telemetry = [telemetry_callback]( + const StreamingProxy::TelemetryData& telemetry) { + if (telemetry_callback) { + telemetry_callback(telemetry); + } + }; + if (frame_transform) { + StreamingProxy::forward_transformed_sse_stream( + url, request_body, sink, frame_transform, report_telemetry, + timeout_seconds, mark_backend_progress, 1000, + mark_client_delivery); + } else { + StreamingProxy::forward_sse_stream( + url, request_body, sink, report_telemetry, + timeout_seconds, mark_backend_progress, 1000, + mark_client_delivery); + } } else { StreamingProxy::forward_byte_stream(url, request_body, sink, timeout_seconds, - mark_stream_progress + mark_byte_stream_progress ); } } catch (const std::exception& e) { diff --git a/test/cpp/test_llamacpp_stream_model.cpp b/test/cpp/test_llamacpp_stream_model.cpp new file mode 100644 index 0000000000..8c7db746f0 --- /dev/null +++ b/test/cpp/test_llamacpp_stream_model.cpp @@ -0,0 +1,892 @@ +#include "lemon/backends/llamacpp/llamacpp_server.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace std::chrono_literals; + +namespace { + +constexpr const char* kRequestedModel = "builtin.K2-Horizon-0.9B-GGUF"; +constexpr const char* kBackendModel = "/private/cache/models/K2-Horizon-1B-BF16.gguf"; +constexpr std::size_t kMaxSseFrameBytes = + lemon::StreamingProxy::kMaxSseFrameBytes; + +int failures = 0; + +void check(bool condition, const char* name) { + if (condition) { + std::printf("[PASS] %s\n", name); + return; + } + std::printf("[FAIL] %s\n", name); + ++failures; +} + +class TestLlamaCppServer : public lemon::backends::LlamaCppServer { +public: + explicit TestLlamaCppServer(int port) + : LlamaCppServer("debug", nullptr, nullptr) { + port_ = port; + } + + bool is_backend_alive() const override { + return true; + } + + void forward_raw_sse(const std::string& endpoint, + const std::string& request_body, + httplib::DataSink& sink) { + lemon::WrappedServer::forward_streaming_request( + endpoint, request_body, sink, true, 5, nullptr); + } +}; + +std::vector data_events(const std::string& stream, + int* malformed_events = nullptr) { + std::vector events; + std::string buffer = stream; + std::string event_data; + bool has_data_line = false; + auto finish_event = [&]() { + if (event_data.empty() || event_data == "[DONE]") { + event_data.clear(); + has_data_line = false; + return; + } + const nlohmann::json parsed = + nlohmann::json::parse(event_data, nullptr, false); + event_data.clear(); + has_data_line = false; + if (parsed.is_discarded()) { + if (malformed_events) { + ++*malformed_events; + } + return; + } + events.push_back(parsed); + }; + lemon::StreamingProxy::process_sse_lines( + buffer, [&event_data, &finish_event, + &has_data_line](const std::string& line) { + if (line.empty()) { + finish_event(); + return; + } + if (line.rfind("data:", 0) != 0) { + return; + } + std::size_t payload_start = std::strlen("data:"); + while (payload_start < line.size() && + (line[payload_start] == ' ' || + line[payload_start] == '\t')) { + ++payload_start; + } + if (has_data_line) { + event_data.push_back('\n'); + } + event_data.append(line, payload_start, std::string::npos); + has_data_line = true; + }); + return events; +} + +std::size_t count_occurrences(const std::string& value, + const std::string& needle) { + std::size_t count = 0; + std::size_t offset = 0; + while ((offset = value.find(needle, offset)) != std::string::npos) { + ++count; + offset += needle.size(); + } + return count; +} + +std::string sized_model_frame(std::size_t frame_size, + const std::string& separator) { + const std::string prefix = + "data: {\"id\":\"sized\",\"model\":\"" + + std::string(kBackendModel) + "\",\"padding\":\""; + const std::string suffix = "\"}" + separator; + if (prefix.size() + suffix.size() > frame_size) { + throw std::runtime_error("test frame size is too small"); + } + return prefix + + std::string(frame_size - prefix.size() - suffix.size(), 'x') + + suffix; +} + +std::string sized_incomplete_model_frame(std::size_t frame_size) { + const std::string prefix = + "data: {\"id\":\"sized\",\"model\":\"" + + std::string(kBackendModel) + "\",\"padding\":\""; + if (prefix.size() > frame_size) { + throw std::runtime_error("test frame size is too small"); + } + return prefix + std::string(frame_size - prefix.size(), 'x'); +} + +struct StreamResult { + std::string output; + bool retryable_reset = false; + int done_calls = 0; +}; + +struct RawStreamResult { + std::string output; + bool interrupted = false; + std::string error_message; + int backend_progress_calls = 0; +}; + +StreamResult forward_stream(TestLlamaCppServer& server, + const std::string& endpoint, + const std::string& fault = "") { + StreamResult result; + httplib::DataSink sink; + sink.write = [&result](const char* data, size_t length) { + result.output.append(data, length); + return true; + }; + sink.done = [&result]() { ++result.done_calls; }; + sink.done_with_trailer = [&result](const httplib::Headers&) { + ++result.done_calls; + }; + sink.is_writable = []() { return true; }; + + nlohmann::json request = { + {"model", kRequestedModel}, + {"stream", true}, + }; + if (!fault.empty()) { + request["fault"] = fault; + } + try { + server.forward_streaming_request( + endpoint, request.dump(), sink, true, 5, nullptr); + } catch (const lemon::BackendStreamRetryableReset&) { + result.retryable_reset = true; + } + return result; +} + +StreamResult forward_wrapped_raw_stream(TestLlamaCppServer& server, + const std::string& endpoint, + const std::string& fault) { + StreamResult result; + httplib::DataSink sink; + sink.write = [&result](const char* data, size_t length) { + result.output.append(data, length); + return true; + }; + sink.done = [&result]() { ++result.done_calls; }; + sink.done_with_trailer = [&result](const httplib::Headers&) { + ++result.done_calls; + }; + sink.is_writable = []() { return true; }; + + const nlohmann::json request = { + {"model", kRequestedModel}, + {"stream", true}, + {"fault", fault}, + }; + try { + server.forward_raw_sse(endpoint, request.dump(), sink); + } catch (const lemon::BackendStreamRetryableReset&) { + result.retryable_reset = true; + } + return result; +} + +RawStreamResult forward_raw_stream(int port, const std::string& fault) { + RawStreamResult result; + httplib::DataSink sink; + sink.write = [&result](const char* data, size_t length) { + result.output.append(data, length); + return true; + }; + sink.done = []() {}; + sink.done_with_trailer = [](const httplib::Headers&) {}; + sink.is_writable = []() { return true; }; + + const nlohmann::json request = { + {"model", kRequestedModel}, + {"stream", true}, + {"fault", fault}, + }; + try { + lemon::StreamingProxy::forward_sse_stream( + "http://127.0.0.1:" + std::to_string(port) + + "/v1/chat/completions", + request.dump(), sink, nullptr, 5, + [&result]() { ++result.backend_progress_calls; }, 0); + } catch (const std::runtime_error& error) { + result.interrupted = true; + result.error_message = error.what(); + } + return result; +} + +int forward_stream_to_disconnected_client(TestLlamaCppServer& server) { + int write_calls = 0; + httplib::DataSink sink; + sink.write = [&write_calls](const char*, size_t) { + ++write_calls; + return false; + }; + sink.done = []() {}; + sink.done_with_trailer = [](const httplib::Headers&) {}; + sink.is_writable = []() { return true; }; + + const nlohmann::json request = { + {"model", kRequestedModel}, + {"stream", true}, + }; + server.forward_streaming_request( + "/v1/chat/completions", request.dump(), sink, true, 5, nullptr); + return write_calls; +} + +} // namespace + +int main() { + httplib::Server backend; + std::atomic partial_before_output_requests{0}; + std::atomic partial_after_output_requests{0}; + const std::vector endpoints = { + "/v1/chat/completions", + "/v1/completions", + "/v1/responses", + }; + + for (const std::string& endpoint : endpoints) { + backend.Post(endpoint, [endpoint, &partial_before_output_requests, + &partial_after_output_requests]( + const httplib::Request& request, + httplib::Response& response) { + const nlohmann::json body = + nlohmann::json::parse(request.body, nullptr, false); + const std::string fault = + body.is_object() ? body.value("fault", "") : ""; + + nlohmann::json event; + if (endpoint == "/v1/responses") { + event = { + {"type", "response.completed"}, + {"response", { + {"id", "resp_mock"}, + {"model", kBackendModel}, + {"status", "completed"}, + }}, + }; + } else { + event = { + {"id", "cmpl_mock"}, + {"model", kBackendModel}, + {"choices", nlohmann::json::array()}, + }; + } + + const std::string separator = + endpoint == "/v1/completions" ? "\r\n\r\n" : "\n\n"; + std::string frame = + ": upstream-comment\n\n" + + std::string("data: ") + event.dump() + separator; + const std::string partial = + "data: {\"id\":\"partial\",\"model\":\"" + + std::string(kBackendModel) + "\""; + + if (fault == "done_literal_partial") { + event["choices"] = nlohmann::json::array({ + {{"delta", {{"content", "data: [DONE]"}}}}, + }); + frame = ": upstream-comment\n\n" + + std::string("data: ") + event.dump() + separator; + } + + if (fault == "multiline_model") { + const std::string multiline_stream = + "id: multiline\n" + "data: {\"id\":\"multi\",\n" + "data: \"model\":\"" + + std::string(kBackendModel) + + "\",\"choices\":[]}\n" + ": retained-comment\n\n" + "data: [DONE]\n\n"; + response.set_content(multiline_stream, "text/event-stream"); + return; + } + if (fault == "bom_model") { + const std::string bom_stream = + std::string("\xEF\xBB\xBF", 3) + "data: " + + event.dump() + separator + "data: [DONE]\n\n"; + response.set_chunked_content_provider( + "text/event-stream", + [bom_stream](size_t, httplib::DataSink& sink) { + for (const char byte : bom_stream) { + if (!sink.write(&byte, 1)) { + return false; + } + } + sink.done(); + return false; + }); + return; + } + if (fault == "bom_done_only") { + response.set_content( + std::string("\xEF\xBB\xBF", 3) + + "data: [DONE]\n\n", + "text/event-stream"); + return; + } + if (fault == "double_bom_model") { + const std::string double_bom_stream = + std::string("\xEF\xBB\xBF\xEF\xBB\xBF", 6) + + "data: " + event.dump() + separator + + "data: [DONE]\n\n"; + response.set_chunked_content_provider( + "text/event-stream", + [double_bom_stream](size_t, httplib::DataSink& sink) { + for (const char byte : double_bom_stream) { + if (!sink.write(&byte, 1)) { + return false; + } + } + sink.done(); + return false; + }); + return; + } + if (fault == "double_bom_done_only") { + response.set_content( + std::string("\xEF\xBB\xBF\xEF\xBB\xBF", 6) + + "data: [DONE]\n\n", + "text/event-stream"); + return; + } + if (fault == "terminal_cr") { + response.set_content( + "data: " + event.dump() + "\r\n\r", + "text/event-stream"); + return; + } + if (fault == "malformed_model") { + response.set_content( + "data: {\"id\":\"broken\",\"model\":\"" + + std::string(kBackendModel) + "\"\n\n", + "text/event-stream"); + return; + } + if (fault == "scalar_model") { + response.set_content( + "data: \"" + std::string(kBackendModel) + "\"\n\n", + "text/event-stream"); + return; + } + if (fault == "frame_at_limit_lf") { + response.set_content( + sized_model_frame(kMaxSseFrameBytes, "\n\n"), + "text/event-stream"); + return; + } + if (fault == "frame_at_limit_crlf") { + response.set_content( + sized_model_frame(kMaxSseFrameBytes, "\r\n\r\n"), + "text/event-stream"); + return; + } + if (fault == "frame_over_limit") { + response.set_content( + sized_model_frame(kMaxSseFrameBytes + 1, "\n\n"), + "text/event-stream"); + return; + } + if (fault == "frame_over_limit_after_frame") { + const std::string safe_frame = + "data: {\"id\":\"safe\",\"model\":\"" + + std::string(kRequestedModel) + "\",\"choices\":[]}\n\n"; + response.set_content( + safe_frame + + sized_model_frame(kMaxSseFrameBytes + 1, "\n\n"), + "text/event-stream"); + return; + } + if (fault == "incomplete_frame_over_limit") { + response.set_content( + sized_incomplete_model_frame(kMaxSseFrameBytes + 1), + "text/event-stream"); + return; + } + if (fault == "incomplete_frame_over_limit_after_frame") { + const std::string safe_frame = + "data: {\"id\":\"safe\",\"model\":\"" + + std::string(kRequestedModel) + "\",\"choices\":[]}\n\n"; + response.set_content( + safe_frame + + sized_incomplete_model_frame(kMaxSseFrameBytes + 1), + "text/event-stream"); + return; + } + if (fault == "one_byte_fragmentation") { + event["padding"] = std::string(4096, 'x'); + const std::string fragmented = + "data: " + event.dump() + "\r\n\r\n" + "data: [DONE]\r\r"; + response.set_chunked_content_provider( + "text/event-stream", + [fragmented](size_t, httplib::DataSink& sink) { + for (const char byte : fragmented) { + if (!sink.write(&byte, 1)) { + return false; + } + } + sink.done(); + return false; + }); + return; + } + if (fault == "incomplete_progress") { + const std::string fragmented = + "data: {\"id\":\"active\",\"model\":\"" + + std::string(kBackendModel) + "\""; + response.set_chunked_content_provider( + "text/event-stream", + [fragmented](size_t, httplib::DataSink& sink) { + for (const char byte : fragmented) { + if (!sink.write(&byte, 1)) { + return false; + } + std::this_thread::sleep_for(1ms); + } + sink.done(); + return false; + }); + return; + } + if (fault.rfind("split_done_", 0) == 0) { + std::string done_separator = "\n\n"; + if (fault == "split_done_crlf") { + done_separator = "\r\n\r\n"; + } else if (fault == "split_done_cr") { + done_separator = "\r\r"; + } + const std::string done = "data: [DONE]" + done_separator; + response.set_chunked_content_provider( + "text/event-stream", + [frame, done](size_t, httplib::DataSink& sink) { + sink.write(frame.data(), frame.size()); + const std::size_t split = done.size() / 2; + sink.write(done.data(), split); + std::this_thread::sleep_for(5ms); + sink.write(done.data() + split, done.size() - split); + sink.done(); + return false; + }); + return; + } + if (fault == "clean_partial_before_output") { + response.set_content(partial, "text/event-stream"); + return; + } + + std::string truncated; + if (fault == "partial_before_output") { + const int attempt = partial_before_output_requests.fetch_add(1) + 1; + if (attempt == 1) { + truncated = partial; + } + } else if (fault == "partial_after_output") { + partial_after_output_requests.fetch_add(1); + truncated = frame + partial; + } else if (fault == "done_literal_partial") { + truncated = frame + partial; + } + + if (!truncated.empty()) { + response.set_content_provider( + truncated.size() + 128, + "text/event-stream", + [truncated = std::move(truncated)]( + size_t offset, size_t, httplib::DataSink& sink) { + if (offset == 0) { + sink.write(truncated.data(), truncated.size()); + } + return false; + }); + return; + } + + response.set_chunked_content_provider( + "text/event-stream", + [frame](size_t, httplib::DataSink& sink) { + const size_t split = frame.size() / 2; + sink.write(frame.data(), split); + std::this_thread::sleep_for(5ms); + sink.write(frame.data() + split, frame.size() - split); + const std::string done = "data: [DONE]\n\n"; + const size_t done_split = done.size() / 2; + sink.write(done.data(), done_split); + sink.write(done.data() + done_split, + done.size() - done_split); + sink.done(); + return false; + }); + }); + } + + const int port = backend.bind_to_any_port("127.0.0.1"); + if (port <= 0) { + std::printf("[FAIL] failed to bind mock backend\n"); + return 1; + } + + std::thread backend_thread([&backend]() { backend.listen_after_bind(); }); + backend.wait_until_ready(); + + TestLlamaCppServer server(port); + + const std::string chat = + forward_stream(server, "/v1/chat/completions").output; + const auto chat_events = data_events(chat); + check(chat_events.size() == 1, "chat stream keeps one JSON event"); + check(!chat_events.empty() && chat_events[0].value("model", "") == kRequestedModel, + "chat stream exposes the requested model identity"); + check(chat.find(kBackendModel) == std::string::npos, + "chat stream does not expose the backend GGUF path"); + check(chat.find(": upstream-comment\n\n") != std::string::npos, + "chat stream preserves complete non-data frames"); + check(count_occurrences(chat, "data: [DONE]\n\n") == 1, + "chat stream preserves one split DONE frame"); + + const std::string completion = + forward_stream(server, "/v1/completions").output; + const auto completion_events = data_events(completion); + check(completion_events.size() == 1, "completion stream keeps one JSON event"); + check(!completion_events.empty() && + completion_events[0].value("model", "") == kRequestedModel, + "completion stream exposes the requested model identity"); + check(completion.find(kBackendModel) == std::string::npos, + "completion stream does not expose the backend GGUF path"); + check(completion.find("\r\n\r\n") != std::string::npos, + "completion stream preserves CRLF frame boundaries"); + + const std::string responses = + forward_stream(server, "/v1/responses").output; + const auto response_events = data_events(responses); + check(response_events.size() == 1, "Responses stream keeps one JSON event"); + check(!response_events.empty() && + response_events[0].at("response").value("model", "") == kRequestedModel, + "Responses stream exposes the requested model identity"); + check(responses.find(kBackendModel) == std::string::npos, + "Responses stream does not expose the backend GGUF path"); + + TestLlamaCppServer multiline_server(port); + const StreamResult multiline = forward_stream( + multiline_server, "/v1/chat/completions", "multiline_model"); + const auto multiline_events = data_events(multiline.output); + check(multiline_events.size() == 1 && + multiline_events[0].value("model", "") == kRequestedModel, + "multi-line SSE data is normalized as one JSON event"); + check(multiline.output.find(kBackendModel) == std::string::npos, + "multi-line SSE data does not expose the backend GGUF path"); + check(multiline.output.find("id: multiline\n") != std::string::npos && + multiline.output.find(": retained-comment\n\n") != + std::string::npos, + "multi-line normalization preserves non-data fields"); + + TestLlamaCppServer bom_server(port); + const StreamResult bom = forward_stream( + bom_server, "/v1/chat/completions", "bom_model"); + const auto bom_events = data_events(bom.output); + check(!bom.retryable_reset && bom_events.size() == 1 && + bom_events[0].value("model", "") == kRequestedModel, + "a stream-leading UTF-8 BOM preserves the public model identity"); + check(bom.output.find(kBackendModel) == std::string::npos, + "a stream-leading UTF-8 BOM does not expose the backend GGUF path"); + + TestLlamaCppServer bom_done_server(port); + const StreamResult bom_done = forward_stream( + bom_done_server, "/v1/chat/completions", "bom_done_only"); + check(!bom_done.retryable_reset && + count_occurrences(bom_done.output, "data: [DONE]\n\n") == 1, + "a BOM-prefixed terminal event produces exactly one DONE frame"); + + const RawStreamResult raw_bom_done = + forward_raw_stream(port, "bom_done_only"); + check(!raw_bom_done.interrupted && + count_occurrences( + raw_bom_done.output, "data: [DONE]\n\n") == 1, + "a raw BOM-prefixed terminal event produces exactly one DONE frame"); + + TestLlamaCppServer double_bom_server(port); + const StreamResult double_bom = forward_stream( + double_bom_server, "/v1/chat/completions", "double_bom_model"); + check(double_bom.retryable_reset && double_bom.output.empty(), + "two fragmented leading UTF-8 BOMs fail before client output"); + check(double_bom.output.find(kBackendModel) == std::string::npos, + "two leading UTF-8 BOMs cannot expose the backend GGUF path"); + + TestLlamaCppServer double_bom_done_server(port); + const StreamResult double_bom_done = forward_stream( + double_bom_done_server, "/v1/chat/completions", + "double_bom_done_only"); + check(double_bom_done.retryable_reset && double_bom_done.output.empty(), + "two leading UTF-8 BOMs cannot duplicate a transformed DONE frame"); + + const RawStreamResult raw_double_bom_done = + forward_raw_stream(port, "double_bom_done_only"); + check(raw_double_bom_done.interrupted && + raw_double_bom_done.output.empty(), + "two leading UTF-8 BOMs cannot duplicate a raw DONE frame"); + + TestLlamaCppServer terminal_cr_server(port); + const StreamResult terminal_cr = forward_stream( + terminal_cr_server, "/v1/chat/completions", "terminal_cr"); + const auto terminal_cr_events = data_events(terminal_cr.output); + check(!terminal_cr.retryable_reset && terminal_cr_events.size() == 1 && + terminal_cr_events[0].value("model", "") == kRequestedModel, + "a terminal bare CR completes a transformed SSE frame at EOF"); + check(terminal_cr.output.find(kBackendModel) == std::string::npos, + "a terminal bare CR frame does not expose the backend GGUF path"); + + TestLlamaCppServer malformed_server(port); + const StreamResult malformed = forward_stream( + malformed_server, "/v1/chat/completions", "malformed_model"); + check(malformed.retryable_reset && malformed.output.empty(), + "a malformed OpenAI SSE object fails before client output"); + check(malformed.output.find(kBackendModel) == std::string::npos, + "a malformed OpenAI SSE object does not expose the backend GGUF path"); + + TestLlamaCppServer scalar_server(port); + const StreamResult scalar = forward_stream( + scalar_server, "/v1/chat/completions", "scalar_model"); + check(scalar.retryable_reset && scalar.output.empty(), + "a non-object OpenAI SSE value fails before client output"); + check(scalar.output.find(kBackendModel) == std::string::npos, + "a non-object OpenAI SSE value does not expose the backend GGUF path"); + + const std::vector exact_limit_faults = { + "frame_at_limit_lf", + "frame_at_limit_crlf", + }; + for (const std::string& fault : exact_limit_faults) { + TestLlamaCppServer exact_limit_server(port); + const StreamResult transformed_exact_limit = forward_stream( + exact_limit_server, "/v1/chat/completions", fault); + check(!transformed_exact_limit.retryable_reset && + transformed_exact_limit.output.find(kRequestedModel) != + std::string::npos && + transformed_exact_limit.output.find(kBackendModel) == + std::string::npos, + ("transformed stream accepts an exact-limit frame for " + + fault).c_str()); + + const RawStreamResult raw_exact_limit = + forward_raw_stream(port, fault); + check(!raw_exact_limit.interrupted, + ("raw stream accepts an exact-limit frame for " + fault).c_str()); + } + + TestLlamaCppServer oversized_server(port); + const StreamResult transformed_oversized = forward_stream( + oversized_server, "/v1/chat/completions", "frame_over_limit"); + check(transformed_oversized.retryable_reset && + transformed_oversized.output.empty(), + "a transformed oversized frame fails before client output"); + check(transformed_oversized.output.find(kBackendModel) == + std::string::npos, + "a transformed oversized frame does not expose the backend path"); + + const RawStreamResult raw_oversized = + forward_raw_stream(port, "frame_over_limit"); + check(raw_oversized.interrupted && raw_oversized.output.empty(), + "a raw oversized frame fails before client output"); + + TestLlamaCppServer oversized_incomplete_server(port); + const StreamResult transformed_oversized_incomplete = forward_stream( + oversized_incomplete_server, "/v1/chat/completions", + "incomplete_frame_over_limit"); + check(transformed_oversized_incomplete.retryable_reset && + transformed_oversized_incomplete.output.empty() && + transformed_oversized_incomplete.output.find(kBackendModel) == + std::string::npos, + "an incomplete transformed oversized frame fails closed"); + + const RawStreamResult raw_oversized_incomplete = + forward_raw_stream(port, "incomplete_frame_over_limit"); + check(raw_oversized_incomplete.interrupted && + raw_oversized_incomplete.output.empty(), + "an incomplete raw oversized frame fails before client output"); + + for (const std::string& fault : { + "frame_over_limit_after_frame", + "incomplete_frame_over_limit_after_frame", + }) { + TestLlamaCppServer raw_wrapper_server(port); + const StreamResult raw_wrapper = forward_wrapped_raw_stream( + raw_wrapper_server, "/v1/chat/completions", fault); + int malformed_events = 0; + const auto events = data_events(raw_wrapper.output, &malformed_events); + check(!raw_wrapper.retryable_reset && raw_wrapper.done_calls == 1 && + malformed_events == 0 && events.size() == 2 && + events[0].value("model", "") == kRequestedModel && + events[1].contains("error"), + ("a wrapped raw oversized frame ends with a standalone error for " + + fault).c_str()); + check(raw_wrapper.output.find(kBackendModel) == std::string::npos, + ("a wrapped raw oversized frame does not expose buffered data for " + + fault).c_str()); + } + + TestLlamaCppServer fragmented_server(port); + const StreamResult transformed_fragmented = forward_stream( + fragmented_server, "/v1/chat/completions", + "one_byte_fragmentation"); + check(!transformed_fragmented.retryable_reset && + transformed_fragmented.output.find(kRequestedModel) != + std::string::npos && + transformed_fragmented.output.find(kBackendModel) == + std::string::npos && + count_occurrences( + transformed_fragmented.output, "data: [DONE]") == 1, + "a transformed stream accepts one-byte SSE fragmentation"); + + const RawStreamResult raw_fragmented = + forward_raw_stream(port, "one_byte_fragmentation"); + check(!raw_fragmented.interrupted && + count_occurrences(raw_fragmented.output, "data: [DONE]") == 1, + "a raw stream accepts one-byte SSE fragmentation"); + + const RawStreamResult active_incomplete = + forward_raw_stream(port, "incomplete_progress"); + check(active_incomplete.interrupted && active_incomplete.output.empty(), + "an active incomplete frame remains retryable before client output"); + check(active_incomplete.backend_progress_calls > 1, + "raw backend chunks refresh progress before an SSE frame completes"); + + const std::vector split_done_faults = { + "split_done_lf", + "split_done_crlf", + "split_done_cr", + }; + for (const std::string& fault : split_done_faults) { + TestLlamaCppServer transformed_done_server(port); + const StreamResult transformed_done = forward_stream( + transformed_done_server, "/v1/chat/completions", fault); + check(!transformed_done.retryable_reset && + count_occurrences( + transformed_done.output, "data: [DONE]") == 1 && + transformed_done.output.find(kRequestedModel) != + std::string::npos && + transformed_done.output.find(kBackendModel) == + std::string::npos, + ("transformed stream preserves one fragmented DONE for " + + fault).c_str()); + + const RawStreamResult raw_done = forward_raw_stream(port, fault); + check(!raw_done.interrupted && + count_occurrences(raw_done.output, "data: [DONE]") == 1, + ("raw stream preserves one fragmented DONE for " + fault).c_str()); + } + + TestLlamaCppServer literal_done_server(port); + const StreamResult literal_done = forward_stream( + literal_done_server, "/v1/chat/completions", + "done_literal_partial"); + int literal_done_malformed_events = 0; + const auto literal_done_events = + data_events(literal_done.output, &literal_done_malformed_events); + check(!literal_done.retryable_reset && + literal_done_malformed_events == 0 && + literal_done_events.size() == 2 && + literal_done_events[0].value("model", "") == kRequestedModel && + literal_done_events[1].contains("error"), + "DONE text inside JSON does not mask a transformed interruption"); + check(literal_done.output.find(kBackendModel) == std::string::npos, + "a false-DONE interruption does not expose the backend GGUF path"); + + const RawStreamResult raw_literal_done = + forward_raw_stream(port, "done_literal_partial"); + check(raw_literal_done.interrupted, + "DONE text inside JSON does not mask a raw-stream interruption"); + + check(forward_stream_to_disconnected_client(server) == 1, + "a disconnected client receives no buffered writes during cleanup"); + + TestLlamaCppServer retry_server(port); + const StreamResult interrupted_before_output = forward_stream( + retry_server, "/v1/chat/completions", "partial_before_output"); + check(interrupted_before_output.retryable_reset, + "a partial first event preserves the pre-output retry signal"); + check(interrupted_before_output.output.empty(), + "a partial first event emits no client bytes"); + check(interrupted_before_output.output.find(kBackendModel) == + std::string::npos, + "a partial first event does not expose the backend GGUF path"); + + TestLlamaCppServer clean_eof_server(port); + const StreamResult clean_eof_partial = forward_stream( + clean_eof_server, "/v1/chat/completions", + "clean_partial_before_output"); + check(clean_eof_partial.retryable_reset, + "a clean HTTP EOF with an incomplete event remains retryable"); + check(clean_eof_partial.output.empty() && + clean_eof_partial.output.find(kBackendModel) == + std::string::npos, + "a clean HTTP EOF does not flush an incomplete event"); + + StreamResult retried; + if (interrupted_before_output.retryable_reset) { + TestLlamaCppServer reloaded_server(port); + retried = forward_stream( + reloaded_server, "/v1/chat/completions", "partial_before_output"); + } + int malformed_retried_events = 0; + const auto retried_events = + data_events(retried.output, &malformed_retried_events); + check(partial_before_output_requests.load() == 2, + "the caller can replay once after a pre-output interruption"); + check(retried_events.size() == 1 && + retried_events[0].value("model", "") == kRequestedModel, + "the replay produces one normalized event"); + check(malformed_retried_events == 0 && + retried.output.find(kBackendModel) == std::string::npos && + retried.output.find("\"error\"") == std::string::npos && + count_occurrences(retried.output, "data: [DONE]\n\n") == 1, + "the replay produces one complete error-free stream"); + + TestLlamaCppServer no_replay_server(port); + const StreamResult interrupted_after_output = forward_stream( + no_replay_server, "/v1/chat/completions", "partial_after_output"); + int malformed_events = 0; + const auto events_after_output = + data_events(interrupted_after_output.output, &malformed_events); + check(!interrupted_after_output.retryable_reset, + "an interruption after client output is not replayed"); + check(partial_after_output_requests.load() == 1, + "an interruption after client output makes one backend request"); + check(malformed_events == 0, + "the post-output interruption emits only valid JSON events"); + check(events_after_output.size() == 2 && + events_after_output[0].value("model", "") == kRequestedModel && + events_after_output[1].contains("error"), + "the partial event is replaced by a separate streaming error"); + check(interrupted_after_output.output.find(kBackendModel) == + std::string::npos, + "the post-output interruption does not expose the backend GGUF path"); + + backend.stop(); + backend_thread.join(); + + return failures == 0 ? 0 : 1; +} diff --git a/test/cpp/test_llamacpp_system_backend.cpp b/test/cpp/test_llamacpp_system_backend.cpp index 7d0ced7bec..6a3401efc4 100644 --- a/test/cpp/test_llamacpp_system_backend.cpp +++ b/test/cpp/test_llamacpp_system_backend.cpp @@ -208,6 +208,95 @@ int main() { ScopedEnvVar path_env("PATH"); ScopedEnvVar hip_env("LEMONADE_GGML_HIP_PATH"); + check(lemon::backends::llamacpp::detail::identifies_k2_horizon_model( + "builtin.K2-Horizon-0.9B-GGUF", "", ""), + "identifies K2-Horizon from the selected model name"); + check(lemon::backends::llamacpp::detail::identifies_k2_horizon_model( + "user.custom", "IFM/K2_Horizon-7B-GGUF:model.gguf", ""), + "identifies K2-Horizon from the checkpoint"); + check(lemon::backends::llamacpp::detail::identifies_k2_horizon_model( + "user.custom", "example/model:model.gguf", "k2-horizon"), + "identifies K2-Horizon from GGUF architecture metadata"); + check(!lemon::backends::llamacpp::detail::identifies_k2_horizon_model( + "Tiny-Test-Model-GGUF", "unsloth/gemma-3-GGUF:model.gguf", "gemma3"), + "does not classify an ordinary GGUF model as K2-Horizon"); + check(!lemon::backends::llamacpp::detail::identifies_k2_horizon_model( + "user.K2-Horizon-Alias", + "IFM/K2-Horizon-7B-GGUF:model.gguf", + "gemma3"), + "uses nonempty GGUF architecture instead of model-name fallbacks"); + check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( + "system", false, "builtin.K2-Horizon-0.9B-GGUF", "", ""), + "requires unsupported-architecture evidence for the K2 diagnostic"); + const std::string unsupported_k2_architecture = + "error loading model: unknown model architecture: 'k2-horizon'"; + check(lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( + "system", + false, + "builtin.K2-Horizon-0.9B-GGUF", + "", + "", + unsupported_k2_architecture), + "reports a K2 diagnostic for concrete unsupported-architecture evidence"); + check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( + "system", + false, + "builtin.K2-Horizon-0.9B-GGUF", + "", + "", + "error loading model: failed to allocate compute buffers"), + "keeps the ordinary diagnostic for a K2 resource failure"); + check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( + "vulkan", false, "builtin.K2-Horizon-0.9B-GGUF", "", ""), + "does not report the system diagnostic for a managed backend"); + check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( + "system", false, "Tiny-Test-Model-GGUF", "", "gemma3"), + "does not report the K2 diagnostic for an ordinary model"); + check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( + "system", true, "builtin.K2-Horizon-0.9B-GGUF", "", ""), + "does not classify a cancelled K2 load as binary incompatibility"); + check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version( + "version: 4242 (incompatible mock)") == "b4242", + "parses the incompatible system binary version"); + check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version( + "version: 0.3.0-dev (build 10671, commit 35999d101)\n" + "built with AppleClang 21.0.0.21000101 for Darwin arm64") == + "b10671", + "parses the build number from the required IFM binary version"); + check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version("") == + "unknown", + "reports an unknown system binary version when probing returns no output"); + check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version( + "Failed to execute: missing-llama-server", 1) == "unknown", + "ignores captured stderr when the version probe fails"); + + const std::string diagnostic = + lemon::backends::llamacpp::detail::k2_horizon_system_startup_error( + "builtin.K2-Horizon-0.9B-GGUF", + "/opt/old-llama/bin/llama-server", + "b4242", + unsupported_k2_architecture); + check(diagnostic.find("builtin.K2-Horizon-0.9B-GGUF") != std::string::npos, + "K2 startup diagnostic names the selected model"); + check(diagnostic.find("/opt/old-llama/bin/llama-server") != std::string::npos, + "K2 startup diagnostic names the selected executable"); + check(diagnostic.find("b4242") != std::string::npos, + "K2 startup diagnostic includes the detected version"); + check(diagnostic.find("Original error: " + unsupported_k2_architecture) != + std::string::npos, + "K2 startup diagnostic preserves the original startup error"); + check(diagnostic.find("model/K2Horizon") != std::string::npos && + diagnostic.find("PATH") != std::string::npos, + "K2 startup diagnostic provides the IFM branch remediation"); + const std::string unknown_version_diagnostic = + lemon::backends::llamacpp::detail::k2_horizon_system_startup_error( + "builtin.K2-Horizon-0.9B-GGUF", + "/opt/old-llama/bin/llama-server", + "detected", + "llama-server failed to start"); + check(unknown_version_diagnostic.find("version unknown") != std::string::npos, + "K2 startup diagnostic reports unknown for an unparseable version"); + // PATH discovery is a direct C++ decision: no lemond is needed to prove it. path_env.set(system_bin.string()); check(!lemon::utils::find_executable_in_path("llama-server").empty(), diff --git a/test/cpp/test_mcp_client_config.cpp b/test/cpp/test_mcp_client_config.cpp index eb022301f3..0e75d86af2 100644 --- a/test/cpp/test_mcp_client_config.cpp +++ b/test/cpp/test_mcp_client_config.cpp @@ -1,5 +1,6 @@ #include "lemon/mcp_client.h" +#include #include #include #include @@ -14,11 +15,98 @@ #include #include +#if defined(LEMONADE_MCP_PROCESS_TEST_HOOK) && !defined(_WIN32) + #include + #include + #include + +using McpProcessTestHook = void (*)(pid_t); +extern "C" void lemonade_test_set_mcp_process_hooks( + McpProcessTestHook exit_observed, McpProcessTestHook group_cleanup, + McpProcessTestHook final_reap); +#endif + namespace fs = std::filesystem; using namespace std::chrono_literals; namespace { +#if defined(LEMONADE_MCP_PROCESS_TEST_HOOK) && \ + defined(LEMONADE_TEST_PYTHON) && \ + defined(LEMONADE_TEST_MCP_STDIO_SERVER) +std::atomic mcp_process_hook_sequence{0}; +std::atomic mcp_exit_observed_order{-1}; +std::atomic mcp_group_cleanup_order{-1}; +std::atomic mcp_final_reap_order{-1}; +std::atomic mcp_leader_reserved{false}; +std::atomic mcp_leader_reserved_after_cleanup{false}; +std::atomic mcp_leader_absent_after_reap{false}; + +bool exited_child_is_reserved(pid_t pid) { + siginfo_t child_info{}; + int result; + do { + result = ::waitid(P_PID, static_cast(pid), &child_info, + WEXITED | WNOHANG | WNOWAIT); + } while (result < 0 && errno == EINTR); + return result == 0 && child_info.si_pid == pid; +} + +void record_mcp_exit_observed(pid_t pid) { + mcp_leader_reserved.store(exited_child_is_reserved(pid)); + mcp_exit_observed_order.store(mcp_process_hook_sequence.fetch_add(1)); +} + +void record_mcp_group_cleanup(pid_t pid) { + mcp_leader_reserved_after_cleanup.store(exited_child_is_reserved(pid)); + mcp_group_cleanup_order.store(mcp_process_hook_sequence.fetch_add(1)); +} + +void record_mcp_final_reap(pid_t pid) { + siginfo_t child_info{}; + errno = 0; + int result; + do { + result = ::waitid(P_PID, static_cast(pid), &child_info, + WEXITED | WNOHANG | WNOWAIT); + } while (result < 0 && errno == EINTR); + mcp_leader_absent_after_reap.store(result < 0 && errno == ECHILD); + mcp_final_reap_order.store(mcp_process_hook_sequence.fetch_add(1)); +} + +class McpProcessHookGuard { +public: + McpProcessHookGuard() { + mcp_process_hook_sequence.store(0); + mcp_exit_observed_order.store(-1); + mcp_group_cleanup_order.store(-1); + mcp_final_reap_order.store(-1); + mcp_leader_reserved.store(false); + mcp_leader_reserved_after_cleanup.store(false); + mcp_leader_absent_after_reap.store(false); + lemonade_test_set_mcp_process_hooks( + record_mcp_exit_observed, record_mcp_group_cleanup, + record_mcp_final_reap); + } + + ~McpProcessHookGuard() { + lemonade_test_set_mcp_process_hooks(nullptr, nullptr, nullptr); + } +}; + +void verify_mcp_process_hook_result() { + const bool order_is_valid = mcp_exit_observed_order.load() == 0 && + mcp_group_cleanup_order.load() == 1 && + mcp_final_reap_order.load() == 2; + if (!mcp_leader_reserved.load() || + !mcp_leader_reserved_after_cleanup.load() || + !mcp_leader_absent_after_reap.load() || !order_is_valid) { + throw std::runtime_error( + "MCP POSIX stop did not retain the leader through group cleanup"); + } +} +#endif + void set_test_env(const std::string& name, const std::string& value) { #ifdef _WIN32 if (_putenv_s(name.c_str(), value.c_str()) != 0) { @@ -234,18 +322,30 @@ int run_tests() { // Reconnect and verify that a child process exiting without a response // wakes the request immediately rather than waiting for its full timeout. manager.connect_server_json(id); - const auto exit_start = std::chrono::steady_clock::now(); - assert(throws([&] { - manager.call_tool_json( - id, json{{"name", "exit"}, - {"arguments", json::object()}, - {"timeout_ms", 10000}}); - })); - const auto exit_elapsed = - std::chrono::steady_clock::now() - exit_start; - assert(exit_elapsed < 5s); - - manager.remove_server_json(id); + { +#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK + McpProcessHookGuard process_hook_guard; +#endif + const auto exit_start = std::chrono::steady_clock::now(); + const bool exit_failed = throws([&] { + manager.call_tool_json( + id, json{{"name", "exit"}, + {"arguments", json::object()}, + {"timeout_ms", 10000}}); + }); + const auto exit_elapsed = + std::chrono::steady_clock::now() - exit_start; + if (!exit_failed || exit_elapsed >= 5s) { + throw std::runtime_error( + "MCP process exit did not wake the pending request"); + } + + manager.remove_server_json(id); +#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK + verify_mcp_process_hook_result(); + std::cout << "MCP POSIX process lifecycle hook checks passed\n"; +#endif + } // Disconnect must also cancel initialization in progress. This covers a // race where disconnect could previously return before connect installed diff --git a/test/cpp/test_mcp_client_process_contract.cpp b/test/cpp/test_mcp_client_process_contract.cpp new file mode 100644 index 0000000000..7a498f263f --- /dev/null +++ b/test/cpp/test_mcp_client_process_contract.cpp @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include + +#ifndef LEMONADE_MCP_CLIENT_SOURCE + #error "LEMONADE_MCP_CLIENT_SOURCE must name mcp_client.cpp" +#endif + +namespace { + +int failures = 0; + +void check(const std::string& name, bool condition) { + if (condition) { + std::cout << "[PASS] " << name << '\n'; + return; + } + std::cerr << "[FAIL] " << name << '\n'; + ++failures; +} + +std::string read_source() { + std::ifstream input(LEMONADE_MCP_CLIENT_SOURCE); + if (!input) { + throw std::runtime_error("failed to open mcp_client.cpp"); + } + return {std::istreambuf_iterator(input), + std::istreambuf_iterator()}; +} + +std::string stop_body(const std::string& source) { + const std::size_t start = source.find("void stop_locked()"); + const std::size_t end = source.find( + "if (stdout_thread_.joinable())", start); + if (start == std::string::npos || end == std::string::npos) { + throw std::runtime_error("failed to locate MCP stop body"); + } + return source.substr(start, end - start); +} + +} // namespace + +int main() { + try { + const std::string body = stop_body(read_source()); + const std::size_t windows_process = + body.find("if (process_info_.hProcess)"); + const std::size_t posix_start = + body.find("#else", windows_process); + if (windows_process == std::string::npos || + posix_start == std::string::npos) { + throw std::runtime_error("failed to locate MCP POSIX stop path"); + } + const std::string posix = body.substr(posix_start); + check("MCP stop uses one POSIX lifecycle path", + posix.find("#ifdef __APPLE__") == std::string::npos); + if (posix.find("#ifdef __APPLE__") != std::string::npos) { + return 1; + } + + const std::size_t observe = posix.find("::waitid("); + const std::size_t terminate = + posix.find("::kill(-pid_, SIGTERM)"); + const std::size_t force_kill = + posix.rfind("::kill(-pid_, SIGKILL)"); + const std::size_t final_reap = posix.find("::waitpid(pid_"); + + check("MCP stop observes the POSIX child without reaping", + observe != std::string::npos && + posix.find("WNOWAIT", observe) != std::string::npos); + check("MCP stop cleans the POSIX process group before final reap", + observe < terminate && terminate < force_kill && + force_kill < final_reap); + check("MCP stop does not use a reaping POSIX status probe", + posix.find("::waitpid(pid_, &status, WNOHANG)") == + std::string::npos && + posix.find("::waitpid(pid_, &status, 0)") == final_reap); + } catch (const std::exception& error) { + std::cerr << "mcp client process contract failed: " << error.what() + << '\n'; + return 1; + } + + if (failures != 0) { + std::cerr << failures << " MCP client process contract check(s) failed\n"; + return 1; + } + std::cout << "MCP client process contract checks passed\n"; + return 0; +} diff --git a/test/cpp/test_process_manager.cpp b/test/cpp/test_process_manager.cpp index f609bb4511..dc68a1224f 100644 --- a/test/cpp/test_process_manager.cpp +++ b/test/cpp/test_process_manager.cpp @@ -1,14 +1,19 @@ #include +#include #include #include #include #include +#include #include +#include #include #include #include +#include +#include #include #include @@ -18,6 +23,55 @@ using lemon::utils::ProcessManager; namespace { int failures = 0; +std::atomic coordinate_forks{false}; +std::atomic first_fork_ready{false}; +std::atomic release_first_fork{false}; +std::atomic second_fork_completed{false}; +std::atomic fork_prepare_count{0}; +std::atomic fork_parent_count{0}; +#ifdef __linux__ +std::atomic run_command_pipe_ready{false}; +std::atomic release_run_command_pipe{false}; +std::atomic run_command_pipe_fd{-1}; +std::atomic run_command_pipe_flags{-1}; +#endif +#ifdef __APPLE__ +std::atomic first_output_pipe_ready{false}; +std::atomic release_first_output_command{false}; +std::atomic output_pipe_hook_count{0}; +std::atomic inspect_run_command_reap_order{false}; +std::atomic run_command_reap_sequence{0}; +std::atomic run_command_exit_observed_order{-1}; +std::atomic run_command_group_cleanup_order{-1}; +std::atomic run_command_final_reap_order{-1}; +std::atomic run_command_leader_reserved{false}; +std::atomic run_command_leader_reserved_after_cleanup{false}; +std::atomic run_command_leader_absent_after_reap{false}; +#endif + +void before_coordinated_fork() { + if (!coordinate_forks.load()) { + return; + } + +#ifdef __linux__ + if (fork_prepare_count.fetch_add(1) == 0) { + first_fork_ready.store(true); + while (coordinate_forks.load() && !release_first_fork.load()) { + std::this_thread::yield(); + } + } +#else + fork_prepare_count.fetch_add(1); +#endif +} + +void after_coordinated_fork_in_parent() { + fork_parent_count.fetch_add(1); + if (coordinate_forks.load() && fork_prepare_count.load() >= 2) { + second_fork_completed.store(true); + } +} void check(const char* name, bool condition) { std::printf("[%s] %s\n", condition ? "PASS" : "FAIL", name); @@ -26,6 +80,7 @@ void check(const char* name, bool condition) { } } +#ifdef __linux__ ProcessHandle make_handle(pid_t pid) { return {nullptr, static_cast(pid)}; } @@ -84,9 +139,750 @@ void kill_and_reap(pid_t pid) { } } +std::string read_fd_target(pid_t pid, int fd) { + const std::string path = "/proc/" + std::to_string(pid) + "/fd/" + + std::to_string(fd); + char target[256]; + const ssize_t length = readlink(path.c_str(), target, sizeof(target) - 1); + if (length < 0) { + return {}; + } + target[length] = '\0'; + return target; +} + +bool wait_for_process_name(pid_t pid, const std::string& expected, + std::chrono::milliseconds timeout) { + const auto deadline = std::chrono::steady_clock::now() + timeout; + do { + std::ifstream comm_file("/proc/" + std::to_string(pid) + "/comm"); + std::string process_name; + if (std::getline(comm_file, process_name) && process_name == expected) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } while (std::chrono::steady_clock::now() < deadline); + return false; +} +#endif + +int capture_with_closed_standard_output() { + const pid_t probe = fork(); + if (probe == 0) { + close(STDOUT_FILENO); + close(STDERR_FILENO); + try { + const ProcessHandle handle = ProcessManager::start_process( + "/bin/sh", + {"-c", "printf 'closed-stdio-error' >&2"}, + "", + false, + false, + {}, + 64); + if (handle.pid <= 0 || + ProcessManager::wait_for_exit(handle, 5) != 0) { + _exit(2); + } + if (ProcessManager::read_output(handle, 64) != + "closed-stdio-error") { + _exit(3); + } + + std::string command_output; + const int result = ProcessManager::run_process_with_output( + "/bin/sh", + {"-c", "printf 'closed-stdio-version' >&2"}, + [&command_output](const std::string& line) { + command_output += line; + return true; + }, + "", + 5); + _exit(result == 0 && command_output == "closed-stdio-version" ? 0 + : 5); + } catch (...) { + _exit(4); + } + } + if (probe <= 0) { + return -1; + } + + int status = 0; + if (waitpid(probe, &status, 0) != probe || !WIFEXITED(status)) { + return -1; + } + return WEXITSTATUS(status); +} + +#ifdef __linux__ +struct ConcurrentCaptureResult { + bool setup_succeeded = false; + int short_exit_code = -1; + long short_read_ms = -1; + std::string short_output; +}; + +struct ConcurrentPopenResult { + bool setup_succeeded = false; + bool close_on_exec = false; + bool descriptor_inherited = false; + int command_status = -1; + std::string command_output; +}; + +ConcurrentPopenResult run_overlapping_popen_and_exec() { + ConcurrentPopenResult result; + char sentinel_path[] = "/tmp/lemonade-popen-overlap-XXXXXX"; + const int sentinel_fd = mkstemp(sentinel_path); + if (sentinel_fd < 0) { + return result; + } + close(sentinel_fd); + unlink(sentinel_path); + + run_command_pipe_ready.store(false); + release_run_command_pipe.store(false); + run_command_pipe_fd.store(-1); + run_command_pipe_flags.store(-1); + + auto command = std::async(std::launch::async, [&]() { + return ProcessManager::run_command("printf popen-output", + result.command_output, 5); + }); + + const auto hook_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(1); + while (!run_command_pipe_ready.load() && + std::chrono::steady_clock::now() < hook_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + ProcessHandle long_handle{nullptr, 0, nullptr}; + bool child_exec_observed = false; + std::string parent_pipe_target; + std::string child_pipe_target; + const int pipe_fd = run_command_pipe_fd.load(); + if (run_command_pipe_ready.load() && pipe_fd >= 0) { + parent_pipe_target = read_fd_target(getpid(), pipe_fd); + try { + long_handle = ProcessManager::start_process( + "/bin/sh", + {"-c", "printf ready > " + std::string(sentinel_path) + + "; exec sleep 3"}, + "", false, false, {}, 64); + child_exec_observed = + wait_for_process_name(long_handle.pid, "sleep", + std::chrono::milliseconds(1000)) && + access(sentinel_path, F_OK) == 0; + if (child_exec_observed) { + child_pipe_target = read_fd_target(long_handle.pid, pipe_fd); + } + } catch (...) { + } + } + + result.close_on_exec = + (run_command_pipe_flags.load() & FD_CLOEXEC) != 0; + result.descriptor_inherited = !parent_pipe_target.empty() && + child_pipe_target == parent_pipe_target; + result.setup_succeeded = run_command_pipe_ready.load() && + parent_pipe_target.rfind("pipe:[", 0) == 0 && + long_handle.pid > 0 && child_exec_observed; + + release_run_command_pipe.store(true); + if (long_handle.pid > 0) { + ProcessManager::kill_process(long_handle); + } + result.command_status = command.get(); + unlink(sentinel_path); + return result; +} + +ConcurrentCaptureResult run_overlapping_captured_spawns() { + ConcurrentCaptureResult result; + if (pthread_atfork(before_coordinated_fork, + after_coordinated_fork_in_parent, nullptr) != 0) { + return result; + } + + coordinate_forks.store(true); + first_fork_ready.store(false); + release_first_fork.store(false); + second_fork_completed.store(false); + fork_prepare_count.store(0); + fork_parent_count.store(0); + + ProcessHandle short_handle{nullptr, 0, nullptr}; + ProcessHandle long_handle{nullptr, 0, nullptr}; + std::exception_ptr short_error; + std::exception_ptr long_error; + std::atomic short_spawn_finished{false}; + std::atomic long_spawn_finished{false}; + std::atomic workers_can_exit{false}; + + std::thread short_thread([&]() { + try { + short_handle = ProcessManager::start_process( + "/bin/sh", {"-c", "printf 'short-capture' >&2"}, "", false, + false, {}, 64); + } catch (...) { + short_error = std::current_exception(); + second_fork_completed.store(true); + } + short_spawn_finished.store(true); + while (!workers_can_exit.load()) { + std::this_thread::yield(); + } + }); + + const auto coordination_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!first_fork_ready.load() && + std::chrono::steady_clock::now() < coordination_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + std::thread long_thread; + if (first_fork_ready.load()) { + long_thread = std::thread([&]() { + try { + long_handle = ProcessManager::start_process( + "/bin/sh", {"-c", "exec sleep 3"}, "", false, false, {}, + 64); + } catch (...) { + long_error = std::current_exception(); + second_fork_completed.store(true); + } + long_spawn_finished.store(true); + while (!workers_can_exit.load()) { + std::this_thread::yield(); + } + }); + } else { + second_fork_completed.store(true); + long_spawn_finished.store(true); + } + + while (first_fork_ready.load() && !second_fork_completed.load() && + std::chrono::steady_clock::now() < coordination_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + const bool forks_overlapped = second_fork_completed.load(); + release_first_fork.store(true); + if (!forks_overlapped) { + second_fork_completed.store(true); + } + + while ((!short_spawn_finished.load() || !long_spawn_finished.load()) && + std::chrono::steady_clock::now() < coordination_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + coordinate_forks.store(false); + + if (!forks_overlapped || !short_spawn_finished.load() || + !long_spawn_finished.load() || short_error || long_error || + short_handle.pid <= 0 || long_handle.pid <= 0) { + ProcessManager::kill_process(short_handle); + ProcessManager::kill_process(long_handle); + workers_can_exit.store(true); + short_thread.join(); + if (long_thread.joinable()) { + long_thread.join(); + } + return result; + } + + const auto exit_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (result.short_exit_code < 0 && + std::chrono::steady_clock::now() < exit_deadline) { + result.short_exit_code = ProcessManager::get_exit_code(short_handle); + if (result.short_exit_code < 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + + const auto read_started = std::chrono::steady_clock::now(); + result.short_output = ProcessManager::read_output(short_handle, 64, 1000); + result.short_read_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - read_started) + .count(); + if (result.short_exit_code < 0) { + ProcessManager::kill_process(short_handle); + } + ProcessManager::kill_process(long_handle); + workers_can_exit.store(true); + short_thread.join(); + long_thread.join(); + result.setup_succeeded = true; + return result; +} +#endif + +#ifdef __APPLE__ +struct ConcurrentRunWithOutputResult { + bool setup_succeeded = false; + bool short_completed_promptly = false; + int short_exit_code = -1; + int long_exit_code = -1; + long short_wait_ms = -1; + std::string short_output; +}; + +ConcurrentRunWithOutputResult run_overlapping_output_commands() { + ConcurrentRunWithOutputResult result; + char sentinel_path[] = "/tmp/lemonade-process-overlap-XXXXXX"; + const int sentinel_fd = mkstemp(sentinel_path); + if (sentinel_fd < 0) { + return result; + } + close(sentinel_fd); + unlink(sentinel_path); + + coordinate_forks.store(true); + first_output_pipe_ready.store(false); + release_first_output_command.store(false); + output_pipe_hook_count.store(0); + + std::string short_output; + auto short_command = std::async(std::launch::async, [&]() { + return ProcessManager::run_process_with_output( + "/bin/sh", {"-c", "printf 'short-capture\\n'"}, + [&short_output](const std::string& line) { + short_output += line; + return true; + }, + "", 5); + }); + + const auto first_pipe_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(1); + while (!first_output_pipe_ready.load() && + std::chrono::steady_clock::now() < first_pipe_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + std::string long_output; + const std::string long_command_text = + "printf ready > " + std::string(sentinel_path) + "; sleep 3"; + auto long_command = std::async(std::launch::async, [&]() { + return ProcessManager::run_command(long_command_text, long_output, 5); + }); + + const auto sentinel_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(1); + while (access(sentinel_path, F_OK) != 0 && + std::chrono::steady_clock::now() < sentinel_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + const bool setup_succeeded = first_output_pipe_ready.load() && + output_pipe_hook_count.load() == 2 && + access(sentinel_path, F_OK) == 0; + release_first_output_command.store(true); + coordinate_forks.store(false); + + const auto wait_started = std::chrono::steady_clock::now(); + result.short_completed_promptly = + short_command.wait_for(std::chrono::milliseconds(1000)) == + std::future_status::ready; + result.short_wait_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - wait_started) + .count(); + result.long_exit_code = long_command.get(); + result.short_exit_code = short_command.get(); + result.short_output = std::move(short_output); + result.setup_succeeded = setup_succeeded; + unlink(sentinel_path); + return result; +} + +bool wait_for_process_gone(pid_t pid, std::chrono::milliseconds timeout) { + const auto deadline = std::chrono::steady_clock::now() + timeout; + do { + errno = 0; + if (::kill(pid, 0) != 0 && errno == ESRCH) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } while (std::chrono::steady_clock::now() < deadline); + + errno = 0; + return ::kill(pid, 0) != 0 && errno == ESRCH; +} + +struct RunCommandTimeoutResult { + bool setup_succeeded = false; + bool descendant_gone = false; + int command_status = 0; +}; + +RunCommandTimeoutResult run_command_with_descendant_timeout() { + RunCommandTimeoutResult result; + char sentinel_path[] = "/tmp/lemonade-command-descendant-XXXXXX"; + const int sentinel_fd = mkstemp(sentinel_path); + if (sentinel_fd < 0) { + return result; + } + close(sentinel_fd); + unlink(sentinel_path); + + std::string output; + result.command_status = ProcessManager::run_command( + "sleep 30 & echo $! > " + std::string(sentinel_path) + "; wait", + output, 1); + + std::ifstream sentinel(sentinel_path); + pid_t descendant = -1; + sentinel >> descendant; + result.setup_succeeded = descendant > 0; + if (result.setup_succeeded) { + result.descendant_gone = + wait_for_process_gone(descendant, std::chrono::seconds(2)); + if (!result.descendant_gone) { + ::kill(descendant, SIGKILL); + } + } + unlink(sentinel_path); + return result; +} + +struct RunCommandBackgroundResult { + bool setup_succeeded = false; + bool descendant_gone = false; + int command_status = -1; +}; + +RunCommandBackgroundResult run_command_with_background_descendant() { + RunCommandBackgroundResult result; + char sentinel_path[] = "/tmp/lemonade-command-background-XXXXXX"; + const int sentinel_fd = mkstemp(sentinel_path); + if (sentinel_fd < 0) { + return result; + } + close(sentinel_fd); + unlink(sentinel_path); + + std::string output; + result.command_status = ProcessManager::run_command( + "sleep 30 /dev/null 2>&1 & echo $! > " + + std::string(sentinel_path), + output, 5); + + std::ifstream sentinel(sentinel_path); + pid_t descendant = -1; + sentinel >> descendant; + result.setup_succeeded = descendant > 0; + if (result.setup_succeeded) { + result.descendant_gone = + wait_for_process_gone(descendant, std::chrono::seconds(2)); + if (!result.descendant_gone) { + ::kill(descendant, SIGKILL); + } + } + unlink(sentinel_path); + return result; +} + +struct RunCommandReapOrderResult { + bool leader_reserved = false; + bool leader_reserved_after_cleanup = false; + bool leader_absent_after_reap = false; + int exit_observed_order = -1; + int group_cleanup_order = -1; + int final_reap_order = -1; + int command_status = -1; + std::string command_output; +}; + +RunCommandReapOrderResult run_command_without_descendants() { + inspect_run_command_reap_order.store(true); + run_command_reap_sequence.store(0); + run_command_exit_observed_order.store(-1); + run_command_group_cleanup_order.store(-1); + run_command_final_reap_order.store(-1); + run_command_leader_reserved.store(false); + run_command_leader_reserved_after_cleanup.store(false); + run_command_leader_absent_after_reap.store(false); + + RunCommandReapOrderResult result; + result.command_status = + ProcessManager::run_command("printf no-descendant; exit 9", + result.command_output, 5); + inspect_run_command_reap_order.store(false); + result.leader_reserved = run_command_leader_reserved.load(); + result.leader_reserved_after_cleanup = + run_command_leader_reserved_after_cleanup.load(); + result.leader_absent_after_reap = + run_command_leader_absent_after_reap.load(); + result.exit_observed_order = run_command_exit_observed_order.load(); + result.group_cleanup_order = run_command_group_cleanup_order.load(); + result.final_reap_order = run_command_final_reap_order.load(); + return result; +} +#endif + } // namespace +#ifdef __linux__ +extern "C" void lemonade_test_run_command_pipe_created(int fd) { + int flags; + do { + flags = fcntl(fd, F_GETFD); + } while (flags < 0 && errno == EINTR); + run_command_pipe_fd.store(fd); + run_command_pipe_flags.store(flags); + run_command_pipe_ready.store(true); + while (!release_run_command_pipe.load()) { + std::this_thread::yield(); + } +} +#endif + +#ifdef __APPLE__ +extern "C" void lemonade_test_run_with_output_pipe_created() { + if (!coordinate_forks.load()) { + return; + } + + const int index = output_pipe_hook_count.fetch_add(1); + if (index == 0) { + first_output_pipe_ready.store(true); + while (coordinate_forks.load() && + !release_first_output_command.load()) { + std::this_thread::yield(); + } + } else if (index == 1) { + return; + } +} + +bool exited_child_is_reserved(pid_t pid) { + siginfo_t child_info{}; + int result; + do { + result = waitid(P_PID, static_cast(pid), &child_info, + WEXITED | WNOHANG | WNOWAIT); + } while (result < 0 && errno == EINTR); + return result == 0 && child_info.si_pid == pid; +} + +extern "C" void lemonade_test_run_command_exit_observed(pid_t pid) { + if (!inspect_run_command_reap_order.load()) { + return; + } + run_command_leader_reserved.store(exited_child_is_reserved(pid)); + run_command_exit_observed_order.store( + run_command_reap_sequence.fetch_add(1)); +} + +extern "C" void lemonade_test_run_command_group_cleanup_complete(pid_t pid) { + if (!inspect_run_command_reap_order.load()) { + return; + } + run_command_leader_reserved_after_cleanup.store( + exited_child_is_reserved(pid)); + run_command_group_cleanup_order.store( + run_command_reap_sequence.fetch_add(1)); +} + +extern "C" void lemonade_test_run_command_final_reap_complete(pid_t pid) { + if (!inspect_run_command_reap_order.load()) { + return; + } + siginfo_t child_info{}; + errno = 0; + int result; + do { + result = waitid(P_PID, static_cast(pid), &child_info, + WEXITED | WNOHANG | WNOWAIT); + } while (result < 0 && errno == EINTR); + run_command_leader_absent_after_reap.store(result < 0 && errno == ECHILD); + run_command_final_reap_order.store(run_command_reap_sequence.fetch_add(1)); +} +#endif + int main() { + { + const ProcessHandle handle = ProcessManager::start_process( + "/bin/sh", + {"-c", "printf 'discarded-prefix:original-startup-error' >&2"}, + "", + false, + false, + {}, + 22); + check("start_process returns a child handle for captured output", + handle.pid > 0); + + if (handle.pid > 0) { + check("captured-output child exits successfully", + ProcessManager::wait_for_exit(handle, 5) == 0); + const std::string output = ProcessManager::read_output(handle, 22); + check("read_output returns a bounded tail from the original child", + output == "original-startup-error"); + } + } + + check("process output capture survives closed stdout and stderr", + capture_with_closed_standard_output() == 0); + +#ifdef __linux__ + { + const ConcurrentPopenResult result = run_overlapping_popen_and_exec(); + check("overlapping popen and exec complete setup", + result.setup_succeeded); + check("popen stream is close-on-exec", result.close_on_exec); + check("exec child cannot inherit the popen stream", + !result.descriptor_inherited); + check("popen command preserves output and wait status", + WIFEXITED(result.command_status) && + WEXITSTATUS(result.command_status) == 0 && + result.command_output == "popen-output"); + } + + { + const ConcurrentCaptureResult result = run_overlapping_captured_spawns(); + check("overlapping captured spawns complete setup", + result.setup_succeeded); + if (result.setup_succeeded) { + std::printf("[INFO] short exit=%d read=%ldms output='%s'\n", + result.short_exit_code, result.short_read_ms, + result.short_output.c_str()); + check("short captured process exits successfully", + result.short_exit_code == 0); + check("long process cannot retain the short process capture pipes", + result.short_read_ms < 750); + check("short process output remains isolated", + result.short_output == "short-capture"); + } + } +#endif + +#ifdef __APPLE__ + { + const RunCommandReapOrderResult result = + run_command_without_descendants(); + check("run_command no-descendant probe preserves output and status", + result.command_output == "no-descendant" && + WIFEXITED(result.command_status) && + WEXITSTATUS(result.command_status) == 9); + check("run_command reserves the exited leader through group cleanup", + result.leader_reserved && + result.leader_reserved_after_cleanup); + check("run_command reaps only after process-group cleanup", + result.exit_observed_order == 0 && + result.group_cleanup_order == 1 && + result.final_reap_order == 2 && + result.leader_absent_after_reap); + } + + { + const ConcurrentRunWithOutputResult result = + run_overlapping_output_commands(); + check("overlapping output commands complete setup", + result.setup_succeeded); + std::printf("[INFO] short exit=%d wait=%ldms output='%s'\n", + result.short_exit_code, result.short_wait_ms, + result.short_output.c_str()); + check("short output command exits successfully", + result.short_exit_code == 0); + check("long output command exits successfully", + result.long_exit_code == 0); + check("long command cannot retain the short command capture pipe", + result.short_completed_promptly); + check("short command output remains isolated", + result.short_output == "short-capture"); + } + + { + std::string output; + const int status = ProcessManager::run_command( + "printf 'line-one\\nunterminated'; exit 7", output, 5); + check("run_command preserves raw output bytes", + output == "line-one\nunterminated"); + check("run_command preserves wait-status semantics", + WIFEXITED(status) && WEXITSTATUS(status) == 7); + } + + { + const RunCommandTimeoutResult result = + run_command_with_descendant_timeout(); + check("run_command timeout descendant test completes setup", + result.setup_succeeded); + check("run_command reports timeout", result.command_status == -1); + check("run_command timeout terminates descendant processes", + result.descendant_gone); + } + + { + std::string output; + const auto started = std::chrono::steady_clock::now(); + const int status = ProcessManager::run_command( + "exec 1>&-; sleep 5", output, 1); + const auto elapsed = std::chrono::steady_clock::now() - started; + check("run_command enforces timeout after output closes", + status == -1); + check("run_command returns promptly after output closes", + elapsed < std::chrono::seconds(3)); + } + + { + const RunCommandBackgroundResult result = + run_command_with_background_descendant(); + check("run_command background descendant test completes setup", + result.setup_succeeded); + check("run_command preserves successful shell status", + WIFEXITED(result.command_status) && + WEXITSTATUS(result.command_status) == 0); + check("run_command success terminates residual process-group members", + result.descendant_gone); + } + + { + std::string output; + const int status = ProcessManager::run_command( + "sleep 1; printf 'no-deadline'", output, 0); + check("run_command zero timeout waits without a deadline", + WIFEXITED(status) && WEXITSTATUS(status) == 0 && + output == "no-deadline"); + } + + { + std::string output; + const int exit_code = ProcessManager::run_process_with_output( + "/bin/pwd", {}, + [&output](const std::string& line) { + output += line; + return true; + }, + "/private/tmp", 5); + check("output command preserves working-directory semantics", + exit_code == 0 && output == "/private/tmp"); + } + + { + const auto started = std::chrono::steady_clock::now(); + const int exit_code = ProcessManager::run_process_with_output( + "/bin/sh", {"-c", "printf 'cancel\\n'; sleep 3"}, + [](const std::string&) { return false; }, "", 5); + const auto elapsed = std::chrono::steady_clock::now() - started; + check("output callback cancellation kills and reaps promptly", + exit_code == -1 && elapsed < std::chrono::seconds(2)); + } +#endif + + { + const ProcessHandle handle = ProcessManager::start_process( + "/bin/sh", {"-c", "printf 'GET /health\\n'"}, "", true, true); + check("filtered output creates a reader-completion tracker", + handle.output_capture != nullptr); + check("filtered-output child exits successfully", + ProcessManager::wait_for_exit(handle, 5) == 0); + } + +#ifdef __linux__ { const pid_t child = spawn_exiting_child(42); check("fork exited child", child > 0); @@ -128,6 +924,7 @@ int main() { check("is_running() rejects non-existent PID", !ProcessManager::is_running( make_handle(std::numeric_limits::max()))); +#endif if (failures == 0) { std::printf("\nAll process_manager tests passed\n"); diff --git a/test/cpp/test_process_manager_windows.cpp b/test/cpp/test_process_manager_windows.cpp new file mode 100644 index 0000000000..67d7b66105 --- /dev/null +++ b/test/cpp/test_process_manager_windows.cpp @@ -0,0 +1,549 @@ +#ifndef _WINSOCKAPI_ +#define _WINSOCKAPI_ +#endif +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using lemon::utils::ProcessHandle; +using lemon::utils::ProcessManager; + +namespace { + +int failures = 0; + +void check(const char* name, bool condition) { + std::printf("[%s] %s\n", condition ? "PASS" : "FAIL", name); + if (!condition) { + ++failures; + } +} + +bool standard_handle_is_valid(DWORD stream) { + const HANDLE handle = GetStdHandle(stream); + if (handle == nullptr || handle == INVALID_HANDLE_VALUE) { + return false; + } + + SetLastError(ERROR_SUCCESS); + const DWORD type = GetFileType(handle); + return type != FILE_TYPE_UNKNOWN || GetLastError() == ERROR_SUCCESS; +} + +int probe_standard_handles() { + int result = 0; + if (!standard_handle_is_valid(STD_INPUT_HANDLE)) { + result |= 1; + } + if (!standard_handle_is_valid(STD_OUTPUT_HANDLE)) { + result |= 2; + } + if (!standard_handle_is_valid(STD_ERROR_HANDLE)) { + result |= 4; + } + + if ((result & 2) == 0) { + constexpr char message[] = "stdout-ready\n"; + DWORD written = 0; + WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), message, + static_cast(sizeof(message) - 1), &written, nullptr); + } + if ((result & 4) == 0) { + constexpr char message[] = "stderr-ready\n"; + DWORD written = 0; + WriteFile(GetStdHandle(STD_ERROR_HANDLE), message, + static_cast(sizeof(message) - 1), &written, nullptr); + } + return result; +} + +std::string current_executable() { + std::vector path(32768); + const DWORD length = GetModuleFileNameA( + nullptr, path.data(), static_cast(path.size())); + if (length == 0 || + static_cast(length) >= path.size()) { + return {}; + } + return std::string(path.data(), length); +} + +std::string unique_sentinel_path() { + char temp_directory[MAX_PATH + 1] = {}; + const DWORD directory_length = GetTempPathA( + static_cast(sizeof(temp_directory)), temp_directory); + if (directory_length == 0 || directory_length >= sizeof(temp_directory)) { + return {}; + } + + char path[MAX_PATH + 1] = {}; + if (GetTempFileNameA(temp_directory, "lpm", 0, path) == 0) { + return {}; + } + if (!DeleteFileA(path)) { + return {}; + } + return path; +} + +bool create_sentinel(const std::string& path) { + const HANDLE file = CreateFileA( + path.c_str(), GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + CREATE_ALWAYS, FILE_ATTRIBUTE_TEMPORARY, nullptr); + if (file == INVALID_HANDLE_VALUE) { + return false; + } + return CloseHandle(file) != FALSE; +} + +bool sentinel_exists(const std::string& path) { + const DWORD attributes = GetFileAttributesA(path.c_str()); + return attributes != INVALID_FILE_ATTRIBUTES && + (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; +} + +bool wait_for_sentinel( + const std::string& path, + std::chrono::milliseconds timeout) { + const auto deadline = std::chrono::steady_clock::now() + timeout; + do { + if (sentinel_exists(path)) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } while (std::chrono::steady_clock::now() < deadline); + return sentinel_exists(path); +} + +bool wait_for_flag( + const std::atomic& flag, + std::chrono::milliseconds timeout) { + const auto deadline = std::chrono::steady_clock::now() + timeout; + do { + if (flag.load(std::memory_order_acquire)) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } while (std::chrono::steady_clock::now() < deadline); + return flag.load(std::memory_order_acquire); +} + +bool pipe_closed_without_marker(HANDLE pipe) { + char marker = '\0'; + DWORD bytes_read = 0; + if (ReadFile(pipe, &marker, 1, &bytes_read, nullptr)) { + return bytes_read == 0; + } + return GetLastError() == ERROR_BROKEN_PIPE; +} + +class TemporarySentinel { +public: + TemporarySentinel() : path_(unique_sentinel_path()) { + } + + ~TemporarySentinel() { + if (!path_.empty()) { + DeleteFileA(path_.c_str()); + } + } + + TemporarySentinel(const TemporarySentinel&) = delete; + TemporarySentinel& operator=(const TemporarySentinel&) = delete; + + const std::string& path() const { + return path_; + } + + bool valid() const { + return !path_.empty(); + } + + bool signal() const { + return valid() && create_sentinel(path_); + } + +private: + std::string path_; +}; + +class ScopedHandle { +public: + ScopedHandle() = default; + + ~ScopedHandle() { + reset(); + } + + ScopedHandle(const ScopedHandle&) = delete; + ScopedHandle& operator=(const ScopedHandle&) = delete; + + HANDLE get() const { + return handle_; + } + + HANDLE* receive() { + reset(); + return &handle_; + } + + void reset(HANDLE handle = nullptr) { + if (handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(handle_); + } + handle_ = handle; + } + +private: + HANDLE handle_ = nullptr; +}; + +int capture_wait_child(const std::string& release_path) { + constexpr char message[] = "capture-ready\n"; + DWORD written = 0; + if (!WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), message, + static_cast(sizeof(message) - 1), &written, + nullptr) || + written != sizeof(message) - 1) { + return 10; + } + return wait_for_sentinel(release_path, std::chrono::seconds(30)) ? 0 : 11; +} + +int long_wait_child( + const std::string& ready_path, + const std::string& release_path, + const std::string& canary_handle_value) { + std::uintptr_t canary_value = 0; + try { + std::size_t parsed = 0; + canary_value = static_cast( + std::stoull(canary_handle_value, &parsed)); + if (parsed != canary_handle_value.size()) { + return 20; + } + } catch (const std::exception&) { + return 20; + } + + constexpr char marker = 'I'; + DWORD written = 0; + WriteFile(reinterpret_cast(canary_value), &marker, 1, &written, + nullptr); + if (!create_sentinel(ready_path)) { + return 21; + } + return wait_for_sentinel(release_path, std::chrono::seconds(30)) ? 0 : 22; +} + +class InvalidStandardHandles { +public: + explicit InvalidStandardHandles( + HANDLE replacement = INVALID_HANDLE_VALUE) + : input_(GetStdHandle(STD_INPUT_HANDLE)), + output_(GetStdHandle(STD_OUTPUT_HANDLE)), + error_(GetStdHandle(STD_ERROR_HANDLE)) { + const BOOL input_set = + SetStdHandle(STD_INPUT_HANDLE, replacement); + const BOOL output_set = + SetStdHandle(STD_OUTPUT_HANDLE, replacement); + const BOOL error_set = + SetStdHandle(STD_ERROR_HANDLE, replacement); + installed_ = input_set && output_set && error_set; + } + + ~InvalidStandardHandles() { + SetStdHandle(STD_INPUT_HANDLE, input_); + SetStdHandle(STD_OUTPUT_HANDLE, output_); + SetStdHandle(STD_ERROR_HANDLE, error_); + } + + bool installed() const { + return installed_; + } + +private: + HANDLE input_; + HANDLE output_; + HANDLE error_; + bool installed_ = false; +}; + +void test_concurrent_capture_handle_isolation(const std::string& executable) { + TemporarySentinel capture_release; + TemporarySentinel long_ready; + TemporarySentinel long_release; + const bool sentinels_ready = capture_release.valid() && + long_ready.valid() && long_release.valid(); + check("create process-overlap sentinels", sentinels_ready); + if (!sentinels_ready) { + return; + } + + std::atomic capture_ready{false}; + std::string captured_output; + std::future capture_future; + bool capture_launched = false; + try { + capture_future = std::async( + std::launch::async, + [&]() { + return ProcessManager::run_process_with_output( + executable, {"--capture-wait", capture_release.path()}, + [&](const std::string& line) { + captured_output += line; + captured_output.push_back('\n'); + if (line == "capture-ready") { + capture_ready.store(true, std::memory_order_release); + } + return true; + }, + "", 20); + }); + capture_launched = true; + } catch (const std::exception&) { + } + check("launch capture child A", capture_launched); + if (!capture_launched) { + return; + } + + const bool capture_announced = + wait_for_flag(capture_ready, std::chrono::seconds(5)); + check("capture child A announces readiness", capture_announced); + + ScopedHandle canary_read; + ScopedHandle canary_write; + bool canary_created = false; + if (capture_announced) { + SECURITY_ATTRIBUTES attributes{}; + attributes.nLength = sizeof(attributes); + attributes.bInheritHandle = TRUE; + if (CreatePipe( + canary_read.receive(), canary_write.receive(), &attributes, + 0) && + SetHandleInformation( + canary_read.get(), HANDLE_FLAG_INHERIT, 0)) { + canary_created = true; + } + } + check("create inheritable overlap canary", canary_created); + + ProcessHandle long_handle{}; + bool long_started = false; + if (canary_created) { + try { + long_handle = ProcessManager::start_process( + executable, + {"--long-wait", long_ready.path(), long_release.path(), + std::to_string(reinterpret_cast( + canary_write.get()))}, "", + false, false); + long_started = long_handle.handle != nullptr; + } catch (const std::exception&) { + } + } + check("start long-lived child B during capture", long_started); + + bool long_announced = false; + if (long_started) { + long_announced = + wait_for_sentinel(long_ready.path(), std::chrono::seconds(5)); + } + check("long-lived child B announces readiness", long_announced); + + const bool long_running_before_release = + long_announced && ProcessManager::is_running(long_handle); + check("long-lived child B is running", long_running_before_release); + + canary_write.reset(); + const bool canary_excluded = + canary_created && long_running_before_release && + pipe_closed_without_marker(canary_read.get()); + check("child B excludes unrelated inheritable pipe writers", + canary_excluded); + + const bool capture_released = capture_release.signal(); + check("release capture child A", capture_released); + + bool capture_completed_promptly = false; + int capture_exit = -1; + bool capture_result_collected = false; + if (capture_released && long_running_before_release) { + capture_completed_promptly = + capture_future.wait_for(std::chrono::seconds(3)) == + std::future_status::ready; + if (capture_completed_promptly) { + try { + capture_exit = capture_future.get(); + capture_result_collected = true; + } catch (const std::exception&) { + } + } + } + + const bool long_running_after_capture = + capture_result_collected && ProcessManager::is_running(long_handle); + check("capture returns promptly while child B remains running", + capture_completed_promptly && long_running_after_capture); + + capture_release.signal(); + const bool long_released = long_release.signal(); + check("release long-lived child B", long_released); + + int long_exit = -1; + if (long_started) { + long_exit = ProcessManager::wait_for_exit(long_handle, 10); + if (long_exit >= 0) { + ProcessManager::reap_process(long_handle); + } else { + ProcessManager::kill_process(long_handle); + } + } + check("long-lived child B exits after release", long_exit == 0); + + if (!capture_result_collected) { + try { + capture_exit = capture_future.get(); + capture_result_collected = true; + } catch (const std::exception&) { + } + } + check("capture child A exits after release", + capture_result_collected && capture_exit == 0); + check("capture child A output is complete", + captured_output.find("capture-ready\n") != std::string::npos); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc == 2 && std::string(argv[1]) == "--probe-stdio") { + return probe_standard_handles(); + } + if (argc == 3 && std::string(argv[1]) == "--capture-wait") { + return capture_wait_child(argv[2]); + } + if (argc == 5 && std::string(argv[1]) == "--long-wait") { + return long_wait_child(argv[2], argv[3], argv[4]); + } + + const std::string executable = current_executable(); + check("resolve test executable", !executable.empty()); + if (executable.empty()) { + return failures; + } + + ProcessHandle quiet_handle{}; + bool quiet_override_installed = false; + bool quiet_started = false; + { + InvalidStandardHandles invalid_stdio; + quiet_override_installed = invalid_stdio.installed(); + if (quiet_override_installed) { + try { + quiet_handle = ProcessManager::start_process( + executable, {"--probe-stdio"}, "", false, false); + quiet_started = quiet_handle.handle != nullptr; + } catch (const std::exception&) { + } + } + } + check("replace parent standard handles", quiet_override_installed); + check("start quiet child without parent standard handles", quiet_started); + + int quiet_exit = -1; + if (quiet_started) { + quiet_exit = ProcessManager::wait_for_exit(quiet_handle, 10); + if (quiet_exit >= 0) { + ProcessManager::reap_process(quiet_handle); + } else { + ProcessManager::kill_process(quiet_handle); + } + } + check("quiet child receives valid standard handles", quiet_exit == 0); + + ProcessHandle stale_handle{}; + bool stale_override_installed = false; + bool stale_started = false; + const HANDLE stale_value = reinterpret_cast( + static_cast(0x1234567)); + { + InvalidStandardHandles stale_stdio(stale_value); + stale_override_installed = stale_stdio.installed(); + if (stale_override_installed) { + try { + stale_handle = ProcessManager::start_process( + executable, {"--probe-stdio"}, "", false, false); + stale_started = stale_handle.handle != nullptr; + } catch (const std::exception&) { + } + } + } + check("replace parent standard handles with stale values", + stale_override_installed); + check("start child with stale parent standard handles", stale_started); + + int stale_exit = -1; + if (stale_started) { + stale_exit = ProcessManager::wait_for_exit(stale_handle, 10); + if (stale_exit >= 0) { + ProcessManager::reap_process(stale_handle); + } else { + ProcessManager::kill_process(stale_handle); + } + } + check("stale standard handles are replaced with valid handles", + stale_exit == 0); + + bool capture_override_installed = false; + int capture_exit = -1; + std::string captured_output; + { + InvalidStandardHandles invalid_stdio; + capture_override_installed = invalid_stdio.installed(); + if (capture_override_installed) { + try { + capture_exit = ProcessManager::run_process_with_output( + executable, {"--probe-stdio"}, + [&captured_output](const std::string& line) { + captured_output += line; + captured_output.push_back('\n'); + return true; + }, + "", 10); + } catch (const std::exception&) { + } + } + } + check("replace standard handles for captured child", + capture_override_installed); + check("captured child receives valid standard handles", capture_exit == 0); + check("captured stdout remains connected", + captured_output.find("stdout-ready\n") != std::string::npos); + check("captured stderr remains connected", + captured_output.find("stderr-ready\n") != std::string::npos); + + test_concurrent_capture_handle_isolation(executable); + + if (failures == 0) { + std::printf("\nAll Windows process_manager tests passed\n"); + return 0; + } + std::printf("\n%d Windows process_manager test(s) failed\n", failures); + return 1; +} diff --git a/test/cpp/test_streaming_heartbeat.cpp b/test/cpp/test_streaming_heartbeat.cpp index f414644db4..34f74362d3 100644 --- a/test/cpp/test_streaming_heartbeat.cpp +++ b/test/cpp/test_streaming_heartbeat.cpp @@ -214,6 +214,105 @@ static void test_heartbeat_emission_during_pause_between_tokens() { check(telemetry_result.error_message.empty(), "stream completed without error"); } +static void test_heartbeat_emission_during_partial_frame_progress() { + httplib::Server backend; + std::atomic partial_frame_in_progress{false}; + std::atomic backend_progress_count{0}; + std::atomic heartbeat_during_partial_frame{0}; + + backend.Post("/v1/chat/completions", + [&](const httplib::Request&, httplib::Response& res) { + res.set_chunked_content_provider( + "text/event-stream", + [&](size_t, httplib::DataSink& sink) { + const std::string prefix = + "data: {\"choices\":[{\"delta\":{\"content\":\""; + partial_frame_in_progress.store( + true, std::memory_order_release); + sink.write(prefix.data(), prefix.size()); + for (int fragment = 0; fragment < 40; ++fragment) { + static constexpr char payload[] = "x"; + if (!sink.write(payload, sizeof(payload) - 1)) { + partial_frame_in_progress.store( + false, std::memory_order_release); + return false; + } + std::this_thread::sleep_for(25ms); + } + partial_frame_in_progress.store( + false, std::memory_order_release); + + const std::string suffix = "\"}}]}\n\n"; + sink.write(suffix.data(), suffix.size()); + const std::string done = "data: [DONE]\n\n"; + sink.write(done.data(), done.size()); + sink.done(); + return false; + }); + }); + + const int port = backend.bind_to_any_port("127.0.0.1"); + if (port <= 0) { + std::printf("[FAIL] failed to bind mock backend\n"); + ++g_failures; + return; + } + + std::thread backend_thread([&backend]() { + backend.listen_after_bind(); + }); + backend.wait_until_ready(); + + int data_chunk_count = 0; + int done_count = 0; + httplib::DataSink downstream; + downstream.write = [&](const char* data, size_t len) { + const std::string write(data, len); + if (write == ": ping\n\n" && + partial_frame_in_progress.load(std::memory_order_acquire)) { + heartbeat_during_partial_frame.fetch_add( + 1, std::memory_order_release); + } + if (write.find("\"content\":\"") != std::string::npos) { + ++data_chunk_count; + } + if (write.find("[DONE]") != std::string::npos) { + ++done_count; + } + return true; + }; + downstream.done = []() {}; + downstream.is_writable = []() { return true; }; + + lemon::StreamingProxy::TelemetryData telemetry_result; + lemon::StreamingProxy::forward_sse_stream( + "http://127.0.0.1:" + std::to_string(port) + + "/v1/chat/completions", + R"({"model":"test-model","stream":true})", + downstream, + [&](const lemon::StreamingProxy::TelemetryData& tel) { + telemetry_result = tel; + }, + 10, + [&]() { + backend_progress_count.fetch_add(1, std::memory_order_release); + }, + 100); + + backend.stop(); + backend_thread.join(); + + check(heartbeat_during_partial_frame.load(std::memory_order_acquire) >= 1, + "heartbeat reaches the client while an SSE frame is incomplete"); + check(backend_progress_count.load(std::memory_order_acquire) > 1, + "partial SSE fragments retain backend-progress signaling"); + check(data_chunk_count == 1, + "partial SSE fragments are forwarded as one complete frame"); + check(done_count == 1, "partial-frame stream preserves one done marker"); + check(telemetry_result.error_message.empty(), + "partial-frame stream completes without error"); +} + static void test_heartbeat_client_disconnect_aborts_upstream() { httplib::Server backend; std::atomic release_backend{false}; @@ -293,6 +392,7 @@ static void test_heartbeat_client_disconnect_aborts_upstream() { int main() { test_heartbeat_emission_during_prefill(); test_heartbeat_emission_during_pause_between_tokens(); + test_heartbeat_emission_during_partial_frame_progress(); test_heartbeat_client_disconnect_aborts_upstream(); if (g_failures == 0) { diff --git a/test/fixtures/llamacpp_validation_artifacts.json b/test/fixtures/llamacpp_validation_artifacts.json new file mode 100644 index 0000000000..e690ce564f --- /dev/null +++ b/test/fixtures/llamacpp_validation_artifacts.json @@ -0,0 +1,26 @@ +{ + "K2-Horizon-0.9B-GGUF": { + "revision": "8496259ac62d33192d0fffe708303ed2ca29a384", + "filename": "K2-Horizon-1B-BF16.gguf", + "size_bytes": 2159424896, + "sha256": "371010db1807bb07b62e738422ee0de26c1e15a347f31108ed2c6e219095a8b8", + "gguf_architecture": "k2-horizon", + "max_context_window": 131072 + }, + "K2-Horizon-3.7B-GGUF": { + "revision": "67b824b8079635b77e19a2a4cd2682a9716478c1", + "filename": "K2-Horizon-4B-BF16.gguf", + "size_bytes": 10128343424, + "sha256": "372ecf9977eb22c92739fec478f2c887daef659d7c55a609ac97b905d1a243b1", + "gguf_architecture": "k2-horizon", + "max_context_window": 524288 + }, + "K2-Horizon-7B-GGUF": { + "revision": "bcb8c25b76112ce96a962f5b8ab624435d1ee0c9", + "filename": "K2-Horizon-7B-BF16.gguf", + "size_bytes": 18010413440, + "sha256": "088c5d0814ef955d137fd1073ee3a68f6a53411fca44b75dc58a320640000444", + "gguf_architecture": "k2-horizon", + "max_context_window": 524288 + } +} diff --git a/test/fixtures/llamacpp_validation_models.json b/test/fixtures/llamacpp_validation_models.json new file mode 100644 index 0000000000..45cfefc156 --- /dev/null +++ b/test/fixtures/llamacpp_validation_models.json @@ -0,0 +1,36 @@ +{ + "K2-Horizon-0.9B-GGUF": { + "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": [ + "chat", + "reasoning", + "tool-calling", + "hot" + ], + "size": 2.16 + }, + "K2-Horizon-3.7B-GGUF": { + "checkpoint": "IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": [ + "chat", + "reasoning", + "tool-calling" + ], + "size": 10.13 + }, + "K2-Horizon-7B-GGUF": { + "checkpoint": "IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": [ + "chat", + "reasoning", + "tool-calling" + ], + "size": 18.01 + } +} diff --git a/test/requirements.txt b/test/requirements.txt index 6b5687e991..d85c9d3551 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -28,3 +28,6 @@ ollama # JSON Schema validator (required for routing policy / decision fixture tests) jsonschema + +# YAML parser (required for workflow supply-chain tests) +pyyaml diff --git a/test/test_debian_version.py b/test/test_debian_version.py deleted file mode 100644 index b4f5e42e0f..0000000000 --- a/test/test_debian_version.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for Debian package version resolution.""" - -from __future__ import annotations - -import os -import shutil -import stat -import subprocess -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -RESOLVER = ROOT / ".github" / "scripts" / "resolve_debian_version.sh" - - -class DebianVersionTests(unittest.TestCase): - def setUp(self) -> None: - self.temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary_directory.cleanup) - self.repo = Path(self.temporary_directory.name) / "repo" - self.repo.mkdir() - self.git_env = os.environ.copy() - self.git_env.update( - { - "GIT_AUTHOR_DATE": "2001-01-01T00:00:00+0000", - "GIT_AUTHOR_EMAIL": "test@example.com", - "GIT_AUTHOR_NAME": "Test Author", - "GIT_COMMITTER_DATE": "2001-01-01T00:00:00+0000", - "GIT_COMMITTER_EMAIL": "test@example.com", - "GIT_COMMITTER_NAME": "Test Committer", - } - ) - self.run_git("init", "--quiet") - - def run_git(self, *args: str) -> str: - return subprocess.run( - ["git", *args], - cwd=self.repo, - env=self.git_env, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - - def write_cmake_version(self, contents: str) -> None: - (self.repo / "CMakeLists.txt").write_text(contents, encoding="utf-8") - - def commit_all(self, message: str) -> None: - self.run_git("add", "--all") - self.run_git("commit", "--quiet", "--allow-empty", "--message", message) - - def move_to_sha_prefix(self, prefix_kind: str) -> str: - for attempt in range(256): - self.run_git( - "commit", - "--quiet", - "--allow-empty", - "--message", - f"sha-prefix-{prefix_kind}-{attempt}", - ) - short_sha = self.run_git("rev-parse", "--short", "HEAD") - if prefix_kind == "alpha" and short_sha[0] in "abcdef": - return short_sha - if prefix_kind == "numeric" and short_sha[0].isdigit(): - return short_sha - self.fail(f"Could not create a {prefix_kind}-leading commit SHA") - - def run_resolver( - self, *, dpkg_exit: int = 0 - ) -> tuple[subprocess.CompletedProcess, Path]: - fake_bin = Path(self.temporary_directory.name) / "fake-bin" - fake_bin.mkdir(exist_ok=True) - validation_log = Path(self.temporary_directory.name) / "dpkg-arguments" - fake_dpkg = fake_bin / "dpkg" - fake_dpkg.write_text( - "#!/bin/sh\n" - 'printf "%s\\n" "$@" > "$DPKG_VALIDATION_LOG"\n' - 'exit "$DPKG_EXIT"\n', - encoding="utf-8", - ) - fake_dpkg.chmod(fake_dpkg.stat().st_mode | stat.S_IXUSR) - - env = os.environ.copy() - env.update( - { - "DPKG_EXIT": str(dpkg_exit), - "DPKG_VALIDATION_LOG": str(validation_log), - "PATH": f"{fake_bin}{os.pathsep}{env['PATH']}", - } - ) - result = subprocess.run( - ["bash", str(RESOLVER), "24.04"], - cwd=self.repo, - env=env, - check=False, - capture_output=True, - text=True, - ) - return result, validation_log - - def assert_tagless_version(self, prefix_kind: str) -> None: - self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") - self.commit_all("initial") - short_sha = self.move_to_sha_prefix(prefix_kind) - commit_count = int(self.run_git("rev-list", "--count", "HEAD")) - - result, validation_log = self.run_resolver() - - expected = f"11.9.0+git{commit_count}.g{short_sha}" - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), expected) - self.assertEqual( - validation_log.read_text(encoding="utf-8").splitlines(), - ["--validate-version", f"{expected}~24.04"], - ) - - def test_tagless_alpha_leading_sha_uses_cmake_version_prefix(self) -> None: - self.assert_tagless_version("alpha") - - def test_tagless_numeric_leading_sha_uses_cmake_version_prefix(self) -> None: - self.assert_tagless_version("numeric") - - def test_tagless_versions_increase_with_commit_ancestry(self) -> None: - self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") - self.commit_all("parent") - parent_sha = self.run_git("rev-parse", "--short", "HEAD") - parent_count = int(self.run_git("rev-list", "--count", "HEAD")) - parent_result, _validation_log = self.run_resolver() - - self.run_git("commit", "--quiet", "--allow-empty", "--message", "child") - child_sha = self.run_git("rev-parse", "--short", "HEAD") - child_count = int(self.run_git("rev-list", "--count", "HEAD")) - child_result, _validation_log = self.run_resolver() - - parent_version = f"11.9.0+git{parent_count}.g{parent_sha}" - child_version = f"11.9.0+git{child_count}.g{child_sha}" - self.assertEqual(parent_result.returncode, 0, parent_result.stderr) - self.assertEqual(child_result.returncode, 0, child_result.stderr) - self.assertEqual(parent_result.stdout.strip(), parent_version) - self.assertEqual(child_result.stdout.strip(), child_version) - self.assertGreater(child_count, parent_count) - - dpkg = shutil.which("dpkg") - if dpkg: - subprocess.run( - [dpkg, "--compare-versions", parent_version, "lt", child_version], - check=True, - ) - subprocess.run( - [dpkg, "--compare-versions", "11.9.0", "lt", parent_version], - check=True, - ) - - def test_reachable_tag_preserves_git_describe_version(self) -> None: - self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") - self.commit_all("tagged release") - self.run_git("tag", "--no-sign", "v4.5.6") - self.run_git("commit", "--quiet", "--allow-empty", "--message", "after tag") - described = self.run_git("describe", "--tags", "--always") - - result, validation_log = self.run_resolver() - - expected = described.removeprefix("v") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), expected) - self.assertEqual( - validation_log.read_text(encoding="utf-8").splitlines(), - ["--validate-version", f"{expected}~24.04"], - ) - - def test_tagless_repo_rejects_missing_or_malformed_cmake_version(self) -> None: - for name, cmake_contents in ( - ("missing", None), - ("malformed", "project(lemon_cpp VERSION next)\n"), - ): - with self.subTest(name=name): - if cmake_contents is None: - (self.repo / "README.md").write_text("test\n", encoding="utf-8") - else: - self.write_cmake_version(cmake_contents) - self.commit_all(name) - - result, validation_log = self.run_resolver() - - self.assertNotEqual(result.returncode, 0) - self.assertEqual(result.stdout, "") - self.assertFalse(validation_log.exists()) - - def test_dpkg_validation_failure_is_propagated(self) -> None: - self.write_cmake_version("project(lemon_cpp VERSION 11.9.0)\n") - self.commit_all("initial") - - result, validation_log = self.run_resolver(dpkg_exit=42) - - self.assertEqual(result.returncode, 42) - self.assertEqual(result.stdout, "") - self.assertTrue(validation_log.exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_llamacpp_capabilities.py b/test/test_llamacpp_capabilities.py index 2ebee6f32f..80e12aeed5 100644 --- a/test/test_llamacpp_capabilities.py +++ b/test/test_llamacpp_capabilities.py @@ -7,6 +7,7 @@ import json import types import unittest +from unittest import mock from test.utils import llamacpp_capability_validation as capabilities @@ -25,6 +26,11 @@ def __init__(self) -> None: self.default_tool_response_is_text = False self.stop_tool_call = False self.tool_reasoning = False + self.reasoning_by_effort = { + "high": "high reasoning", + "medium": "medium reasoning", + "low": None, + } self.tool_arguments = json.dumps( capabilities.EXPECTED_TOOL_ARGUMENTS, separators=(",", ":"), @@ -120,10 +126,12 @@ def request_json(self, method, url, timeout, **kwargs): ) if "reasoning_effort" in kwargs_value: effort = kwargs_value["reasoning_effort"] - reasoning = f"{effort} reasoning" - if self.reasoning_marker == effort: + reasoning = self.reasoning_by_effort[effort] + if reasoning is not None and self.reasoning_marker == effort: reasoning += " " - message = {"content": "42", "reasoning_content": reasoning} + message = {"content": "42"} + if reasoning is not None: + message["reasoning_content"] = reasoning if self.stop_tool_call: message["tool_calls"] = [self.tool_call()] return self.response(), self.openai(message) @@ -267,6 +275,39 @@ def run_profile(self, harness=None): ) return active, matrix, passed, summary + def test_marker_scan_lazily_visits_a_wide_array(self) -> None: + marker = "" + + class GuardedWideList(list): + def __len__(self): + return 200_000 + + def __getitem__(self, index): + if index == 0: + return marker + raise AssertionError("marker scan eagerly accessed a later sibling") + + def __iter__(self): + yield marker + raise AssertionError("marker scan continued after the first marker") + + found = capabilities.find_raw_ifm_control_marker( + GuardedWideList(), + "response.items", + ) + + self.assertEqual(found, (marker, "response.items[0]")) + + def test_marker_scan_preserves_depth_first_path_order(self) -> None: + found = capabilities.find_raw_ifm_control_marker( + { + "first": ["safe", {"text": ""}], + "second": "", + } + ) + + self.assertEqual(found, ("", "response.first[1].text")) + def test_complete_profile_exercises_the_exact_contract(self) -> None: harness, matrix, passed, summary = self.run_profile() @@ -278,6 +319,8 @@ def test_complete_profile_exercises_the_exact_contract(self) -> None: [case["id"] for case in matrix["cases"]], list(capabilities.K2_HORIZON_CASE_IDS), ) + self.assertEqual(len(capabilities.K2_HORIZON_CASE_IDS), 14) + self.assertIn("openai_tool_xml_typed", capabilities.K2_HORIZON_CASE_IDS) capabilities.validate_capability_matrix_evidence( matrix, capabilities.K2_HORIZON_PROFILE, @@ -359,6 +402,21 @@ def test_reasoning_marker_fails_the_named_case_and_matrix(self) -> None: MODEL, ) + def test_low_reasoning_requires_the_deterministic_blank_think_faster_result( + self, + ) -> None: + harness = CapabilityHarness() + harness.reasoning_by_effort["low"] = "unexpected low reasoning" + + _harness, matrix, passed, summary = self.run_profile(harness) + + self.assertFalse(passed) + self.assertEqual( + [case["id"] for case in matrix["cases"] if not case["pass"]], + ["openai_reasoning_low"], + ) + self.assertIn("reasoning_content", summary) + def test_stream_requires_a_terminal_done_frame(self) -> None: harness = CapabilityHarness() harness.omit_stream_done = True @@ -408,6 +466,22 @@ def test_nonstream_responses_are_bound_to_the_requested_model(self) -> None: ) self.assertNotIn("openai_plain_off_stream", failed_ids) + def test_stream_responses_are_bound_to_the_requested_model(self) -> None: + harness = CapabilityHarness() + harness.stream_response_models = ["builtin.Not-K2", "builtin.Not-K2"] + + _harness, matrix, passed, _summary = self.run_profile(harness) + + self.assertFalse(passed) + self.assertEqual( + {case["id"] for case in matrix["cases"] if not case["pass"]}, + { + "openai_plain_off_stream", + "openai_reasoning_high_stream", + "openai_tool_stream_xml", + }, + ) + def test_stream_reasoning_requires_separate_marker_free_content(self) -> None: for stream_reasoning in ("", "high reasoning"): with self.subTest(stream_reasoning=stream_reasoning): @@ -422,11 +496,11 @@ def test_stream_reasoning_requires_separate_marker_free_content(self) -> None: {"openai_reasoning_high_stream"}, ) - def test_stream_accepts_a_stable_backend_model_identity(self) -> None: + def test_stream_accepts_the_canonical_requested_model_identity(self) -> None: harness = CapabilityHarness() harness.stream_response_models = [ - "K2-Horizon-1B-BF16.gguf", - "K2-Horizon-1B-BF16.gguf", + MODEL.removeprefix("builtin."), + MODEL.removeprefix("builtin."), ] _harness, _matrix, passed, summary = self.run_profile(harness) @@ -434,11 +508,12 @@ def test_stream_accepts_a_stable_backend_model_identity(self) -> None: self.assertTrue(passed, summary) def test_stream_rejects_changed_or_missing_backend_model_identity(self) -> None: + canonical_model = MODEL.removeprefix("builtin.") cases = ( - ["K2-Horizon-1B-BF16.gguf", "different.gguf"], - [None, "K2-Horizon-1B-BF16.gguf"], - ["K2-Horizon-1B-BF16.gguf", None], - ["K2-Horizon-1B-BF16.gguf", " "], + [canonical_model, "different.gguf"], + [None, canonical_model], + [canonical_model, None], + [canonical_model, " "], ) for stream_models in cases: with self.subTest(stream_models=stream_models): @@ -480,6 +555,185 @@ def test_stream_rejects_duplicate_json_members(self) -> None: ): capabilities._decode_openai_stream([event, "data: [DONE]"]) + def test_stream_enforces_an_independent_wall_clock_deadline(self) -> None: + events = [ + "data: " + + json.dumps( + { + "model": MODEL, + "choices": [ + { + "index": 0, + "delta": {"content": "42"}, + "finish_reason": "stop", + } + ], + } + ), + "data: [DONE]", + ] + + with ( + mock.patch("time.monotonic", side_effect=[0.0, 10**9]), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "wall-clock deadline", + ), + ): + capabilities._decode_openai_stream(events) + + def test_stream_rejects_an_event_count_above_the_bound(self) -> None: + terminal_events = [ + "data: " + + json.dumps( + { + "model": MODEL, + "choices": [ + { + "index": 0, + "delta": {"content": "42"}, + "finish_reason": "stop", + } + ], + } + ), + "data: [DONE]", + ] + events = [": keep-alive"] + terminal_events + + with ( + mock.patch.object(capabilities, "MAX_STREAM_EVENTS", 1), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "event count", + ), + ): + capabilities._decode_openai_stream(events) + + def test_stream_event_bound_ignores_blank_lines_and_heartbeats(self) -> None: + terminal_events = [ + "data: " + + json.dumps( + { + "model": MODEL, + "choices": [ + { + "index": 0, + "delta": {"content": "42"}, + "finish_reason": "stop", + } + ], + } + ), + "data: [DONE]", + ] + events = [item for _index in range(2500) for item in ("", ": heartbeat")] + events.extend(terminal_events) + + with mock.patch.object(capabilities, "MAX_STREAM_EVENTS", 2): + message, finish_reason, saw_done, model = ( + capabilities._decode_openai_stream(events) + ) + + self.assertEqual(message["content"], "42") + self.assertEqual(finish_reason, "stop") + self.assertTrue(saw_done) + self.assertEqual(model, MODEL) + + def test_stream_rejects_total_sse_bytes_above_the_bound(self) -> None: + events = [ + ":xxxx", + "data: " + + json.dumps( + { + "model": MODEL, + "choices": [ + { + "index": 0, + "delta": {"content": "42"}, + "finish_reason": "stop", + } + ], + } + ), + "data: [DONE]", + ] + + with ( + mock.patch.object(capabilities, "MAX_STREAM_SSE_BYTES", 5), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "SSE byte count", + ), + ): + capabilities._decode_openai_stream(events) + + def test_stream_rejects_accumulated_content_above_the_bound(self) -> None: + events = [ + "data: " + + json.dumps( + { + "model": MODEL, + "choices": [ + { + "index": 0, + "delta": {"content": "xxx"}, + "finish_reason": "stop", + } + ], + } + ), + "data: [DONE]", + ] + + with ( + mock.patch.object(capabilities, "MAX_STREAM_TEXT_BYTES", 2), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "content and reasoning", + ), + ): + capabilities._decode_openai_stream(events) + + def test_stream_rejects_tool_arguments_above_the_bound(self) -> None: + events = [ + "data: " + + json.dumps( + { + "model": MODEL, + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_large", + "type": "function", + "function": { + "name": capabilities.TOOL_NAME, + "arguments": "xxx", + }, + } + ] + }, + "finish_reason": "tool_calls", + } + ], + } + ), + "data: [DONE]", + ] + + with ( + mock.patch.object(capabilities, "MAX_STREAM_TOOL_ARGUMENT_BYTES", 2), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "tool arguments", + ), + ): + capabilities._decode_openai_stream(events) + def test_stream_rejects_choice_chunks_after_finish_reason(self) -> None: later_choices = ( {"index": 0, "delta": {"content": "2"}}, @@ -531,8 +785,8 @@ def test_tool_arguments_require_exact_json_keys_and_types(self) -> None: arguments = ( '{"city":"London","city":"Paris","unit":"celsius",' '"days":2,"include_hourly":false}', - '{"city":"Paris","unit":"celsius","days":2.0,' '"include_hourly":false}', - '{"city":"Paris","unit":"celsius","days":2,' '"include_hourly":0}', + '{"city":"Paris","unit":"celsius","days":2.0,"include_hourly":false}', + '{"city":"Paris","unit":"celsius","days":2,"include_hourly":0}', ) for tool_arguments in arguments: @@ -554,6 +808,32 @@ def test_tool_arguments_require_exact_json_keys_and_types(self) -> None: }.issubset(failed_ids) ) + def test_malformed_xml_typed_tool_response_is_rejected_hard(self) -> None: + harness = CapabilityHarness() + payload = capabilities._tool_payload(MODEL, "xml_typed", stream=False) + + for finish_reason, arguments in ( + ("stop", harness.tool_arguments), + ("tool_calls", '{"city":'), + ): + with self.subTest(finish_reason=finish_reason, arguments=arguments): + harness.tool_arguments = arguments + + def request_json(_method, _url, timeout, **_kwargs): + self.assertEqual(timeout, 60) + return harness.response(), harness.openai( + {"content": None, "tool_calls": [harness.tool_call()]}, + finish_reason, + ) + + with self.assertRaises(capabilities.CapabilityValidationError): + capabilities._openai_tool_case( + request_json, + "http://localhost:13305/api/v1/chat/completions", + 60, + payload, + ) + def test_omitted_tool_format_requires_the_xml_default(self) -> None: harness = CapabilityHarness() harness.default_tool_response_is_text = True diff --git a/test/test_llamacpp_check_reporting.py b/test/test_llamacpp_check_reporting.py new file mode 100644 index 0000000000..35bb8f317e --- /dev/null +++ b/test/test_llamacpp_check_reporting.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Regression tests for llama.cpp validation check reporting.""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CORE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" +PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" +MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" +SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" + + +def workflow_job(workflow: Path, job_name: str, next_job_name: str | None) -> str: + text = workflow.read_text(encoding="utf-8") + job = text.split(f" {job_name}:\n", 1)[1] + if next_job_name is not None: + job = job.split(f" {next_job_name}:\n", 1)[0] + return job + + +class LlamaCppCheckReportingTests(unittest.TestCase): + def test_reusable_validation_callers_grant_check_write_permission(self) -> None: + callers = ( + (PR_WORKFLOW, "validate", "reconcile"), + (MANUAL_WORKFLOW, "validate", None), + (SCHEDULE_WORKFLOW, "validate", "publish"), + ) + + for workflow, job_name, next_job_name in callers: + with self.subTest(workflow=workflow.name): + job = workflow_job(workflow, job_name, next_job_name) + self.assertIn( + " checks: write\n", + job, + "The caller must grant each permission requested by the " + "reusable validation workflow.", + ) + + def test_reusable_workflow_starts_or_resets_check_before_validation(self) -> None: + core = CORE_WORKFLOW.read_text(encoding="utf-8") + invocation = workflow_job(CORE_WORKFLOW, "validate-invocation", "plan") + existing_branch = ' if [ -z "$check_run_id" ]; then\n' + invalid_id_guard = ' if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then\n' + self.assertIn(existing_branch, invocation) + self.assertIn(invalid_id_guard, invocation) + existing_check = invocation.split(existing_branch, 1)[1].split( + invalid_id_guard, 1 + )[0] + + self.assertIn( + " check_run_id:\n" + " value: ${{ jobs.validate-invocation.outputs.check_run_id }}\n", + core.split("permissions:\n", 1)[0], + ) + self.assertIn(" checks: write\n", invocation) + self.assertIn( + " check_run_id: ${{ steps.start-check.outputs.check_run_id }}\n", + invocation, + ) + self.assertIn(" id: start-check\n", invocation) + self.assertIn("github.event_name == 'pull_request_target'", invocation) + self.assertIn("github.event_name == 'merge_group'", invocation) + self.assertIn( + "${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}", + invocation, + ) + self.assertIn( + "AUTHORIZATION_EPOCH: ${{ inputs.authorization_epoch }}", invocation + ) + self.assertIn("RUN_NUMBER: ${{ github.run_number }}", invocation) + self.assertIn("-f status=in_progress", invocation) + self.assertIn( + 'echo "check_run_id=$check_run_id" >> "$GITHUB_OUTPUT"', invocation + ) + self.assertIn(" else\n", existing_check) + reset = existing_check.split(" else\n", 1)[1] + self.assertIn('"repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}"', reset) + self.assertIn("-f status=in_progress", reset) + self.assertNotIn("-f conclusion=", reset) + self.assertGreaterEqual(invocation.count("--paginate --slurp"), 2) + + def test_pr_wrapper_reconciles_the_check_started_by_reusable_validation( + self, + ) -> None: + wrapper = PR_WORKFLOW.read_text(encoding="utf-8") + policy = workflow_job(PR_WORKFLOW, "policy", "validate") + validate = workflow_job(PR_WORKFLOW, "validate", "reconcile") + reconcile = workflow_job(PR_WORKFLOW, "reconcile", None) + + self.assertNotIn("\n start-check:\n", wrapper) + self.assertNotIn("\n authorize-invocation:\n", wrapper) + self.assertIn( + " run_validation: ${{ steps.decision.outputs.run_validation }}", policy + ) + self.assertIn(" needs: policy\n", validate) + self.assertNotIn("start-check", validate) + self.assertIn(" needs: [policy, validate]\n", reconcile) + self.assertNotIn("needs.start-check", reconcile) + self.assertIn( + "CHECK_RUN_ID: ${{ needs.validate.outputs.check_run_id }}", reconcile + ) + + exact_check_lookup = reconcile.split( + ' if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then\n', 1 + )[1].split(' if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then\n', 1)[0] + self.assertIn("commits/${MERGE_SHA}/check-runs", reconcile) + self.assertIn('--arg external_id "$external_id"', exact_check_lookup) + self.assertIn('.app.slug == "github-actions"', exact_check_lookup) + self.assertIn(".external_id == $external_id", exact_check_lookup) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_llamacpp_pr_file_classification.py b/test/test_llamacpp_pr_file_classification.py new file mode 100644 index 0000000000..083240fd5a --- /dev/null +++ b/test/test_llamacpp_pr_file_classification.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Tests for exact-commit llama.cpp pull-request file classification.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +POLICY_HELPER = ROOT / ".github" / "scripts" / "llamacpp_required_check_policy.py" +PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" +BASE_SHA = "a" * 40 +CANDIDATE_A_SHA = "b" * 40 +CANDIDATE_B_SHA = "c" * 40 + + +def load_policy_helper(): + spec = importlib.util.spec_from_file_location( + "llamacpp_required_check_policy", POLICY_HELPER + ) + if spec is None or spec.loader is None: + raise RuntimeError("could not load llama.cpp required-check policy") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def compare_response( + *, + target_sha: str, + files: list[dict[str, object]], + base_sha: str = BASE_SHA, + merge_base_sha: str = BASE_SHA, +) -> dict[str, object]: + return { + "ahead_by": 1, + "base_commit": {"sha": base_sha}, + "behind_by": 0, + "commits": [{"sha": target_sha}], + "files": files, + "merge_base_commit": {"sha": merge_base_sha}, + "status": "ahead", + "total_commits": 1, + } + + +def extract_classify_step_script() -> str: + workflow = PR_WORKFLOW.read_text(encoding="utf-8") + step = workflow.split( + " - name: Classify the complete pull request file set\n", 1 + )[1].split(" - name:", 1)[0] + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +class TestPullRequestFileClassification(unittest.TestCase): + def test_exact_compare_identity_can_be_classified_as_neutral(self) -> None: + policy = load_policy_helper() + + self.assertFalse( + policy.classify_compare_response( + compare_response( + target_sha=CANDIDATE_B_SHA, + files=[{"filename": "docs/dev/testing.md", "status": "modified"}], + ), + expected_base_sha=BASE_SHA, + expected_target_sha=CANDIDATE_B_SHA, + expected_count=1, + ) + ) + + def test_same_count_neutral_payload_for_other_candidate_is_rejected( + self, + ) -> None: + policy = load_policy_helper() + + with self.assertRaises(ValueError): + policy.classify_compare_response( + compare_response( + target_sha=CANDIDATE_A_SHA, + files=[{"filename": "docs/dev/testing.md", "status": "modified"}], + ), + expected_base_sha=BASE_SHA, + expected_target_sha=CANDIDATE_B_SHA, + expected_count=1, + ) + + def test_compare_file_cap_is_conservatively_relevant(self) -> None: + policy = load_policy_helper() + neutral_files = [ + {"filename": f"docs/file-{index}.md", "status": "modified"} + for index in range(300) + ] + + self.assertFalse( + policy.classify_compare_response( + compare_response( + target_sha=CANDIDATE_B_SHA, + files=neutral_files, + ), + expected_base_sha=BASE_SHA, + expected_target_sha=CANDIDATE_B_SHA, + expected_count=300, + ) + ) + self.assertTrue( + policy.classify_compare_response( + {}, + expected_base_sha=BASE_SHA, + expected_target_sha=CANDIDATE_B_SHA, + expected_count=301, + ) + ) + + def test_incomplete_compare_file_list_is_conservatively_relevant(self) -> None: + policy = load_policy_helper() + + self.assertTrue( + policy.classify_compare_response( + compare_response( + target_sha=CANDIDATE_B_SHA, + files=[{"filename": "docs/dev/testing.md", "status": "modified"}], + ), + expected_base_sha=BASE_SHA, + expected_target_sha=CANDIDATE_B_SHA, + expected_count=2, + ) + ) + + def test_compare_identity_and_file_schema_errors_fail_closed(self) -> None: + policy = load_policy_helper() + valid = compare_response( + target_sha=CANDIDATE_B_SHA, + files=[{"filename": "docs/dev/testing.md", "status": "modified"}], + ) + invalid_responses = { + "base": {**valid, "base_commit": {"sha": CANDIDATE_A_SHA}}, + "merge-base": { + **valid, + "merge_base_commit": {"sha": CANDIDATE_A_SHA}, + }, + "commit-count": {**valid, "total_commits": 2}, + "commit-list": {**valid, "commits": []}, + "file-path": { + **valid, + "files": [{"filename": "/docs/testing.md", "status": "modified"}], + }, + "file-status": { + **valid, + "files": [{"filename": "docs/testing.md", "status": "moved"}], + }, + } + + for name, response in invalid_responses.items(): + with self.subTest(name=name), self.assertRaises(ValueError): + policy.classify_compare_response( + response, + expected_base_sha=BASE_SHA, + expected_target_sha=CANDIDATE_B_SHA, + expected_count=1, + ) + + def test_compare_rename_from_protected_path_is_relevant(self) -> None: + policy = load_policy_helper() + + self.assertTrue( + policy.classify_compare_response( + compare_response( + target_sha=CANDIDATE_B_SHA, + files=[ + { + "filename": "docs/old-build-input.md", + "previous_filename": "CMakeLists.txt", + "status": "renamed", + } + ], + ), + expected_base_sha=BASE_SHA, + expected_target_sha=CANDIDATE_B_SHA, + expected_count=1, + ) + ) + + def test_identical_comparison_cannot_contain_changed_files(self) -> None: + policy = load_policy_helper() + invalid = { + "ahead_by": 0, + "base_commit": {"sha": BASE_SHA}, + "behind_by": 0, + "commits": [], + "files": [{"filename": "docs/testing.md", "status": "modified"}], + "merge_base_commit": {"sha": BASE_SHA}, + "status": "identical", + "total_commits": 0, + } + + with self.assertRaises(ValueError): + policy.classify_compare_response( + invalid, + expected_base_sha=BASE_SHA, + expected_target_sha=BASE_SHA, + expected_count=1, + ) + + def test_workflow_does_not_accept_stale_same_count_neutral_payload( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + trusted_helper = ( + temporary + / "trusted" + / ".github" + / "scripts" + / "llamacpp_required_check_policy.py" + ) + trusted_helper.parent.mkdir(parents=True) + shutil.copyfile(POLICY_HELPER, trusted_helper) + response_path = temporary / "response.json" + response_path.write_text( + json.dumps( + compare_response( + target_sha=CANDIDATE_A_SHA, + files=[ + { + "filename": "docs/dev/testing.md", + "status": "modified", + } + ], + ) + ), + encoding="utf-8", + ) + calls_path = temporary / "gh-calls.txt" + output_path = temporary / "github-output.txt" + + fake_git = temporary / "git" + fake_git.write_text( + "#!/bin/sh\nprintf '%s\\n' \"$BASE_SHA\"\n", encoding="utf-8" + ) + fake_git.chmod(0o755) + fake_gh = temporary / "gh" + fake_gh.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +Path(os.environ["FAKE_GH_CALLS"]).write_text("\\n".join(sys.argv[1:])) +response = json.loads(Path(os.environ["FAKE_GH_RESPONSE"]).read_text()) +if any("/pulls/7/files" in argument for argument in sys.argv): + response = [response["files"]] +print(json.dumps(response)) +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "BASE_SHA": BASE_SHA, + "EXPECTED_COUNT": "1", + "FAKE_GH_CALLS": str(calls_path), + "FAKE_GH_RESPONSE": str(response_path), + "GH_TOKEN": "test-token", + "GITHUB_OUTPUT": str(output_path), + "GITHUB_REPOSITORY": "LLM360/lemonade", + "MERGE_SHA": CANDIDATE_B_SHA, + "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", + "PR_NUMBER": "7", + "RUNNER_TEMP": str(temporary), + } + ) + + completed = subprocess.run( + ["bash", "-c", extract_classify_step_script()], + cwd=temporary, + env=environment, + check=False, + capture_output=True, + text=True, + ) + outputs = ( + output_path.read_text(encoding="utf-8") if output_path.exists() else "" + ) + calls = calls_path.read_text(encoding="utf-8") + + self.assertNotEqual(completed.returncode, 0) + self.assertNotIn("relevant=false", outputs) + self.assertIn(f"compare/{BASE_SHA}...{CANDIDATE_B_SHA}", calls) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_llamacpp_release_assets.py b/test/test_llamacpp_release_assets.py index 51073854d2..bc630764bd 100644 --- a/test/test_llamacpp_release_assets.py +++ b/test/test_llamacpp_release_assets.py @@ -7,6 +7,22 @@ from test.utils import llamacpp_release_assets as assets +ROCM_ASSET_FAMILIES = { + "gfx1200": "gfx120X", + "gfx1201": "gfx120X", + "gfx1100": "gfx110X", + "gfx1101": "gfx110X", + "gfx1102": "gfx110X", + "gfx1103": "gfx110X", + "gfx1030": "gfx103X", + "gfx1031": "gfx103X", + "gfx1032": "gfx103X", + "gfx1033": "gfx103X", + "gfx1034": "gfx103X", + "gfx1035": "gfx103X", + "gfx1036": "gfx103X", +} + class LlamaCppReleaseAssetTests(unittest.TestCase): def setUp(self) -> None: @@ -16,7 +32,7 @@ def setUp(self) -> None: lemonade_release="b300", backend_versions={ "therock": {"version": "7.14.0"}, - "rocm_asset_families": {}, + "rocm_asset_families": ROCM_ASSET_FAMILIES, }, ) @@ -57,8 +73,6 @@ def test_rocm_nightly_covers_advertised_platform_targets(self) -> None: "gfx120X", "gfx110X", "gfx103X", - "gfx90a", - "gfx908", "gfx1152", ) @@ -67,10 +81,67 @@ def test_rocm_nightly_covers_advertised_platform_targets(self) -> None: self.assertIn(f"llama-b200-windows-rocm-{target}-x64.zip", required) self.assertIn(f"llama-b200-ubuntu-rocm-{target}-x64.zip", required) - self.assertIn("llama-b200-ubuntu-rocm-gfx942-x64.zip", required) - self.assertNotIn("llama-b200-windows-rocm-gfx942-x64.zip", required) + for target in ("gfx908", "gfx90a", "gfx942"): + with self.subTest(target=target): + self.assertIn(f"llama-b200-ubuntu-rocm-{target}-x64.zip", required) + self.assertNotIn(f"llama-b200-windows-rocm-{target}-x64.zip", required) self.assertFalse(any("gfx950" in name for name in required)) - self.assertEqual(len(required), 17) + self.assertEqual(len(required), 15) + + def test_rocm_requirements_bind_each_asset_to_concrete_build_targets(self) -> None: + required = assets.build_rocm_asset_target_requirements( + rocm_release="b200", + backend_versions={"rocm_asset_families": ROCM_ASSET_FAMILIES}, + ) + + expected_gfx103x = { + "gfx1030", + "gfx1031", + "gfx1032", + "gfx1033", + "gfx1034", + "gfx1035", + "gfx1036", + } + self.assertEqual( + set(required["llama-b200-windows-rocm-gfx103X-x64.zip"]), + expected_gfx103x, + ) + self.assertEqual( + set(required["llama-b200-ubuntu-rocm-gfx103X-x64.zip"]), + expected_gfx103x, + ) + self.assertEqual( + required["llama-b200-ubuntu-rocm-gfx90a-x64.zip"], + ("gfx90a",), + ) + self.assertNotIn("llama-b200-windows-rocm-gfx90a-x64.zip", required) + self.assertFalse( + any( + "gfx950" in target + for targets in required.values() + for target in targets + ) + ) + self.assertEqual(set(required), set(self.requirements["rocm"]["rocm-nightly"])) + + def test_rocm_target_grouping_tracks_asset_family_mapping_drift(self) -> None: + changed_families = dict(ROCM_ASSET_FAMILIES) + changed_families["gfx1033"] = "gfx1033" + + required = assets.build_rocm_asset_target_requirements( + rocm_release="b200", + backend_versions={"rocm_asset_families": changed_families}, + ) + + self.assertEqual( + required["llama-b200-windows-rocm-gfx1033-x64.zip"], + ("gfx1033",), + ) + self.assertNotIn( + "gfx1033", + required["llama-b200-windows-rocm-gfx103X-x64.zip"], + ) def test_rocm_targets_use_resource_family_mapping_without_duplicates(self) -> None: requirements = assets.build_asset_requirements( @@ -80,6 +151,7 @@ def test_rocm_targets_use_resource_family_mapping_without_duplicates(self) -> No backend_versions={ "therock": {"version": "7.14.0"}, "rocm_asset_families": { + **ROCM_ASSET_FAMILIES, "gfx1151": "gfx115X", "gfx1152": "gfx115X", }, @@ -90,6 +162,32 @@ def test_rocm_targets_use_resource_family_mapping_without_duplicates(self) -> No self.assertEqual(required.count("llama-b200-windows-rocm-gfx115X-x64.zip"), 1) self.assertEqual(required.count("llama-b200-ubuntu-rocm-gfx115X-x64.zip"), 1) + def test_rocm_canonical_family_names_cannot_be_remapped(self) -> None: + collapsing_mapping = { + target: "gfx1151" + for target in ( + "gfx1150", + "gfx120X", + "gfx110X", + "gfx103X", + "gfx90a", + "gfx908", + "gfx1152", + "gfx942", + ) + } + + with self.assertRaisesRegex(ValueError, "concrete ROCm ISA"): + assets.build_asset_requirements( + ggml_release="b100", + rocm_release="b200", + lemonade_release="b300", + backend_versions={ + "therock": {"version": "7.14.0"}, + "rocm_asset_families": collapsing_mapping, + }, + ) + def test_missing_new_target_keeps_rocm_nightly_pin_ineligible(self) -> None: required = self.requirements["rocm"]["rocm-nightly"] available = set(required) @@ -104,6 +202,87 @@ def test_missing_new_target_keeps_rocm_nightly_pin_ineligible(self) -> None: self.assertEqual(eligible, []) self.assertEqual(missing, {"rocm-nightly": [missing_asset]}) + def test_rocm_eligibility_requires_attested_targets_but_allows_extras( + self, + ) -> None: + required_assets = self.requirements["rocm"]["rocm-nightly"] + required_targets = assets.build_rocm_asset_target_requirements( + rocm_release="b200", + backend_versions={"rocm_asset_families": ROCM_ASSET_FAMILIES}, + ) + available = {name: 1 for name in required_assets} + attested = { + name: (*targets, "gfx999") for name, targets in required_targets.items() + } + + eligible, missing = assets.evaluate_asset_group( + available, + self.requirements["rocm"], + required_asset_targets=required_targets, + attested_asset_targets=attested, + ) + + self.assertEqual(eligible, ["rocm-nightly"]) + self.assertEqual(missing, {"rocm-nightly": []}) + + incomplete = dict(attested) + target_asset = "llama-b200-windows-rocm-gfx103X-x64.zip" + incomplete[target_asset] = tuple( + target for target in incomplete[target_asset] if target != "gfx1035" + ) + eligible, missing = assets.evaluate_asset_group( + available, + self.requirements["rocm"], + required_asset_targets=required_targets, + attested_asset_targets=incomplete, + ) + self.assertEqual(eligible, []) + self.assertEqual(missing, {"rocm-nightly": [target_asset]}) + + def test_missing_rocm_attestation_does_not_affect_independent_groups(self) -> None: + ggml_available = { + name: 1 + for required in self.requirements["ggml"].values() + for name in required + } + rocm_available = {name: 1 for name in self.requirements["rocm"]["rocm-nightly"]} + required_targets = assets.build_rocm_asset_target_requirements( + rocm_release="b200", + backend_versions={"rocm_asset_families": ROCM_ASSET_FAMILIES}, + ) + + ggml_eligible, _ = assets.evaluate_asset_group( + ggml_available, + self.requirements["ggml"], + ) + rocm_eligible, missing = assets.evaluate_asset_group( + rocm_available, + self.requirements["rocm"], + required_asset_targets=required_targets, + attested_asset_targets={}, + ) + + self.assertEqual(set(ggml_eligible), {"cpu", "metal", "vulkan"}) + self.assertEqual(rocm_eligible, []) + self.assertEqual( + missing["rocm-nightly"], + self.requirements["rocm"]["rocm-nightly"], + ) + + def test_zero_sized_required_asset_keeps_backend_ineligible(self) -> None: + required = self.requirements["ggml"]["cpu"] + available = {name: 1 for name in required} + zero_sized_asset = "llama-b100-bin-win-cpu-x64.zip" + available[zero_sized_asset] = 0 + + eligible, missing = assets.evaluate_asset_group( + available, + self.requirements["ggml"], + ) + + self.assertNotIn("cpu", eligible) + self.assertEqual(missing["cpu"], [zero_sized_asset]) + def test_invalid_release_tag_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "Invalid GGML_RELEASE"): assets.build_asset_requirements( diff --git a/test/test_llamacpp_release_manifest.py b/test/test_llamacpp_release_manifest.py index a1f60bea94..e0450f6f03 100644 --- a/test/test_llamacpp_release_manifest.py +++ b/test/test_llamacpp_release_manifest.py @@ -10,6 +10,7 @@ import unittest from pathlib import Path +from test.utils import llamacpp_release_assets as release_assets from test.utils import llamacpp_release_manifest as manifest @@ -124,6 +125,41 @@ def _immutable_source_manifest_fixture(self) -> tuple[dict, bytes, str]: ) return release, evidence, source_commit + def _rocm_source_manifest_fixture( + self, + ) -> tuple[dict, bytes, str, list[dict]]: + release, evidence, source_commit = self._immutable_source_manifest_fixture() + document = json.loads(evidence) + document["release_repository"] = "lemonade-sdk/llamacpp-rocm" + document["schema_version"] = 2 + targets_by_name = { + "a-first.zip": ["gfx1036", "gfx1033", "gfx1035"], + "z-last.zip": ["gfx942"], + } + for binding in document["assets"]: + binding["build_targets"] = targets_by_name[binding["name"]] + evidence = json.dumps( + document, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + self._bind_source_evidence(release, evidence) + expected = [ + { + "build_targets": ["gfx1033", "gfx1035", "gfx1036"], + "digest": document["assets"][0]["digest"], + "name": "a-first.zip", + "size": 100, + }, + { + "build_targets": ["gfx942"], + "digest": document["assets"][1]["digest"], + "name": "z-last.zip", + "size": 200, + }, + ] + return release, evidence, source_commit, expected + @staticmethod def _bind_source_evidence(release: dict, evidence: bytes) -> None: source_asset = next( @@ -134,6 +170,86 @@ def _bind_source_evidence(release: dict, evidence: bytes) -> None: source_asset["size"] = len(evidence) source_asset["digest"] = "sha256:" + hashlib.sha256(evidence).hexdigest() + @staticmethod + def _complete_managed_manifest( + versions: dict, + *, + ggml_tag: str = "b1", + rocm_tag: str = "b3", + lemonade_tag: str = "b2", + extra_rocm_target: str | None = None, + ) -> str: + requirements = release_assets.build_asset_requirements( + ggml_release=ggml_tag, + rocm_release=rocm_tag, + lemonade_release=lemonade_tag, + backend_versions=versions, + ) + rocm_targets = release_assets.build_rocm_asset_target_requirements( + rocm_release=rocm_tag, + backend_versions=versions, + ) + releases = [] + for release_index, (repository, tag, group) in enumerate( + ( + ("ggml-org/llama.cpp", ggml_tag, "ggml"), + ("lemonade-sdk/llamacpp-rocm", rocm_tag, "rocm"), + ("lemonade-sdk/llama.cpp", lemonade_tag, "lemonade"), + ), + start=1, + ): + payload = release_payload( + tag, + release_index, + release_index * 100, + repository="ggml-org/llama.cpp", + publisher_claim_type=( + "source-release-tag" + if repository == "ggml-org/llama.cpp" + else "immutable-source-manifest" + ), + ) + source_assets = [ + asset + for asset in payload["assets"] + if asset["name"] == manifest.SOURCE_MANIFEST_ASSET_NAME + ] + names = [ + name + for backend_names in requirements[group].values() + for name in backend_names + ] + binary_assets = [ + { + "digest": "sha256:" + f"{release_index * 1000 + index:064x}", + "id": release_index * 1000 + index, + "name": name, + "size": index, + "state": "uploaded", + "updated_at": "2026-09-05T12:40:00Z", + } + for index, name in enumerate(names, start=1) + ] + payload["assets"] = binary_assets + source_assets + if repository == "lemonade-sdk/llamacpp-rocm": + assets_by_name = {asset["name"]: asset for asset in binary_assets} + payload["build_target_attestations"] = [] + for name, required_targets in rocm_targets.items(): + target_asset = assets_by_name[name] + build_targets = list(required_targets) + if extra_rocm_target is not None: + build_targets.append(extra_rocm_target) + payload["build_target_attestations"].append( + { + "build_targets": build_targets, + "digest": target_asset["digest"], + "name": name, + "size": target_asset["size"], + } + ) + releases.append((repository, tag, payload)) + return manifest.build_release_asset_manifest(releases) + def test_immutable_source_manifest_binds_release_source_and_assets(self) -> None: release, evidence, source_commit = self._immutable_source_manifest_fixture() @@ -153,6 +269,119 @@ def test_immutable_source_manifest_binds_release_source_and_assets(self) -> None source_commit, ) + def test_rocm_source_manifest_normalizes_asset_bound_build_targets(self) -> None: + release, evidence, source_commit, expected = ( + self._rocm_source_manifest_fixture() + ) + + actual_commit, actual_attestations = ( + manifest.validate_immutable_source_manifest_with_attestation( + evidence, + release, + "lemonade-sdk/llamacpp-rocm", + "b4321", + "b" * 40, + ) + ) + + self.assertEqual(actual_commit, source_commit) + self.assertEqual(actual_attestations, expected) + + def test_legacy_rocm_source_manifest_has_no_build_target_attestation( + self, + ) -> None: + release, evidence, source_commit, _expected = ( + self._rocm_source_manifest_fixture() + ) + document = json.loads(evidence) + document["schema_version"] = 1 + for binding in document["assets"]: + binding.pop("build_targets") + legacy = json.dumps( + document, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + self._bind_source_evidence(release, legacy) + + actual_commit, actual_attestations = ( + manifest.validate_immutable_source_manifest_with_attestation( + legacy, + release, + "lemonade-sdk/llamacpp-rocm", + "b4321", + "b" * 40, + ) + ) + + self.assertEqual(actual_commit, source_commit) + self.assertEqual(actual_attestations, []) + + def test_rocm_build_target_attestation_rejects_malformed_or_duplicate_claims( + self, + ) -> None: + release, evidence, _source_commit, _expected = ( + self._rocm_source_manifest_fixture() + ) + document = json.loads(evidence) + cases = ( + ( + "concrete ROCm ISA", + lambda value: value["assets"][0]["build_targets"].append("gfx103X"), + ), + ( + "duplicate build target", + lambda value: value["assets"][0]["build_targets"].append("gfx1033"), + ), + ( + "build_targets must be an array", + lambda value: value["assets"][0].update(build_targets="gfx1033"), + ), + ( + "unexpected fields", + lambda value: value["assets"][0].update(unexpected=True), + ), + ) + for expected_error, mutate in cases: + with self.subTest(expected_error=expected_error): + changed = copy.deepcopy(document) + mutate(changed) + changed_bytes = json.dumps( + changed, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + changed_release = copy.deepcopy(release) + self._bind_source_evidence(changed_release, changed_bytes) + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.validate_immutable_source_manifest_with_attestation( + changed_bytes, + changed_release, + "lemonade-sdk/llamacpp-rocm", + "b4321", + "b" * 40, + ) + + def test_source_manifest_schema_two_is_reserved_for_rocm(self) -> None: + release, evidence, _source_commit, _expected = ( + self._rocm_source_manifest_fixture() + ) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "schema_version 2 is only valid", + ): + manifest.validate_immutable_source_manifest_with_attestation( + evidence, + release, + "lemonade-sdk/llama.cpp", + "b4321", + "b" * 40, + ) + def test_immutable_source_manifest_rejects_unbound_or_ambiguous_input(self) -> None: release, evidence, _source_commit = self._immutable_source_manifest_fixture() document = json.loads(evidence) @@ -411,7 +640,7 @@ def test_manifest_is_canonical_across_release_and_asset_order(self) -> None: self.assertEqual(first, second) parsed = json.loads(first) - self.assertEqual(parsed["schema_version"], 4) + self.assertEqual(parsed["schema_version"], 5) self.assertEqual( [release["repository"] for release in parsed["releases"]], ["ggml-org/llama.cpp", "lemonade-sdk/llamacpp-rocm"], @@ -425,6 +654,70 @@ def test_manifest_is_canonical_across_release_and_asset_order(self) -> None: "sha256:" + "b" * 64, ) + def test_release_manifest_carries_normalized_rocm_build_targets(self) -> None: + payload = copy.deepcopy(self.rocm) + target_asset = payload["assets"][0] + payload["build_target_attestations"] = [ + { + "build_targets": ["gfx1036", "gfx1033", "gfx1035"], + "digest": target_asset["digest"].lower(), + "name": target_asset["name"], + "size": target_asset["size"], + } + ] + + canonical = manifest.build_release_asset_manifest( + [("lemonade-sdk/llamacpp-rocm", "b1235", payload)] + ) + release = json.loads(canonical)["releases"][0] + + self.assertEqual( + release["build_target_attestations"], + [ + { + "build_targets": ["gfx1033", "gfx1035", "gfx1036"], + "digest": target_asset["digest"].lower(), + "name": target_asset["name"], + "size": target_asset["size"], + } + ], + ) + self.assertEqual( + manifest.canonicalize_release_asset_manifest(canonical), canonical + ) + + def test_release_manifest_rejects_tampered_rocm_build_target_binding(self) -> None: + payload = copy.deepcopy(self.rocm) + target_asset = payload["assets"][0] + base_attestation = { + "build_targets": ["gfx1033"], + "digest": target_asset["digest"].lower(), + "name": target_asset["name"], + "size": target_asset["size"], + } + cases = ( + ("does not match release asset", {**base_attestation, "size": 999}), + ( + "duplicate build target", + {**base_attestation, "build_targets": ["gfx1033", "gfx1033"]}, + ), + ( + "concrete ROCm ISA", + {**base_attestation, "build_targets": ["gfx103X"]}, + ), + ) + for expected_error, attestation in cases: + with self.subTest(expected_error=expected_error): + changed = copy.deepcopy(payload) + changed["build_target_attestations"] = [attestation] + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.build_release_asset_manifest( + [("lemonade-sdk/llamacpp-rocm", "b1235", changed)] + ) + def test_every_asset_requires_a_sha256_digest(self) -> None: for digest in (None, "", "md5:" + "a" * 32, "sha256:abc"): with self.subTest(digest=digest): @@ -668,6 +961,147 @@ def test_unchanged_managed_pins_do_not_require_a_manifest(self) -> None: ) ) + def test_rocm_selector_changes_require_their_managed_manifest_groups(self) -> None: + base = { + "llamacpp": { + "cpu": "b1", + "cuda": "b2", + "metal": "b1", + "rocm-nightly": "b3", + "rocm-stable": "b2", + "vulkan": "b1", + }, + "rocm_asset_families": {"gfx1033": "gfx103X"}, + "therock": {"version": "7.14.0"}, + } + + mapping_change = copy.deepcopy(base) + mapping_change["rocm_asset_families"]["gfx1033"] = "gfx1033" + self.assertEqual( + manifest.managed_llamacpp_pin_changes(base, mapping_change), + {"rocm-nightly": ("b3", "b3")}, + ) + + therock_change = copy.deepcopy(base) + therock_change["therock"]["version"] = "7.15.0" + self.assertEqual( + manifest.managed_llamacpp_pin_changes(base, therock_change), + {"rocm-stable": ("b2", "b2")}, + ) + + for field, value in ( + ("architectures", ["gfx103X", "gfx110X"]), + ("url_mapping", {"windows": "runtime.zip"}), + ): + with self.subTest(field=field): + changed = copy.deepcopy(base) + changed["therock"][field] = value + self.assertEqual( + manifest.managed_llamacpp_pin_changes(base, changed), + {"rocm-stable": ("b2", "b2")}, + ) + + comment_only = copy.deepcopy(base) + comment_only["therock"]["comment"] = "documentation only" + self.assertEqual( + manifest.managed_llamacpp_pin_changes(base, comment_only), + {}, + ) + + def test_rocm_pin_manifest_requires_complete_candidate_target_attestation( + self, + ) -> None: + base = { + "llamacpp": { + "cpu": "b1", + "cuda": "b2", + "metal": "b1", + "rocm-nightly": "b3", + "rocm-stable": "b2", + "vulkan": "b1", + }, + "rocm_asset_families": { + "gfx1033": "gfx103X", + "gfx1035": "gfx103X", + "gfx1036": "gfx103X", + }, + "therock": {"version": "7.14.0"}, + } + candidate = copy.deepcopy(base) + candidate["llamacpp"]["rocm-nightly"] = "b9" + complete = self._complete_managed_manifest( + candidate, + rocm_tag="b9", + extra_rocm_target="gfx999", + ) + self.assertTrue( + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + complete, + ) + ) + + incomplete = json.loads(complete) + rocm_release = next( + release + for release in incomplete["releases"] + if release["repository"] == "lemonade-sdk/llamacpp-rocm" + ) + target_attestation = next( + attestation + for attestation in rocm_release["build_target_attestations"] + if attestation["name"] == "llama-b9-windows-rocm-gfx103X-x64.zip" + ) + target_attestation["build_targets"].remove("gfx1035") + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "incomplete build-target attestation", + ): + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + json.dumps(incomplete), + ) + + rocm_release["build_target_attestations"] = [] + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "incomplete build-target attestation", + ): + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + json.dumps(incomplete), + ) + + def test_rocm_mapping_only_change_rejects_manifest_for_old_mapping(self) -> None: + base = { + "llamacpp": { + "cpu": "b1", + "cuda": "b2", + "metal": "b1", + "rocm-nightly": "b3", + "rocm-stable": "b2", + "vulkan": "b1", + }, + "rocm_asset_families": {"gfx1033": "gfx103X"}, + "therock": {"version": "7.14.0"}, + } + candidate = copy.deepcopy(base) + candidate["rocm_asset_families"]["gfx1033"] = "gfx1033" + old_mapping_manifest = self._complete_managed_manifest(base) + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + "llamacpp.rocm-nightly", + ): + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + old_mapping_manifest, + ) + def test_changed_managed_pin_requires_a_matching_release_manifest(self) -> None: base = { "llamacpp": { @@ -677,13 +1111,36 @@ def test_changed_managed_pin_requires_a_matching_release_manifest(self) -> None: "rocm-nightly": "b3", "rocm-stable": "b2", "vulkan": "b1", - } + }, + "rocm_asset_families": {}, + "therock": {"version": "7.14.0"}, } candidate = copy.deepcopy(base) candidate["llamacpp"]["vulkan"] = "b1234" + requirements = release_assets.build_asset_requirements( + ggml_release="b1234", + rocm_release="b3", + lemonade_release="b2", + backend_versions=candidate, + ) + ggml = copy.deepcopy(self.ggml) + for offset, asset_name in enumerate( + requirements["ggml"]["vulkan"], + start=1, + ): + ggml["assets"].append( + { + "digest": "sha256:" + f"{offset:064x}", + "id": 1000 + offset, + "name": asset_name, + "size": offset, + "state": "uploaded", + "updated_at": "2026-09-05T12:40:00Z", + } + ) release_manifest = manifest.build_release_asset_manifest( [ - ("ggml-org/llama.cpp", "b1234", self.ggml), + ("ggml-org/llama.cpp", "b1234", ggml), ( "lemonade-sdk/llamacpp-rocm", "b3", @@ -725,6 +1182,41 @@ def test_changed_managed_pin_requires_a_matching_release_manifest(self) -> None: ) ) + required_asset = requirements["ggml"]["vulkan"][-1] + for defect in ("missing", "zero-size"): + with self.subTest(defect=defect): + incomplete = json.loads(release_manifest) + ggml_release = next( + release + for release in incomplete["releases"] + if release["repository"] == "ggml-org/llama.cpp" + ) + if defect == "missing": + ggml_release["assets"] = [ + asset + for asset in ggml_release["assets"] + if asset["name"] != required_asset + ] + expected_error = "missing required asset" + else: + target_asset = next( + asset + for asset in ggml_release["assets"] + if asset["name"] == required_asset + ) + target_asset["size"] = 0 + expected_error = "positive size" + + with self.assertRaisesRegex( + manifest.ReleaseManifestError, + expected_error, + ): + manifest.require_release_manifest_for_managed_pin_changes( + base, + candidate, + json.dumps(incomplete), + ) + mismatched = json.loads(release_manifest) mismatched["releases"][0]["tag_name"] = "b9999" with self.assertRaisesRegex( @@ -803,7 +1295,7 @@ def test_canonical_manifest_rejects_duplicate_json_members(self) -> None: canonical = manifest.build_release_asset_manifest( [("ggml-org/llama.cpp", "b1234", self.ggml)] ) - duplicate = canonical[:-1] + ',"schema_version":4}' + duplicate = canonical[:-1] + ',"schema_version":5}' with self.assertRaisesRegex( manifest.ReleaseManifestError, @@ -817,14 +1309,14 @@ def test_canonical_manifest_requires_integer_schema_version(self) -> None: [("ggml-org/llama.cpp", "b1234", self.ggml)] ) ) - for schema_version in (True, 4.0, "4"): + for schema_version in (True, 5.0, "5"): with self.subTest(schema_version=schema_version): changed = copy.deepcopy(canonical) changed["schema_version"] = schema_version with self.assertRaisesRegex( manifest.ReleaseManifestError, - "schema_version must be 4", + "schema_version must be 5", ): manifest.canonicalize_release_asset_manifest(json.dumps(changed)) diff --git a/test/test_llamacpp_system_backend.py b/test/test_llamacpp_system_backend.py index 86a0cf3cff..04b69c4457 100644 --- a/test/test_llamacpp_system_backend.py +++ b/test/test_llamacpp_system_backend.py @@ -55,8 +55,8 @@ def get_arg(flag, default): # lemond detects the system llama-server version by running -# `llama-server --version` and reading one line of stdout from the llamacpp -# backend resolver in src/cpp/server/backends/llamacpp/llamacpp_server.cpp. +# `llama-server --version` and reading bounded output from the llamacpp backend +# resolver in src/cpp/server/backends/llamacpp/llamacpp_server.cpp. # The real binary prints a version line and exits immediately. We must mirror # that: otherwise the probe blocks forever on our long-lived HTTP server and # /internal/set hangs until the client times out. @@ -168,6 +168,69 @@ def log_message(self, format, *args): ReusableHTTPServer(("127.0.0.1", port), Handler).serve_forever() """ +MOCK_INCOMPATIBLE_LLAMA_SERVER_PYTHON = """#!/usr/bin/env python3 +import os +import sys + +if "--version" in sys.argv or "--help" in sys.argv: + print("version: 0.3.0-dev (build 4242, commit incompatible)") + sys.exit(0) + +invocation_path = os.environ.get("MOCK_LLAMA_INVOCATION_PATH", "") +if invocation_path: + with open(invocation_path, "a", encoding="utf-8") as handle: + handle.write("model\\n") + +if "--log-file" in sys.argv: + print("error: unrecognized argument: --log-file", file=sys.stderr) + sys.exit(2) + +message = "error loading model: unknown model architecture: 'k2-horizon'" +print(message, file=sys.stderr) +sys.exit(1) +""" + +MOCK_RESOURCE_FAILURE_LLAMA_SERVER_PYTHON = """#!/usr/bin/env python3 +import os +import sys + +if "--version" in sys.argv or "--help" in sys.argv: + print("version: 0.3.0-dev (build 4242, commit incompatible)") + sys.exit(0) + +invocation_path = os.environ.get("MOCK_LLAMA_INVOCATION_PATH", "") +if invocation_path: + with open(invocation_path, "a", encoding="utf-8") as handle: + handle.write("model\\n") + +message = "error loading model: failed to allocate compute buffers" +print(message, file=sys.stderr) +sys.exit(1) +""" + +MOCK_CANCELLABLE_LLAMA_SERVER_PYTHON = """#!/usr/bin/env python3 +import os +import sys +import time + +if "--version" in sys.argv or "--help" in sys.argv: + print("version: 0.3.0-dev (build 4242, commit incompatible)") + sys.exit(0) + +invocation_path = os.environ.get("MOCK_LLAMA_INVOCATION_PATH", "") +if invocation_path: + with open(invocation_path, "a", encoding="utf-8") as handle: + handle.write("model\\n") + +print( + "error loading model: unknown model architecture: 'k2-horizon'", + file=sys.stderr, + flush=True, +) +while True: + time.sleep(1) +""" + def _is_server_running(port=PORT): """Check if the server is running on the given port.""" @@ -333,15 +396,24 @@ def setUpClass(cls): cls.temp_bin_dir = tempfile.mkdtemp(prefix="lemonade_llamacpp_mock_bin_") cls.cache_dir = tempfile.mkdtemp(prefix="lemonade_llamacpp_test_") cls.dummy_llama_server_path = os.path.join(cls.temp_bin_dir, "llama-server") + cls.k2_fixture_path = os.path.join(cls.temp_bin_dir, "k2-system-fixture.bin") cls.capture_path = os.path.join(cls.temp_bin_dir, "captured_chat_request.json") cls.control_path = os.path.join(cls.temp_bin_dir, "mock_control.json") + cls.invocation_path = os.path.join(cls.temp_bin_dir, "model_invocations.txt") cls.original_env = { name: os.environ.get(name) - for name in ("PATH", "MOCK_LLAMA_REQUEST_PATH", "MOCK_LLAMA_CONTROL_PATH") + for name in ( + "PATH", + "MOCK_LLAMA_REQUEST_PATH", + "MOCK_LLAMA_CONTROL_PATH", + "MOCK_LLAMA_INVOCATION_PATH", + ) } try: cls._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) + with open(cls.k2_fixture_path, "wb") as handle: + handle.write(b"GGUF") # Keep the external-system integration deterministic on AMD hosts as # well: production accepts the plugin beside a PATH llama-server. @@ -359,6 +431,7 @@ def setUpClass(cls): os.environ["PATH"] = cls.temp_bin_dir + os.pathsep + original_path os.environ["MOCK_LLAMA_REQUEST_PATH"] = cls.capture_path os.environ["MOCK_LLAMA_CONTROL_PATH"] = cls.control_path + os.environ["MOCK_LLAMA_INVOCATION_PATH"] = cls.invocation_path cls._write_mock_control({}) # One lemond and one spawned mock llama-server are enough for all @@ -417,10 +490,64 @@ def _write_mock_control(cls, control): with open(cls.control_path, "w", encoding="utf-8") as handle: json.dump(control, handle) + @classmethod + def _model_invocation_count(cls): + try: + with open(cls.invocation_path, "r", encoding="utf-8") as handle: + return sum(1 for line in handle if line.strip() == "model") + except FileNotFoundError: + return 0 + + @classmethod + def _wait_for_model_invocations(cls, expected, timeout=10): + deadline = time.time() + timeout + while time.time() < deadline: + if cls._model_invocation_count() >= expected: + return True + time.sleep(0.05) + return cls._model_invocation_count() >= expected + + @staticmethod + def _wait_for_job_status(job_id, expected, timeout=20): + deadline = time.time() + timeout + last_status = None + while time.time() < deadline: + response = requests.get( + f"http://localhost:{PORT}/api/v1/jobs/{job_id}", + timeout=TIMEOUT_DEFAULT, + ) + if response.status_code == 200: + last_status = response.json().get("status") + if last_status == expected: + return response.json() + time.sleep(0.1) + raise AssertionError( + f"job {job_id} did not reach {expected}; last status={last_status}" + ) + + def _register_k2_system_fixture(self): + k2_fixture = "user.K2-Horizon-System-Binary-Fixture" + registration = requests.post( + f"http://localhost:{PORT}/api/v1/models/register", + json={ + "model_name": k2_fixture, + "recipe": "llamacpp", + "checkpoint": self.k2_fixture_path, + "source": "local_path", + "labels": ["chat", "reasoning", "tool-calling"], + }, + timeout=TIMEOUT_DEFAULT, + ) + self.assertEqual(registration.status_code, 200, registration.text) + self.assertTrue(registration.json()["model"]["downloaded"]) + return k2_fixture + def setUp(self): print(f"\n=== Starting test: {self._testMethodName} ===") if os.path.exists(self.capture_path): os.remove(self.capture_path) + if os.path.exists(self.invocation_path): + os.remove(self.invocation_path) self._write_mock_control({}) @unittest.skipUnless( @@ -552,6 +679,160 @@ def test_008_backend_context_error_preserves_http_status(self): self.assertEqual(error["status_code"], 400) self.assertIn("exceeds the available context size", error["message"]) + @unittest.skipUnless( + sys.platform.startswith("linux"), "System backend only supported on Linux" + ) + def test_009_incompatible_system_binary_has_actionable_k2_diagnostic(self): + k2_fixture = self._register_k2_system_fixture() + + unload_response = requests.post( + f"http://localhost:{PORT}/api/v1/unload", + json={}, + timeout=TIMEOUT_DEFAULT, + ) + self.assertEqual(unload_response.status_code, 200, unload_response.text) + + self._write_llama_server(MOCK_INCOMPATIBLE_LLAMA_SERVER_PYTHON) + try: + ordinary_response = requests.post( + f"http://localhost:{PORT}/api/v1/load", + json={ + "model_name": ENDPOINT_TEST_MODEL, + "llamacpp_backend": "system", + }, + timeout=TIMEOUT_MODEL_OPERATION, + ) + self.assertNotEqual(ordinary_response.status_code, 200) + ordinary_message = ordinary_response.json()["error"]["message"] + self.assertTrue( + ordinary_message.endswith(": llama-server failed to start"), + ordinary_message, + ) + self.assertNotIn("K2-Horizon-capable", ordinary_message) + self.assertEqual(self._model_invocation_count(), 2) + + os.remove(self.invocation_path) + + k2_response = requests.post( + f"http://localhost:{PORT}/api/v1/load", + json={"model_name": k2_fixture, "llamacpp_backend": "system"}, + timeout=TIMEOUT_MODEL_OPERATION, + ) + self.assertNotEqual(k2_response.status_code, 200) + message = k2_response.json()["error"]["message"] + self.assertIn(k2_fixture, message) + self.assertIn(self.dummy_llama_server_path, message) + self.assertIn("b4242", message) + self.assertIn( + "Original error: error loading model: unknown model architecture: " + "'k2-horizon'", + message, + ) + self.assertIn("K2-Horizon-capable", message) + self.assertIn("model/K2Horizon", message) + self.assertIn("PATH", message) + self.assertEqual( + self._model_invocation_count(), + 2, + "diagnostic collection must not re-launch the model command", + ) + finally: + self._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) + + @unittest.skipUnless( + sys.platform.startswith("linux"), "System backend only supported on Linux" + ) + def test_010_k2_resource_failure_keeps_ordinary_startup_diagnostic(self): + k2_fixture = self._register_k2_system_fixture() + unload_response = requests.post( + f"http://localhost:{PORT}/api/v1/unload", + json={}, + timeout=TIMEOUT_DEFAULT, + ) + self.assertEqual(unload_response.status_code, 200, unload_response.text) + + self._write_llama_server(MOCK_RESOURCE_FAILURE_LLAMA_SERVER_PYTHON) + try: + response = requests.post( + f"http://localhost:{PORT}/api/v1/load", + json={"model_name": k2_fixture, "llamacpp_backend": "system"}, + timeout=TIMEOUT_MODEL_OPERATION, + ) + self.assertNotEqual(response.status_code, 200) + message = response.json()["error"]["message"] + self.assertTrue( + message.endswith(": llama-server failed to start"), + message, + ) + self.assertNotIn("K2-Horizon-capable", message) + self.assertNotIn("model/K2Horizon", message) + self.assertEqual( + self._model_invocation_count(), + 2, + "resource failure diagnosis must not repeat the model command", + ) + finally: + self._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) + + @unittest.skipUnless( + sys.platform.startswith("linux"), "System backend only supported on Linux" + ) + def test_011_cancelled_k2_load_does_not_relaunch_model_command(self): + k2_fixture = self._register_k2_system_fixture() + unload_response = requests.post( + f"http://localhost:{PORT}/api/v1/unload", + json={}, + timeout=TIMEOUT_DEFAULT, + ) + self.assertEqual(unload_response.status_code, 200, unload_response.text) + + job_id = None + self._write_llama_server(MOCK_CANCELLABLE_LLAMA_SERVER_PYTHON) + try: + response = requests.post( + f"http://localhost:{PORT}/api/v1/jobs", + json={ + "name": "cancel-k2-system-load", + "definition": { + "steps": [ + { + "id": "load", + "op": "load", + "params": { + "model": k2_fixture, + "llamacpp_backend": "system", + }, + } + ] + }, + }, + timeout=TIMEOUT_DEFAULT, + ) + self.assertEqual(response.status_code, 202, response.text) + job_id = response.json()["id"] + self.assertTrue(self._wait_for_model_invocations(1)) + + interrupt_response = requests.post( + f"http://localhost:{PORT}/api/v1/jobs/{job_id}/interrupt", + timeout=TIMEOUT_DEFAULT, + ) + self.assertEqual( + interrupt_response.status_code, 200, interrupt_response.text + ) + self._wait_for_job_status(job_id, "interrupted") + self.assertEqual( + self._model_invocation_count(), + 1, + "a cancelled load must not retry or replay the model command", + ) + finally: + if job_id is not None: + requests.delete( + f"http://localhost:{PORT}/api/v1/jobs/{job_id}", + timeout=TIMEOUT_DEFAULT, + ) + self._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) + def _run_tests(): """Run llamacpp system backend tests.""" diff --git a/test/test_llamacpp_validation_args.ps1 b/test/test_llamacpp_validation_args.ps1 deleted file mode 100644 index 8731c6a7e7..0000000000 --- a/test/test_llamacpp_validation_args.ps1 +++ /dev/null @@ -1,152 +0,0 @@ -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/../.github/scripts/llamacpp_validation_args.ps1" - -function Assert-Arguments { - param( - [string[]] $Actual, - [string[]] $Expected, - [string] $Case - ) - - if ($Actual.Count -ne $Expected.Count) { - throw "$Case returned $($Actual.Count) arguments; expected $($Expected.Count)." - } - - for ($index = 0; $index -lt $Expected.Count; $index++) { - if ($Actual[$index] -ne $Expected[$index]) { - throw "$Case argument $index was '$($Actual[$index])'; expected '$($Expected[$index])'." - } - } -} - -Assert-Arguments ` - -Actual @(Get-LlamaCppValidationModelArguments -Models "" -Lite $false) ` - -Expected @() ` - -Case "default mode" - -Assert-Arguments ` - -Actual @( - Get-LlamaCppValidationModelArguments ` - -Models "" ` - -Lite $false ` - -CapabilityProfile "k2-horizon-v1" ` - -CapabilityModels "K2-Horizon-0.9B-GGUF" - ) ` - -Expected @( - "--capability-profile", - "k2-horizon-v1", - "--capability-model", - "K2-Horizon-0.9B-GGUF" - ) ` - -Case "hot selection capability profile" - -Assert-Arguments ` - -Actual @( - Get-LlamaCppValidationModelArguments ` - -Models "K2-Horizon-0.9B-GGUF,K2-Horizon-3.7B-GGUF" ` - -Lite $false ` - -CapabilityProfile "k2-horizon-v1" ` - -CapabilityModels "K2-Horizon-0.9B-GGUF" - ) ` - -Expected @( - "--model", - "K2-Horizon-0.9B-GGUF", - "--model", - "K2-Horizon-3.7B-GGUF", - "--capability-profile", - "k2-horizon-v1", - "--capability-model", - "K2-Horizon-0.9B-GGUF" - ) ` - -Case "capability profile" - -Assert-Arguments ` - -Actual @( - Get-LlamaCppValidationModelArguments ` - -Models " K2-Horizon-0.9B-GGUF, ,K2-Horizon-3.7B-GGUF " ` - -Lite $false - ) ` - -Expected @( - "--model", - "K2-Horizon-0.9B-GGUF", - "--model", - "K2-Horizon-3.7B-GGUF" - ) ` - -Case "explicit models" - -Assert-Arguments ` - -Actual @( - Get-LlamaCppValidationModelArguments ` - -Models "K2-Horizon-0.9B-GGUF,K2-Horizon-0.9B-GGUF" ` - -Lite $false - ) ` - -Expected @( - "--model", - "K2-Horizon-0.9B-GGUF", - "--model", - "K2-Horizon-0.9B-GGUF" - ) ` - -Case "duplicate models" - -Assert-Arguments ` - -Actual @(Get-LlamaCppValidationModelArguments -Models " , " -Lite $true) ` - -Expected @("--lite") ` - -Case "lite mode" - -$conflict = $null -try { - Get-LlamaCppValidationModelArguments ` - -Models "K2-Horizon-0.9B-GGUF" ` - -Lite $true | Out-Null -} catch { - $conflict = $_ -} - -if ($null -eq $conflict) { - throw "Explicit models and lite mode must be rejected." -} - -if ($conflict.Exception.Message -notmatch "mutually exclusive") { - throw "Conflict error must explain that explicit models and lite mode are mutually exclusive." -} - -$missingProfile = $null -try { - Get-LlamaCppValidationModelArguments ` - -Models "K2-Horizon-0.9B-GGUF" ` - -CapabilityModels "K2-Horizon-0.9B-GGUF" | Out-Null -} catch { - $missingProfile = $_ -} - -if ($null -eq $missingProfile) { - throw "Capability models without a profile must be rejected." -} - -$missingModel = $null -try { - Get-LlamaCppValidationModelArguments ` - -Models "K2-Horizon-0.9B-GGUF" ` - -CapabilityProfile "k2-horizon-v1" | Out-Null -} catch { - $missingModel = $_ -} - -if ($null -eq $missingModel) { - throw "A capability profile without capability models must be rejected." -} - -$unselectedModel = $null -try { - Get-LlamaCppValidationModelArguments ` - -Models "K2-Horizon-0.9B-GGUF" ` - -CapabilityProfile "k2-horizon-v1" ` - -CapabilityModels "K2-Horizon-3.7B-GGUF" | Out-Null -} catch { - $unselectedModel = $_ -} - -if ($null -eq $unselectedModel) { - throw "Capability models outside the explicit selection must be rejected." -} diff --git a/test/test_llamacpp_validation_artifacts.py b/test/test_llamacpp_validation_artifacts.py new file mode 100644 index 0000000000..7e6062ea2e --- /dev/null +++ b/test/test_llamacpp_validation_artifacts.py @@ -0,0 +1,314 @@ +import hashlib +import json +import os +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from test.utils import llamacpp_validation_artifacts as artifacts +from test.utils.llamacpp_validation_artifacts import ( + ValidationArtifactError, + verify_validation_artifacts, +) + +MODEL_ID = "K2-Horizon-Test-GGUF" +REPOSITORY = "IFM/K2-Horizon-Test-GGUF" +REVISION = "a" * 40 +FILENAME = "model.gguf" + + +def gguf_string(value): + encoded = value.encode("utf-8") + return struct.pack(" dict: - planned_models = models or [K2_SMALL] - return { - "include": [ - { - "target": "windows-vulkan", - "backend": "vulkan", - "channel": "", - "models": [], - "expected_models": planned_models, - "capability_profile": capabilities.K2_HORIZON_PROFILE, - "capability_models": [K2_SMALL], - }, - { - "target": "rocm-stable", - "backend": "rocm", - "channel": "stable", - "models": [], - "expected_models": planned_models, - "capability_profile": capabilities.K2_HORIZON_PROFILE, - "capability_models": [K2_SMALL], - }, - { - "target": "rocm-nightly", - "backend": "rocm", - "channel": "nightly", - "models": [], - "expected_models": planned_models, - "capability_profile": capabilities.K2_HORIZON_PROFILE, - "capability_models": [K2_SMALL], - }, - ] - } +def expected_matrix( + models_by_target: dict[str, list[str]] | None = None, +) -> dict: + matrix = planning.create_validation_plan("schedule") + for row in matrix["include"]: + if models_by_target is None or row["target"] not in models_by_target: + continue + planned_models = models_by_target[row["target"]] + row["models"] = list(planned_models) + row["expected_models"] = list(planned_models) + return matrix def passing_capability_matrix(model: str = K2_SMALL) -> dict: @@ -69,9 +49,10 @@ def passing_capability_matrix(model: str = K2_SMALL) -> dict: "openai_tool_default_xml", "openai_tool_stream_xml", } - reasoning_case = case_id.startswith("openai_reasoning_") or case_id == ( - "ollama_thinking" - ) + reasoning_case = ( + case_id.startswith("openai_reasoning_") + and case_id != "openai_reasoning_low" + ) or case_id == "ollama_thinking" content_case = case_id in { "openai_plain_off_nonstream", "openai_plain_off_stream", @@ -133,10 +114,22 @@ def passing_result(model: str = K2_SMALL, *, capability: bool | None = None) -> class LlamaCppValidationEvidenceTests(unittest.TestCase): - def write_expected_results(self, root: Path) -> None: - for filename in evidence.EXPECTED_RESULT_FILES: + def write_expected_results(self, root: Path, matrix: dict | None = None) -> None: + selected_matrix = matrix or expected_matrix() + for row in selected_matrix["include"]: + filename = f"llamacpp_validation_{row['target']}.json" + records = [ + passing_result(model, capability=model == K2_SMALL) + for model in row["expected_models"] + ] (root / filename).write_text( - json.dumps([passing_result()]), + json.dumps(records), + encoding="utf-8", + ) + restart_filename = f"llamacpp_restart_validation_{row['target']}.json" + restart_model = row["capability_models"][0] + (root / restart_filename).write_text( + json.dumps([passing_result(restart_model, capability=False)]), encoding="utf-8", ) @@ -147,8 +140,22 @@ def test_all_expected_nonempty_passing_results_are_accepted(self) -> None: results = evidence.load_and_validate_results(root, expected_matrix()) - self.assertEqual(set(results), set(evidence.EXPECTED_RESULT_FILES)) - self.assertTrue(all(len(records) == 1 for records in results.values())) + self.assertEqual( + set(results), + set(evidence.EXPECTED_RESULT_FILES) + | set(evidence.EXPECTED_RESTART_RESULT_FILES), + ) + self.assertGreater( + len(results["llamacpp_validation_windows-vulkan.json"]), + 1, + ) + self.assertTrue( + all( + len(records) == 1 + for filename, records in results.items() + if filename != "llamacpp_validation_windows-vulkan.json" + ) + ) def test_missing_empty_and_invalid_json_files_fail_closed(self) -> None: cases = { @@ -170,6 +177,38 @@ def test_missing_empty_and_invalid_json_files_fail_closed(self) -> None: with self.assertRaises(evidence.ValidationEvidenceError): evidence.load_and_validate_results(root, expected_matrix()) + def test_restart_evidence_must_be_present_exact_and_passing(self) -> None: + mutations = { + "missing": None, + "empty": "", + "invalid": "not json", + "empty list": "[]", + "failed": json.dumps([{**passing_result(capability=False), "pass": False}]), + "wrong model": json.dumps( + [passing_result("Other-Llama", capability=False)] + ), + "extra model": json.dumps( + [ + passing_result(capability=False), + passing_result("Other-Llama", capability=False), + ] + ), + "unexpected capabilities": json.dumps([passing_result()]), + } + target_name = "llamacpp_restart_validation_linux-vulkan.json" + for name, replacement in mutations.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + target = root / target_name + if replacement is None: + target.unlink() + else: + target.write_text(replacement, encoding="utf-8") + + with self.assertRaises(evidence.ValidationEvidenceError): + evidence.load_and_validate_results(root, expected_matrix()) + def test_failed_or_malformed_records_fail_closed(self) -> None: cases = { "failed": [{**passing_result(), "pass": False}], @@ -199,11 +238,120 @@ def test_failed_or_malformed_records_fail_closed(self) -> None: with self.assertRaises(evidence.ValidationEvidenceError): evidence.load_and_validate_results(root, expected_matrix()) + def test_exact_producer_record_schema_is_accepted(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "result.json" + expected = passing_result() + path.write_text(json.dumps([expected]), encoding="utf-8") + + records = evidence.load_and_validate_result_file( + path, + [K2_SMALL], + capability_profile=capabilities.K2_HORIZON_PROFILE, + capability_models=[K2_SMALL], + ) + + self.assertEqual(records, [expected]) + + def test_result_file_above_byte_limit_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "result.json" + path.write_text( + json.dumps([passing_result(capability=False)]), + encoding="utf-8", + ) + + with ( + mock.patch.object( + evidence, + "MAX_VALIDATION_EVIDENCE_BYTES", + path.stat().st_size - 1, + ), + self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "byte limit", + ), + ): + evidence.load_and_validate_result_file(path, [K2_SMALL]) + + def test_result_file_at_byte_limit_is_accepted(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "result.json" + expected = passing_result(capability=False) + path.write_text(json.dumps([expected]), encoding="utf-8") + + with mock.patch.object( + evidence, + "MAX_VALIDATION_EVIDENCE_BYTES", + path.stat().st_size, + ): + records = evidence.load_and_validate_result_file(path, [K2_SMALL]) + + self.assertEqual(records, [expected]) + + def test_result_file_symlink_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source.json" + source.write_text( + json.dumps([passing_result(capability=False)]), + encoding="utf-8", + ) + path = root / "result.json" + try: + path.symlink_to(source.name) + except OSError as exc: + self.skipTest(f"symlinks unavailable: {exc}") + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "regular file", + ): + evidence.load_and_validate_result_file(path, [K2_SMALL]) + + def test_result_and_capability_schemas_reject_unknown_fields(self) -> None: + def add_result_field(records: list[dict]) -> None: + records[0]["schema_version"] = 999 + + def add_matrix_field(records: list[dict]) -> None: + capability_record = next( + record for record in records if "capability_matrix" in record + ) + capability_record["capability_matrix"]["verification"] = "failed" + + def add_case_field(records: list[dict]) -> None: + capability_record = next( + record for record in records if "capability_matrix" in record + ) + capability_record["capability_matrix"]["cases"][0][ + "verification" + ] = "failed" + + mutations = { + "result": add_result_field, + "capability matrix": add_matrix_field, + "capability case": add_case_field, + } + for name, mutate in mutations.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + target = root / evidence.EXPECTED_RESULT_FILES[0] + records = json.loads(target.read_text(encoding="utf-8")) + mutate(records) + target.write_text(json.dumps(records), encoding="utf-8") + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "unexpected.*fields", + ): + evidence.load_and_validate_results(root, expected_matrix()) + def test_runtime_models_must_exactly_match_each_planned_lane(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) self.write_expected_results(root) - target = root / "llamacpp_validation_rocm-stable.json" + target = root / "llamacpp_validation_windows-rocm-stable.json" target.write_text( json.dumps([passing_result("some-other-model")]), encoding="utf-8", @@ -215,7 +363,24 @@ def test_runtime_models_must_exactly_match_each_planned_lane(self) -> None: ): evidence.load_and_validate_results(root, expected_matrix()) + def test_selected_models_must_exactly_match_expected_models(self) -> None: + matrix = expected_matrix() + matrix["include"][0]["models"] = [] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root, matrix) + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "selected models", + ): + evidence.load_and_validate_results(root, matrix) + def test_every_planned_lane_must_include_k2_small(self) -> None: + matrix = expected_matrix() + for row in matrix["include"]: + row["models"] = ["some-other-model"] + row["expected_models"] = ["some-other-model"] with tempfile.TemporaryDirectory() as directory: root = Path(directory) for filename in evidence.EXPECTED_RESULT_FILES: @@ -230,25 +395,67 @@ def test_every_planned_lane_must_include_k2_small(self) -> None: ): evidence.load_and_validate_results( root, - expected_matrix(["some-other-model"]), + matrix, ) - def test_inconsistent_planned_lane_sets_fail_closed(self) -> None: + def test_exact_per_lane_planned_model_sets_are_accepted(self) -> None: matrix = expected_matrix() - matrix["include"][2]["expected_models"] = [K2_SMALL, "another-model"] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root, matrix) + + results = evidence.load_and_validate_results(root, matrix) + + self.assertEqual( + len(results["llamacpp_validation_windows-vulkan.json"]), + len(matrix["include"][0]["expected_models"]), + ) + + def test_exact_lane_mapping_is_required(self) -> None: + mutations = {} + missing = expected_matrix() + missing["include"].pop() + mutations["missing"] = missing + wrong_backend = expected_matrix() + wrong_backend["include"][2]["backend"] = "cuda" + mutations["backend"] = wrong_backend + wrong_pin = expected_matrix() + wrong_pin["include"][3]["managed_pin"] = "cpu" + mutations["pin"] = wrong_pin + wrong_platform = expected_matrix() + wrong_platform["include"][1]["build_platform"] = "windows" + mutations["platform"] = wrong_platform + wrong_runner = expected_matrix() + wrong_runner["include"][0]["runner"] = ["windows-latest"] + mutations["runner"] = wrong_runner + with tempfile.TemporaryDirectory() as directory: root = Path(directory) self.write_expected_results(root) - (root / "llamacpp_validation_rocm-nightly.json").write_text( - json.dumps([passing_result(K2_SMALL), passing_result("another-model")]), - encoding="utf-8", - ) + for name, matrix in mutations.items(): + with self.subTest(name=name): + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "scheduled validation lanes|mapping", + ): + evidence.load_and_validate_results(root, matrix) - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "same planned model set", - ): - evidence.load_and_validate_results(root, matrix) + def test_large_k2_models_require_the_128gb_vulkan_lane(self) -> None: + for model in ( + planning.K2_MEDIUM, + planning.K2_LARGE, + f"builtin.{planning.K2_MEDIUM}", + f"builtin.{planning.K2_LARGE}", + ): + matrix = expected_matrix({"linux-vulkan": [K2_SMALL, model]}) + with self.subTest(model=model), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root, matrix) + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "128gb", + ): + evidence.load_and_validate_results(root, matrix) def test_capability_evidence_is_required_and_validated(self) -> None: mutations = {} @@ -296,17 +503,107 @@ def test_duplicate_json_keys_in_evidence_fail_closed(self) -> None: ): evidence.load_and_validate_results(root, expected_matrix()) + def test_nonstandard_json_numeric_constants_fail_closed(self) -> None: + for constant in ("NaN", "Infinity", "-Infinity"): + with ( + self.subTest(constant=constant), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + self.write_expected_results(root) + target = root / evidence.EXPECTED_RESULT_FILES[0] + raw = target.read_text(encoding="utf-8").replace( + '"tokens_per_second": 4.0', + f'"tokens_per_second": {constant}', + 1, + ) + target.write_text(raw, encoding="utf-8") + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "nonstandard JSON numeric constant", + ): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_overflowed_numeric_metrics_fail_closed(self) -> None: + for field, value in ( + ("time_to_first_token", "1e400"), + ("tokens_per_second", "1e400"), + ): + with self.subTest(field=field), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + target = root / evidence.EXPECTED_RESULT_FILES[0] + raw = target.read_text(encoding="utf-8") + raw = raw.replace( + f'"{field}": {passing_result()[field]}', + f'"{field}": {value}', + 1, + ) + target.write_text(raw, encoding="utf-8") + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "non-finite JSON number", + ): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_overflowed_numeric_extra_field_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + target = root / evidence.EXPECTED_RESULT_FILES[0] + raw = target.read_text(encoding="utf-8").replace( + "{", '{"untrusted_extra": 1e400,', 1 + ) + target.write_text(raw, encoding="utf-8") + + with self.assertRaisesRegex( + evidence.ValidationEvidenceError, + "non-finite JSON number", + ): + evidence.load_and_validate_results(root, expected_matrix()) + + def test_overflowed_numeric_extra_matrix_field_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_expected_results(root) + raw_matrix = json.dumps(expected_matrix()).replace( + "{", '{"untrusted_extra": 1e400,', 1 + ) + + with ( + mock.patch( + "sys.argv", + [ + "llamacpp_validation_evidence.py", + "--directory", + str(root), + "--expected-matrix-json", + raw_matrix, + ], + ), + self.assertRaisesRegex(SystemExit, "1"), + ): + evidence.main() + def test_non_designated_model_cannot_attach_capability_evidence(self) -> None: - other_model = "Other-Llama" - matrix = expected_matrix([K2_SMALL, other_model]) + matrix = expected_matrix() + planned_models = matrix["include"][0]["expected_models"] + other_model = next(model for model in planned_models if model != K2_SMALL) records = [ - passing_result(), - passing_result(other_model, capability=True), + passing_result( + model, + capability=model in {K2_SMALL, other_model}, + ) + for model in planned_models ] with tempfile.TemporaryDirectory() as directory: root = Path(directory) - for filename in evidence.EXPECTED_RESULT_FILES: - (root / filename).write_text(json.dumps(records), encoding="utf-8") + self.write_expected_results(root, matrix) + (root / "llamacpp_validation_windows-vulkan.json").write_text( + json.dumps(records), encoding="utf-8" + ) with self.assertRaisesRegex( evidence.ValidationEvidenceError, diff --git a/test/test_process_inheritance_contract.py b/test/test_process_inheritance_contract.py new file mode 100644 index 0000000000..2ae049f2bf --- /dev/null +++ b/test/test_process_inheritance_contract.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Source contracts for subprocess handle inheritance.""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WINDOWS_PROCESS_SOURCE = ( + ROOT / "src" / "cpp" / "server" / "utils" / "platform" / "process_windows.cpp" +) +WINDOWS_PROCESS_TEST_SOURCE = ROOT / "test" / "cpp" / "test_process_manager_windows.cpp" +CMAKE_SOURCE = ROOT / "CMakeLists.txt" +CPP_WORKFLOW_SOURCE = ( + ROOT / ".github" / "workflows" / "cpp_server_build_test_release.yml" +) +LINUX_PROCESS_SOURCE = ( + ROOT / "src" / "cpp" / "server" / "utils" / "platform" / "process_linux.cpp" +) +MACOS_PROCESS_SOURCE = ( + ROOT / "src" / "cpp" / "server" / "utils" / "platform" / "process_macos.cpp" +) +MCP_CLIENT_SOURCE = ROOT / "src" / "cpp" / "server" / "mcp_client.cpp" +SYSTEM_INFO_SOURCE = ROOT / "src" / "cpp" / "server" / "system_info.cpp" + + +class ProcessInheritanceContractTests(unittest.TestCase): + def test_windows_process_creation_uses_one_restricted_handle_gateway(self): + source = WINDOWS_PROCESS_SOURCE.read_text(encoding="utf-8") + + self.assertEqual(source.count("CreateProcessA("), 1) + self.assertEqual( + source.count("create_process_with_restricted_handles("), + 4, + ) + for required in ( + "STARTUPINFOEXA", + "PROC_THREAD_ATTRIBUTE_HANDLE_LIST", + "EXTENDED_STARTUPINFO_PRESENT", + "UpdateProcThreadAttribute", + "DuplicateHandle", + "DUPLICATE_SAME_ACCESS", + ): + with self.subTest(required=required): + self.assertIn(required, source) + + def test_windows_gateway_normalizes_missing_standard_handles(self): + source = WINDOWS_PROCESS_SOURCE.read_text(encoding="utf-8") + helper_start = source.index( + "static BOOL create_process_with_restricted_handles(" + ) + helper_end = source.index("\n}\n\n// Helper function", helper_start) + 2 + helper = source[helper_start:helper_end] + + for required in ( + "STARTF_USESTDHANDLES", + "fallback_handles", + '"NUL", access', + "GENERIC_READ", + "GENERIC_WRITE", + "GetFileType(source_handles[i])", + "source_handles[i] = fallback", + "CloseHandle(fallback_handles[i])", + ): + with self.subTest(required=required): + self.assertIn(required, helper) + + self.assertLess(helper.index('"NUL", access'), helper.index("DuplicateHandle(")) + + def test_windows_missing_standard_handle_runtime_test_is_wired_into_ci(self): + runtime_test = WINDOWS_PROCESS_TEST_SOURCE.read_text(encoding="utf-8") + cmake_source = CMAKE_SOURCE.read_text(encoding="utf-8") + workflow = CPP_WORKFLOW_SOURCE.read_text(encoding="utf-8") + + for required in ( + "InvalidStandardHandles", + "INVALID_HANDLE_VALUE", + "stale_value", + "GetFileType(handle)", + "--probe-stdio", + "quiet child receives valid standard handles", + "captured child receives valid standard handles", + ): + with self.subTest(required=required): + self.assertIn(required, runtime_test) + + self.assertIn("BUILD_TESTING AND WIN32", cmake_source) + self.assertIn("add_cpp_ci_test(ProcessManagerWindowsTest CI ON", cmake_source) + self.assertIn( + "--target test_mcp_client_config test_process_manager_windows", workflow + ) + self.assertIn("mcp_client_config|ProcessManagerWindowsTest", workflow) + + def test_other_windows_process_creation_cannot_inherit_unlisted_handles(self): + mcp_source = MCP_CLIENT_SOURCE.read_text(encoding="utf-8") + system_info_source = SYSTEM_INFO_SOURCE.read_text(encoding="utf-8") + + create_process_sites = {} + for source_path in (ROOT / "src" / "cpp").rglob("*.cpp"): + source = source_path.read_text(encoding="utf-8") + call_count = source.count("CreateProcessA(") + source.count( + "CreateProcessW(" + ) + if call_count: + create_process_sites[source_path.relative_to(ROOT).as_posix()] = ( + call_count + ) + self.assertEqual( + create_process_sites, + { + "src/cpp/server/mcp_client.cpp": 1, + "src/cpp/server/system_info.cpp": 1, + "src/cpp/server/utils/platform/process_windows.cpp": 1, + }, + ) + + self.assertEqual(mcp_source.count("CreateProcessW("), 1) + for required in ( + "STARTUPINFOEXW", + "PROC_THREAD_ATTRIBUTE_HANDLE_LIST", + "EXTENDED_STARTUPINFO_PRESENT", + ): + with self.subTest(required=required): + self.assertIn(required, mcp_source) + + self.assertEqual(system_info_source.count("CreateProcessA("), 1) + self.assertIn( + "nullptr, nullptr, FALSE, CREATE_NO_WINDOW", + system_info_source, + ) + + def test_linux_capture_pipes_are_atomically_close_on_exec(self): + source = LINUX_PROCESS_SOURCE.read_text(encoding="utf-8") + system_info_source = SYSTEM_INFO_SOURCE.read_text(encoding="utf-8") + helper_start = source.index("static bool create_pipe_above_standard_streams") + helper_end = source.index("\n}\n", helper_start) + 2 + helper = source[helper_start:helper_end] + + self.assertIn("pipe2(pipe_fds, O_CLOEXEC)", helper) + self.assertNotRegex(helper, r"(?", run_command_start + ) + run_command = process_source[run_command_start:run_command_end] + self.assertIn("spawn_capturing_output(", run_command) + self.assertNotIn("popen(", run_command) + reserved_options = run_command.index("int wait_options = WEXITED | WNOWAIT;") + reserved_exit = run_command.index("waitid(", reserved_options) + group_cleanup = run_command.index( + "kill_process_group(pid, false);", reserved_exit + ) + final_reap = run_command.index("waitpid(pid, &status, 0)", group_cleanup) + self.assertNotIn("waitpid(pid, &status, WNOHANG)", run_command) + self.assertLess(reserved_options, reserved_exit) + self.assertLess(reserved_exit, group_cleanup) + self.assertLess(group_cleanup, final_reap) + + spawn_start = process_source.index("ProcessHandle MacOSProcessPlatform::spawn(") + spawn_end = process_source.index( + "void MacOSProcessPlatform::terminate(", spawn_start + ) + spawn = process_source[spawn_start:spawn_end] + for standard_stream in ("STDIN_FILENO", "STDOUT_FILENO", "STDERR_FILENO"): + with self.subTest(standard_stream=standard_stream): + self.assertIn( + f"file_actions.add_inherit_if_open({standard_stream})", spawn + ) + + apple_branch_start = mcp_source.index( + "#ifdef __APPLE__", mcp_source.index("if (pipe_creation_failed)") + ) + apple_branch_end = mcp_source.index("#else", apple_branch_start) + apple_branch = mcp_source[apple_branch_start:apple_branch_end] + self.assertNotIn("::fork()", apple_branch) + for required in ( + "posix_spawn(", + "posix_spawn_file_actions_adddup2", + "posix_spawn_file_actions_addclose", + "posix_spawn_file_actions_addchdir_np", + "posix_spawnattr_setpgroup(&attributes, 0)", + "POSIX_SPAWN_CLOEXEC_DEFAULT", + "POSIX_SPAWN_SETPGROUP", + ): + with self.subTest(required=required): + self.assertIn(required, apple_branch) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_publish_llamacpp_update.py b/test/test_publish_llamacpp_update.py index 21fa79c352..ca218cf7a0 100644 --- a/test/test_publish_llamacpp_update.py +++ b/test/test_publish_llamacpp_update.py @@ -13,10 +13,14 @@ import unittest from pathlib import Path +from test.utils import llamacpp_release_assets as release_assets from test.utils import llamacpp_release_manifest as release_manifest ROOT = Path(__file__).resolve().parents[1] PUBLISHER = ROOT / ".github" / "scripts" / "publish_llamacpp_update.sh" +SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" +CHECKOUT_SHA = "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09" +DOWNLOAD_ARTIFACT_SHA = "37930b1c2abaa49bbe596cd826c3c89aef350131" VERSIONS_PATH = Path("src/cpp/resources/backend_versions.json") MANIFEST_PATH = Path(".github/llamacpp_release_manifest.json") @@ -32,6 +36,10 @@ def setUp(self) -> None: self.event_log = self.temp_root / "events.log" self.snapshot_count = self.temp_root / "snapshot-count.txt" self.closed_prs = self.temp_root / "closed-prs.txt" + self.dynamic_open_prs = self.temp_root / "dynamic-open-prs.txt" + self.pr_create_failed = self.temp_root / "pr-create-failed.txt" + self.pr_draft_state = self.temp_root / "pr-draft-state.txt" + self.rocm_latest_query_count = self.temp_root / "rocm-latest-count.txt" self.real_git = shutil.which("git") if not self.real_git: self.skipTest("git is required") @@ -41,13 +49,47 @@ def setUp(self) -> None: self.run_git("init", "--quiet") self.run_git("config", "user.name", "Test Author") self.run_git("config", "user.email", "test@example.com") + self.base_versions = { + "llamacpp": { + "cpu": "b1", + "cuda": "b1", + "metal": "b1", + "rocm-nightly": "b1", + "rocm-stable": "b1", + "vulkan": "b1", + }, + "rocm_asset_families": {}, + "therock": {"version": "7.14.0"}, + } + self.candidate_versions = { + **self.base_versions, + "llamacpp": { + "cpu": "b100", + "cuda": "b200", + "metal": "b100", + "rocm-nightly": "b300", + "rocm-stable": "b200", + "vulkan": "b100", + }, + } versions = self.repo / VERSIONS_PATH versions.parent.mkdir(parents=True) versions.write_text( - '{"llamacpp":{"vulkan":"b1","cpu":"b1"}}\n', encoding="utf-8" + json.dumps(self.base_versions, separators=(",", ":")) + "\n", + encoding="utf-8", ) self.upstream_head = "f" * 40 self.live_upstream_head = "9" * 40 + requirements = release_assets.build_asset_requirements( + ggml_release="b100", + rocm_release="b300", + lemonade_release="b200", + backend_versions=self.candidate_versions, + ) + rocm_target_requirements = release_assets.build_rocm_asset_target_requirements( + rocm_release="b300", + backend_versions=self.candidate_versions, + ) self.release_payloads = { "ggml": self.release_payload( repository="ggml-org/llama.cpp", @@ -55,6 +97,7 @@ def setUp(self) -> None: release_id=100, release_tag_commit="a" * 40, claimed_source_commit="a" * 40, + asset_names=self.asset_names(requirements["ggml"]), ), "rocm": self.release_payload( repository="lemonade-sdk/llamacpp-rocm", @@ -62,6 +105,8 @@ def setUp(self) -> None: release_id=300, release_tag_commit="b" * 40, claimed_source_commit="c" * 40, + asset_names=self.asset_names(requirements["rocm"]), + build_targets_by_asset=rocm_target_requirements, ), "lemonade": self.release_payload( repository="lemonade-sdk/llama.cpp", @@ -69,6 +114,7 @@ def setUp(self) -> None: release_id=200, release_tag_commit="d" * 40, claimed_source_commit="e" * 40, + asset_names=self.asset_names(requirements["lemonade"]), ), } self.expected_manifest = self.build_expected_manifest() @@ -101,31 +147,46 @@ def release_payload( release_id: int, release_tag_commit: str, claimed_source_commit: str, + asset_names: list[str], + build_targets_by_asset: dict[str, tuple[str, ...]] | None = None, ) -> dict: - binary_asset = { - "digest": "sha256:" + f"{release_id:064x}"[-64:], - "id": release_id * 10, - "name": f"asset-{release_id}.zip", - "size": release_id, - "state": "uploaded", - "updated_at": "2026-09-06T12:00:00Z", - } - assets = [binary_asset] + assets = [ + { + "digest": "sha256:" + + hashlib.sha256( + f"{repository}@{tag}:{asset_name}".encode("utf-8") + ).hexdigest(), + "id": release_id * 1000 + index, + "name": asset_name, + "size": release_id + index, + "state": "uploaded", + "updated_at": "2026-09-06T12:00:00Z", + } + for index, asset_name in enumerate(asset_names, start=1) + ] source_manifest = None + build_target_attestations = [] if repository != "ggml-org/llama.cpp": + source_bindings = [] + for asset in assets: + binding = { + "digest": asset["digest"], + "name": asset["name"], + "size": asset["size"], + } + if build_targets_by_asset is not None: + binding["build_targets"] = list( + build_targets_by_asset.get(asset["name"], ()) + ) + build_target_attestations.append(dict(binding)) + source_bindings.append(binding) source_manifest = json.dumps( { - "assets": [ - { - "digest": binary_asset["digest"], - "name": binary_asset["name"], - "size": binary_asset["size"], - } - ], + "assets": source_bindings, "release_repository": repository, "release_tag": tag, "release_tag_commit": release_tag_commit, - "schema_version": 1, + "schema_version": 2 if build_targets_by_asset is not None else 1, "source_commit": claimed_source_commit, "source_repository": "ggml-org/llama.cpp", }, @@ -151,10 +212,19 @@ def release_payload( "immutable": True, "tag_name": tag, "_claimed_source_commit": claimed_source_commit, + "_build_target_attestations": build_target_attestations, "_release_tag_commit": release_tag_commit, "_source_manifest": source_manifest, } + @staticmethod + def asset_names(requirements: dict[str, list[str]]) -> list[str]: + return [ + asset_name + for backend_assets in requirements.values() + for asset_name in backend_assets + ] + def build_expected_manifest(self) -> str: releases = [] for key, repository, tag in ( @@ -164,6 +234,7 @@ def build_expected_manifest(self) -> str: ): payload = dict(self.release_payloads[key]) claimed_commit = payload.pop("_claimed_source_commit") + build_target_attestations = payload.pop("_build_target_attestations") release_tag_commit = payload.pop("_release_tag_commit") payload.pop("_source_manifest") payload.update( @@ -174,6 +245,7 @@ def build_expected_manifest(self) -> str: else "immutable-source-manifest" ), "publisher_claimed_source_commit": claimed_commit, + "build_target_attestations": build_target_attestations, "release_tag_commit": release_tag_commit, "source_repository": "ggml-org/llama.cpp", "upstream_reference": "refs/heads/master", @@ -214,6 +286,24 @@ def install_command_wrappers(self) -> None: "printf 'git' >> \"$EVENT_LOG\"\n" 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' "printf '\\n' >> \"$EVENT_LOG\"\n" + 'if [ "$1" = push ]; then\n' + ' if [ "${GIT_TERMINAL_PROMPT:-}" != 0 ] || ' + '[ "${GIT_CONFIG_KEY_0:-}" != credential.helper ] || ' + '[ "${GIT_CONFIG_VALUE_0+x}" != x ] || ' + '[ "${GIT_CONFIG_VALUE_0}" != "" ] || ' + '[ ! -x "${GIT_ASKPASS:-}" ]; then\n' + " printf 'missing ephemeral authenticated push configuration\\n' >&2\n" + " exit 96\n" + " fi\n" + ' username=$("$GIT_ASKPASS" "Username for https://github.com")\n' + ' password=$("$GIT_ASKPASS" "Password for https://github.com")\n' + ' if [ "$username" != x-access-token ] || ' + '[ "$password" != "$GH_TOKEN" ]; then\n' + " printf 'invalid ephemeral authenticated push configuration\\n' >&2\n" + " exit 97\n" + " fi\n" + " printf 'authenticated-push\\n' >> \"$EVENT_LOG\"\n" + "fi\n" 'if [ "$1" = push ] && ' '[ "$MUTATE_BRANCH_AFTER_PUSH" = true ]; then\n' ' "$REAL_GIT" "$@" || exit $?\n' @@ -232,18 +322,82 @@ def install_command_wrappers(self) -> None: "printf 'gh' >> \"$EVENT_LOG\"\n" 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' "printf '\\n' >> \"$EVENT_LOG\"\n" + 'if [ "$1" = api ] && [ "$2" = graphql ]; then\n' + " printf '%s\\n' \"$AUTHENTICATED_ACTOR\"\n" + " exit 0\n" + "fi\n" 'if [ "$1" = pr ] && [ "$2" = create ]; then\n' + " requested_draft=false\n" + ' case " $* " in *" --draft "*) requested_draft=true ;; esac\n' + ' if [ "$MUTATE_BRANCH_BEFORE_PR_CREATE_FAILURE" = true ]; then\n' + ' "$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" update-ref ' + '"refs/heads/$EXPECTED_BRANCH" "$VALIDATED_BASE_SHA"\n' + " fi\n" + ' if [ "$PR_CREATE_EXIT_CODE" -ne 0 ]; then\n' + ' if [ "$CREATE_PR_BEFORE_FAILURE" = true ]; then\n' + ' head_sha=$("$REAL_GIT" ' + '--git-dir="$REMOTE_REPOSITORY_PATH" show-ref --verify --hash ' + '"refs/heads/$EXPECTED_BRANCH")\n' + " printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' " + '"$CREATED_PR_NUMBER" "$EXPECTED_BRANCH" "LLM360/lemonade" ' + '"$EXPECTED_PUBLICATION_ACTOR" "main" "$head_sha" ' + '> "$DYNAMIC_OPEN_PRS_FILE"\n' + ' if [ "$requested_draft" = true ]; then\n' + " printf 'draft\\n' > \"$PR_DRAFT_STATE_FILE\"\n" + " fi\n" + ' if [ "$DUPLICATE_PR_BEFORE_FAILURE" = true ]; then\n' + " printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' " + '"42" "$EXPECTED_BRANCH" "LLM360/lemonade" ' + '"$EXPECTED_PUBLICATION_ACTOR" "main" "$head_sha" ' + '>> "$DYNAMIC_OPEN_PRS_FILE"\n' + " fi\n" + " fi\n" + " printf 'failed\\n' > \"$PR_CREATE_FAILED_FILE\"\n" + " printf 'simulated pull request creation failure\\n' >&2\n" + ' exit "$PR_CREATE_EXIT_CODE"\n' + " fi\n" + ' if [ "$MUTATE_BRANCH_AFTER_PR_CREATE" = true ]; then\n' + ' "$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" update-ref ' + '"refs/heads/$EXPECTED_BRANCH" "$VALIDATED_BASE_SHA"\n' + " fi\n" + ' if [ "$requested_draft" = true ]; then\n' + " printf 'draft\\n' > \"$PR_DRAFT_STATE_FILE\"\n" + " fi\n" " printf 'https://github.com/LLM360/lemonade/pull/%s\\n' " '"$CREATED_PR_NUMBER"\n' " exit 0\n" "fi\n" + 'if [ "$1" = pr ] && [ "$2" = ready ]; then\n' + ' if [ "$PR_READY_EXIT_CODE" -ne 0 ]; then\n' + " printf 'simulated pull request ready failure\\n' >&2\n" + ' exit "$PR_READY_EXIT_CODE"\n' + " fi\n" + ' if [ "$PR_READY_CLOSES_CURRENT" = true ]; then\n' + ' printf \'%s\\n\' "$3" >> "$CLOSED_PRS_FILE"\n' + " fi\n" + ' if [ "$PR_READY_NOOP" != true ]; then\n' + ' rm -f "$PR_DRAFT_STATE_FILE"\n' + " fi\n" + " exit 0\n" + "fi\n" 'if [ "$1" = api ]; then\n' ' case "$2" in\n' " repos/ggml-org/llama.cpp) printf '%s\\n' master ;;\n" " repos/ggml-org/llama.cpp/releases) " "printf '%s\\n' \"$FRESH_GGML_RELEASE\" ;;\n" - " repos/lemonade-sdk/llamacpp-rocm/releases/latest) " - "printf '%s\\n' \"$FRESH_ROCM_RELEASE\" ;;\n" + " repos/lemonade-sdk/llamacpp-rocm/releases/latest)\n" + " count=0\n" + ' if [ -f "$ROCM_LATEST_QUERY_COUNT_FILE" ]; then ' + 'count=$(cat "$ROCM_LATEST_QUERY_COUNT_FILE"); fi\n' + " count=$((count + 1))\n" + " printf '%s\\n' \"$count\" > " + '"$ROCM_LATEST_QUERY_COUNT_FILE"\n' + ' if [ "$ROCM_RELEASE_DRIFT_AT" -gt 0 ] && ' + '[ "$count" -ge "$ROCM_RELEASE_DRIFT_AT" ]; then\n' + " printf 'b301\\n'\n" + " else\n" + " printf '%s\\n' \"$FRESH_ROCM_RELEASE\"\n" + " fi ;;\n" " repos/lemonade-sdk/llama.cpp/releases/latest) " "printf '%s\\n' \"$FRESH_LEMONADE_RELEASE\" ;;\n" " repos/ggml-org/llama.cpp/releases/tags/b100)\n" @@ -266,14 +420,16 @@ def install_command_wrappers(self) -> None: "printf '%s' \"$ROCM_SOURCE_MANIFEST\" ;;\n" " repos/lemonade-sdk/llama.cpp/releases/assets/2001) " "printf '%s' \"$LEMONADE_SOURCE_MANIFEST\" ;;\n" - " repos/ggml-org/llama.cpp/commits/b100) " + " repos/ggml-org/llama.cpp/commits/tags/b100) " "printf '%s\\n' \"$GGML_RELEASE_TAG_COMMIT\" ;;\n" - " repos/lemonade-sdk/llamacpp-rocm/commits/b300) " + " repos/lemonade-sdk/llamacpp-rocm/commits/tags/b300) " "printf '%s\\n' \"$ROCM_RELEASE_TAG_COMMIT\" ;;\n" - " repos/lemonade-sdk/llama.cpp/commits/b200) " + " repos/lemonade-sdk/llama.cpp/commits/tags/b200) " "printf '%s\\n' \"$LEMONADE_RELEASE_TAG_COMMIT\" ;;\n" - " repos/ggml-org/llama.cpp/commits/master) " + " repos/ggml-org/llama.cpp/commits/heads/master) " "printf '%s\\n' \"$LIVE_UPSTREAM_HEAD\" ;;\n" + " repos/*/commits/b*|repos/*/commits/master) " + "printf '%040d\\n' 7 ;;\n" " repos/ggml-org/llama.cpp/compare/*)\n" " comparison=${2##*/}; base=${comparison%%...*}; " "head=${comparison#*...}\n" @@ -287,8 +443,13 @@ def install_command_wrappers(self) -> None: '"merge_base_commit":{"sha":"%s"},' '"url":"https://api.github.com/repos/ggml-org/llama.cpp/compare/%s...%s"}\\n\' ' '"$status" "$base" "$merge_base" "$base" "$head" ;;\n' - " repos/LLM360/lemonade/pulls) " - "printf '%s\\n' \"$OPEN_PR_RECORDS\" ;;\n" + " repos/LLM360/lemonade/pulls)\n" + ' if [ -f "$PR_CREATE_FAILED_FILE" ] && ' + '[ "$FAIL_POST_CREATE_PULLS_QUERY" = true ]; then exit 92; fi\n' + " printf '%s\\n' \"$OPEN_PR_RECORDS\"\n" + ' if [ -f "$DYNAMIC_OPEN_PRS_FILE" ]; then ' + "sed -n '1,$p' \"$DYNAMIC_OPEN_PRS_FILE\"; fi ;;\n" + " repos/LLM360/lemonade/actions/permissions/workflow) exit 91 ;;\n" " repos/LLM360/lemonade/pulls/[0-9]*)\n" " number=${2##*/}\n" ' case " $* " in\n' @@ -297,7 +458,12 @@ def install_command_wrappers(self) -> None: " printf '{}\\n'\n" " exit 0 ;;\n" " esac\n" - " record=$(printf '%s\\n' \"$OPEN_PR_RECORDS\" | " + " records=$(printf '%s\\n' \"$OPEN_PR_RECORDS\")\n" + ' if [ -f "$DYNAMIC_OPEN_PRS_FILE" ]; then\n' + " records=$(printf '%s\\n%s\\n' \"$records\" " + '"$(sed -n \'1,$p\' "$DYNAMIC_OPEN_PRS_FILE")")\n' + " fi\n" + " record=$(printf '%s\\n' \"$records\" | " "awk -F '\\t' -v number=\"$number\" '$1 == number { print; exit }')\n" " head_ref=$(printf '%s' \"$record\" | cut -f2)\n" " head_repository=$(printf '%s' \"$record\" | cut -f3)\n" @@ -305,13 +471,17 @@ def install_command_wrappers(self) -> None: " base_ref=$(printf '%s' \"$record\" | cut -f5)\n" " head_ref=${head_ref:-$EXPECTED_BRANCH}\n" " head_repository=${head_repository:-LLM360/lemonade}\n" - " author=${author:-github-actions[bot]}\n" + " author=${author:-$EXPECTED_PUBLICATION_ACTOR}\n" " base_ref=${base_ref:-main}\n" " head_ref=${PR_DETAIL_HEAD_REF:-$head_ref}\n" " head_repository=${PR_DETAIL_HEAD_REPOSITORY:-$head_repository}\n" " author=${PR_DETAIL_AUTHOR:-$author}\n" " base_ref=${PR_DETAIL_BASE_REF:-$base_ref}\n" " state=open\n" + " draft=false\n" + ' if [ "$number" = "$CREATED_PR_NUMBER" ] && ' + '[ -f "$PR_DRAFT_STATE_FILE" ]; then draft=true; fi\n' + " draft=${PR_DETAIL_DRAFT:-$draft}\n" ' if [ -f "$CLOSED_PRS_FILE" ] && ' 'grep -qx "$number" "$CLOSED_PRS_FILE"; then state=closed; fi\n' ' head_sha=$("$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" ' @@ -322,11 +492,11 @@ def install_command_wrappers(self) -> None: 'show-ref --verify --hash "refs/heads/$base_ref" 2>/dev/null || ' "printf '%040d' 0)\n" " printf " - '\'{"state":"%s","head":{"ref":"%s",' + '\'{"state":"%s","draft":%s,"head":{"ref":"%s",' '"sha":"%s","repo":{"full_name":"%s"}},' '"user":{"login":"%s"},"base":{"ref":"%s",' '"sha":"%s"}}\\n\' ' - '"$state" "$head_ref" "$head_sha" "$head_repository" ' + '"$state" "$draft" "$head_ref" "$head_sha" "$head_repository" ' '"$author" "$base_ref" "$base_sha" ;;\n' " esac\n" "fi\n" @@ -338,14 +508,28 @@ def install_command_wrappers(self) -> None: def publisher_env( self, *, + authenticated_actor: str = "github-actions[bot]", + create_pr_before_failure: bool = False, divergent_commit: str = "", + duplicate_pr_before_failure: bool = False, + fail_post_create_pulls_query: bool = False, include_expected_manifest: bool = True, + include_publication_token: bool = True, mutate_on_snapshot: int = 0, open_pr_records: str = "", fresh_ggml_release: str = "b100", mutate_branch_after_push: bool = False, + mutate_branch_after_pr_create: bool = False, + mutate_branch_before_pr_create_failure: bool = False, + pr_create_exit_code: int = 0, pr_detail_base_ref: str = "", pr_detail_head_ref: str = "", + pr_detail_draft: str = "", + pr_ready_closes_current: bool = False, + pr_ready_exit_code: int = 0, + pr_ready_noop: bool = False, + rocm_release_drift_at: int = 0, + expected_publication_actor: str | None = "github-actions[bot]", validated_base_sha: str | None = None, ) -> dict[str, str]: api_payloads = {} @@ -359,7 +543,18 @@ def publisher_env( env = os.environ.copy() env.update( { + "AUTHENTICATED_ACTOR": authenticated_actor, + "CREATE_PR_BEFORE_FAILURE": ( + "true" if create_pr_before_failure else "false" + ), "DIVERGENT_COMMIT": divergent_commit, + "DYNAMIC_OPEN_PRS_FILE": str(self.dynamic_open_prs), + "DUPLICATE_PR_BEFORE_FAILURE": ( + "true" if duplicate_pr_before_failure else "false" + ), + "FAIL_POST_CREATE_PULLS_QUERY": ( + "true" if fail_post_create_pulls_query else "false" + ), "CLOSED_PRS_FILE": str(self.closed_prs), "CREATED_PR_NUMBER": "41", "EVENT_LOG": str(self.event_log), @@ -389,17 +584,34 @@ def publisher_env( "MUTATE_BRANCH_AFTER_PUSH": ( "true" if mutate_branch_after_push else "false" ), + "MUTATE_BRANCH_AFTER_PR_CREATE": ( + "true" if mutate_branch_after_pr_create else "false" + ), + "MUTATE_BRANCH_BEFORE_PR_CREATE_FAILURE": ( + "true" if mutate_branch_before_pr_create_failure else "false" + ), "OPEN_PR_RECORDS": open_pr_records, "PATH": f"{self.fake_bin}{os.pathsep}{env['PATH']}", "PR_BODY_FILE": "pr_body.md", + "PR_CREATE_EXIT_CODE": str(pr_create_exit_code), + "PR_CREATE_FAILED_FILE": str(self.pr_create_failed), + "PR_DRAFT_STATE_FILE": str(self.pr_draft_state), "PR_DETAIL_AUTHOR": "", "PR_DETAIL_BASE_REF": pr_detail_base_ref, "PR_DETAIL_HEAD_REF": pr_detail_head_ref, + "PR_DETAIL_DRAFT": pr_detail_draft, "PR_DETAIL_HEAD_REPOSITORY": "", "PR_DETAIL_HEAD_SHA": "", + "PR_READY_CLOSES_CURRENT": ( + "true" if pr_ready_closes_current else "false" + ), + "PR_READY_EXIT_CODE": str(pr_ready_exit_code), + "PR_READY_NOOP": "true" if pr_ready_noop else "false", "REAL_GIT": self.real_git, "REMOTE_REPOSITORY_PATH": str(self.remote), "ROCM_RELEASE_PAYLOAD": json.dumps(api_payloads["rocm"]), + "ROCM_LATEST_QUERY_COUNT_FILE": str(self.rocm_latest_query_count), + "ROCM_RELEASE_DRIFT_AT": str(rocm_release_drift_at), "ROCM_SOURCE_MANIFEST": self.release_payloads["rocm"][ "_source_manifest" ], @@ -411,6 +623,14 @@ def publisher_env( or self.run_git("rev-parse", "HEAD"), } ) + if expected_publication_actor is None: + env.pop("EXPECTED_PUBLICATION_ACTOR", None) + else: + env["EXPECTED_PUBLICATION_ACTOR"] = expected_publication_actor + if include_publication_token: + env["GH_TOKEN"] = "dedicated-publication-token" + else: + env.pop("GH_TOKEN", None) if include_expected_manifest: env["EXPECTED_RELEASE_ASSET_MANIFEST"] = self.expected_manifest return env @@ -418,28 +638,58 @@ def publisher_env( def run_publisher( self, *, + authenticated_actor: str = "github-actions[bot]", + create_pr_before_failure: bool = False, divergent_commit: str = "", + duplicate_pr_before_failure: bool = False, + fail_post_create_pulls_query: bool = False, include_expected_manifest: bool = True, + include_publication_token: bool = True, mutate_on_snapshot: int = 0, open_pr_records: str = "", fresh_ggml_release: str = "b100", mutate_branch_after_push: bool = False, + mutate_branch_after_pr_create: bool = False, + mutate_branch_before_pr_create_failure: bool = False, + pr_create_exit_code: int = 0, pr_detail_base_ref: str = "", pr_detail_head_ref: str = "", + pr_detail_draft: str = "", + pr_ready_closes_current: bool = False, + pr_ready_exit_code: int = 0, + pr_ready_noop: bool = False, + rocm_release_drift_at: int = 0, + expected_publication_actor: str | None = "github-actions[bot]", validated_base_sha: str | None = None, ) -> subprocess.CompletedProcess[str]: return self.run_command( ["bash", str(PUBLISHER)], cwd=self.repo, env=self.publisher_env( + authenticated_actor=authenticated_actor, + create_pr_before_failure=create_pr_before_failure, divergent_commit=divergent_commit, + duplicate_pr_before_failure=duplicate_pr_before_failure, + fail_post_create_pulls_query=fail_post_create_pulls_query, include_expected_manifest=include_expected_manifest, + include_publication_token=include_publication_token, mutate_on_snapshot=mutate_on_snapshot, open_pr_records=open_pr_records, fresh_ggml_release=fresh_ggml_release, mutate_branch_after_push=mutate_branch_after_push, + mutate_branch_after_pr_create=mutate_branch_after_pr_create, + mutate_branch_before_pr_create_failure=( + mutate_branch_before_pr_create_failure + ), + pr_create_exit_code=pr_create_exit_code, pr_detail_base_ref=pr_detail_base_ref, pr_detail_head_ref=pr_detail_head_ref, + pr_detail_draft=pr_detail_draft, + pr_ready_closes_current=pr_ready_closes_current, + pr_ready_exit_code=pr_ready_exit_code, + pr_ready_noop=pr_ready_noop, + rocm_release_drift_at=rocm_release_drift_at, + expected_publication_actor=expected_publication_actor, validated_base_sha=validated_base_sha, ), check=False, @@ -447,10 +697,23 @@ def run_publisher( def change_versions(self) -> None: (self.repo / VERSIONS_PATH).write_text( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n', + json.dumps(self.candidate_versions, separators=(",", ":")) + "\n", encoding="utf-8", ) + def advance_base_versions(self, versions: dict[str, str]) -> None: + document = { + **self.base_versions, + "llamacpp": {**self.base_versions["llamacpp"], **versions}, + } + (self.repo / VERSIONS_PATH).write_text( + json.dumps(document, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + self.run_git("add", str(VERSIONS_PATH)) + self.run_git("commit", "--quiet", "--message", "advance backend pins") + self.run_git("push", "--quiet", "origin", "main") + def trusted_open_pr_record(self, number: int = 17) -> str: return ( f"{number}\t{self.branch_name}\tLLM360/lemonade\t" @@ -464,7 +727,18 @@ def create_remote_update_branch( unrelated_contents: str | None = None, ) -> tuple[str, str, str]: base_oid = self.run_git("rev-parse", "HEAD") - (self.repo / VERSIONS_PATH).write_text(versions_contents, encoding="utf-8") + overrides = json.loads(versions_contents) + document = { + **self.candidate_versions, + "llamacpp": { + **self.candidate_versions["llamacpp"], + **overrides["llamacpp"], + }, + } + (self.repo / VERSIONS_PATH).write_text( + json.dumps(document, separators=(",", ":")) + "\n", + encoding="utf-8", + ) if unrelated_contents is not None: (self.repo / "unrelated.txt").write_text( unrelated_contents, encoding="utf-8" @@ -487,7 +761,114 @@ def events(self) -> list[str]: return [] return self.event_log.read_text(encoding="utf-8").splitlines() - def test_unchanged_versions_reconcile_before_exiting(self) -> None: + def assert_no_remote_mutation(self) -> None: + events = self.events() + self.assertFalse(any(event.startswith("git ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertFalse( + any( + event.startswith("gh ") + and any( + marker in event + for marker in ( + "<--method> ", + "<--method> ", + "<--method> ", + "<-f> ", + ) + ) + for event in events + ) + ) + + def test_missing_publication_token_fails_before_any_mutation(self) -> None: + self.change_versions() + body_before = (self.repo / "pr_body.md").read_text(encoding="utf-8") + manifest_before = (self.repo / MANIFEST_PATH).read_text(encoding="utf-8") + + result = self.run_publisher(include_publication_token=False) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("GH_TOKEN", result.stderr) + self.assertEqual(self.events(), []) + self.assertEqual( + (self.repo / "pr_body.md").read_text(encoding="utf-8"), body_before + ) + self.assertEqual( + (self.repo / MANIFEST_PATH).read_text(encoding="utf-8"), manifest_before + ) + + def test_missing_expected_publication_actor_fails_before_any_mutation( + self, + ) -> None: + self.change_versions() + body_before = (self.repo / "pr_body.md").read_text(encoding="utf-8") + + result = self.run_publisher(expected_publication_actor=None) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("EXPECTED_PUBLICATION_ACTOR", result.stderr) + self.assertEqual(self.events(), []) + self.assertEqual( + (self.repo / "pr_body.md").read_text(encoding="utf-8"), body_before + ) + + def test_authenticated_actor_mismatch_fails_before_any_mutation(self) -> None: + self.change_versions() + body_before = (self.repo / "pr_body.md").read_text(encoding="utf-8") + + result = self.run_publisher( + authenticated_actor="octocat", + expected_publication_actor="lemonade-release[bot]", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("authenticated publication actor", result.stderr) + self.assertEqual( + self.events(), + [ + "gh <-f> " + "<--jq> <.data.viewer.login>" + ], + ) + self.assert_no_remote_mutation() + self.assertEqual( + (self.repo / "pr_body.md").read_text(encoding="utf-8"), body_before + ) + + def test_configured_publication_actor_owns_commit_and_pull_request( + self, + ) -> None: + actor = "lemonade-release[bot]" + self.change_versions() + + result = self.run_publisher( + authenticated_actor=actor, + expected_publication_actor=actor, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.run_git("show", "-s", "--format=%an", "HEAD"), actor) + self.assertEqual( + self.run_git("show", "-s", "--format=%ae", "HEAD"), + f"{actor}@users.noreply.github.com", + ) + self.assertTrue( + any(event.startswith("gh ") for event in self.events()) + ) + self.assertTrue( + any(event.startswith("gh ") for event in self.events()) + ) + + def test_push_uses_ephemeral_askpass_without_persisting_the_token(self) -> None: + self.change_versions() + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("authenticated-push", self.events()) + + def test_unchanged_versions_report_stale_pr_for_manual_cleanup(self) -> None: stale_record = ( "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" "LLM360/lemonade\tgithub-actions[bot]\tmain" @@ -497,16 +878,36 @@ def test_unchanged_versions_reconcile_before_exiting(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("nothing to update", result.stdout) + self.assertIn( + "Superseded pull request #8 " + "(auto/llamacpp-update-b90-b190-b290-deadbeef1234) " + "requires manual cleanup.", + result.stdout, + ) events = self.events() - self.assertTrue( - any( - event.startswith("gh ") - and "<-f> " in event - for event in events - ) + self.assertFalse( + any("" in event for event in events) ) + self.assertFalse(any("<-f> " in event for event in events)) self.assertFalse(any(event.startswith("git ") for event in events)) + def test_unchanged_versions_preserve_stale_pr_when_release_tags_drift( + self, + ) -> None: + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + open_pr_records=stale_record, + rocm_release_drift_at=2, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("candidate releases changed", result.stderr) + self.assertFalse(any("<-f> " in event for event in self.events())) + def test_untrusted_stale_pull_request_is_not_closed(self) -> None: stale_record = ( "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" @@ -546,6 +947,56 @@ def test_stale_candidate_fails_before_query_or_push(self) -> None: ) self.assertFalse(any(event.startswith("git ") for event in events)) + def test_scheduled_publication_rejects_all_managed_pin_downgrades(self) -> None: + self.advance_base_versions({"vulkan": "b6000", "cpu": "b6000"}) + self.change_versions() + + result = self.run_publisher() + + self.assertNotEqual(result.returncode, 0) + self.assertIn("would downgrade llamacpp.cpu from b6000 to b100", result.stderr) + events = self.events() + self.assertFalse(any(event.startswith("git ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + + def test_scheduled_publication_rejects_a_mixed_backend_downgrade(self) -> None: + self.advance_base_versions({"vulkan": "b50", "cpu": "b150"}) + self.change_versions() + + result = self.run_publisher() + + self.assertNotEqual(result.returncode, 0) + self.assertIn("would downgrade llamacpp.cpu from b150 to b100", result.stderr) + events = self.events() + self.assertFalse(any(event.startswith("git ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + + def test_scheduled_publication_rejects_arbitrary_precision_downgrade( + self, + ) -> None: + self.advance_base_versions({"cpu": "b9223372036854775808"}) + candidate = { + **self.candidate_versions, + "llamacpp": { + **self.candidate_versions["llamacpp"], + "cpu": "b9223372036854775807", + }, + } + (self.repo / VERSIONS_PATH).write_text( + json.dumps(candidate, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + + result = self.run_publisher() + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "would downgrade llamacpp.cpu from b9223372036854775808 " + "to b9223372036854775807", + result.stderr, + ) + self.assert_no_remote_mutation() + def test_missing_release_manifest_fails_before_external_queries(self) -> None: self.change_versions() @@ -574,6 +1025,75 @@ def test_release_mutation_at_final_recheck_prevents_remote_mutation(self) -> Non self.assertFalse(any(event.startswith("gh ") for event in events)) self.assertFalse(any("<-f> " in event for event in events)) + def test_missing_required_asset_prevents_remote_mutation(self) -> None: + self.change_versions() + missing_asset = "llama-b100-bin-win-vulkan-x64.zip" + self.release_payloads["ggml"]["assets"] = [ + asset + for asset in self.release_payloads["ggml"]["assets"] + if asset["name"] != missing_asset + ] + self.expected_manifest = self.build_expected_manifest() + + result = self.run_publisher() + + self.assertNotEqual(result.returncode, 0) + self.assertIn("llamacpp.vulkan", result.stderr) + self.assertIn(missing_asset, result.stderr) + self.assert_no_remote_mutation() + + def test_zero_size_required_asset_prevents_remote_mutation(self) -> None: + self.change_versions() + zero_size_asset = "llama-b100-bin-ubuntu-arm64.tar.gz" + for asset in self.release_payloads["ggml"]["assets"]: + if asset["name"] == zero_size_asset: + asset["size"] = 0 + break + else: + self.fail(f"fixture is missing required asset {zero_size_asset}") + self.expected_manifest = self.build_expected_manifest() + + result = self.run_publisher() + + self.assertNotEqual(result.returncode, 0) + self.assertIn("llamacpp.cpu", result.stderr) + self.assertIn("positive size", result.stderr) + self.assertIn(zero_size_asset, result.stderr) + self.assert_no_remote_mutation() + + def test_capture_uses_namespaced_refs_when_branch_and_tag_names_collide( + self, + ) -> None: + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + events = self.events() + self.assertTrue( + any( + "" in event + for event in events + ) + ) + for repository, tag in ( + ("ggml-org/llama.cpp", "b100"), + ("lemonade-sdk/llamacpp-rocm", "b300"), + ("lemonade-sdk/llama.cpp", "b200"), + ): + self.assertTrue( + any( + f"" in event + for event in events + ) + ) + self.assertFalse( + any(f"" in event for event in events) + ) + self.assertFalse( + any( + "" in event for event in events + ) + ) + def test_mutable_release_prevents_remote_mutation(self) -> None: self.change_versions() self.release_payloads["rocm"]["immutable"] = False @@ -615,6 +1135,423 @@ def test_fork_network_manifest_anchor_fails_against_live_upstream(self) -> None: self.assertFalse(any(event.startswith("git ") for event in events)) self.assertFalse(any(event.startswith("gh ") for event in events)) + def test_new_pull_request_does_not_depend_on_actions_admin_permission( + self, + ) -> None: + self.change_versions() + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + events = self.events() + self.assertFalse( + any("actions/permissions/workflow" in event for event in events) + ) + self.assertTrue(any(event.startswith("gh ") for event in events)) + + def test_new_pull_request_is_promoted_only_after_final_identity_checks( + self, + ) -> None: + self.change_versions() + + result = self.run_publisher() + + self.assertEqual(result.returncode, 0, result.stderr) + events = self.events() + create_index = next( + index + for index, event in enumerate(events) + if event.startswith("gh ") + ) + ready_index = next( + index + for index, event in enumerate(events) + if event.startswith("gh ") + ) + self.assertIn("<--draft>", events[create_index]) + self.assertGreater(ready_index, create_index) + self.assertTrue( + any( + create_index < index < ready_index + and "" in event + for index, event in enumerate(events) + ) + ) + self.assertFalse(self.pr_draft_state.exists()) + + def test_branch_move_after_draft_creation_leaves_no_ready_pull_request( + self, + ) -> None: + self.change_versions() + + result = self.run_publisher(mutate_branch_after_pr_create=True) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("identity changed before publication", result.stderr) + events = self.events() + create = next(event for event in events if event.startswith("gh ")) + self.assertIn("<--draft>", create) + self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertTrue(self.pr_draft_state.exists()) + + def test_latest_release_drift_before_push_prevents_remote_mutation(self) -> None: + self.change_versions() + + result = self.run_publisher(rocm_release_drift_at=2) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("candidate releases changed", result.stderr) + self.assert_no_remote_mutation() + + def test_latest_release_drift_after_draft_creation_prevents_promotion( + self, + ) -> None: + self.change_versions() + + result = self.run_publisher(rocm_release_drift_at=3) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("candidate releases changed", result.stderr) + events = self.events() + self.assertTrue(any(event.startswith("gh ") for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertTrue(self.pr_draft_state.exists()) + + def test_late_release_tag_drift_preserves_superseded_pull_request(self) -> None: + self.change_versions() + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + open_pr_records=stale_record, + rocm_release_drift_at=3, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("candidate releases changed", result.stderr) + events = self.events() + self.assertFalse(any("<-f> " in event for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertTrue(self.pr_draft_state.exists()) + + def test_late_release_manifest_drift_preserves_superseded_pull_request( + self, + ) -> None: + self.change_versions() + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + mutate_on_snapshot=4, + open_pr_records=stale_record, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("changed since validation", result.stderr) + events = self.events() + self.assertFalse(any("<-f> " in event for event in events)) + self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertTrue(self.pr_draft_state.exists()) + + def test_ready_api_failure_preserves_superseded_pull_request(self) -> None: + self.change_versions() + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + open_pr_records=stale_record, + pr_ready_exit_code=92, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("simulated pull request ready failure", result.stderr) + events = self.events() + self.assertTrue(any(event.startswith("gh ") for event in events)) + self.assertFalse(any("<-f> " in event for event in events)) + self.assertTrue(self.pr_draft_state.exists()) + + def test_ready_noop_preserves_superseded_pull_request(self) -> None: + self.change_versions() + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + open_pr_records=stale_record, + pr_ready_noop=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("remained a draft", result.stderr) + self.assertFalse(any("<-f> " in event for event in self.events())) + self.assertTrue(self.pr_draft_state.exists()) + + def test_concurrent_current_pr_closure_preserves_superseded_pull_request( + self, + ) -> None: + self.change_versions() + stale_record = ( + "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" + "LLM360/lemonade\tgithub-actions[bot]\tmain" + ) + + result = self.run_publisher( + open_pr_records=stale_record, + pr_ready_closes_current=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("identity changed before publication", result.stderr) + close_events = [ + event for event in self.events() if "<-f> " in event + ] + self.assertFalse( + any("" in event for event in close_events) + ) + + def test_latest_release_drift_before_failed_create_reconciliation_is_safe( + self, + ) -> None: + self.change_versions() + + result = self.run_publisher( + create_pr_before_failure=True, + pr_create_exit_code=42, + rocm_release_drift_at=3, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("candidate releases changed", result.stderr) + events = self.events() + self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertTrue(self.pr_draft_state.exists()) + + def test_failed_pull_request_creation_preserves_new_branch_after_zero_recheck( + self, + ) -> None: + self.change_versions() + branch_name = self.branch_name + + result = self.run_publisher(pr_create_exit_code=42) + + self.assertNotEqual(result.returncode, 0) + published_oid = self.run_git("rev-parse", "HEAD") + branch_lookup = self.run_command( + [ + self.real_git, + "--git-dir", + str(self.remote), + "show-ref", + "--verify", + f"refs/heads/{branch_name}", + ], + check=False, + ) + self.assertEqual(branch_lookup.returncode, 0) + self.assertEqual(branch_lookup.stdout.split()[0], published_oid) + events = self.events() + create_index = next( + index + for index, event in enumerate(events) + if event.startswith("gh ") + ) + recheck_index = next( + index + for index, event in enumerate(events) + if index > create_index + and event.startswith("gh ") + ) + self.assertIn(f"<-f> ", events[recheck_index]) + self.assertIn("<-f> ", events[recheck_index]) + self.assertLess(create_index, recheck_index) + self.assertFalse( + any( + event.startswith("git ") + and f"<:refs/heads/{branch_name}>" in event + for event in events + ) + ) + self.assertIn("preserving", result.stderr) + + def test_failed_pull_request_creation_preserves_preexisting_branch(self) -> None: + _base_oid, branch_name, branch_oid = self.create_remote_update_branch( + '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' + ) + self.change_versions() + + result = self.run_publisher(pr_create_exit_code=42) + + self.assertNotEqual(result.returncode, 0) + remote_oid = self.run_command( + [ + self.real_git, + "--git-dir", + str(self.remote), + "show-ref", + "--verify", + "--hash", + f"refs/heads/{branch_name}", + ] + ).stdout.strip() + self.assertEqual(remote_oid, branch_oid) + self.assertFalse( + any( + event.startswith("git ") + and f"<:refs/heads/{branch_name}>" in event + for event in self.events() + ) + ) + + def test_failed_pull_request_creation_adopts_trusted_concurrent_pr(self) -> None: + self.change_versions() + branch_name = self.branch_name + + result = self.run_publisher( + create_pr_before_failure=True, + pr_create_exit_code=42, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.run_command( + [ + self.real_git, + "--git-dir", + str(self.remote), + "show-ref", + "--verify", + "--hash", + f"refs/heads/{branch_name}", + ] + ).stdout.strip(), + self.run_git("rev-parse", "HEAD"), + ) + events = self.events() + self.assertFalse( + any( + event.startswith("git ") + and f"<:refs/heads/{branch_name}>" in event + for event in events + ) + ) + self.assertTrue( + any("" in event for event in events) + ) + create = next(event for event in events if event.startswith("gh ")) + self.assertIn("<--draft>", create) + self.assertTrue(any(event.startswith("gh ") for event in events)) + self.assertFalse(self.pr_draft_state.exists()) + + def test_failed_pull_request_creation_preserves_branch_when_recheck_fails( + self, + ) -> None: + self.change_versions() + branch_name = self.branch_name + + result = self.run_publisher( + fail_post_create_pulls_query=True, + pr_create_exit_code=42, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + self.run_command( + [ + self.real_git, + "--git-dir", + str(self.remote), + "show-ref", + "--verify", + f"refs/heads/{branch_name}", + ], + check=False, + ).returncode, + 0, + ) + self.assertFalse( + any( + event.startswith("git ") + and f"<:refs/heads/{branch_name}>" in event + for event in self.events() + ) + ) + + def test_failed_pull_request_creation_preserves_branch_when_recheck_is_ambiguous( + self, + ) -> None: + self.change_versions() + branch_name = self.branch_name + + result = self.run_publisher( + create_pr_before_failure=True, + duplicate_pr_before_failure=True, + pr_create_exit_code=42, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("multiple matching pull requests", result.stderr) + self.assertEqual( + self.run_command( + [ + self.real_git, + "--git-dir", + str(self.remote), + "show-ref", + "--verify", + "--hash", + f"refs/heads/{branch_name}", + ] + ).stdout.strip(), + self.run_git("rev-parse", "HEAD"), + ) + self.assertFalse( + any( + event.startswith("git ") + and f"<:refs/heads/{branch_name}>" in event + for event in self.events() + ) + ) + + def test_failed_pull_request_creation_preserves_concurrently_moved_branch( + self, + ) -> None: + validated_base = self.run_git("rev-parse", "HEAD") + self.change_versions() + branch_name = self.branch_name + + result = self.run_publisher( + mutate_branch_before_pr_create_failure=True, + pr_create_exit_code=42, + ) + + self.assertNotEqual(result.returncode, 0) + remote_oid = self.run_command( + [ + self.real_git, + "--git-dir", + str(self.remote), + "show-ref", + "--verify", + "--hash", + f"refs/heads/{branch_name}", + ] + ).stdout.strip() + self.assertEqual(remote_oid, validated_base) + self.assertFalse( + any( + event.startswith("git ") + and f"<:refs/heads/{branch_name}>" in event + for event in self.events() + ) + ) + def test_publication_binds_manifest_to_branch_commit_and_pr_body(self) -> None: self.change_versions() (self.repo / MANIFEST_PATH).write_text("tampered\n", encoding="utf-8") @@ -720,6 +1657,9 @@ def test_trusted_open_pr_is_refreshed_on_the_same_validated_base(self) -> None: self.assertIn("<--repo> ", events[edit_index]) self.assertIn("<--body-file> ", events[edit_index]) self.assertFalse(any(event.startswith("gh ") for event in events)) + self.assertFalse( + any("actions/permissions/workflow" in event for event in events) + ) def test_retargeted_open_pr_fails_before_refresh(self) -> None: self.create_remote_update_branch( @@ -753,7 +1693,7 @@ def test_remote_branch_move_after_push_fails_before_pr_mutation(self) -> None: ) ) - def test_retargeted_stale_pr_is_not_closed(self) -> None: + def test_concurrently_retargeted_stale_pr_is_only_reported(self) -> None: stale_record = ( "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" "LLM360/lemonade\tgithub-actions[bot]\tmain" @@ -764,8 +1704,16 @@ def test_retargeted_stale_pr_is_not_closed(self) -> None: pr_detail_head_ref="retargeted-branch", ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("identity changed before publication", result.stderr) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + "Superseded pull request #8 " + "(auto/llamacpp-update-b90-b190-b290-deadbeef1234) " + "requires manual cleanup.", + result.stdout, + ) + self.assertFalse( + any("" in event for event in self.events()) + ) self.assertFalse(any("<-f> " in event for event in self.events())) def test_trusted_open_pr_with_unrelated_changes_is_not_overwritten(self) -> None: @@ -854,7 +1802,7 @@ def test_evolved_orphan_uses_a_new_content_addressed_branch(self) -> None: any(event.startswith("gh ") for event in self.events()) ) - def test_base_advance_uses_a_new_branch_and_closes_the_old_pull_request( + def test_base_advance_uses_a_new_branch_and_reports_the_old_pull_request( self, ) -> None: _base_oid, old_branch, _branch_oid = self.create_remote_update_branch( @@ -873,9 +1821,19 @@ def test_base_advance_uses_a_new_branch_and_closes_the_old_pull_request( self.assertEqual(result.returncode, 0, result.stderr) self.assertNotEqual(old_branch, new_branch) events = self.events() - self.assertTrue( + self.assertFalse( any("" in event for event in events) ) + self.assertFalse(any("<-f> " in event for event in events)) + ready_message = "Marked pull request #41 ready for review." + report_message = ( + f"Superseded pull request #17 ({old_branch}) requires manual cleanup." + ) + self.assertIn(ready_message, result.stdout) + self.assertIn(report_message, result.stdout) + self.assertLess( + result.stdout.index(ready_message), result.stdout.index(report_message) + ) push = next(event for event in events if event.startswith("git ")) self.assertIn(f"", push) @@ -967,5 +1925,58 @@ def test_shallow_checkout_can_refresh_an_existing_pull_request(self) -> None: ) +class PublishLlamaCppWorkflowTests(unittest.TestCase): + def test_publication_uses_only_a_dedicated_token_and_expected_actor( + self, + ) -> None: + schedule = SCHEDULE_WORKFLOW.read_text(encoding="utf-8") + publication = schedule.split(" publish:\n", 1)[1] + preflight_marker = " - name: Require dedicated publication identity\n" + checkout_marker = f" - uses: actions/checkout@{CHECKOUT_SHA} # v5.1.0\n" + self.assertIn(preflight_marker, publication) + self.assertIn(checkout_marker, publication) + preflight = publication.split(preflight_marker, 1)[1].split(checkout_marker, 1)[ + 0 + ] + checkout = publication.split(checkout_marker, 1)[1].split("\n - name:", 1)[ + 0 + ] + manifest = publication.split( + " - name: Verify release asset manifest\n", 1 + )[1].split(" - name: Publish update pull request\n", 1)[0] + publisher = publication.split(" - name: Publish update pull request\n", 1)[ + 1 + ] + + self.assertLess( + publication.index(" - name: Require dedicated publication identity"), + publication.index(f" - uses: actions/checkout@{CHECKOUT_SHA}"), + ) + self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", preflight) + self.assertIn( + "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", + preflight, + ) + self.assertIn("gh api graphql", preflight) + self.assertIn(".data.viewer.login", preflight) + self.assertIn("token: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", checkout) + self.assertIn("persist-credentials: false", checkout) + self.assertEqual( + publication.count( + f"uses: actions/download-artifact@{DOWNLOAD_ARTIFACT_SHA} # v7.0.0" + ), + 2, + ) + self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", manifest) + self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", publisher) + self.assertIn( + "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", + publisher, + ) + self.assertNotIn("secrets.GITHUB_TOKEN", publication) + self.assertNotIn("contents: write", publication) + self.assertNotIn("pull-requests: write", publication) + + if __name__ == "__main__": unittest.main() diff --git a/test/test_run_llamacpp_validation.py b/test/test_run_llamacpp_validation.py new file mode 100644 index 0000000000..22e5f76175 --- /dev/null +++ b/test/test_run_llamacpp_validation.py @@ -0,0 +1,4466 @@ +#!/usr/bin/env python3 +"""Tests for the portable llama.cpp validation process runner.""" + +from __future__ import annotations + +import argparse +import ctypes +import importlib.util +import json +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +from test.utils import llamacpp_capability_validation as capabilities +from test.utils.llamacpp_validation_plan import create_validation_plan + +ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = ROOT / ".github" / "scripts" / "run_llamacpp_validation.py" +K2_SMALL = "K2-Horizon-0.9B-GGUF" +K2_MEDIUM = "K2-Horizon-3.7B-GGUF" +K2_SMALL_FILENAME = "K2-Horizon-1B-BF16.gguf" + + +def _passing_capability_matrix(model: str) -> dict: + cases = [] + for case_id in capabilities.K2_HORIZON_CASE_IDS: + protocol = "openai" + if case_id == "ollama_thinking": + protocol = "ollama" + elif case_id == "anthropic_translation": + protocol = "anthropic" + stream = case_id in { + "openai_plain_off_stream", + "openai_reasoning_high_stream", + "openai_tool_stream_xml", + } + tool_case = case_id in { + "openai_tool_json", + "openai_tool_xml", + "openai_tool_xml_typed", + "openai_tool_default_xml", + "openai_tool_stream_xml", + } + reasoning_case = ( + case_id.startswith("openai_reasoning_") + and case_id != "openai_reasoning_low" + ) or case_id == "ollama_thinking" + content_case = case_id in { + "openai_plain_off_nonstream", + "openai_plain_off_stream", + "openai_tool_result_followup", + "anthropic_translation", + "ollama_thinking", + } or case_id.startswith("openai_reasoning_") + finish_reason = "stop" + if tool_case: + finish_reason = "tool_calls" + elif case_id == "anthropic_translation": + finish_reason = "end_turn" + cases.append( + { + "id": case_id, + "protocol": protocol, + "stream": stream, + "pass": True, + "status_code": 200, + "finish_reason": finish_reason, + "content_chars": 2 if content_case else 0, + "reasoning_chars": 8 if reasoning_case else 0, + "tool_call_count": 1 if tool_case else 0, + "terminal_frame": True, + "tool_name": capabilities.TOOL_NAME if tool_case else None, + "tool_arguments": ( + capabilities.EXPECTED_TOOL_ARGUMENTS if tool_case else None + ), + "parity_with": ( + "openai_tool_xml" + if case_id in {"openai_tool_default_xml", "openai_tool_stream_xml"} + else None + ), + "raw_ifm_marker": None, + "error": None, + } + ) + return { + "profile": capabilities.K2_HORIZON_PROFILE, + "model": f"builtin.{model.removeprefix('builtin.')}", + "pass": True, + "cases": cases, + } + + +def log_line(tag: str, message: str) -> str: + if tag == "Process": + message = f"[49972] 0.13.484.059 I {message}" + return f"2026-09-06 12:00:00.000 [Info] ({tag}) {message}" + + +def backend_log_line(backend: str) -> str: + return log_line("LlamaCpp", f"Using LlamaCpp Backend: {backend}") + + +def verified_artifacts(runner, root: Path | None = None): + artifact_path = Path("/cache") / K2_SMALL_FILENAME + if root is not None: + artifact_path = root / "verified-artifacts" / K2_SMALL_FILENAME + artifact_path.parent.mkdir(parents=True, exist_ok=True) + artifact_path.touch() + artifact_path = artifact_path.resolve() + return { + K2_SMALL: runner.VerifiedValidationArtifact( + model_id=K2_SMALL, + resolved_path=artifact_path, + size_bytes=0, + sha256="0" * 64, + ) + } + + +def bind_verified_artifact_path(contents: str, expected_artifacts: dict) -> str: + expected_path = expected_artifacts[K2_SMALL].resolved_path + return contents.replace( + f"/cache/{K2_SMALL_FILENAME}", + str(expected_path), + ) + + +def load_runner(): + spec = importlib.util.spec_from_file_location( + "run_llamacpp_validation", RUNNER_PATH + ) + if spec is None or spec.loader is None: + raise RuntimeError("could not load llama.cpp validation runner") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeProcess: + next_pid = 4100 + + def __init__(self): + self.pid = FakeProcess.next_pid + FakeProcess.next_pid += 1 + + @staticmethod + def poll(): + return None + + +def _pid_is_active(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + +def _pid_is_running(pid: int) -> bool: + try: + import psutil + except ModuleNotFoundError: + return _pid_is_active(pid) + try: + return psutil.Process(pid).status() not in { + psutil.STATUS_DEAD, + psutil.STATUS_ZOMBIE, + } + except psutil.Error: + return False + + +def _process_tree_inspection_available() -> bool: + if sys.platform != "darwin": + return True + try: + import psutil + + psutil.Process(os.getpid()).children(recursive=True) + except ModuleNotFoundError: + return False + except (OSError, psutil.Error): + return False + return True + + +DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE = _process_tree_inspection_available() + + +class LlamaCppValidationRunnerTests(unittest.TestCase): + def test_direct_help_invocation_imports_repository_modules(self): + result = subprocess.run( + [sys.executable, str(RUNNER_PATH), "--help"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_restart_command_is_bounded_cached_smoke_with_distinct_evidence(self): + runner = load_runner() + output_path = Path("isolated") / "restart.json" + + command = runner.build_restart_validation_command( + python_executable=Path("python"), + backend="vulkan", + channel="", + target="windows-vulkan", + model=K2_SMALL, + port=13305, + output_path=output_path, + ) + + self.assertEqual(command[0], "python") + self.assertEqual( + Path(command[1]), (ROOT / "test/validate_llamacpp.py").resolve() + ) + self.assertTrue(runner.VALIDATION_MODEL_CATALOG.is_absolute()) + self.assertTrue(runner.VALIDATION_ARTIFACT_LOCKS.is_absolute()) + runner.VALIDATION_MODEL_CATALOG.relative_to(ROOT.resolve()) + runner.VALIDATION_ARTIFACT_LOCKS.relative_to(ROOT.resolve()) + self.assertIn("--skip-install", command) + self.assertEqual(command.count("--model"), 1) + self.assertIn(K2_SMALL, command) + self.assertNotIn("--capability-profile", command) + self.assertNotIn("--capability-model", command) + self.assertNotIn("--logs-dir", command) + self.assertIn(str(output_path), command) + self.assertNotIn("server-restart-logs-windows-vulkan", command) + + def test_protected_validation_scrubs_ambient_download_credentials(self): + runner = load_runner() + + class FakeContainment: + @staticmethod + def popen_kwargs(): + return {} if os.name == "nt" else {"start_new_session": True} + + @staticmethod + def assign(_process): + return None + + @staticmethod + def close(): + return True + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + github_output = root / "github-output" + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="metal", + channel="", + target="macos-metal", + models=K2_SMALL, + lite=False, + capability_profile="k2-horizon-v1", + capability_models=K2_SMALL, + allow_huggingface_download_credentials=False, + port=13305, + ) + + old_directory = Path.cwd() + os.chdir(root) + try: + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object(runner.time, "sleep"), + mock.patch.object( + runner, + "_create_process_containment", + side_effect=lambda **_kwargs: FakeContainment(), + ), + mock.patch.object( + runner.subprocess, + "Popen", + side_effect=(FakeProcess(), FakeProcess(), FakeProcess()), + ) as popen, + mock.patch.object( + runner, + "_run_contained_command", + side_effect=lambda command, **_kwargs: ( + self._complete_command(command).returncode + ), + ) as run, + mock.patch.object( + runner, + "_shutdown_server", + side_effect=(True, True, True), + ) as shutdown, + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=verified_artifacts(runner), + ) as verify_artifacts, + mock.patch.object( + runner, + "require_accelerator_attestation", + ) as attest_accelerator, + mock.patch.object( + runner, + "revalidate_validation_cache", + wraps=runner.revalidate_validation_cache, + ) as revalidate_cache, + mock.patch.dict( + os.environ, + { + "RUNNER_TEMP": str(runner_temp), + "HF_ENDPOINT": "https://example.invalid", + "hF_ToKeN": "download-token", + "hUgGiNg_FaCe_HuB_tOkEn": "legacy-download-token", + "HuGgInGfAcE_ToKeN": "alternate-download-token", + "LEMONADE_API_KEY": "api-key", + "LEMONADE_DEFAULTS_PATH": "/host/defaults.json", + "LEMONADE_LLAMACPP_METAL_BIN": "/host/llama-server", + "lemonade_rocm_install_method": "wheel", + "LEMONADE_BACKEND_WATCHDOG_POLL_SECONDS": "1", + "LEMONADE_CACHE_DIR": "/host/lemonade-cache", + "LEMONADE_GGML_HIP_PATH": "/host/libggml-hip.so", + "HF_HOME": "/host/huggingface", + "LEMONADE_ALLOWED_ORIGINS": "https://example.invalid", + "ROCM_PATH": "/host/rocm", + "LLAMA_ARG_MODEL": "/host/model.gguf", + "LLAMA_ARG_CTX_SIZE": "1", + "HTTP_PROXY": "http://proxy.invalid:3128", + "all_proxy": "socks5://proxy.invalid:1080", + "NO_PROXY": "example.invalid", + "GITHUB_OUTPUT": str(github_output), + "GITHUB_ENV": "/runner/command/environment", + "ACTIONS_RUNTIME_TOKEN": "actions-token", + "LD_PRELOAD": "/attacker/inject.so", + "PYTHONPATH": "/attacker/python", + "UNKNOWN_DEPLOYMENT_SECRET": "secret", + }, + ), + ): + result = runner.run_validation(args) + finally: + os.chdir(old_directory) + github_output_contents = github_output.read_text(encoding="utf-8") + + self.assertEqual(result, 0) + self.assertEqual(popen.call_count, 3) + self.assertEqual(run.call_count, 3) + self.assertEqual(shutdown.call_count, 3) + self.assertEqual(verify_artifacts.call_count, 3) + self.assertEqual(attest_accelerator.call_count, 2) + self.assertEqual(revalidate_cache.call_count, 10) + self.assertEqual( + attest_accelerator.call_args_list[0].args[3], + verified_artifacts(runner), + ) + self.assertEqual( + attest_accelerator.call_args_list[1].args[3], + verified_artifacts(runner), + ) + self.assertEqual( + attest_accelerator.call_args_list[0].kwargs["selected_models"], + [K2_SMALL], + ) + self.assertEqual( + attest_accelerator.call_args_list[1].kwargs["selected_models"], + [K2_SMALL], + ) + + first_server = popen.call_args_list[0] + second_server = popen.call_args_list[1] + third_server = popen.call_args_list[2] + self.assertEqual(first_server.args[0][1], second_server.args[0][1]) + self.assertEqual(first_server.args[0][1], third_server.args[0][1]) + self.assertFalse(first_server.args[0][2].startswith("--")) + self.assertEqual(first_server.args[0][2], second_server.args[0][2]) + self.assertEqual(first_server.args[0][2], third_server.args[0][2]) + self.assertEqual(first_server.args[0][3:], second_server.args[0][3:]) + self.assertEqual(first_server.args[0][3:], third_server.args[0][3:]) + expected_hf_cache = first_server.kwargs["env"]["HF_HUB_CACHE"] + hf_cache_path = Path(expected_hf_cache) + hf_cache_path.relative_to(runner_temp.resolve()) + self.assertEqual(hf_cache_path.name, "hf-hub") + self.assertEqual(hf_cache_path.parent.name, "lemonade-cache") + self.assertTrue( + hf_cache_path.parents[1].name.startswith("llamacpp-validation-macos-metal-") + ) + private_temp = first_server.kwargs["env"]["TMPDIR"] + self.assertEqual(first_server.kwargs["env"]["TMP"], private_temp) + self.assertEqual(first_server.kwargs["env"]["TEMP"], private_temp) + Path(private_temp).relative_to(runner_temp.resolve()) + self.assertEqual(Path(private_temp).name, "private-temp") + download_credential_keys = { + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "HUGGINGFACE_TOKEN", + } + for call in popen.call_args_list: + self.assertFalse( + download_credential_keys & {key.upper() for key in call.kwargs["env"]} + ) + for call in popen.call_args_list: + if os.name != "nt": + self.assertIs(call.kwargs.get("start_new_session"), True) + self.assertEqual(call.kwargs["env"]["HF_HUB_CACHE"], expected_hf_cache) + self.assertEqual(call.kwargs["env"]["TMPDIR"], private_temp) + self.assertEqual(call.kwargs["env"]["TMP"], private_temp) + self.assertEqual(call.kwargs["env"]["TEMP"], private_temp) + self.assertEqual(call.kwargs["env"]["HOME"], private_temp) + self.assertEqual(call.kwargs["env"]["USERPROFILE"], private_temp) + self.assertEqual(call.kwargs["env"]["APPDATA"], private_temp) + self.assertEqual(call.kwargs["env"]["LOCALAPPDATA"], private_temp) + self.assertNotIn("HF_ENDPOINT", call.kwargs["env"]) + self.assertNotIn("LEMONADE_API_KEY", call.kwargs["env"]) + self.assertNotIn("LEMONADE_LLAMACPP_METAL_BIN", call.kwargs["env"]) + self.assertNotIn("lemonade_rocm_install_method", call.kwargs["env"]) + self.assertNotIn( + "LEMONADE_BACKEND_WATCHDOG_POLL_SECONDS", call.kwargs["env"] + ) + self.assertNotIn("LEMONADE_CACHE_DIR", call.kwargs["env"]) + self.assertNotIn("LEMONADE_GGML_HIP_PATH", call.kwargs["env"]) + self.assertNotIn("HF_HOME", call.kwargs["env"]) + self.assertNotIn("LEMONADE_ALLOWED_ORIGINS", call.kwargs["env"]) + self.assertNotIn("ROCM_PATH", call.kwargs["env"]) + self.assertEqual( + { + key.upper(): value + for key, value in call.kwargs["env"].items() + if key.upper().startswith("LLAMA_ARG_") + }, + {"LLAMA_ARG_LOG_VERBOSITY": "4"}, + ) + self.assertNotIn("HTTP_PROXY", call.kwargs["env"]) + self.assertNotIn("all_proxy", call.kwargs["env"]) + self.assertNotIn("GITHUB_ENV", call.kwargs["env"]) + self.assertNotIn("GITHUB_OUTPUT", call.kwargs["env"]) + self.assertNotIn("ACTIONS_RUNTIME_TOKEN", call.kwargs["env"]) + self.assertNotIn("LD_PRELOAD", call.kwargs["env"]) + self.assertNotIn("PYTHONPATH", call.kwargs["env"]) + self.assertNotIn("UNKNOWN_DEPLOYMENT_SECRET", call.kwargs["env"]) + self.assertEqual(call.kwargs["env"]["NO_PROXY"], "127.0.0.1") + self.assertEqual(call.kwargs["env"]["no_proxy"], "127.0.0.1") + self.assertEqual(call.kwargs["env"]["LEMONADE_CI_MODE"], "True") + self.assertEqual( + call.kwargs["env"]["LEMONADE_DEFAULTS_PATH"], + str((resources / "defaults.json").resolve()), + ) + + prepare_command = run.call_args_list[0].args[0] + primary_command = run.call_args_list[1].args[0] + restart_command = run.call_args_list[2].args[0] + self.assertIn("--prepare-only", prepare_command) + self.assertNotIn("--skip-install", prepare_command) + self.assertNotIn("--prepare-only", primary_command) + self.assertIn("--skip-install", primary_command) + self.assertIn("--skip-install", restart_command) + for call in run.call_args_list: + self.assertFalse( + download_credential_keys & {key.upper() for key in call.kwargs["env"]} + ) + self.assertEqual( + Path(primary_command[primary_command.index("--output") + 1]).name, + "llamacpp_validation_macos-metal.json", + ) + self.assertEqual( + Path(restart_command[restart_command.index("--output") + 1]).name, + "llamacpp_restart_validation_macos-metal.json", + ) + for call in run.call_args_list: + self.assertEqual(call.kwargs["env"]["HF_HUB_CACHE"], expected_hf_cache) + self.assertEqual(call.kwargs["env"]["TMPDIR"], private_temp) + self.assertEqual(call.kwargs["env"]["TMP"], private_temp) + self.assertEqual(call.kwargs["env"]["TEMP"], private_temp) + self.assertEqual(call.kwargs["env"]["HOME"], private_temp) + self.assertEqual(call.kwargs["env"]["USERPROFILE"], private_temp) + self.assertEqual(call.kwargs["env"]["APPDATA"], private_temp) + self.assertEqual(call.kwargs["env"]["LOCALAPPDATA"], private_temp) + self.assertNotIn("HF_ENDPOINT", call.kwargs["env"]) + self.assertNotIn("LEMONADE_API_KEY", call.kwargs["env"]) + self.assertNotIn("LEMONADE_LLAMACPP_METAL_BIN", call.kwargs["env"]) + self.assertEqual( + { + key.upper(): value + for key, value in call.kwargs["env"].items() + if key.upper().startswith("LLAMA_ARG_") + }, + {"LLAMA_ARG_LOG_VERBOSITY": "4"}, + ) + self.assertNotIn("GITHUB_ENV", call.kwargs["env"]) + self.assertNotIn("GITHUB_OUTPUT", call.kwargs["env"]) + self.assertNotIn("ACTIONS_RUNTIME_TOKEN", call.kwargs["env"]) + self.assertNotIn("LD_PRELOAD", call.kwargs["env"]) + self.assertNotIn("PYTHONPATH", call.kwargs["env"]) + self.assertNotIn("UNKNOWN_DEPLOYMENT_SECRET", call.kwargs["env"]) + + server_log_names = { + Path(call.kwargs[stream].name).name + for call in popen.call_args_list + for stream in ("stdout", "stderr") + } + self.assertEqual( + server_log_names, + { + "lemond.prepare.stdout.log", + "lemond.prepare.stderr.log", + "lemond.stdout.log", + "lemond.stderr.log", + "lemond.restart.stdout.log", + "lemond.restart.stderr.log", + }, + ) + for call in popen.call_args_list: + Path(call.kwargs["stdout"].name).relative_to(runner_temp.resolve()) + Path(call.kwargs["stderr"].name).relative_to(runner_temp.resolve()) + + workflow_outputs = dict( + line.split("=", 1) for line in github_output_contents.splitlines() + ) + self.assertEqual( + Path(workflow_outputs["validation_result"]).name, + "llamacpp_validation_macos-metal.json", + ) + self.assertEqual( + Path(workflow_outputs["restart_result"]).name, + "llamacpp_restart_validation_macos-metal.json", + ) + self.assertEqual(len(workflow_outputs), 8) + for output_path in workflow_outputs.values(): + Path(output_path).relative_to(runner_temp.resolve()) + + @staticmethod + def _complete_command(command): + if "--prepare-only" not in command: + output_path = Path(command[command.index("--output") + 1]) + models = [ + command[index + 1] + for index, argument in enumerate(command) + if argument == "--model" + ] + capability_models = { + command[index + 1].removeprefix("builtin.") + for index, argument in enumerate(command) + if argument == "--capability-model" + } + records = [] + for model in models: + record = { + "model": model, + "pass": True, + "response": "Four.", + "input_tokens": 10, + "output_tokens": 2, + "time_to_first_token": 0.5, + "tokens_per_second": 4.0, + } + if model.removeprefix("builtin.") in capability_models: + record["capability_matrix"] = _passing_capability_matrix(model) + records.append(record) + output_path.write_text(json.dumps(records), encoding="utf-8") + return argparse.Namespace(returncode=0) + + def _complete_process(self, **kwargs): + kwargs["stdout_path"].touch() + kwargs["stderr_path"].touch() + return self._complete_command(kwargs["command"]).returncode, True + + def test_validation_attests_locked_model_with_selected_unlocked_schedule_model( + self, + ): + runner = load_runner() + schedule_models = create_validation_plan("schedule")["include"][0]["models"] + artifact_locks = json.loads( + (ROOT / "test/fixtures/llamacpp_validation_artifacts.json").read_text( + encoding="utf-8" + ) + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + expected_artifacts = {} + artifact_root = root / "verified-artifacts" + artifact_root.mkdir() + for model, lock in artifact_locks.items(): + artifact_path = artifact_root / lock["filename"] + artifact_path.touch() + expected_artifacts[model] = runner.VerifiedValidationArtifact( + model_id=model, + resolved_path=artifact_path.resolve(), + size_bytes=0, + sha256="0" * 64, + ) + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="windows-vulkan", + models=",".join(schedule_models), + lite=False, + capability_profile="k2-horizon-v1", + capability_models=K2_SMALL, + port=13305, + ) + + def completed_model_load(model: str, gguf_path: Path) -> str: + return "\n".join( + ( + log_line("LlamaCpp", f"Loading model: builtin.{model}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: {gguf_path}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line( + "Process", + "load_tensors: offloaded 12/24 layers to GPU", + ), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + + def complete_with_attestation_logs(**kwargs): + result = self._complete_process(**kwargs) + command = kwargs["command"] + if "--prepare-only" in command: + return result + models = [ + command[index + 1].removeprefix("builtin.") + for index, argument in enumerate(command) + if argument == "--model" + ] + sessions = [ + completed_model_load( + model, + ( + expected_artifacts[model].resolved_path + if model in expected_artifacts + else Path("/cache") / f"{model}.gguf" + ), + ) + for model in models + ] + kwargs["stderr_path"].write_text( + "\n".join(sessions) + "\n", encoding="utf-8" + ) + return result + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=complete_with_attestation_logs, + ), + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=expected_artifacts, + ), + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + ): + try: + result = runner.run_validation(args) + except RuntimeError as exc: + self.fail(f"selected unlocked schedule model was rejected: {exc}") + + self.assertEqual(result, 0) + + def test_validation_rejects_empty_primary_evidence_before_restart(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="linux-vulkan", + models=K2_SMALL, + lite=False, + capability_profile="k2-horizon-v1", + capability_models=K2_SMALL, + port=13305, + ) + phases = 0 + + def write_empty_evidence(**kwargs): + nonlocal phases + phases += 1 + kwargs["stdout_path"].touch() + kwargs["stderr_path"].touch() + command = kwargs["command"] + if "--prepare-only" not in command: + output_path = Path(command[command.index("--output") + 1]) + output_path.write_text("[]", encoding="utf-8") + return 0, True + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=write_empty_evidence, + ), + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value={}, + ), + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + self.assertRaisesRegex(RuntimeError, "nonempty array"), + ): + runner.run_validation(args) + + self.assertEqual(phases, 2) + + def test_validation_rejects_restart_evidence_for_another_model(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="linux-vulkan", + models=K2_SMALL, + lite=False, + capability_profile="k2-horizon-v1", + capability_models=K2_SMALL, + port=13305, + ) + + def write_wrong_restart_evidence(**kwargs): + result = self._complete_process(**kwargs) + command = kwargs["command"] + output_path = Path(command[command.index("--output") + 1]) + if output_path.name.startswith("llamacpp_restart_validation_"): + output_path.write_text( + json.dumps( + [ + { + "model": "Other-Llama", + "pass": True, + "response": "Four.", + "input_tokens": 10, + "output_tokens": 2, + "time_to_first_token": 0.5, + "tokens_per_second": 4.0, + } + ] + ), + encoding="utf-8", + ) + return result + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=write_wrong_restart_evidence, + ) as run_process, + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value={}, + ), + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + self.assertRaisesRegex(RuntimeError, "do not match planned models"), + ): + runner.run_validation(args) + + self.assertEqual(run_process.call_count, 3) + + def test_accelerator_attestation_requires_ordered_device_and_offload_markers(self): + runner = load_runner() + positive = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + log = root / "lemond.stderr.log" + empty_companion = root / "lemond.stdout.log" + empty_companion.touch() + log.write_text( + bind_verified_artifact_path(positive, expected_artifacts), + encoding="utf-8", + ) + identities = runner.require_accelerator_attestation( + [empty_companion, log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + self.assertEqual( + [identity.path for identity in identities], [empty_companion, log] + ) + + failures = { + "missing backend": positive.replace( + backend_log_line("vulkan") + "\n", "" + ), + "wrong backend": positive.replace( + backend_log_line("vulkan"), backend_log_line("cuda") + ), + "duplicate backend": positive.replace( + backend_log_line("vulkan"), + backend_log_line("vulkan") + "\n" + backend_log_line("vulkan"), + ), + "malformed backend": positive.replace( + backend_log_line("vulkan"), + log_line("LlamaCpp", "Using LlamaCpp Backend: vulkan extra"), + ), + "backend after artifact identity": positive.replace( + backend_log_line("vulkan") + + "\n" + + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}") + + "\n" + + backend_log_line("vulkan"), + ), + "backend after completed load": positive + + "\n" + + backend_log_line("vulkan"), + "zero": positive.replace("12/24", "0/24"), + "greater than total": positive.replace("12/24", "25/24"), + "malformed offload": positive.replace("12/24", "many/all"), + "missing offload": positive.replace( + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU") + + "\n", + "", + ), + "missing child success": positive.replace( + log_line("Process", "srv llama_server: model loaded"), "" + ), + "offload before device": positive.replace( + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ) + + "\n" + + log_line( + "Process", "load_tensors: offloaded 12/24 layers to GPU" + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU") + + "\n" + + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + ), + "missing device": positive.replace( + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ) + + "\n", + "", + ), + "missing artifact identity": positive.replace( + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}") + + "\n", + "", + ), + "artifact suffix is not exact": positive.replace( + K2_SMALL_FILENAME, f"{K2_SMALL_FILENAME}.evil" + ), + "device before artifact identity": positive.replace( + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}") + + "\n" + + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ) + + "\n" + + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + ), + "other model cannot mask target": positive.replace( + K2_SMALL, "Other-Llama" + ), + "later draft masks target": positive.replace( + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ) + + "\n" + + log_line( + "Process", "load_tensors: offloaded 12/24 layers to GPU" + ), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ) + + "\n" + + log_line("Process", "load_tensors: offloaded 0/24 layers to GPU") + + "\n" + + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan1 (GPU)", + ) + + "\n" + + log_line( + "Process", "load_tensors: offloaded 24/24 layers to GPU" + ), + ), + "deceptive child success text": positive.replace( + log_line("Process", "srv llama_server: model loaded"), + log_line("Process", "did not see srv llama_server: model loaded"), + ), + "second zero offload on same line": positive.replace( + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line( + "Process", + "load_tensors: offloaded 12/24 layers to GPU " + "load_tensors: offloaded 0/24 layers to GPU", + ), + ), + "malformed then valid offload on same line": positive.replace( + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line( + "Process", + "load_tensors: offloaded many/all layers to GPU " + "load_tensors: offloaded 12/24 layers to GPU", + ), + ), + } + for name, contents in failures.items(): + with self.subTest(name=name): + log.write_text( + bind_verified_artifact_path(contents, expected_artifacts), + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "accelerator"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + + unexpected_model = ( + positive + + "\n" + + log_line("LlamaCpp", "Loading model: builtin.Unexpected-Llama") + ) + log.write_text( + bind_verified_artifact_path(unexpected_model, expected_artifacts), + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "unexpected model"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + + def test_selected_unlocked_load_cannot_replace_locked_model_attestation(self): + runner = load_runner() + schedule_models = create_validation_plan("schedule")["include"][0]["models"] + unlocked_model = next( + model + for model in schedule_models + if model not in {K2_SMALL, "K2-Horizon-3.7B-GGUF", "K2-Horizon-7B-GGUF"} + ) + unlocked_load = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: builtin.{unlocked_model}"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "lemond.stderr.log" + log.write_text(unlocked_load, encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "missing models"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + verified_artifacts(runner, root), + "vulkan", + selected_models=[unlocked_model, K2_SMALL], + ) + + def test_accelerator_attestation_rejects_same_basename_from_another_path(self): + runner = load_runner() + positive = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /unverified/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + unverified_artifact = root / "unverified" / K2_SMALL_FILENAME + unverified_artifact.parent.mkdir() + unverified_artifact.touch() + log = root / "lemond.stderr.log" + log.write_text( + positive.replace( + f"/unverified/{K2_SMALL_FILENAME}", + str(unverified_artifact.resolve()), + ), + encoding="utf-8", + ) + + with self.assertRaisesRegex(RuntimeError, "wrong artifact"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + + def test_accelerator_attestation_binds_device_to_requested_backend(self): + runner = load_runner() + backend_devices = { + "cuda": "CUDA0", + "metal": "MTL0", + "rocm-stable": "ROCm0", + "vulkan": "Vulkan0", + } + + def attestation(backend, device): + return "\n".join( + ( + log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), + backend_log_line(backend), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + f"llama_prepare_model_devices: using device {device} (GPU)", + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + log = root / "lemond.stderr.log" + for backend, device in backend_devices.items(): + with self.subTest(backend=backend, device=device): + log.write_text( + bind_verified_artifact_path( + attestation(backend, device), expected_artifacts + ), + encoding="utf-8", + ) + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + expected_backend=backend, + ) + + wrong_device = next( + candidate + for other_backend, candidate in backend_devices.items() + if other_backend != backend + ) + with self.subTest(backend=backend, wrong_device=wrong_device): + log.write_text( + bind_verified_artifact_path( + attestation(backend, wrong_device), expected_artifacts + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "backend"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + expected_backend=backend, + ) + + def test_rocm_attestation_binds_exact_release_channel(self): + runner = load_runner() + + def attestation(channel): + return "\n".join( + ( + log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), + log_line("LlamaCpp", f"Using LlamaCpp Backend: rocm-{channel}"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device ROCm0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + log = root / "lemond.stderr.log" + for channel in ("stable", "nightly"): + expected_backend = f"rocm-{channel}" + log.write_text( + bind_verified_artifact_path( + attestation(channel), expected_artifacts + ), + encoding="utf-8", + ) + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + expected_backend=expected_backend, + ) + + swapped_channel = "nightly" if channel == "stable" else "stable" + with self.assertRaisesRegex(RuntimeError, "backend"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + expected_backend=f"rocm-{swapped_channel}", + ) + + def test_attestation_backend_resolution_rejects_channel_ambiguity(self): + runner = load_runner() + self.assertEqual( + runner.resolve_attestation_backend("rocm", "stable"), "rocm-stable" + ) + self.assertEqual( + runner.resolve_attestation_backend("rocm", "nightly"), "rocm-nightly" + ) + self.assertEqual(runner.resolve_attestation_backend("metal", ""), "metal") + + for backend, channel in ( + ("rocm", ""), + ("rocm", "candidate"), + ("metal", "stable"), + ): + with self.subTest(backend=backend, channel=channel): + with self.assertRaisesRegex(RuntimeError, "channel"): + runner.resolve_attestation_backend(backend, channel) + + def test_cpu_attestation_requires_cpu_only_build_markers(self): + runner = load_runner() + positive = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), + backend_log_line("cpu"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "load_tensors: CPU_Mapped model buffer size = 2056.83 MiB", + ), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + accelerator_markers = ( + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 0/24 layers to GPU"), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + log = root / "lemond.stderr.log" + log.write_text( + bind_verified_artifact_path(positive, expected_artifacts), + encoding="utf-8", + ) + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + expected_backend="cpu", + ) + + missing_cpu_buffer = positive.replace( + log_line( + "Process", + "load_tensors: CPU_Mapped model buffer size = 2056.83 MiB", + ) + + "\n", + "", + ) + log.write_text( + bind_verified_artifact_path(missing_cpu_buffer, expected_artifacts), + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "CPU|cpu"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + expected_backend="cpu", + ) + + for marker in accelerator_markers: + with self.subTest(marker=marker): + invalid_cpu_log = positive.replace( + log_line("Process", "srv llama_server: model loaded"), + marker + + "\n" + + log_line("Process", "srv llama_server: model loaded"), + ) + log.write_text( + bind_verified_artifact_path( + invalid_cpu_log, expected_artifacts + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "CPU|cpu"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + expected_backend="cpu", + ) + + def test_accelerator_attestation_does_not_carry_state_across_log_streams(self): + runner = load_runner() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + stdout_log = root / "stdout.log" + stderr_log = root / "stderr.log" + partial_load = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + ) + ) + stdout_log.write_text( + bind_verified_artifact_path(partial_load, expected_artifacts), + encoding="utf-8", + ) + stderr_log.write_text( + "\n".join( + ( + log_line( + "Process", "load_tensors: offloaded 12/24 layers to GPU" + ), + log_line("Process", "srv llama_server: model loaded"), + ) + ), + encoding="utf-8", + ) + + with self.assertRaisesRegex(RuntimeError, "incomplete"): + runner.require_accelerator_attestation( + [stdout_log, stderr_log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + + def test_accelerator_attestation_rejects_every_bad_duplicate_or_late_offload(self): + runner = load_runner() + positive = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + bad_second_loads = { + "zero": "\n".join( + ( + log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 0/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ), + "missing": "\n".join( + ( + log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "srv llama_server: model loaded"), + ) + ), + "incomplete": log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), + "late zero outside a load": log_line( + "Process", "load_tensors: offloaded 0/24 layers to GPU" + ), + "late malformed outside a load": ( + log_line("Process", "load_tensors: offloaded many/all layers to GPU") + ), + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + log = root / "lemond.stdout.log" + for name, second_load in bad_second_loads.items(): + with self.subTest(name=name): + contents = bind_verified_artifact_path( + f"{positive}\n{second_load}\n", expected_artifacts + ) + log.write_text(contents, encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "accelerator|incomplete"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + + def test_accelerator_attestation_rejects_oversized_log_lines(self): + runner = load_runner() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + log = root / "lemond.stdout.log" + log.write_bytes(b"x" * (runner.MAX_ATTESTATION_LINE_BYTES + 1)) + + with self.assertRaisesRegex(RuntimeError, "line is too long"): + runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + + def test_accelerator_attestation_enforces_aggregate_log_budget(self): + runner = load_runner() + positive = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + positive = bind_verified_artifact_path(positive, expected_artifacts) + first_log = root / "lemond.stdout.log" + second_log = root / "lemond.stderr.log" + first_log.write_text(positive, encoding="utf-8") + second_log.write_text("12345", encoding="utf-8") + budget = len(positive.encode("utf-8")) + 4 + + with ( + mock.patch.object(runner, "MAX_ATTESTATION_LOG_BYTES", budget), + self.assertRaisesRegex(RuntimeError, "byte budget"), + ): + runner.require_accelerator_attestation( + [first_log, second_log], + root, + "validation", + expected_artifacts, + "vulkan", + ) + + def test_attested_log_replacement_fails_final_identity_check(self): + runner = load_runner() + contents = "\n".join( + ( + log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), + backend_log_line("vulkan"), + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), + log_line( + "Process", + "llama_prepare_model_devices: using device Vulkan0 (GPU)", + ), + log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), + log_line("Process", "srv llama_server: model loaded"), + ) + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected_artifacts = verified_artifacts(runner, root) + contents = bind_verified_artifact_path(contents, expected_artifacts) + log = root / "lemond.stdout.log" + log.write_text(contents, encoding="utf-8") + identity = runner.require_accelerator_attestation( + [log], + root, + "validation", + expected_artifacts, + "vulkan", + )[0] + log.rename(root / "original.log") + log.write_text(contents, encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "changed"): + runner._revalidate_regular_file( + identity, + root, + require_nonempty=False, + ) + + def test_restart_is_not_attempted_when_primary_validation_fails(self): + runner = load_runner() + + class FakeContainment: + @staticmethod + def popen_kwargs(): + return {} + + @staticmethod + def assign(_process): + return None + + @staticmethod + def close(): + return True + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="linux-vulkan", + models=K2_SMALL, + lite=False, + capability_profile="k2-horizon-v1", + capability_models=K2_SMALL, + port=13305, + ) + + old_directory = Path.cwd() + os.chdir(root) + try: + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object(runner.time, "sleep"), + mock.patch.object( + runner, + "_create_process_containment", + side_effect=lambda **_kwargs: FakeContainment(), + ), + mock.patch.object( + runner.subprocess, + "Popen", + side_effect=(FakeProcess(), FakeProcess()), + ) as popen, + mock.patch.object( + runner, + "_run_contained_command", + side_effect=(0, 7), + ) as run, + mock.patch.object( + runner, + "_shutdown_server", + side_effect=(True, True), + ) as shutdown, + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=verified_artifacts(runner), + ) as verify, + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + ): + result = runner.run_validation(args) + finally: + os.chdir(old_directory) + + self.assertEqual(result, 7) + self.assertEqual(popen.call_count, 2) + self.assertEqual(run.call_count, 2) + self.assertEqual(shutdown.call_count, 2) + verify.assert_called_once() + + def test_direct_locked_model_is_verified_without_a_capability_profile(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="linux-vulkan", + models=f"builtin.{K2_SMALL}", + lite=False, + capability_profile="", + capability_models="", + port=13305, + ) + + old_directory = Path.cwd() + os.chdir(root) + try: + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=self._complete_process, + ) as run_process, + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=verified_artifacts(runner), + ) as verify, + mock.patch.object( + runner, + "require_accelerator_attestation", + return_value=[], + ) as attest, + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + ): + result = runner.run_validation(args) + finally: + os.chdir(old_directory) + + self.assertEqual(result, 0) + self.assertEqual(run_process.call_count, 3) + self.assertEqual(verify.call_count, 3) + self.assertEqual(attest.call_count, 2) + self.assertEqual( + attest.call_args_list[0].args[3], + verified_artifacts(runner), + ) + self.assertEqual( + attest.call_args_list[1].args[3], + verified_artifacts(runner), + ) + for call in verify.call_args_list: + self.assertEqual(call.args[3], [f"builtin.{K2_SMALL}"]) + + def test_direct_mixed_models_restart_the_selected_locked_candidate(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="linux-vulkan", + models=f"Other-Model,builtin.{K2_SMALL}", + lite=False, + capability_profile="", + capability_models="", + port=13305, + ) + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=self._complete_process, + ) as run_process, + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=verified_artifacts(runner), + ), + mock.patch.object( + runner, + "require_accelerator_attestation", + return_value=[], + ) as attest, + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + ): + result = runner.run_validation(args) + + self.assertEqual(result, 0) + restart_command = run_process.call_args_list[2].kwargs["command"] + restart_model_index = restart_command.index("--model") + 1 + self.assertEqual(restart_command[restart_model_index], f"builtin.{K2_SMALL}") + self.assertEqual(attest.call_count, 2) + self.assertEqual( + attest.call_args_list[0].kwargs["selected_models"], + ["Other-Model", f"builtin.{K2_SMALL}"], + ) + self.assertEqual( + attest.call_args_list[1].kwargs["selected_models"], + [f"builtin.{K2_SMALL}"], + ) + + def test_capability_model_is_the_restart_target_when_other_models_are_locked(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="windows-vulkan", + models=f"{K2_MEDIUM},{K2_SMALL}", + lite=False, + capability_profile="k2-horizon-v1", + capability_models=K2_SMALL, + port=13305, + ) + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=self._complete_process, + ) as run_process, + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value={}, + ), + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + ): + result = runner.run_validation(args) + + self.assertEqual(result, 0) + restart_command = run_process.call_args_list[2].kwargs["command"] + restart_model_index = restart_command.index("--model") + 1 + self.assertEqual(restart_command[restart_model_index], K2_SMALL) + + def test_cpu_lane_verifies_locked_artifacts_with_cpu_attestation(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="cpu", + channel="", + target="linux-cpu", + models=K2_SMALL, + lite=False, + capability_profile="", + capability_models="", + port=13305, + ) + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=self._complete_process, + ), + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=verified_artifacts(runner), + ) as verify, + mock.patch.object( + runner, + "require_accelerator_attestation", + return_value=[], + ) as attest, + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + ): + result = runner.run_validation(args) + + self.assertEqual(result, 0) + self.assertEqual(verify.call_count, 3) + self.assertEqual(attest.call_count, 2) + for call in attest.call_args_list: + self.assertEqual(call.args[4], "cpu") + + def test_rocm_lane_attests_exact_channel_for_primary_and_restart(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + + for channel in ("stable", "nightly"): + with self.subTest(channel=channel): + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="rocm", + channel=channel, + target=f"linux-rocm-{channel}", + models=K2_SMALL, + lite=False, + capability_profile="", + capability_models="", + port=13305, + ) + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=self._complete_process, + ), + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=verified_artifacts(runner), + ), + mock.patch.object( + runner, + "require_accelerator_attestation", + return_value=[], + ) as attest, + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + ): + result = runner.run_validation(args) + + self.assertEqual(result, 0) + self.assertEqual(attest.call_count, 2) + for call in attest.call_args_list: + self.assertEqual(call.args[4], f"rocm-{channel}") + + def test_artifact_change_during_full_validation_stops_before_restart(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + state = root / "artifact-state" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + state.write_text("trusted", encoding="ascii") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="metal", + channel="", + target="macos-metal", + models=K2_SMALL, + lite=False, + capability_profile="", + capability_models="", + port=13305, + ) + + process_count = 0 + + def run_process(**kwargs): + nonlocal process_count + process_count += 1 + self._complete_process(**kwargs) + if process_count == 2: + state.write_text("swapped", encoding="ascii") + return 0, True + + def verify(*_args): + if state.read_text(encoding="ascii") != "trusted": + raise ValueError("artifact changed across inference") + return verified_artifacts(runner) + + old_directory = Path.cwd() + os.chdir(root) + try: + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=run_process, + ), + mock.patch.object( + runner, + "verify_validation_artifacts", + side_effect=verify, + ) as verify_artifacts, + mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), + self.assertRaisesRegex(ValueError, "across inference"), + ): + runner.run_validation(args) + finally: + os.chdir(old_directory) + + self.assertEqual(process_count, 2) + self.assertEqual(verify_artifacts.call_count, 2) + + def test_attested_log_replacement_stops_workflow_output_emission(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lemond = root / "lemond" + python = root / "python" + runner_temp = root / "runner-temp" + github_output = root / "github-output" + lemond.touch() + python.touch() + runner_temp.mkdir() + resources = root / "resources" + resources.mkdir() + (resources / "defaults.json").write_text("{}", encoding="utf-8") + args = argparse.Namespace( + lemond=lemond, + python=python, + backend="vulkan", + channel="", + target="linux-vulkan", + models=K2_SMALL, + lite=False, + capability_profile="", + capability_models="", + port=13305, + ) + validation_identity = None + + def attest( + log_paths, + allowed_root, + _phase, + _expected_artifacts, + _expected_backend, + **_kwargs, + ): + nonlocal validation_identity + identities = [ + runner._capture_regular_file( + path, + allowed_root, + require_nonempty=False, + ) + for path in log_paths + ] + if validation_identity is None: + validation_identity = identities[0] + else: + attacked_path = validation_identity.path + attacked_path.rename(attacked_path.with_suffix(".original")) + attacked_path.touch() + return identities + + with ( + mock.patch.object(runner, "_require_idle_runner"), + mock.patch.object(runner, "_require_unused_port"), + mock.patch.object( + runner, + "_run_validation_process", + side_effect=self._complete_process, + ), + mock.patch.object( + runner, + "verify_validation_artifacts", + return_value=verified_artifacts(runner), + ), + mock.patch.object( + runner, + "require_accelerator_attestation", + side_effect=attest, + ), + mock.patch.dict( + os.environ, + { + "RUNNER_TEMP": str(runner_temp), + "GITHUB_OUTPUT": str(github_output), + }, + ), + self.assertRaisesRegex(RuntimeError, "changed"), + ): + runner.run_validation(args) + + self.assertFalse(github_output.exists()) + + def test_cache_root_rejects_a_preexisting_symlink(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + real_runner_temp = root / "real-runner-temp" + runner_temp = root / "runner-temp" + real_runner_temp.mkdir() + runner_temp.symlink_to(real_runner_temp, target_is_directory=True) + + create_cache = getattr(runner, "create_validation_cache", None) + self.assertIsNotNone(create_cache) + with self.assertRaisesRegex(RuntimeError, "symlink or junction"): + create_cache(runner_temp, "linux-vulkan") + + def test_validation_cache_is_unpredictable_and_has_a_revalidation_contract(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + runner_temp = Path(directory) + first = runner.create_validation_cache(runner_temp, "linux-vulkan") + runner.revalidate_validation_cache(first) + for identity in first: + self.assertTrue(identity.path.is_dir()) + try: + second = runner.create_validation_cache(runner_temp, "linux-vulkan") + except (OSError, RuntimeError) as exc: + self.fail(f"cache path was predictable or reusable: {exc}") + + self.assertNotEqual(first, second) + revalidate = getattr(runner, "revalidate_validation_cache", None) + self.assertIsNotNone(revalidate) + + def test_server_log_files_are_created_exclusively(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cache = root / "cache" + cache.mkdir() + stdout_path = root / "stdout.log" + stdout_path.write_text("stale", encoding="utf-8") + with ( + mock.patch.object(runner.subprocess, "Popen") as popen, + self.assertRaises(FileExistsError), + ): + runner._run_validation_process( + lemond=root / "lemond", + cache_directory=cache, + config_directory=cache, + command=["validate"], + environment={}, + port=13305, + pid_path=root / "lemond.pid", + stdout_path=stdout_path, + stderr_path=root / "stderr.log", + ) + + popen.assert_not_called() + + def test_containment_assignment_precedes_pid_and_close_failure_keeps_pid(self): + runner = load_runner() + events = [] + + class FakeContainment: + @staticmethod + def popen_kwargs(): + return {} + + @staticmethod + def assign(_process): + events.append("assign") + + @staticmethod + def close(): + events.append("close") + return False + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cache = root / "cache" + cache.mkdir() + pid_path = root / "lemond.pid" + + def run_client( + selected_command, + *, + env, + containment, + timeout_seconds, + ): + self.assertTrue(pid_path.is_file()) + self.assertEqual(selected_command, ["validate"]) + self.assertEqual(env, {}) + self.assertIsInstance(containment, FakeContainment) + self.assertEqual( + timeout_seconds, + runner.VALIDATION_PHASE_TIMEOUT_SECONDS, + ) + events.append("client") + return 0 + + def shutdown(*_args): + events.append("shutdown") + return True + + with ( + mock.patch.object( + runner, + "_create_process_containment", + return_value=FakeContainment(), + ), + mock.patch.object( + runner.subprocess, + "Popen", + side_effect=lambda *_args, **_kwargs: ( + events.append("popen") or FakeProcess() + ), + ), + mock.patch.object(runner.time, "sleep"), + mock.patch.object( + runner, + "_run_contained_command", + side_effect=run_client, + ), + mock.patch.object( + runner, + "_shutdown_server", + side_effect=shutdown, + ), + ): + exit_code, cleanup_succeeded = runner._run_validation_process( + lemond=root / "lemond", + cache_directory=cache, + config_directory=cache, + command=["validate"], + environment={}, + port=13305, + pid_path=pid_path, + stdout_path=root / "stdout.log", + stderr_path=root / "stderr.log", + ) + pid_evidence_retained = pid_path.is_file() + + self.assertEqual(exit_code, 0) + self.assertFalse(cleanup_succeeded) + self.assertEqual(events, ["popen", "assign", "client", "shutdown", "close"]) + self.assertTrue(pid_evidence_retained) + + def test_validation_cache_rejects_directory_replacement_and_symlink_swap(self): + runner = load_runner() + + for cache_name in ( + "hf_cache_directory", + "private_temp_directory", + "results_directory", + "logs_directory", + ): + for attack in ("directory", "symlink"): + self._assert_cache_directory_attack_is_rejected( + runner, cache_name, attack + ) + + def _assert_cache_directory_attack_is_rejected( + self, runner, cache_name: str, attack: str + ) -> None: + with ( + self.subTest(cache_name=cache_name, attack=attack), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + cache = runner.create_validation_cache(root, "linux-vulkan") + attacked_path = getattr(cache, cache_name).path + original = attacked_path.with_name(f"{attacked_path.name}-original") + attacked_path.rename(original) + if attack == "directory": + attacked_path.mkdir() + else: + outside = root / "outside" + outside.mkdir() + try: + attacked_path.symlink_to(outside, target_is_directory=True) + except OSError as exc: + self.skipTest(f"directory symlinks are unavailable: {exc}") + + with self.assertRaisesRegex(RuntimeError, "changed|symlink or junction"): + runner.revalidate_validation_cache(cache) + + def test_pid_file_failure_still_stops_the_started_server(self): + runner = load_runner() + + class FakeContainment: + @staticmethod + def popen_kwargs(): + return {} + + @staticmethod + def assign(_process): + return None + + @staticmethod + def close(): + return True + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cache = root / "cache" + cache.mkdir() + pid_path = root / "pid-as-directory" + pid_path.mkdir() + with ( + mock.patch.object( + runner, + "_create_process_containment", + return_value=FakeContainment(), + ), + mock.patch.object( + runner.subprocess, "Popen", return_value=FakeProcess() + ), + mock.patch.object(runner, "_run_contained_command") as run, + mock.patch.object( + runner, "_shutdown_server", return_value=True + ) as shutdown, + self.assertRaises(OSError), + ): + runner._run_validation_process( + lemond=root / "lemond", + cache_directory=cache, + config_directory=cache, + command=["validate"], + environment={}, + port=13305, + pid_path=pid_path, + stdout_path=root / "stdout.log", + stderr_path=root / "stderr.log", + ) + + run.assert_not_called() + shutdown.assert_called_once() + + def test_already_exited_server_is_an_unclean_validation_failure(self): + runner = load_runner() + + class ExitedProcess: + pid = 999 + + @staticmethod + def poll(): + return 17 + + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "cache" + cache.mkdir() + with mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([], True), + ): + clean = runner._shutdown_server(ExitedProcess(), 13305, cache) + + self.assertFalse(clean) + + def test_nonzero_clean_shutdown_exit_is_a_validation_failure(self): + runner = load_runner() + + class NonzeroShutdownProcess: + pid = 1001 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout): + del timeout + self.returncode = 139 + return self.returncode + + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "cache" + cache.mkdir() + with ( + mock.patch.object(runner, "_open_local_url"), + mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([], True), + ), + ): + clean = runner._shutdown_server(NonzeroShutdownProcess(), 13305, cache) + + self.assertFalse(clean) + + def test_clean_shutdown_uses_http_and_returns_true(self): + runner = load_runner() + wait_timeouts = [] + + class CleanShutdownProcess: + pid = 1003 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout): + wait_timeouts.append(timeout) + self.returncode = 0 + return self.returncode + + process = CleanShutdownProcess() + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "cache" + cache.mkdir() + with ( + mock.patch.object(runner, "_open_local_url") as open_url, + mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([], True), + ), + ): + clean = runner._shutdown_server(process, 13305, cache) + + self.assertTrue(clean) + self.assertTrue(wait_timeouts) + self.assertTrue(all(timeout > 0 for timeout in wait_timeouts)) + open_url.assert_called_once() + self.assertEqual(open_url.call_args.kwargs, {"timeout": 10}) + + def test_shutdown_cancellation_rescans_cache_before_bounded_abort(self): + runner = load_runner() + cancellation_deadline = time.monotonic() + 4 + cancellation = runner._ValidationCancelled( + signal.SIGINT, + cancellation_deadline, + ) + orphan = argparse.Namespace(pid=2002) + process = argparse.Namespace(pid=2001) + process.poll = mock.Mock(return_value=None) + containment = object() + + with ( + mock.patch.object( + runner, + "_open_local_url_with_cancellation", + side_effect=cancellation, + ), + mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([orphan], True), + ) as scan, + mock.patch.object(runner, "_force_kill_processes") as force_kill, + mock.patch.object(runner, "_abort_process_containment") as abort, + mock.patch.object(runner, "_reap_direct_process") as reap, + self.assertRaises(runner._ValidationCancelled) as raised, + ): + runner._shutdown_server( + process, + 13305, + Path("/validation-cache"), + containment, + ) + + self.assertIs(raised.exception, cancellation) + scan.assert_called_once_with(Path("/validation-cache"), process.pid) + force_kill.assert_called_once_with([orphan], cancellation_deadline) + abort.assert_called_once_with(containment, cancellation_deadline) + reap.assert_called_once_with(process, cancellation_deadline) + + def test_cleaned_orphan_process_still_invalidates_the_lane(self): + runner = load_runner() + + class CleanShutdownProcess: + pid = 1002 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout): + del timeout + self.returncode = 0 + return self.returncode + + orphan = object() + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "cache" + cache.mkdir() + with ( + mock.patch.object(runner, "_open_local_url"), + mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([orphan], True), + ), + mock.patch.object(runner, "_stop_processes", return_value=True) as stop, + ): + clean = runner._shutdown_server(CleanShutdownProcess(), 13305, cache) + + self.assertFalse(clean) + stop.assert_called_once_with([orphan]) + + def test_orphan_cleanup_waits_for_term_exit_before_using_kill(self): + runner = load_runner() + events = [] + + class FakePsutilError(Exception): + pass + + class ExitsOnTermProcess: + @staticmethod + def terminate(): + events.append("terminate") + + @staticmethod + def kill(): + events.append("kill") + + def wait_procs(processes, timeout): + events.append(("wait", timeout)) + return processes, [] + + fake_psutil = argparse.Namespace( + Error=FakePsutilError, + wait_procs=wait_procs, + ) + with mock.patch.dict(sys.modules, {"psutil": fake_psutil}): + clean = runner._stop_processes([ExitsOnTermProcess()]) + + self.assertTrue(clean) + self.assertEqual(events[0], "terminate") + self.assertEqual(events[1][0], "wait") + self.assertGreater(events[1][1], 0) + self.assertLessEqual(events[1][1], runner.PROCESS_WAIT_POLL_SECONDS) + self.assertNotIn("kill", events) + + @unittest.skipIf(os.name == "nt", "POSIX process-group test") + def test_process_group_probe_falls_back_when_psutil_enumeration_fails(self): + runner = load_runner() + + for enumeration_error in ( + PermissionError("denied"), + OSError("enumeration failed"), + ): + fake_psutil = mock.Mock() + fake_psutil.STATUS_DEAD = "dead" + fake_psutil.STATUS_ZOMBIE = "zombie" + fake_psutil.Error = RuntimeError + fake_psutil.process_iter.side_effect = enumeration_error + with ( + self.subTest(error=type(enumeration_error).__name__), + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + mock.patch.object(runner.os, "killpg", return_value=None) as killpg, + ): + self.assertTrue( + runner._PosixProcessGroupContainment._has_live_members(4321) + ) + killpg.assert_called_once_with(4321, 0) + + fake_psutil = mock.Mock() + fake_psutil.STATUS_DEAD = "dead" + fake_psutil.STATUS_ZOMBIE = "zombie" + fake_psutil.Error = RuntimeError + fake_psutil.process_iter.side_effect = PermissionError("denied") + with ( + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + mock.patch.object( + runner.os, + "killpg", + side_effect=ProcessLookupError, + ), + ): + self.assertFalse( + runner._PosixProcessGroupContainment._has_live_members(4321) + ) + + dead_process = mock.Mock() + dead_process.pid = 9875 + dead_process.info = {"status": "dead"} + + def partial_enumeration(_attributes): + yield dead_process + raise OSError("enumeration failed") + + fake_psutil = mock.Mock() + fake_psutil.STATUS_DEAD = "dead" + fake_psutil.STATUS_ZOMBIE = "zombie" + fake_psutil.Error = RuntimeError + fake_psutil.process_iter.side_effect = partial_enumeration + with ( + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + mock.patch.object(runner.os, "killpg", return_value=None) as killpg, + ): + self.assertTrue( + runner._PosixProcessGroupContainment._has_live_members(4321) + ) + killpg.assert_called_once_with(4321, 0) + + @unittest.skipIf(os.name == "nt", "POSIX process-group test") + def test_normal_process_group_close_force_kills_when_cancellation_is_pending(self): + runner = load_runner() + containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" + ) + containment._process_groups = [4324] + cancellation = argparse.Namespace(signum=signal.SIGINT) + + with ( + mock.patch.object( + runner, + "_ACTIVE_CANCELLATION_STATE", + cancellation, + ), + mock.patch.object( + runner._PosixProcessGroupContainment, + "_has_live_members", + return_value=True, + ), + mock.patch.object(runner.os, "killpg") as killpg, + ): + clean = containment.close() + + self.assertFalse(clean) + self.assertEqual( + killpg.call_args_list, + [ + mock.call(4324, signal.SIGTERM), + mock.call(4324, signal.SIGKILL), + ], + ) + + def test_cache_process_scan_fails_closed_when_psutil_enumeration_fails(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + for enumeration_error in ( + PermissionError("denied"), + OSError("enumeration failed"), + ): + fake_psutil = mock.Mock() + fake_psutil.Error = RuntimeError + fake_psutil.process_iter.side_effect = enumeration_error + with ( + self.subTest(error=type(enumeration_error).__name__), + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + ): + scan = runner._cache_processes(Path(directory), os.getpid()) + + self.assertFalse(scan.complete) + self.assertEqual(scan.processes, []) + + def test_partial_cache_process_scan_retains_known_matches(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory).resolve() + known_process = mock.Mock() + known_process.pid = 9876 + known_process.info = {"exe": str(cache / "llama-server")} + + def partial_enumeration(_attributes): + yield known_process + raise PermissionError("enumeration failed") + + fake_psutil = mock.Mock() + fake_psutil.Error = RuntimeError + fake_psutil.process_iter.side_effect = partial_enumeration + with mock.patch.dict(sys.modules, {"psutil": fake_psutil}): + scan = runner._cache_processes(cache, os.getpid()) + + self.assertFalse(scan.complete) + self.assertEqual(scan.processes, [known_process]) + + def test_incomplete_cache_process_scan_invalidates_cleanup_and_stops_matches(self): + runner = load_runner() + + class ExitedProcess: + pid = 9877 + + @staticmethod + def poll(): + return 0 + + known_process = object() + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) + with ( + mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([known_process], False), + ), + mock.patch.object(runner, "_stop_processes", return_value=True) as stop, + ): + clean = runner._shutdown_server(ExitedProcess(), 13305, cache) + + self.assertFalse(clean) + stop.assert_called_once_with([known_process]) + + def test_empty_incomplete_cache_process_scan_invalidates_cleanup(self): + runner = load_runner() + + class ExitedProcess: + pid = 9878 + + @staticmethod + def poll(): + return 0 + + with tempfile.TemporaryDirectory() as directory: + with mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([], False), + ): + clean = runner._shutdown_server( + ExitedProcess(), + 13305, + Path(directory), + ) + + self.assertFalse(clean) + + def test_contained_command_timeout_closes_containment(self): + runner = load_runner() + events = [] + + class TimedOutProcess: + pid = 4322 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout): + events.append(("wait", timeout)) + if self.returncode is None: + time.sleep(timeout) + raise subprocess.TimeoutExpired(["validate"], timeout) + return self.returncode + + def terminate(self): + events.append("terminate") + self.returncode = -signal.SIGTERM + + def kill(self): + events.append("kill") + self.returncode = -signal.SIGKILL + + class FakeContainment: + @staticmethod + def popen_kwargs(): + return {} + + @staticmethod + def assign(_process): + events.append("assign") + + @staticmethod + def abort(_deadline): + events.append("abort") + return False + + @staticmethod + def close(): + events.append("close") + return False + + process = TimedOutProcess() + with ( + mock.patch.object(runner.subprocess, "Popen", return_value=process), + self.assertRaisesRegex(RuntimeError, "timed out after 0.01 seconds"), + ): + runner._run_contained_command( + ["validate"], + env={}, + containment=FakeContainment(), + timeout_seconds=0.01, + ) + + self.assertIn("abort", events) + self.assertLess(events.index("abort"), events.index("kill")) + self.assertNotIn("terminate", events) + self.assertNotIn(("wait", 5), events) + + def test_contained_command_preserves_nonzero_exit_code(self): + runner = load_runner() + + class ExitedProcess: + pid = 4323 + + @staticmethod + def wait(timeout): + del timeout + return 23 + + class FakeContainment: + @staticmethod + def popen_kwargs(): + return {} + + @staticmethod + def assign(_process): + return None + + @staticmethod + def close(): + raise AssertionError("normal process exit must not close containment") + + with mock.patch.object( + runner.subprocess, + "Popen", + return_value=ExitedProcess(), + ): + exit_code = runner._run_contained_command( + ["validate"], + env={}, + containment=FakeContainment(), + timeout_seconds=30, + ) + + self.assertEqual(exit_code, 23) + + def test_darwin_containment_requires_the_trusted_ephemeral_runner_contract(self): + runner = load_runner() + trusted_environment = { + "GITHUB_ACTIONS": "true", + runner.DARWIN_EPHEMERAL_RUNNER_CONTEXT_KEY: ( + runner.DARWIN_EPHEMERAL_RUNNER_CONTEXT + ), + } + + with mock.patch.object(runner.sys, "platform", "darwin"): + with self.assertRaisesRegex( + RuntimeError, "Darwin cannot contain session-escaping descendants" + ): + runner._create_process_containment() + with ( + mock.patch.dict( + os.environ, + {"GITHUB_ACTIONS": "true"}, + clear=True, + ), + self.assertRaisesRegex(RuntimeError, "requires the trusted"), + ): + runner._create_process_containment( + allow_darwin_github_hosted_ephemeral_runner=True, + backend="metal", + target="macos-metal", + ) + with ( + mock.patch.dict(os.environ, trusted_environment, clear=True), + self.assertRaisesRegex(RuntimeError, "restricted to macos-metal"), + ): + runner._create_process_containment( + allow_darwin_github_hosted_ephemeral_runner=True, + backend="vulkan", + target="macos-metal", + ) + with mock.patch.dict(os.environ, trusted_environment, clear=True): + containment = runner._create_process_containment( + allow_darwin_github_hosted_ephemeral_runner=True, + backend="metal", + target="macos-metal", + ) + + self.assertTrue(containment.close()) + + def test_linux_subreaper_lease_restores_only_the_original_disabled_state(self): + runner = load_runner() + + class RunnerProcess: + @staticmethod + def children(recursive): + del recursive + return [] + + fake_psutil = argparse.Namespace( + DEAD_STATUS="dead", + Error=OSError, + NoSuchProcess=ProcessLookupError, + Process=lambda _pid: RunnerProcess(), + STATUS_DEAD="dead", + STATUS_ZOMBIE="zombie", + ZombieProcess=ProcessLookupError, + ) + with ( + mock.patch.object(runner.sys, "platform", "linux"), + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + mock.patch.object( + runner, "_linux_child_subreaper_state", return_value=False + ) as get_state, + mock.patch.object(runner, "_set_linux_child_subreaper") as set_state, + ): + first = runner._PosixProcessGroupContainment() + second = runner._PosixProcessGroupContainment() + self.assertTrue(first.close()) + self.assertTrue(second.close()) + + get_state.assert_called_once_with() + self.assertEqual(set_state.call_args_list, [mock.call(True), mock.call(False)]) + + def test_linux_subreaper_initialization_fails_before_process_launch(self): + runner = load_runner() + + class RunnerProcess: + @staticmethod + def children(recursive): + del recursive + return [] + + fake_psutil = argparse.Namespace( + Error=OSError, + Process=lambda _pid: RunnerProcess(), + ) + with ( + mock.patch.object(runner.sys, "platform", "linux"), + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + mock.patch.object( + runner, + "_acquire_linux_child_subreaper", + side_effect=OSError("prctl denied"), + ), + mock.patch.object(runner.subprocess, "Popen") as popen, + self.assertRaisesRegex(OSError, "prctl denied"), + ): + runner._PosixProcessGroupContainment() + + popen.assert_not_called() + + def test_linux_subreaper_rejects_unsafe_preexisting_process_state(self): + runner = load_runner() + + class RunnerProcess: + descendants = [] + + @classmethod + def children(cls, recursive): + del recursive + return cls.descendants + + fake_psutil = argparse.Namespace( + Error=OSError, + Process=lambda _pid: RunnerProcess(), + ) + with ( + mock.patch.object(runner.sys, "platform", "linux"), + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + mock.patch.object( + runner.signal, + "getsignal", + return_value=signal.SIG_IGN, + ), + mock.patch.object(runner, "_acquire_linux_child_subreaper") as acquire, + self.assertRaisesRegex(RuntimeError, "default SIGCHLD"), + ): + runner._PosixProcessGroupContainment() + acquire.assert_not_called() + + RunnerProcess.descendants = [object()] + with ( + mock.patch.object(runner.sys, "platform", "linux"), + mock.patch.dict(sys.modules, {"psutil": fake_psutil}), + mock.patch.object( + runner.signal, + "getsignal", + return_value=signal.SIG_DFL, + ), + mock.patch.object(runner, "_acquire_linux_child_subreaper") as acquire, + self.assertRaisesRegex(RuntimeError, "pre-existing runner descendants"), + ): + runner._PosixProcessGroupContainment() + acquire.assert_not_called() + + @unittest.skipIf(os.name == "nt", "POSIX descendant tracking test") + @unittest.skipIf( + not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, + "Darwin process-tree inspection is unavailable", + ) + def test_clean_contained_process_exit_remains_clean_with_tracking(self): + runner = load_runner() + containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" + ) + + exit_code = runner._run_contained_command( + [sys.executable, "-c", "pass"], + env=dict(os.environ), + containment=containment, + timeout_seconds=30, + ) + + self.assertEqual(exit_code, 0) + self.assertTrue(containment.close()) + + @unittest.skipIf(os.name == "nt", "POSIX gated process launch test") + @unittest.skipIf( + not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, + "Darwin process-tree inspection is unavailable", + ) + def test_posix_gate_assigns_containment_before_candidate_code_runs(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "candidate-ran.txt" + containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" + ) + original_assign = containment.assign + + def assert_candidate_is_blocked(process): + self.assertFalse(marker.exists()) + original_assign(process) + + try: + with mock.patch.object( + containment, + "assign", + side_effect=assert_candidate_is_blocked, + ): + process = containment.start_process( + [ + sys.executable, + "-c", + ( + "import pathlib, sys; " + "pathlib.Path(sys.argv[1]).write_text(" + "'ran', encoding='ascii')" + ), + str(marker), + ], + env=dict(os.environ), + ) + self.assertEqual(process.wait(timeout=10), 0) + self.assertEqual(marker.read_text(encoding="ascii"), "ran") + finally: + self.assertTrue(containment.close()) + + @unittest.skipUnless( + sys.platform.startswith("linux"), "Linux child-subreaper containment test" + ) + def test_linux_subreaper_reaps_adopted_child_without_stealing_root_status(self): + runner = load_runner() + daemon_program = """ +import os +import pathlib +import sys + +first_child = os.fork() +if first_child: + os._exit(0) +os.setsid() +second_child = os.fork() +if second_child: + os._exit(0) +pathlib.Path(sys.argv[1]).write_text(str(os.getpid()), encoding="ascii") +""" + root_program = f""" +import pathlib +import subprocess +import sys +import time + +subprocess.Popen( + [sys.executable, "-c", {daemon_program!r}, sys.argv[1]], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, +) +while not pathlib.Path(sys.argv[1]).is_file(): + time.sleep(0.01) +time.sleep(0.3) +raise SystemExit(23) +""" + + with tempfile.TemporaryDirectory() as directory: + daemon_pid_path = Path(directory) / "daemon.pid" + containment = runner._PosixProcessGroupContainment() + try: + exit_code = runner._run_contained_command( + [ + sys.executable, + "-c", + root_program, + str(daemon_pid_path), + ], + env=dict(os.environ), + containment=containment, + timeout_seconds=30, + ) + daemon_pid = int(daemon_pid_path.read_text(encoding="ascii")) + self.assertEqual(exit_code, 23) + self.assertFalse(_pid_is_active(daemon_pid)) + finally: + self.assertTrue(containment.close()) + + def test_cancellation_reuses_one_cleanup_deadline(self): + runner = load_runner() + cancellation = argparse.Namespace(signum=signal.SIGINT) + + with ( + mock.patch.object( + runner, + "_ACTIVE_CANCELLATION_STATE", + cancellation, + ), + mock.patch.object(runner.time, "monotonic", return_value=100.0) as clock, + ): + with self.assertRaises(runner._ValidationCancelled) as first: + runner._raise_if_cancelled() + with self.assertRaises(runner._ValidationCancelled) as second: + runner._raise_if_cancelled() + + self.assertEqual(first.exception.cleanup_deadline, 104.0) + self.assertEqual( + second.exception.cleanup_deadline, + first.exception.cleanup_deadline, + ) + clock.assert_called_once_with() + + @unittest.skipIf(os.name == "nt", "POSIX signal containment test") + def test_sigint_and_sigterm_kill_contained_command_descendants(self): + for selected_signal in (signal.SIGINT, signal.SIGTERM): + with self.subTest(selected_signal=selected_signal): + self._assert_runner_signal_kills_descendants(selected_signal) + + @unittest.skipIf(os.name == "nt", "POSIX signal containment test") + def test_sigint_force_kills_term_ignoring_tree_before_escalation(self): + elapsed = self._assert_runner_signal_kills_descendants( + signal.SIGINT, + ignore_sigterm=True, + ) + + self.assertLess(elapsed, 6.0) + + @unittest.skipIf(os.name == "nt", "POSIX signal containment test") + @unittest.skipIf( + not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, + "Darwin process-tree inspection is unavailable", + ) + def test_sigint_and_sigterm_kill_session_escaping_descendants(self): + for selected_signal in (signal.SIGINT, signal.SIGTERM): + with self.subTest(selected_signal=selected_signal): + elapsed = self._assert_runner_signal_kills_descendants( + selected_signal, + ignore_sigterm=True, + escape_session=True, + ) + self.assertLess(elapsed, 6.0) + + @unittest.skipUnless( + sys.platform.startswith("linux"), "Linux child-subreaper containment test" + ) + def test_sigint_and_sigterm_kill_fast_double_fork_session_escape(self): + for selected_signal in (signal.SIGINT, signal.SIGTERM): + with self.subTest(selected_signal=selected_signal): + elapsed = self._assert_runner_signal_kills_descendants( + selected_signal, + ignore_sigterm=True, + double_fork=True, + ) + self.assertLess(elapsed, 6.0) + + def _assert_runner_signal_kills_descendants( + self, + selected_signal: int, + *, + ignore_sigterm: bool = False, + escape_session: bool = False, + double_fork: bool = False, + ) -> float: + descendant_setup = [] + if double_fork: + descendant_setup.extend( + [ + "first_child = os.fork()", + "if first_child:", + " os._exit(0)", + "os.setsid()", + "second_child = os.fork()", + "if second_child:", + " os._exit(0)", + ] + ) + elif escape_session: + descendant_setup.append("os.setsid()") + if ignore_sigterm: + descendant_setup.append("signal.signal(signal.SIGTERM, signal.SIG_IGN)") + descendant_program = f""" +import os +import pathlib +import signal +import sys +import time + +{chr(10).join(descendant_setup)} +pathlib.Path(sys.argv[1]).write_text(str(os.getpid()), encoding="ascii") +time.sleep(30) +""" + parent_signal_setup = "" + if ignore_sigterm: + parent_signal_setup = "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + child_program = f""" +import os +import pathlib +import signal +import subprocess +import sys +import time + +{parent_signal_setup} +child = subprocess.Popen( + [sys.executable, "-c", {descendant_program!r}, sys.argv[2]], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, +) +while not pathlib.Path(sys.argv[2]).is_file(): + time.sleep(0.01) +descendant_pid = pathlib.Path(sys.argv[2]).read_text(encoding="ascii") +pathlib.Path(sys.argv[1]).write_text( + f"{{os.getpid()}},{{descendant_pid}}", + encoding="ascii", +) +time.sleep(30) +""" + with tempfile.TemporaryDirectory() as directory: + pid_path = Path(directory) / "contained-pids.txt" + descendant_ready_path = Path(directory) / "descendant-ready.txt" + helper_program = f""" +import importlib.util +import os +import sys + +spec = importlib.util.spec_from_file_location("signal_runner", {str(RUNNER_PATH)!r}) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) +command = [ + sys.executable, + "-c", + {child_program!r}, + {str(pid_path)!r}, + {str(descendant_ready_path)!r}, +] + +def run_validation(_args): + containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" + ) + try: + return runner._run_contained_command( + command, + env=dict(os.environ), + containment=containment, + timeout_seconds=30, + ) + finally: + runner._close_process_containment(containment) + +runner.run_validation = run_validation +sys.argv = [ + "run_llamacpp_validation.py", + "--lemond", + sys.executable, + "--python", + sys.executable, + "--backend", + "cpu", + "--target", + "linux-cpu", +] +runner.main() +""" + helper = subprocess.Popen( + [sys.executable, "-c", helper_program], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + command_pid = None + descendant_pid = None + try: + deadline = time.monotonic() + 10 + pid_values = None + while pid_values is None and time.monotonic() < deadline: + if helper.poll() is not None: + break + try: + values = pid_path.read_text(encoding="ascii").split(",") + if len(values) == 2: + pid_values = tuple(int(value) for value in values) + except (FileNotFoundError, ValueError): + pass + time.sleep(0.05) + if pid_values is None: + stdout, stderr = helper.communicate(timeout=5) + self.fail( + "signal test helper did not start its descendants: " + f"stdout={stdout!r}, stderr={stderr!r}" + ) + command_pid, descendant_pid = pid_values + signal_started = time.monotonic() + helper.send_signal(selected_signal) + stdout, stderr = helper.communicate(timeout=15) + signal_elapsed = time.monotonic() - signal_started + deadline = time.monotonic() + 10 + while ( + _pid_is_running(command_pid) or _pid_is_running(descendant_pid) + ) and time.monotonic() < deadline: + time.sleep(0.05) + + self.assertEqual( + helper.returncode, + 128 + selected_signal, + f"stdout={stdout!r}, stderr={stderr!r}", + ) + self.assertFalse(_pid_is_running(command_pid)) + self.assertFalse(_pid_is_running(descendant_pid)) + finally: + if helper.poll() is None: + helper.kill() + helper.wait(timeout=5) + if command_pid is not None: + try: + if ( + os.getpgid(command_pid) == command_pid + and command_pid != os.getpgrp() + ): + os.killpg(command_pid, signal.SIGKILL) + except OSError: + pass + if descendant_pid is not None and _pid_is_active(descendant_pid): + os.kill(descendant_pid, signal.SIGKILL) + return signal_elapsed + + @unittest.skipIf(os.name == "nt", "POSIX signal containment test") + def test_sigint_during_blocked_shutdown_aborts_term_ignoring_server_tree(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + pid_path = root / "server-pids.txt" + descendant_ready_path = root / "server-descendant-ready.txt" + shutdown_blocked_path = root / "shutdown-blocked.txt" + descendant_program = ( + "import pathlib, signal, sys, time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "pathlib.Path(sys.argv[1]).write_text('ready', encoding='ascii'); " + "time.sleep(30)" + ) + server_program = f""" +import os +import pathlib +import signal +import subprocess +import sys +import time + +signal.signal(signal.SIGTERM, signal.SIG_IGN) +child = subprocess.Popen([sys.executable, "-c", {descendant_program!r}, sys.argv[2]]) +while not pathlib.Path(sys.argv[2]).is_file(): + time.sleep(0.01) +pathlib.Path(sys.argv[1]).write_text( + f"{{os.getpid()}},{{child.pid}}", + encoding="ascii", +) +time.sleep(30) +""" + helper_program = f""" +import contextlib +import importlib.util +import os +import pathlib +import sys +import time + +spec = importlib.util.spec_from_file_location("shutdown_runner", {str(RUNNER_PATH)!r}) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) +containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" +) +server = runner.subprocess.Popen( + [ + sys.executable, + "-c", + {server_program!r}, + {str(pid_path)!r}, + {str(descendant_ready_path)!r}, + ], + **containment.popen_kwargs(), +) +containment.assign(server) + +def blocked_shutdown(_request, timeout): + del timeout + pathlib.Path({str(shutdown_blocked_path)!r}).write_text( + "blocked", + encoding="ascii", + ) + time.sleep(30) + return contextlib.nullcontext() + +runner._open_local_url = blocked_shutdown +try: + with runner._CancellationSignalGuard(): + try: + runner._shutdown_server( + server, + 13305, + pathlib.Path({str(root)!r}), + containment, + ) + runner._raise_if_cancelled() + finally: + runner._close_process_containment(containment) +except runner._ValidationCancelled as exc: + raise SystemExit(128 + exc.signum) from None +""" + helper = subprocess.Popen( + [sys.executable, "-c", helper_program], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + server_pid = None + descendant_pid = None + try: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + pid_values = tuple( + int(value) + for value in pid_path.read_text(encoding="ascii").split(",") + ) + except (FileNotFoundError, ValueError): + pid_values = () + if len(pid_values) == 2 and shutdown_blocked_path.is_file(): + server_pid, descendant_pid = pid_values + break + if helper.poll() is not None: + break + time.sleep(0.05) + if server_pid is None or descendant_pid is None: + stdout, stderr = helper.communicate(timeout=5) + self.fail( + "shutdown signal helper did not reach the blocked request: " + f"stdout={stdout!r}, stderr={stderr!r}" + ) + + signal_started = time.monotonic() + helper.send_signal(signal.SIGINT) + try: + stdout, stderr = helper.communicate(timeout=6) + except subprocess.TimeoutExpired: + self.fail("blocked shutdown ignored cancellation for 6 seconds") + signal_elapsed = time.monotonic() - signal_started + + self.assertEqual( + helper.returncode, + 128 + signal.SIGINT, + f"stdout={stdout!r}, stderr={stderr!r}", + ) + self.assertLess(signal_elapsed, 6.0) + self.assertFalse(_pid_is_running(server_pid)) + self.assertFalse(_pid_is_running(descendant_pid)) + finally: + if helper.poll() is None: + helper.kill() + if server_pid is not None: + try: + if ( + os.getpgid(server_pid) == server_pid + and server_pid != os.getpgrp() + ): + os.killpg(server_pid, signal.SIGKILL) + except OSError: + pass + try: + helper.communicate(timeout=5) + except subprocess.TimeoutExpired: + helper.kill() + helper.wait(timeout=5) + + @unittest.skipIf(os.name == "nt", "POSIX signal cleanup test") + @unittest.skipIf( + not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, + "Darwin process-tree inspection is unavailable", + ) + def test_sigint_during_orphan_cleanup_aborts_containment_and_orphan(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cache = root / "cache" + cache.mkdir() + contained_pids_path = root / "contained-pids.txt" + contained_descendant_ready_path = root / "contained-ready.txt" + orphan_pid_path = root / "orphan.pid" + orphan_ready_path = root / "orphan-ready.txt" + orphan_term_path = root / "orphan-term.txt" + contained_descendant_program = ( + "import pathlib, signal, sys, time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "pathlib.Path(sys.argv[1]).write_text('ready', encoding='ascii'); " + "time.sleep(30)" + ) + contained_program = f""" +import os +import pathlib +import signal +import subprocess +import sys +import time + +signal.signal(signal.SIGTERM, signal.SIG_IGN) +child = subprocess.Popen( + [sys.executable, "-c", {contained_descendant_program!r}, sys.argv[2]], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, +) +while not pathlib.Path(sys.argv[2]).is_file(): + time.sleep(0.01) +pathlib.Path(sys.argv[1]).write_text( + f"{{os.getpid()}},{{child.pid}}", + encoding="ascii", +) +time.sleep(30) +""" + orphan_program = """ +import pathlib +import signal +import sys +import time + +term_path = pathlib.Path(sys.argv[2]) + +def ignore_term(_signum, _frame): + term_path.write_text("term", encoding="ascii") + +signal.signal(signal.SIGTERM, ignore_term) +pathlib.Path(sys.argv[1]).write_text("ready", encoding="ascii") +while True: + time.sleep(1) +""" + helper_program = f""" +import importlib.util +import os +import pathlib +import signal +import subprocess +import sys +import time +import psutil as real_psutil + +spec = importlib.util.spec_from_file_location("orphan_runner", {str(RUNNER_PATH)!r}) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) +actual_containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" +) +contained = subprocess.Popen( + [ + sys.executable, + "-c", + {contained_program!r}, + {str(contained_pids_path)!r}, + {str(contained_descendant_ready_path)!r}, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + **actual_containment.popen_kwargs(), +) +actual_containment.assign(contained) +orphan = subprocess.Popen( + [ + sys.executable, + "-c", + {orphan_program!r}, + {str(orphan_ready_path)!r}, + {str(orphan_term_path)!r}, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, +) +while not pathlib.Path({str(orphan_ready_path)!r}).is_file(): + time.sleep(0.01) +pathlib.Path({str(orphan_pid_path)!r}).write_text(str(orphan.pid), encoding="ascii") + +class OrphanProcess: + pid = orphan.pid + + @staticmethod + def terminate(): + os.kill(orphan.pid, signal.SIGTERM) + + @staticmethod + def kill(): + os.kill(orphan.pid, signal.SIGKILL) + +def wait_procs(processes, timeout): + deadline = time.monotonic() + timeout + while True: + alive = [] + for process in processes: + try: + waited_pid, _status = os.waitpid(process.pid, os.WNOHANG) + except ChildProcessError: + waited_pid = process.pid + if waited_pid == 0: + alive.append(process) + if not alive: + return processes, [] + if time.monotonic() >= deadline: + return [], alive + time.sleep(0.01) + +real_psutil.wait_procs = wait_procs +sys.modules["psutil"] = real_psutil +runner._cache_processes = lambda _cache, _excluded: runner._CacheProcessScan( + [OrphanProcess()], + True, +) + +class DeferredContainment: + @staticmethod + def close(): + return False + + @staticmethod + def abort(deadline): + return actual_containment.abort(deadline) + +class ExitedProcess: + pid = os.getpid() + + @staticmethod + def poll(): + return 0 + + @staticmethod + def wait(timeout): + del timeout + return 0 + +try: + with runner._CancellationSignalGuard(): + runner._shutdown_server( + ExitedProcess(), + 13305, + pathlib.Path({str(cache)!r}), + DeferredContainment(), + ) + runner._raise_if_cancelled() +except runner._ValidationCancelled as exc: + raise SystemExit(128 + exc.signum) from None +""" + helper = subprocess.Popen( + [sys.executable, "-c", helper_program], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + orphan_pid = None + contained_pid = None + contained_descendant_pid = None + try: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + orphan_pid = int(orphan_pid_path.read_text(encoding="ascii")) + except (FileNotFoundError, ValueError): + orphan_pid = None + try: + contained_pids = tuple( + int(value) + for value in contained_pids_path.read_text( + encoding="ascii" + ).split(",") + ) + except (FileNotFoundError, ValueError): + contained_pids = () + if len(contained_pids) == 2: + contained_pid, contained_descendant_pid = contained_pids + if ( + orphan_pid is not None + and contained_pid is not None + and contained_descendant_pid is not None + and orphan_term_path.is_file() + ): + break + if helper.poll() is not None: + break + time.sleep(0.05) + if ( + orphan_pid is None + or contained_pid is None + or contained_descendant_pid is None + or not orphan_term_path.is_file() + ): + stdout, stderr = helper.communicate(timeout=5) + self.fail( + "shutdown helper did not reach orphan cleanup: " + f"stdout={stdout!r}, stderr={stderr!r}" + ) + + signal_started = time.monotonic() + helper.send_signal(signal.SIGINT) + try: + stdout, stderr = helper.communicate(timeout=3) + except subprocess.TimeoutExpired: + self.fail("orphan cleanup ignored cancellation for 3 seconds") + signal_elapsed = time.monotonic() - signal_started + + self.assertEqual( + helper.returncode, + 128 + signal.SIGINT, + f"stdout={stdout!r}, stderr={stderr!r}", + ) + self.assertLess(signal_elapsed, 3.0) + self.assertFalse( + _pid_is_running(orphan_pid), + f"stdout={stdout!r}, stderr={stderr!r}", + ) + self.assertFalse( + _pid_is_running(contained_pid), + f"stdout={stdout!r}, stderr={stderr!r}", + ) + self.assertFalse( + _pid_is_running(contained_descendant_pid), + f"stdout={stdout!r}, stderr={stderr!r}", + ) + finally: + if helper.poll() is None: + helper.kill() + if orphan_pid is not None and _pid_is_active(orphan_pid): + os.kill(orphan_pid, signal.SIGKILL) + if contained_pid is not None: + try: + if ( + os.getpgid(contained_pid) == contained_pid + and contained_pid != os.getpgrp() + ): + os.killpg(contained_pid, signal.SIGKILL) + except OSError: + pass + try: + helper.communicate(timeout=5) + except subprocess.TimeoutExpired: + helper.kill() + helper.wait(timeout=5) + + @unittest.skipIf(os.name == "nt", "POSIX process-group test") + def test_shutdown_terminates_an_external_descendant_in_the_server_session(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + child_pid_path = root / "child.pid" + containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" + ) + parent = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import pathlib, subprocess, sys; " + "child = subprocess.Popen(['/bin/sleep', '30']); " + "pathlib.Path(sys.argv[1]).write_text(str(child.pid), " + "encoding='ascii')" + ), + str(child_pid_path), + ], + **containment.popen_kwargs(), + ) + containment.assign(parent) + parent.wait(timeout=5) + child_pid = int(child_pid_path.read_text(encoding="ascii")) + try: + self.assertTrue(_pid_is_active(child_pid)) + with mock.patch.object( + runner, + "_cache_processes", + return_value=runner._CacheProcessScan([], True), + ): + clean = runner._shutdown_server( + parent, + 13305, + root, + containment, + ) + + self.assertFalse(clean) + deadline = time.monotonic() + 5 + while _pid_is_active(child_pid) and time.monotonic() < deadline: + time.sleep(0.05) + self.assertFalse(_pid_is_active(child_pid)) + self.assertFalse(containment.close()) + finally: + if _pid_is_active(child_pid): + os.kill(child_pid, signal.SIGKILL) + + @unittest.skipIf(os.name == "nt", "POSIX process-group test") + def test_validation_command_descendants_are_in_the_contained_session(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + child_pid_path = Path(directory) / "child.pid" + containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" + ) + exit_code = runner._run_contained_command( + [ + sys.executable, + "-c", + ( + "import pathlib, subprocess, sys; " + "child = subprocess.Popen(['/bin/sleep', '30']); " + "pathlib.Path(sys.argv[1]).write_text(str(child.pid), " + "encoding='ascii')" + ), + str(child_pid_path), + ], + env=dict(os.environ), + containment=containment, + timeout_seconds=30, + ) + child_pid = int(child_pid_path.read_text(encoding="ascii")) + try: + self.assertEqual(exit_code, 0) + self.assertTrue(_pid_is_active(child_pid)) + self.assertFalse(containment.close()) + deadline = time.monotonic() + 5 + while _pid_is_active(child_pid) and time.monotonic() < deadline: + time.sleep(0.05) + self.assertFalse(_pid_is_active(child_pid)) + finally: + if _pid_is_active(child_pid): + os.kill(child_pid, signal.SIGKILL) + + @unittest.skipIf(os.name == "nt", "POSIX descendant tracking test") + @unittest.skipIf( + not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, + "Darwin process-tree inspection is unavailable", + ) + def test_close_kills_retained_session_escape_after_parent_exit(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + child_pid_path = root / "child.pid" + child_ready_path = root / "child-ready.txt" + descendant_program = ( + "import os, pathlib, sys, time; " + "os.setsid(); " + "pathlib.Path(sys.argv[1]).write_text('ready', encoding='ascii'); " + "time.sleep(30)" + ) + parent_program = f""" +import pathlib +import subprocess +import sys +import time + +child = subprocess.Popen( + [sys.executable, "-c", {descendant_program!r}, sys.argv[2]], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, +) +ready = pathlib.Path(sys.argv[2]) +deadline = time.monotonic() + 5 +while not ready.is_file() and time.monotonic() < deadline: + time.sleep(0.01) +pathlib.Path(sys.argv[1]).write_text(str(child.pid), encoding="ascii") +time.sleep(0.5) +""" + containment = runner._PosixProcessGroupContainment( + allow_untracked_descendants=sys.platform == "darwin" + ) + exit_code = runner._run_contained_command( + [ + sys.executable, + "-c", + parent_program, + str(child_pid_path), + str(child_ready_path), + ], + env=dict(os.environ), + containment=containment, + timeout_seconds=30, + ) + child_pid = int(child_pid_path.read_text(encoding="ascii")) + try: + self.assertEqual(exit_code, 0) + self.assertTrue(_pid_is_running(child_pid)) + self.assertFalse(containment.close()) + deadline = time.monotonic() + 5 + while _pid_is_running(child_pid) and time.monotonic() < deadline: + time.sleep(0.05) + self.assertFalse(_pid_is_running(child_pid)) + finally: + if _pid_is_active(child_pid): + os.kill(child_pid, signal.SIGKILL) + + def test_windows_job_object_sets_kill_on_close_and_closes_once(self): + runner = load_runner() + events = [] + + class FakeKernel32: + @staticmethod + def CreateJobObjectW(_attributes, _name): + events.append("create") + return 123 + + @staticmethod + def SetInformationJobObject(handle, info_class, pointer, size): + information = ctypes.cast( + pointer, + ctypes.POINTER(runner._JobObjectExtendedLimitInformation), + ).contents + events.append( + ( + "configure", + handle, + info_class, + information.basic_limit_information.limit_flags, + size, + ) + ) + return 1 + + @staticmethod + def AssignProcessToJobObject(handle, process_handle): + events.append(("assign", handle, process_handle)) + return 1 + + @staticmethod + def QueryInformationJobObject( + handle, info_class, pointer, size, _return_size + ): + information = ctypes.cast( + pointer, + ctypes.POINTER(runner._JobObjectBasicAccountingInformation), + ).contents + information.active_processes = 0 + events.append(("query", handle, info_class, size)) + return 1 + + @staticmethod + def CloseHandle(handle): + events.append(("close", handle)) + return 1 + + job = runner._WindowsKillOnCloseJob.create(FakeKernel32()) + process = argparse.Namespace(_handle=456) + job.assign(process) + + self.assertTrue(job.abort(time.monotonic() + 4)) + self.assertTrue(job.close()) + self.assertEqual(events[0], "create") + self.assertEqual( + events[1][0:4], + ( + "configure", + 123, + 9, + 0x00002000, + ), + ) + self.assertEqual( + events[1][4], ctypes.sizeof(runner._JobObjectExtendedLimitInformation) + ) + self.assertEqual(events[2], ("assign", 123, 456)) + self.assertEqual( + events[3:], + [ + ( + "query", + 123, + 1, + ctypes.sizeof(runner._JobObjectBasicAccountingInformation), + ), + ("close", 123), + ], + ) + + def test_windows_gate_assigns_before_fast_descendants_and_job_close_kills_tree( + self, + ): + runner = load_runner() + events = [] + assigned = False + descendants = {"child": False, "grandchild": False} + + class FakeStartupInfo: + pass + + class FakeProcess: + _handle = 456 + pid = 654 + + class FakeKernel32: + @staticmethod + def CreateEventW(_attributes, _manual_reset, _initial_state, _name): + events.append("create-gate") + return 789 + + @staticmethod + def SetHandleInformation(handle, mask, flags): + events.append(("inherit", handle, mask, flags)) + return 1 + + @staticmethod + def AssignProcessToJobObject(handle, process_handle): + nonlocal assigned + events.append(("assign", handle, process_handle)) + assigned = True + return 1 + + @staticmethod + def SetEvent(handle): + events.append(("release", handle)) + if not assigned: + raise AssertionError( + "candidate gate released before Job assignment" + ) + descendants["child"] = True + descendants["grandchild"] = True + return 1 + + @staticmethod + def QueryInformationJobObject( + _handle, _info_class, pointer, _size, _return_size + ): + information = ctypes.cast( + pointer, + ctypes.POINTER(runner._JobObjectBasicAccountingInformation), + ).contents + information.active_processes = sum(descendants.values()) + 1 + return 1 + + @staticmethod + def CloseHandle(handle): + events.append(("close", handle)) + if handle == 123: + descendants["child"] = False + descendants["grandchild"] = False + return 1 + + def popen(command, **kwargs): + events.append("popen-gate") + self.assertEqual( + command[0:5], + [sys.executable, "-I", "-S", "-c", runner._WINDOWS_GATE_PROGRAM], + ) + self.assertEqual(command[5:], ["789", "candidate", "spawn-now"]) + self.assertTrue(kwargs["close_fds"]) + self.assertEqual( + kwargs["startupinfo"].lpAttributeList, + {"handle_list": [789]}, + ) + self.assertFalse(any(descendants.values())) + return FakeProcess() + + job = runner._WindowsKillOnCloseJob(FakeKernel32(), 123) + with ( + mock.patch.object( + runner.subprocess, + "STARTUPINFO", + FakeStartupInfo, + create=True, + ), + mock.patch.object(runner.subprocess, "Popen", side_effect=popen), + ): + process = runner._start_contained_process( + job, + ["candidate", "spawn-now"], + env={}, + ) + + self.assertEqual(process.pid, 654) + self.assertTrue(all(descendants.values())) + self.assertLess( + events.index(("assign", 123, 456)), events.index(("release", 789)) + ) + self.assertLess( + events.index(("inherit", 789, 1, 0)), events.index(("release", 789)) + ) + self.assertFalse(job.close()) + self.assertFalse(any(descendants.values())) + + def test_windows_gate_assignment_failure_kills_wrapper_and_closes_raw_handles( + self, + ): + runner = load_runner() + events = [] + + class FakeStartupInfo: + pass + + class FakeProcess: + _handle = 456 + pid = 654 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + def kill(self): + events.append("kill-wrapper") + self.returncode = 1 + + def wait(self, timeout): + events.append(("wait-wrapper", timeout)) + return self.returncode + + class FakeKernel32: + @staticmethod + def CreateEventW(_attributes, _manual_reset, _initial_state, _name): + return 789 + + @staticmethod + def SetHandleInformation(_handle, _mask, _flags): + return 1 + + @staticmethod + def AssignProcessToJobObject(_handle, _process_handle): + events.append("assign-failed") + return 0 + + @staticmethod + def SetEvent(_handle): + events.append("released") + return 1 + + @staticmethod + def QueryInformationJobObject( + _handle, _info_class, pointer, _size, _return_size + ): + information = ctypes.cast( + pointer, + ctypes.POINTER(runner._JobObjectBasicAccountingInformation), + ).contents + information.active_processes = 0 + return 1 + + @staticmethod + def CloseHandle(handle): + events.append(("close", handle)) + return 1 + + process = FakeProcess() + job = runner._WindowsKillOnCloseJob(FakeKernel32(), 123) + with ( + mock.patch.object( + runner.subprocess, + "STARTUPINFO", + FakeStartupInfo, + create=True, + ), + mock.patch.object(runner.subprocess, "Popen", return_value=process), + mock.patch.object(runner, "_windows_last_error", return_value=5), + self.assertRaisesRegex(OSError, "AssignProcessToJobObject"), + ): + job.start_process(["candidate"], env={}) + + self.assertNotIn("released", events) + self.assertIn("kill-wrapper", events) + self.assertIn(("close", 123), events) + self.assertIn(("close", 789), events) + + @unittest.skipUnless(os.name == "nt", "Windows Job Object integration test") + def test_windows_job_close_kills_real_gated_child_and_grandchild(self): + runner = load_runner() + grandchild_program = "import time; time.sleep(60)" + child_program = f""" +import os +import pathlib +import subprocess +import sys +import time + +grandchild = subprocess.Popen([sys.executable, "-c", {grandchild_program!r}]) +pathlib.Path(sys.argv[1]).write_text( + f"{{os.getpid()}},{{grandchild.pid}}", encoding="ascii" +) +time.sleep(60) +""" + candidate_program = f""" +import subprocess +import sys +import time + +subprocess.Popen([sys.executable, "-c", {child_program!r}, sys.argv[1]]) +time.sleep(60) +""" + + with tempfile.TemporaryDirectory() as directory: + pid_path = Path(directory) / "descendants.txt" + job = runner._WindowsKillOnCloseJob.create() + process = None + descendant_pids = [] + try: + process = job.start_process( + [sys.executable, "-c", candidate_program, str(pid_path)], + env=dict(os.environ), + ) + deadline = time.monotonic() + 10 + while not pid_path.is_file() and time.monotonic() < deadline: + time.sleep(0.05) + self.assertTrue(pid_path.is_file()) + descendant_pids = [ + int(value) + for value in pid_path.read_text(encoding="ascii").split(",") + ] + self.assertTrue(all(_pid_is_running(pid) for pid in descendant_pids)) + self.assertFalse(job.close()) + deadline = time.monotonic() + 5 + while ( + any(_pid_is_running(pid) for pid in descendant_pids) + and time.monotonic() < deadline + ): + time.sleep(0.05) + self.assertFalse(any(_pid_is_running(pid) for pid in descendant_pids)) + finally: + job.close() + if process is not None: + runner._reap_direct_process(process, time.monotonic() + 5) + try: + import psutil + except ModuleNotFoundError: + psutil = None + if psutil is not None: + for pid in descendant_pids: + try: + psutil.Process(pid).kill() + except psutil.Error: + pass + + def test_windows_job_query_failure_closes_the_job_and_fails_closed(self): + runner = load_runner() + closed_handles = [] + + class FakeKernel32: + @staticmethod + def QueryInformationJobObject( + _handle, _info_class, _pointer, _size, _return_size + ): + return 0 + + @staticmethod + def CloseHandle(handle): + closed_handles.append(handle) + return 1 + + job = runner._WindowsKillOnCloseJob(FakeKernel32(), 123) + + self.assertFalse(job.close()) + self.assertFalse(job.close()) + self.assertEqual(closed_handles, [123]) + + def test_windows_job_configuration_failure_closes_the_raw_handle(self): + runner = load_runner() + closed_handles = [] + + class FakeKernel32: + @staticmethod + def CreateJobObjectW(_attributes, _name): + return 123 + + @staticmethod + def SetInformationJobObject(_handle, _info_class, _pointer, _size): + return 0 + + @staticmethod + def CloseHandle(handle): + closed_handles.append(handle) + return 1 + + with self.assertRaisesRegex(OSError, "SetInformationJobObject"): + runner._WindowsKillOnCloseJob.create(FakeKernel32()) + + self.assertEqual(closed_handles, [123]) + + def test_server_environment_inherits_only_minimal_runtime_values(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + defaults = Path(directory) / "defaults.json" + defaults.write_text("{}", encoding="utf-8") + environment = runner.build_server_environment( + Path(directory), + inherited={ + "LEMONADE_API_KEY": "user-key", + "LEMONADE_ADMIN_API_KEY": "admin-key", + "LEMONADE_DEFAULTS_PATH": "/host/defaults.json", + "LEMONADE_LLAMACPP_VULKAN_BIN": "/host/llama-server", + "lemonade_rocm_install_method": "wheel", + "LEMONADE_BACKEND_WATCHDOG": "0", + "lemonade_backend_watchdog_poll_seconds": "999", + "LEMONADE_CACHE_DIR": "/host/lemonade-cache", + "LEMONADE_GGML_HIP_PATH": "/host/libggml-hip.so", + "HF_HOME": "/host/huggingface", + "HF_TOKEN": "download-token", + "hugging_face_hub_token": "legacy-download-token", + "HUGGINGFACE_TOKEN": "alternate-download-token", + "LEMONADE_ALLOWED_ORIGINS": "https://example.invalid", + "rocm_path": "/host/rocm", + "HF_ENDPOINT": "https://example.invalid", + "HTTP_PROXY": "http://proxy.invalid:3128", + "https_proxy": "http://proxy.invalid:3128", + "ALL_PROXY": "socks5://proxy.invalid:1080", + "NO_PROXY": "example.invalid", + "LLAMA_ARG_MODEL": "/host/model.gguf", + "LLAMA_ARG_N_GPU_LAYERS": "999", + "llama_arg_ctx_size": "1", + "lemonade_ci_mode": "false", + "PATH": "/trusted/bin", + "LANG": "en_US.UTF-8", + "LC_ALL": "C.UTF-8", + "LC_CTYPE": "UTF-8", + "TZ": "UTC", + "SSL_CERT_FILE": "/trusted/ca.pem", + "SSL_CERT_DIR": "/trusted/ca", + "CURL_CA_BUNDLE": "/trusted/curl-ca.pem", + "REQUESTS_CA_BUNDLE": "/trusted/requests-ca.pem", + "SystemRoot": "C:\\Windows", + "windir": "C:\\Windows", + "COMSPEC": "C:\\Windows\\System32\\cmd.exe", + "PATHEXT": ".COM;.EXE;.BAT;.CMD", + "SYSTEMDRIVE": "C:", + "PROCESSOR_ARCHITECTURE": "AMD64", + "NUMBER_OF_PROCESSORS": "8", + "CUDA_VISIBLE_DEVICES": "0", + "NVIDIA_VISIBLE_DEVICES": "GPU-123", + "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", + "__NV_PRIME_RENDER_OFFLOAD": "1", + "HIP_VISIBLE_DEVICES": "0", + "ROCR_VISIBLE_DEVICES": "0", + "GPU_DEVICE_ORDINAL": "0", + "HSA_OVERRIDE_GFX_VERSION": "11.5.1", + "VK_ICD_FILENAMES": "/trusted/vulkan/icd.json", + "VK_DRIVER_FILES": "/trusted/vulkan/driver.json", + "VK_ADD_DRIVER_FILES": "/trusted/vulkan/additional.json", + "GGML_METAL_NO_RESIDENCY": "1", + "HOME": "/runner/profile", + "USERPROFILE": "C:\\Users\\runner", + "APPDATA": "C:\\Users\\runner\\AppData\\Roaming", + "LOCALAPPDATA": "C:\\Users\\runner\\AppData\\Local", + "GITHUB_OUTPUT": "/runner/command/output", + "GITHUB_ENV": "/runner/command/environment", + "GITHUB_PATH": "/runner/command/path", + "GITHUB_STEP_SUMMARY": "/runner/command/summary", + "ACTIONS_RUNTIME_TOKEN": "actions-token", + "LD_PRELOAD": "/attacker/inject.so", + "DYLD_INSERT_LIBRARIES": "/attacker/inject.dylib", + "DYLD_LIBRARY_PATH": "/attacker/dylibs", + "PYTHONPATH": "/attacker/python", + "PYTHONSTARTUP": "/attacker/startup.py", + "BASH_ENV": "/attacker/bashrc", + "UNKNOWN_DEPLOYMENT_SECRET": "secret", + "PRESERVED": "value", + }, + private_temp_directory=Path(directory) / "private-temp", + candidate_defaults_path=defaults, + ) + + expected_private_temp = str((Path(directory) / "private-temp").resolve()) + self.assertNotIn("LEMONADE_API_KEY", environment) + self.assertNotIn("LEMONADE_ADMIN_API_KEY", environment) + self.assertNotIn("LEMONADE_LLAMACPP_VULKAN_BIN", environment) + self.assertNotIn("lemonade_rocm_install_method", environment) + self.assertNotIn("LEMONADE_BACKEND_WATCHDOG", environment) + self.assertNotIn("lemonade_backend_watchdog_poll_seconds", environment) + self.assertNotIn("LEMONADE_CACHE_DIR", environment) + self.assertNotIn("LEMONADE_GGML_HIP_PATH", environment) + self.assertNotIn("HF_HOME", environment) + self.assertFalse( + { + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "HUGGINGFACE_TOKEN", + } + & {key.upper() for key in environment} + ) + self.assertNotIn("LEMONADE_ALLOWED_ORIGINS", environment) + self.assertNotIn("rocm_path", environment) + self.assertNotIn("HF_ENDPOINT", environment) + self.assertEqual( + { + key.upper(): value + for key, value in environment.items() + if key.upper().startswith("LLAMA_ARG_") + }, + {"LLAMA_ARG_LOG_VERBOSITY": "4"}, + ) + self.assertNotIn("HTTP_PROXY", environment) + self.assertNotIn("https_proxy", environment) + self.assertNotIn("ALL_PROXY", environment) + self.assertEqual(environment["NO_PROXY"], "127.0.0.1") + self.assertEqual(environment["no_proxy"], "127.0.0.1") + self.assertEqual(environment["TMPDIR"], expected_private_temp) + self.assertEqual(environment["TMP"], expected_private_temp) + self.assertEqual(environment["TEMP"], expected_private_temp) + self.assertEqual(environment["LEMONADE_DEFAULTS_PATH"], str(defaults.resolve())) + self.assertEqual(environment["LEMONADE_CI_MODE"], "True") + self.assertNotIn("lemonade_ci_mode", environment) + self.assertEqual(environment["PATH"], "/trusted/bin") + for key, value in { + "LANG": "en_US.UTF-8", + "LC_ALL": "C.UTF-8", + "LC_CTYPE": "UTF-8", + "TZ": "UTC", + "SSL_CERT_FILE": "/trusted/ca.pem", + "SSL_CERT_DIR": "/trusted/ca", + "CURL_CA_BUNDLE": "/trusted/curl-ca.pem", + "REQUESTS_CA_BUNDLE": "/trusted/requests-ca.pem", + "COMSPEC": "C:\\Windows\\System32\\cmd.exe", + "PATHEXT": ".COM;.EXE;.BAT;.CMD", + "SYSTEMDRIVE": "C:", + "PROCESSOR_ARCHITECTURE": "AMD64", + "NUMBER_OF_PROCESSORS": "8", + "CUDA_VISIBLE_DEVICES": "0", + "NVIDIA_VISIBLE_DEVICES": "GPU-123", + "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", + "__NV_PRIME_RENDER_OFFLOAD": "1", + "HIP_VISIBLE_DEVICES": "0", + "ROCR_VISIBLE_DEVICES": "0", + "GPU_DEVICE_ORDINAL": "0", + "HSA_OVERRIDE_GFX_VERSION": "11.5.1", + "VK_ICD_FILENAMES": "/trusted/vulkan/icd.json", + "VK_DRIVER_FILES": "/trusted/vulkan/driver.json", + "VK_ADD_DRIVER_FILES": "/trusted/vulkan/additional.json", + "GGML_METAL_NO_RESIDENCY": "1", + }.items(): + self.assertEqual(environment.get(key), value) + self.assertEqual(environment.get("SYSTEMROOT"), "C:\\Windows") + self.assertEqual(environment.get("WINDIR"), "C:\\Windows") + for key in ("HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA"): + self.assertEqual(environment[key], expected_private_temp) + for key in ( + "ACTIONS_RUNTIME_TOKEN", + "BASH_ENV", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "GITHUB_ENV", + "GITHUB_OUTPUT", + "GITHUB_PATH", + "GITHUB_STEP_SUMMARY", + "LD_PRELOAD", + "PRESERVED", + "PYTHONPATH", + "PYTHONSTARTUP", + "UNKNOWN_DEPLOYMENT_SECRET", + ): + self.assertNotIn(key, environment) + + def test_server_environment_maps_only_explicit_canonical_download_token(self): + runner = load_runner() + inherited = { + "HF_TOKEN": "download-token", + "hf_token": "ambient-case-variant", + "hugging_face_hub_token": "legacy-download-token", + "HuGgInGfAcE_ToKeN": "alternate-download-token", + } + + with tempfile.TemporaryDirectory() as directory: + environment = runner.build_server_environment( + Path(directory), + inherited=inherited, + private_temp_directory=Path(directory) / "private-temp", + allow_download_credentials=True, + ) + + self.assertEqual( + { + key.upper(): value + for key, value in environment.items() + if key.upper() + in { + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "HUGGINGFACE_TOKEN", + } + }, + {"HF_TOKEN": "download-token"}, + ) + + def test_candidate_defaults_must_be_a_real_adjacent_build_resource(self): + runner = load_runner() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + build = root / "build" + outside = root / "outside-resources" + build.mkdir() + outside.mkdir() + (outside / "defaults.json").write_text("{}", encoding="utf-8") + (build / "lemond").touch() + try: + (build / "resources").symlink_to( + outside, + target_is_directory=True, + ) + except OSError as exc: + self.skipTest(f"directory symlinks are unavailable: {exc}") + + with self.assertRaisesRegex(RuntimeError, "build resource"): + runner.resolve_candidate_defaults(build / "lemond") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validate_llamacpp_action_pins.py b/test/test_validate_llamacpp_action_pins.py new file mode 100644 index 0000000000..ce9bb2e57e --- /dev/null +++ b/test/test_validate_llamacpp_action_pins.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Supply-chain contract for protected llama.cpp validation workflows.""" + +from __future__ import annotations + +import re +import tempfile +import unittest +from pathlib import Path + +import yaml +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode + +ROOT = Path(__file__).resolve().parents[1] +CORE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" +DOCS_AND_STYLE_WORKFLOW = ROOT / ".github" / "workflows" / "docs_and_style.yml" +MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" +PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" +SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" +SERVER_BASE = ROOT / "test" / "utils" / "server_base.py" +PROTECTED_WORKFLOWS = ( + PR_WORKFLOW, + CORE_WORKFLOW, + MANUAL_WORKFLOW, + SCHEDULE_WORKFLOW, +) +PINNED_ACTION_PATTERN = re.compile( + r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*@[0-9a-f]{40}$" +) +REVIEWED_ACTION_PINS = { + "actions/checkout": "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09", + "actions/download-artifact": "37930b1c2abaa49bbe596cd826c3c89aef350131", + "actions/setup-python": "ece7cb06caefa5fff74198d8649806c4678c61a1", + "actions/upload-artifact": "bbbca2ddaa5d8feaa63e36b76fdaad77386f024f", +} + + +def read_uses_references(path: Path) -> list[str]: + references: list[str] = [] + visited_nodes: set[int] = set() + + def visit(node: Node | None) -> None: + if node is None or id(node) in visited_nodes: + return + visited_nodes.add(id(node)) + + if isinstance(node, MappingNode): + for key, value in node.value: + if isinstance(key, ScalarNode) and key.value == "uses": + if not isinstance(value, ScalarNode): + raise ValueError(f"uses value must be a scalar in {path}") + references.append(value.value) + else: + visit(key) + visit(value) + elif isinstance(node, SequenceNode): + for value in node.value: + visit(value) + + try: + for document in yaml.compose_all( + path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader + ): + visit(document) + except yaml.YAMLError as error: + raise ValueError(f"invalid YAML in {path}: {error}") from error + return references + + +def resolve_local_uses(repository_root: Path, reference: str) -> Path: + target = (repository_root / reference.removeprefix("./")).resolve() + try: + target.relative_to(repository_root.resolve()) + except ValueError as error: + raise ValueError( + f"local action escapes repository root: {reference}" + ) from error + if target.is_dir(): + manifests = [ + candidate + for candidate in (target / "action.yml", target / "action.yaml") + if candidate.is_file() + ] + if len(manifests) != 1: + raise ValueError(f"local action has no unique manifest: {reference}") + return manifests[0] + if not target.is_file(): + raise ValueError(f"local uses target does not exist: {reference}") + return target + + +def collect_external_actions( + repository_root: Path, entrypoints: tuple[Path, ...] +) -> list[tuple[Path, str]]: + active: set[Path] = set() + visited: set[Path] = set() + external: list[tuple[Path, str]] = [] + + def visit(path: Path) -> None: + path = path.resolve() + try: + path.relative_to(repository_root.resolve()) + except ValueError as error: + raise ValueError(f"uses target escapes repository root: {path}") from error + if path in active: + raise ValueError(f"local uses cycle detected at {path}") + if path in visited: + return + active.add(path) + for reference in read_uses_references(path): + if reference.startswith("./"): + visit(resolve_local_uses(repository_root, reference)) + else: + external.append((path, reference)) + active.remove(path) + visited.add(path) + + for entrypoint in entrypoints: + visit(entrypoint) + return external + + +class LlamaCppProtectedActionPinTests(unittest.TestCase): + def test_all_protected_entrypoints_are_scanned(self) -> None: + self.assertEqual( + set(PROTECTED_WORKFLOWS), + {PR_WORKFLOW, CORE_WORKFLOW, MANUAL_WORKFLOW, SCHEDULE_WORKFLOW}, + ) + + def test_uses_are_read_from_yaml_structure(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository_root = Path(directory) / "repository" + workflow = repository_root / ".github" / "workflows" / "entry.yml" + workflow.parent.mkdir(parents=True) + workflow.write_text( + """\ +metadata: !untrusted-tag ignored +jobs: + quoted: + "uses" : "actions/checkout@main" + flow: {name: flow, 'uses': 'actions/setup-python@main'} + spaced: + uses : actions/upload-artifact@main + script: + run: | + uses: actions/download-artifact@main +""", + encoding="utf-8", + ) + + self.assertEqual( + collect_external_actions(repository_root, (workflow,)), + [ + (workflow.resolve(), "actions/checkout@main"), + (workflow.resolve(), "actions/setup-python@main"), + (workflow.resolve(), "actions/upload-artifact@main"), + ], + ) + + def test_manual_and_schedule_entrypoints_find_mutable_actions(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository_root = Path(directory) / "repository" + workflows = repository_root / ".github" / "workflows" + action = repository_root / ".github" / "actions" / "nested" + workflows.mkdir(parents=True) + action.mkdir(parents=True) + manual = workflows / "validate_llamacpp_manual.yml" + schedule = workflows / "validate_llamacpp_schedule.yml" + core = workflows / "validate_llamacpp_core.yml" + manual.write_text( + "jobs: {direct: {uses: actions/checkout@main}}\n", + encoding="utf-8", + ) + schedule.write_text( + "jobs: {delegated: {'uses': './.github/workflows/validate_llamacpp_core.yml'}}\n", + encoding="utf-8", + ) + core.write_text( + "jobs:\n nested:\n uses : ./.github/actions/nested\n", + encoding="utf-8", + ) + (action / "action.yml").write_text( + "runs: {using: composite, steps: [{uses: actions/setup-python@v6}]}\n", + encoding="utf-8", + ) + + external_actions = collect_external_actions( + repository_root, (manual, schedule) + ) + + self.assertEqual( + {reference for _, reference in external_actions}, + {"actions/checkout@main", "actions/setup-python@v6"}, + ) + for _, reference in external_actions: + self.assertNotRegex(reference, PINNED_ACTION_PATTERN) + + def test_every_external_action_uses_an_immutable_commit(self) -> None: + external_actions = collect_external_actions(ROOT, PROTECTED_WORKFLOWS) + self.assertTrue(external_actions) + for source, reference in external_actions: + with self.subTest(source=source.relative_to(ROOT), reference=reference): + self.assertRegex(reference, PINNED_ACTION_PATTERN) + action, commit = reference.rsplit("@", 1) + if action in REVIEWED_ACTION_PINS: + self.assertEqual(commit, REVIEWED_ACTION_PINS[action]) + + def test_local_action_recursion_rejects_cycles_and_root_escapes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository_root = Path(directory) / "repository" + first = repository_root / ".github" / "actions" / "first" + second = repository_root / ".github" / "actions" / "second" + first.mkdir(parents=True) + second.mkdir(parents=True) + first_manifest = first / "action.yml" + second_manifest = second / "action.yml" + first_manifest.write_text( + "runs:\n using: composite\n steps:\n - uses: " + "./.github/actions/second\n", + encoding="utf-8", + ) + second_manifest.write_text( + "runs:\n using: composite\n steps:\n - uses: " + "./.github/actions/first\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "cycle"): + collect_external_actions(repository_root, (first_manifest,)) + + outside = Path(directory) / "outside.yml" + outside.write_text( + "runs:\n using: composite\n steps: []\n", encoding="utf-8" + ) + first_manifest.write_text( + "runs:\n using: composite\n steps:\n - uses: ./../outside.yml\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "escapes repository root"): + collect_external_actions(repository_root, (first_manifest,)) + + def test_focused_test_dependencies_are_installed_explicitly(self) -> None: + workflow = DOCS_AND_STYLE_WORKFLOW.read_text(encoding="utf-8") + pre_commit_job = workflow.split(" pre-commit:\n", 1)[1].split( + " markdown-link-check:\n", 1 + )[0] + install_step = pre_commit_job.split( + " - name: Run focused Python unit tests\n", 1 + )[0] + self.assertIn( + "pip install pre-commit psutil requests openai pyyaml", install_step + ) + self.assertIn( + 'python -c "import test.utils.server_base"', + pre_commit_job, + ) + server_base = SERVER_BASE.read_text(encoding="utf-8") + self.assertIn("from openai import OpenAI, AsyncOpenAI", server_base) + self.assertIn("import requests", server_base) + + def test_cross_platform_validation_step_explicitly_uses_bash(self) -> None: + workflow = CORE_WORKFLOW.read_text(encoding="utf-8") + validate_job = workflow.split(" validate:\n", 1)[1].split( + " validation-gate:\n", 1 + )[0] + step_marker = " - name: Run validation with lemond\n" + self.assertEqual(validate_job.count(step_marker), 1) + step = validate_job.split(step_marker, 1)[1].split(" - name:", 1)[0] + + self.assertIn(" runs-on: ${{ matrix.runner }}\n", validate_job) + self.assertIn("CANDIDATE_LEMOND:", step) + self.assertIn("runner.os == 'Windows'", step) + self.assertIn( + "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py", step + ) + self.assertIn('--lemond "$CANDIDATE_LEMOND"', step) + self.assertIn(" shell: bash\n", step) + self.assertNotIn(" if:", step) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validate_llamacpp_selection.py b/test/test_validate_llamacpp_selection.py index 9c64745644..b97f6f2b0a 100644 --- a/test/test_validate_llamacpp_selection.py +++ b/test/test_validate_llamacpp_selection.py @@ -8,8 +8,12 @@ import importlib.util import io import json +import socket +import subprocess import sys import tempfile +import threading +import time import types import unittest from pathlib import Path @@ -25,14 +29,41 @@ / "resources" / "server_models.json" ) +VALIDATION_MODEL_CATALOG = ( + Path(__file__).resolve().parent / "fixtures" / "llamacpp_validation_models.json" +) +VALIDATION_ARTIFACTS = ( + Path(__file__).resolve().parent / "fixtures" / "llamacpp_validation_artifacts.json" +) +LLAMACPP_GUIDE = ( + Path(__file__).resolve().parents[1] + / "docs" + / "guide" + / "configuration" + / "llamacpp.md" +) def load_validation_module(): requests_module = types.ModuleType("requests") requests_module.RequestException = RuntimeError + class StubSession: + trust_env = True + + @staticmethod + def request(*args, **kwargs): + return requests_module.request(*args, **kwargs) + + @staticmethod + def close(): + return None + + requests_module.Session = StubSession + server_base_module = types.ModuleType("utils.server_base") server_base_module._auth_headers = lambda: {} + server_base_module.pull_model_with_retry = lambda *_args, **_kwargs: None server_base_module.unload_all_models = lambda **_kwargs: None server_base_module.wait_for_server = lambda **_kwargs: None @@ -66,6 +97,277 @@ def load_validation_module(): VALIDATION = load_validation_module() +@contextlib.contextmanager +def stalled_http_response(prefix: bytes, suffix: bytes, stall_seconds=10): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(30) + address = listener.getsockname() + release = threading.Event() + prefix_sent = threading.Event() + stall_elapsed = threading.Event() + + def serve(): + try: + connection, _address = listener.accept() + with connection: + request = b"" + while b"\r\n\r\n" not in request: + chunk = connection.recv(4096) + if not chunk: + return + request += chunk + connection.sendall(prefix) + prefix_sent.set() + if not release.wait(stall_seconds): + stall_elapsed.set() + connection.sendall(suffix) + except OSError: + pass + + server = threading.Thread(target=serve) + server.start() + try: + yield ( + f"http://127.0.0.1:{address[1]}/stream", + prefix_sent, + stall_elapsed, + ) + finally: + release.set() + try: + socket.create_connection(address, timeout=0.1).close() + except OSError: + pass + listener.close() + server.join(timeout=5) + if server.is_alive(): + raise AssertionError("local test server did not stop") + + +@contextlib.contextmanager +def trickling_http_response(): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(30) + address = listener.getsockname() + release = threading.Event() + body_started = threading.Event() + trickle_elapsed = threading.Event() + + def serve(): + try: + connection, _address = listener.accept() + with connection: + request = b"" + while b"\r\n\r\n" not in request: + chunk = connection.recv(4096) + if not chunk: + return + request += chunk + connection.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" + b"Connection: close\r\n\r\n" + ) + body_started.set() + automatic_deadline = time.monotonic() + 30 + while not release.wait(0.01): + if time.monotonic() >= automatic_deadline: + trickle_elapsed.set() + return + connection.sendall(b"x") + except OSError: + pass + + server = threading.Thread(target=serve) + server.start() + try: + yield ( + f"http://127.0.0.1:{address[1]}/stream", + body_started, + trickle_elapsed, + ) + finally: + release.set() + try: + socket.create_connection(address, timeout=0.1).close() + except OSError: + pass + listener.close() + server.join(timeout=5) + if server.is_alive(): + raise AssertionError("local trickle server did not stop") + + +@contextlib.contextmanager +def trickling_http_headers(): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(30) + address = listener.getsockname() + release = threading.Event() + headers_started = threading.Event() + trickle_elapsed = threading.Event() + + def serve(): + try: + connection, _address = listener.accept() + with connection: + request = b"" + while b"\r\n\r\n" not in request: + chunk = connection.recv(4096) + if not chunk: + return + request += chunk + connection.sendall(b"HTTP/1.1 200 OK\r\nX-Padding: ") + headers_started.set() + automatic_deadline = time.monotonic() + 30 + while not release.wait(0.01): + if time.monotonic() >= automatic_deadline: + trickle_elapsed.set() + return + connection.sendall(b"x") + except OSError: + pass + + server = threading.Thread(target=serve) + server.start() + try: + yield ( + f"http://127.0.0.1:{address[1]}/json", + headers_started, + trickle_elapsed, + ) + finally: + release.set() + try: + socket.create_connection(address, timeout=0.1).close() + except OSError: + pass + listener.close() + server.join(timeout=5) + if server.is_alive(): + raise AssertionError("local header trickle server did not stop") + + +@contextlib.contextmanager +def fixed_http_response(body: bytes): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(30) + address = listener.getsockname() + response_sent = threading.Event() + + def serve(): + try: + connection, _address = listener.accept() + with connection: + request = b"" + while b"\r\n\r\n" not in request: + chunk = connection.recv(4096) + if not chunk: + return + request += chunk + connection.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + + f"Content-Length: {len(body)}\r\n".encode("ascii") + + b"X-Test: bounded\r\nConnection: close\r\n\r\n" + + body + ) + response_sent.set() + except OSError: + pass + + server = threading.Thread(target=serve) + server.start() + try: + yield f"http://127.0.0.1:{address[1]}/json", response_sent + finally: + try: + socket.create_connection(address, timeout=0.1).close() + except OSError: + pass + listener.close() + server.join(timeout=5) + if server.is_alive(): + raise AssertionError("local fixed-response server did not stop") + + +@contextlib.contextmanager +def held_open_pull_response(body: bytes): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(30) + address = listener.getsockname() + terminal_sent = threading.Event() + release = threading.Event() + + def serve(): + try: + connection, _address = listener.accept() + with connection: + request = b"" + while b"\r\n\r\n" not in request: + chunk = connection.recv(4096) + if not chunk: + return + request += chunk + connection.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/event-stream\r\n" + b"Connection: close\r\n\r\n" + body + ) + terminal_sent.set() + release.wait(30) + except OSError: + pass + + server = threading.Thread(target=serve) + server.start() + try: + yield ( + f"http://127.0.0.1:{address[1]}/api/v1/pull", + terminal_sent, + release, + ) + finally: + release.set() + try: + socket.create_connection(address, timeout=0.1).close() + except OSError: + pass + listener.close() + server.join(timeout=5) + if server.is_alive(): + raise AssertionError("held-open pull response server did not stop") + + +class ChunkedResponse: + def __init__(self, chunks, status_code=200, content_type="text/event-stream"): + self.chunks = chunks + self.status_code = status_code + self.headers = {"Content-Type": content_type} + self.encoding = "utf-8" + self.closed = False + + def iter_content(self, **_kwargs): + yield from self.chunks + + def close(self): + self.closed = True + + class LlamaCppValidationSelectionTests(unittest.TestCase): def setUp(self) -> None: self.catalog = [ @@ -174,6 +476,26 @@ def resolve_builtin_model(canonical_model_id): "IFM/K2-Horizon-0.9B-GGUF:model.gguf", ) + def test_explicit_selection_rejects_resolver_response_for_another_model( + self, + ) -> None: + wrong_model = { + "id": "Different-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/different:model.gguf", + } + + with self.assertRaisesRegex( + selection.ModelSelectionError, + "requested.*K2-Horizon-0.9B-GGUF.*Different-Llama", + ): + selection.select_llamacpp_models( + [], + requested_model_ids=["K2-Horizon-0.9B-GGUF"], + builtin_model_resolver=lambda _model_id: wrong_model, + ) + def test_shadowed_builtin_keeps_canonical_id_for_validation_evidence(self) -> None: built_in = { "id": "builtin.Shadowed-Llama", @@ -344,25 +666,25 @@ def test_model_and_lite_arguments_are_mutually_exclusive(self) -> None: class K2HorizonCatalogTests(unittest.TestCase): - def test_k2_horizon_bf16_models_have_exact_catalog_contract(self) -> None: - models = json.loads(MODEL_REGISTRY.read_text(encoding="utf-8")) - expected = { + @staticmethod + def expected_models() -> dict: + return { "K2-Horizon-0.9B-GGUF": { - "checkpoint": ("IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf"), + "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf", "recipe": "llamacpp", "suggested": True, "labels": ["chat", "reasoning", "tool-calling", "hot"], "size": 2.16, }, "K2-Horizon-3.7B-GGUF": { - "checkpoint": ("IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf"), + "checkpoint": "IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf", "recipe": "llamacpp", "suggested": True, "labels": ["chat", "reasoning", "tool-calling"], "size": 10.13, }, "K2-Horizon-7B-GGUF": { - "checkpoint": ("IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf"), + "checkpoint": "IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf", "recipe": "llamacpp", "suggested": True, "labels": ["chat", "reasoning", "tool-calling"], @@ -370,27 +692,75 @@ def test_k2_horizon_bf16_models_have_exact_catalog_contract(self) -> None: }, } - for model_id, expected_record in expected.items(): + def test_product_catalog_does_not_advertise_k2_horizon(self) -> None: + product_models = json.loads(MODEL_REGISTRY.read_text(encoding="utf-8")) + + for model_id in self.expected_models(): with self.subTest(model_id=model_id): - self.assertEqual(models.get(model_id), expected_record) - - def test_model_registry_has_no_duplicate_ids(self) -> None: - duplicate_ids = [] - - def unique_object(pairs): - parsed = {} - for key, value in pairs: - if key in parsed: - duplicate_ids.append(key) - parsed[key] = value - return parsed - - json.loads( - MODEL_REGISTRY.read_text(encoding="utf-8"), - object_pairs_hook=unique_object, - ) + self.assertNotIn(model_id, product_models) + + def test_public_guide_does_not_advertise_validation_candidates(self) -> None: + guide = LLAMACPP_GUIDE.read_text(encoding="utf-8") + + self.assertNotIn("## K2-Horizon", guide) + for model_id in self.expected_models(): + with self.subTest(model_id=model_id): + self.assertNotIn(model_id, guide) + + def test_k2_horizon_validation_models_have_exact_catalog_contract(self) -> None: + models = json.loads(VALIDATION_MODEL_CATALOG.read_text(encoding="utf-8")) + self.assertEqual(models, self.expected_models()) + + def test_k2_horizon_validation_artifacts_are_immutably_bound(self) -> None: + artifacts = json.loads(VALIDATION_ARTIFACTS.read_text(encoding="utf-8")) + expected = { + "K2-Horizon-0.9B-GGUF": { + "revision": "8496259ac62d33192d0fffe708303ed2ca29a384", + "filename": "K2-Horizon-1B-BF16.gguf", + "size_bytes": 2159424896, + "sha256": "371010db1807bb07b62e738422ee0de26c1e15a347f31108ed2c6e219095a8b8", + "gguf_architecture": "k2-horizon", + "max_context_window": 131072, + }, + "K2-Horizon-3.7B-GGUF": { + "revision": "67b824b8079635b77e19a2a4cd2682a9716478c1", + "filename": "K2-Horizon-4B-BF16.gguf", + "size_bytes": 10128343424, + "sha256": "372ecf9977eb22c92739fec478f2c887daef659d7c55a609ac97b905d1a243b1", + "gguf_architecture": "k2-horizon", + "max_context_window": 524288, + }, + "K2-Horizon-7B-GGUF": { + "revision": "bcb8c25b76112ce96a962f5b8ab624435d1ee0c9", + "filename": "K2-Horizon-7B-BF16.gguf", + "size_bytes": 18010413440, + "sha256": "088c5d0814ef955d137fd1073ee3a68f6a53411fca44b75dc58a320640000444", + "gguf_architecture": "k2-horizon", + "max_context_window": 524288, + }, + } + + self.assertEqual(artifacts, expected) + + def test_model_catalogs_have_no_duplicate_ids(self) -> None: + for path in (MODEL_REGISTRY, VALIDATION_MODEL_CATALOG): + duplicate_ids = [] + + def unique_object(pairs): + parsed = {} + for key, value in pairs: + if key in parsed: + duplicate_ids.append(key) + parsed[key] = value + return parsed - self.assertEqual(duplicate_ids, []) + json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=unique_object, + ) + + with self.subTest(path=path): + self.assertEqual(duplicate_ids, []) class LlamaCppValidationRuntimeTests(unittest.TestCase): @@ -400,28 +770,691 @@ def response(status_code=200): def test_request_json_rejects_duplicate_response_members(self) -> None: response_text = '{"model":"wrong","model":"right"}' - response = types.SimpleNamespace( - status_code=200, - content=response_text.encode("utf-8"), - text=response_text, - json=lambda **kwargs: json.loads(response_text, **kwargs), - ) - - with mock.patch.object( - VALIDATION.requests, - "request", - return_value=response, - create=True, - ): + with fixed_http_response(response_text.encode("utf-8")) as (url, _sent): _response, body = VALIDATION.request_json( "GET", - "http://localhost:13305/api/v1/health", - timeout=1, + url, + timeout=5, ) self.assertEqual(body, {"raw_text": response_text}) - def request_json_for_chat(self, chat_message, operations=None): + def test_json_transport_deadline_interrupts_unterminated_trickle(self) -> None: + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with trickling_http_response() as (url, body_started, trickle_elapsed): + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "wall-clock deadline", + ), + ): + VALIDATION.request_json("GET", url, timeout=10) + + self.assertTrue(body_started.is_set()) + self.assertFalse(trickle_elapsed.is_set()) + + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_json_transport_deadline_covers_response_header_trickle(self) -> None: + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with trickling_http_headers() as (url, headers_started, trickle_elapsed): + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "wall-clock deadline", + ), + ): + VALIDATION.request_json("GET", url, timeout=5) + + self.assertTrue(headers_started.is_set()) + self.assertFalse(trickle_elapsed.is_set()) + + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_json_transport_accepts_exact_response_byte_bound(self) -> None: + body = b"x" * VALIDATION.MAX_STREAM_SSE_BYTES + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with fixed_http_response(body) as (url, response_sent): + with mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ): + response, parsed = VALIDATION.request_json("GET", url, timeout=30) + + self.assertTrue(response_sent.is_set()) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, body) + self.assertEqual(response.text, body.decode("utf-8")) + self.assertEqual(response.headers["X-Test"], "bounded") + self.assertEqual(parsed, {"raw_text": body.decode("utf-8")}) + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_json_transport_rejects_response_above_byte_bound(self) -> None: + body = b"x" * (VALIDATION.MAX_STREAM_SSE_BYTES + 1) + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with fixed_http_response(body) as (url, _response_sent): + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "response byte count exceeds bounds", + ), + ): + VALIDATION.request_json("GET", url, timeout=30) + + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_json_transport_rejects_malformed_worker_framing_after_reap(self) -> None: + class CompletedProcess: + returncode = 0 + + def __init__(self): + self.communicate_calls = 0 + + def communicate(self, input=None, timeout=None): + del input, timeout + self.communicate_calls += 1 + return b"invalid framing", b"" + + def poll(self): + return self.returncode + + process = CompletedProcess() + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + return_value=process, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "invalid framing", + ), + ): + VALIDATION.request_json( + "GET", + "http://127.0.0.1:13305/json", + timeout=10, + ) + + self.assertEqual(process.communicate_calls, 1) + self.assertIsNotNone(process.poll()) + + def test_http_worker_is_reaped_when_post_spawn_clock_read_is_interrupted( + self, + ) -> None: + class StartedProcess: + def __init__(self): + self.returncode = None + self.killed = False + self.communicate_calls = [] + + def communicate(self, input=None, timeout=None): + self.communicate_calls.append((input, timeout)) + self.returncode = -9 + return b"", b"" + + def kill(self): + self.killed = True + + def poll(self): + return self.returncode + + interruption = KeyboardInterrupt() + process = StartedProcess() + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + return_value=process, + ), + mock.patch.object( + VALIDATION.time, + "monotonic", + side_effect=(0.0, interruption), + ), + self.assertRaises(KeyboardInterrupt) as raised, + ): + VALIDATION._execute_http_worker( + "GET", + "http://127.0.0.1:13305/json", + 60, + {}, + None, + "json", + 60, + "HTTP response", + ) + + self.assertIs(raised.exception, interruption) + self.assertTrue(process.killed) + self.assertEqual(process.communicate_calls, [(None, None)]) + self.assertIsNotNone(process.poll()) + + def test_http_worker_preserves_control_exception_during_spawn(self) -> None: + interruption = KeyboardInterrupt() + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=interruption, + ), + mock.patch.object( + VALIDATION, + "_kill_and_reap_stream_worker", + ) as cleanup, + mock.patch.object(VALIDATION.time, "monotonic", return_value=0.0), + self.assertRaises(KeyboardInterrupt) as raised, + ): + VALIDATION._execute_http_worker( + "GET", + "http://127.0.0.1:13305/json", + 60, + {}, + None, + "json", + 60, + "HTTP response", + ) + + self.assertIs(raised.exception, interruption) + cleanup.assert_not_called() + + def test_stream_transport_deadline_covers_slow_response_headers(self) -> None: + prefix = b"HTTP/1.1 " + suffix = ( + b"200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n" + ) + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with stalled_http_response(prefix, suffix) as ( + url, + prefix_sent, + stall_elapsed, + ): + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "wall-clock deadline", + ), + ): + _response, lines = VALIDATION.request_stream( + "POST", + url, + timeout=5, + wall_clock_deadline=time.monotonic() + 5, + json={"stream": True}, + ) + list(lines) + + self.assertTrue(prefix_sent.is_set()) + self.assertFalse(stall_elapsed.is_set()) + + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_stream_transport_deadline_covers_blocked_body_cleanup(self) -> None: + prefix = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" + b"Connection: close\r\n\r\n" + ) + suffix = b"data: [DONE]\n" + with stalled_http_response(prefix, suffix) as ( + url, + prefix_sent, + stall_elapsed, + ): + with self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "wall-clock deadline", + ): + _response, lines = VALIDATION.request_stream( + "POST", + url, + timeout=5, + wall_clock_deadline=time.monotonic() + 5, + json={"stream": True}, + ) + list(lines) + + self.assertTrue(prefix_sent.is_set()) + self.assertFalse(stall_elapsed.is_set()) + + def test_stream_transport_timeout_kills_and_reaps_worker(self) -> None: + class TimedOutProcess: + def __init__(self): + self.returncode = None + self.killed = False + self.communicate_calls = [] + + def communicate(self, input=None, timeout=None): + self.communicate_calls.append((input, timeout)) + if timeout is not None: + raise subprocess.TimeoutExpired(["stream-worker"], timeout) + self.returncode = -9 + return b"", b"" + + def kill(self): + self.killed = True + + def poll(self): + return self.returncode + + process = TimedOutProcess() + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + return_value=process, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "wall-clock deadline", + ), + ): + _response, lines = VALIDATION.request_stream( + "POST", + "http://127.0.0.1:13305/stream", + timeout=60, + wall_clock_deadline=time.monotonic() + 60, + json={"stream": True}, + ) + list(lines) + + self.assertTrue(process.killed) + self.assertEqual(len(process.communicate_calls), 2) + self.assertIsNotNone(process.poll()) + + def test_stream_transport_deadline_interrupts_unterminated_trickle(self) -> None: + with trickling_http_response() as (url, body_started, trickle_elapsed): + with self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "wall-clock deadline", + ): + _response, lines = VALIDATION.request_stream( + "POST", + url, + timeout=2, + wall_clock_deadline=time.monotonic() + 5, + json={"stream": True}, + ) + list(lines) + + self.assertTrue(body_started.is_set()) + self.assertFalse(trickle_elapsed.is_set()) + + def test_stream_transport_reassembles_lines_and_bounds_worker_input(self) -> None: + class CompletedProcess: + def __init__(self, output): + self.output = output + self.returncode = 0 + self.communicate_calls = [] + + def communicate(self, input=None, timeout=None): + self.communicate_calls.append((input, timeout)) + return self.output, b"" + + def poll(self): + return self.returncode + + output = VALIDATION._encode_stream_worker_output( + 200, + b"data: one\r\ndata: two\n\n", + ) + process = CompletedProcess(output) + authorization = "Bearer test-secret" + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + return_value=process, + ) as popen, + mock.patch.object( + VALIDATION, + "_auth_headers", + return_value={"Authorization": authorization}, + ), + ): + returned, lines = VALIDATION.request_stream( + "POST", + "http://localhost:13305/api/v1/chat/completions", + timeout=60, + json={"stream": True}, + ) + + self.assertEqual( + list(lines), + [b"data: one\r", b"data: two"], + ) + + self.assertEqual(returned.status_code, 200) + request_spec = json.loads(process.communicate_calls[0][0]) + self.assertNotIn("wall_clock_deadline", request_spec) + self.assertEqual( + request_spec["max_response_bytes"], + VALIDATION.MAX_STREAM_SSE_BYTES, + ) + self.assertEqual(request_spec["headers"]["Authorization"], authorization) + self.assertNotIn(authorization, popen.call_args.args[0]) + + def test_stream_transport_worker_success_is_reaped(self) -> None: + prefix = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" + b"Connection: close\r\n\r\n" + ) + suffix = b"data: one\ndata: [DONE]\n" + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with stalled_http_response(prefix, suffix, stall_seconds=0) as ( + url, + prefix_sent, + _stall_elapsed, + ): + with mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ): + response, lines = VALIDATION.request_stream( + "POST", + url, + timeout=5, + wall_clock_deadline=time.monotonic() + 10, + json={"stream": True}, + ) + + self.assertTrue(prefix_sent.is_set()) + + self.assertEqual(response.status_code, 200) + self.assertEqual(list(lines), [b"data: one", b"data: [DONE]"]) + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_stream_transport_rejects_worker_event_count_above_bound(self) -> None: + output = VALIDATION._encode_stream_worker_output( + 200, + b"data: {}\n\ndata: {}\n\ndata: {}\n\n", + ) + + with ( + mock.patch.object(VALIDATION, "MAX_STREAM_EVENTS", 2), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "event count", + ), + ): + VALIDATION._decode_stream_worker_output(output) + + def test_stream_transport_rejects_a_line_above_its_independent_bound( + self, + ) -> None: + output = VALIDATION._encode_stream_worker_output(200, b"data: 123456\n\n") + + with ( + mock.patch.object(VALIDATION, "MAX_STREAM_SSE_LINE_BYTES", 8), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "SSE line exceeds bounds", + ), + ): + VALIDATION._decode_stream_worker_output(output) + + def test_stream_transport_counts_semantic_frames_not_blank_lines(self) -> None: + chunk_count = 2500 + chunk = json.dumps( + { + "model": "backend-model", + "choices": [ + { + "index": 0, + "delta": {"content": "x"}, + "finish_reason": None, + } + ], + }, + separators=(",", ":"), + ).encode("utf-8") + terminal = json.dumps( + { + "model": "backend-model", + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": "stop", + } + ], + }, + separators=(",", ":"), + ).encode("utf-8") + body = (b"data: " + chunk + b"\n\n") * chunk_count + body += b"data: " + terminal + b"\n\ndata: [DONE]\n\n" + + lines = VALIDATION._split_stream_lines(body) + message, finish_reason, saw_done, model = capabilities._decode_openai_stream( + lines + ) + + self.assertEqual(len(lines), chunk_count + 2) + self.assertEqual(message["content"], "x" * chunk_count) + self.assertEqual(finish_reason, "stop") + self.assertTrue(saw_done) + self.assertEqual(model, "backend-model") + self.assertGreaterEqual(capabilities.MAX_STREAM_EVENTS, 3072 + 2) + + def test_stream_transport_rejects_unterminated_body_above_byte_bound(self) -> None: + class OversizedResponse: + def __init__(self): + self.closed = False + + def iter_content(self, **_kwargs): + yield b"abc" + yield b"def" + + def close(self): + self.closed = True + + response = OversizedResponse() + with ( + mock.patch.object( + VALIDATION.requests, + "request", + return_value=response, + create=True, + ), + mock.patch.object( + VALIDATION, + "_load_stream_worker_request", + return_value=( + "stream", + "POST", + "http://127.0.0.1:13305/stream", + 60, + {}, + {"stream": True}, + 5, + ), + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "SSE byte count", + ), + ): + VALIDATION._run_stream_worker() + + self.assertTrue(response.closed) + + def test_json_worker_closes_response_above_byte_bound(self) -> None: + class OversizedResponse: + def __init__(self): + self.closed = False + + def iter_content(self, **_kwargs): + yield b"abc" + yield b"def" + + def close(self): + self.closed = True + + response = OversizedResponse() + with ( + mock.patch.object( + VALIDATION.requests, + "request", + return_value=response, + create=True, + ), + mock.patch.object( + VALIDATION, + "_load_stream_worker_request", + return_value=( + "json", + "GET", + "http://127.0.0.1:13305/json", + 60, + {}, + None, + 5, + ), + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "HTTP response byte count", + ), + ): + VALIDATION._run_stream_worker() + + self.assertTrue(response.closed) + + def test_json_worker_accepts_exact_custom_response_bound(self) -> None: + class BoundedResponse: + status_code = 200 + headers = {"Content-Type": "application/json"} + encoding = "utf-8" + + def __init__(self): + self.closed = False + + def iter_content(self, **_kwargs): + yield b"abc" + yield b"de" + + def close(self): + self.closed = True + + response = BoundedResponse() + captured_bodies = [] + stdout = types.SimpleNamespace(buffer=io.BytesIO()) + + def encode(_status, body, _headers, _encoding): + captured_bodies.append(body) + return b"encoded" + + with ( + mock.patch.object( + VALIDATION.requests, + "request", + return_value=response, + create=True, + ), + mock.patch.object( + VALIDATION, + "_load_stream_worker_request", + return_value=( + "json", + "GET", + "http://127.0.0.1:13305/json", + 60, + {}, + None, + 5, + ), + ), + mock.patch.object( + VALIDATION, + "_encode_stream_worker_output", + side_effect=encode, + ), + mock.patch.object(VALIDATION.sys, "stdout", stdout), + ): + VALIDATION._run_stream_worker() + + self.assertEqual(captured_bodies, [b"abcde"]) + self.assertEqual(stdout.buffer.getvalue(), b"encoded") + self.assertTrue(response.closed) + + def request_json_for_chat( + self, + chat_message, + operations=None, + load_payloads=None, + response_model="builtin.Test-Llama", + unload_status_code=200, + ): loaded = False def request_json(method, url, timeout, **kwargs): @@ -432,10 +1465,12 @@ def request_json(method, url, timeout, **kwargs): operations.append(operation) if operation == "load": loaded = True + if load_payloads is not None: + load_payloads.append(kwargs["json"]) return self.response(), {} if operation == "unload": loaded = False - return self.response(), {} + return self.response(unload_status_code), {} if operation == "health": return self.response(), { "all_models_loaded": ( @@ -455,13 +1490,665 @@ def request_json(method, url, timeout, **kwargs): if callable(chat_message) else chat_message ) - return self.response(), {"choices": [{"message": message}]} + return self.response(), { + "model": response_model, + "choices": [{"message": message}], + } if operation == "stats": return self.response(), {"output_tokens": 4} return self.response(), {} return request_json + def test_validation_uses_ipv4_loopback_for_server_readiness(self) -> None: + with ( + mock.patch.object(VALIDATION, "wait_for_server") as wait, + mock.patch.object( + VALIDATION, + "request_json", + return_value=(self.response(), {"status": "ok"}), + ) as request, + contextlib.redirect_stdout(io.StringIO()), + ): + VALIDATION.require_running_server("http://127.0.0.1:13305/api/v1", 13305) + + wait.assert_called_once_with( + port=13305, + timeout=VALIDATION.TIMEOUT_HEALTH, + host="127.0.0.1", + ) + self.assertTrue(request.call_args.args[1].startswith("http://127.0.0.1:")) + + def test_server_readiness_bounds_each_connect_attempt(self) -> None: + clock = types.SimpleNamespace(now=0.0) + connect_timeouts = [] + + def connect(_address, timeout): + connect_timeouts.append(timeout) + clock.now += timeout + raise OSError("not ready") + + def sleep(delay): + clock.now += delay + + with ( + mock.patch.object(VALIDATION.time, "monotonic", lambda: clock.now), + mock.patch.object(VALIDATION.time, "sleep", side_effect=sleep), + mock.patch.object( + VALIDATION.socket, + "create_connection", + side_effect=connect, + ), + self.assertRaisesRegex(TimeoutError, "2.5 seconds"), + ): + VALIDATION.wait_for_server(port=13305, timeout=2.5, host="127.0.0.1") + + self.assertGreater(len(connect_timeouts), 1) + self.assertTrue( + all( + 0 < timeout <= VALIDATION.SERVER_CONNECT_ATTEMPT_TIMEOUT_SECONDS + for timeout in connect_timeouts + ) + ) + + def test_unload_all_models_uses_bounded_worker_request_and_retries(self) -> None: + failed = self.response(500) + succeeded = self.response(404) + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=[(failed, {}), (succeeded, {})], + ) as request, + mock.patch.object(VALIDATION.time, "sleep") as sleep, + ): + response = VALIDATION.unload_all_models( + port=13305, + attempts=3, + host="127.0.0.1", + ) + + self.assertIs(response, succeeded) + self.assertEqual(request.call_count, 2) + request.assert_called_with( + "POST", + "http://127.0.0.1:13305/api/v1/unload", + timeout=VALIDATION.TIMEOUT_DEFAULT, + json={}, + ) + sleep.assert_called_once_with(1) + + def test_http_worker_ignores_ambient_proxy_configuration(self) -> None: + response = ChunkedResponse( + (b"ok",), + status_code=503, + content_type="text/plain", + ) + session = mock.Mock() + session.request.return_value = response + stdout = types.SimpleNamespace(buffer=io.BytesIO()) + with ( + mock.patch.object( + VALIDATION.requests, + "Session", + return_value=session, + ), + mock.patch.object( + VALIDATION, + "_load_stream_worker_request", + return_value=( + "pull_sse", + "POST", + "http://127.0.0.1:13305/api/v1/pull", + 60, + {}, + {"model_name": "builtin.Test-Llama", "stream": True}, + VALIDATION.MAX_PULL_SSE_WIRE_BYTES, + ), + ), + mock.patch.object(VALIDATION.sys, "stdout", stdout), + ): + VALIDATION._run_stream_worker() + + self.assertFalse(session.trust_env) + session.request.assert_called_once() + session.close.assert_called_once() + self.assertTrue(response.closed) + + def test_pull_rejects_non_loopback_host_before_spawning_worker(self) -> None: + with ( + mock.patch.object(VALIDATION, "_execute_http_worker") as execute, + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "host must be loopback", + ), + ): + VALIDATION._pull_model_streaming( + "builtin.Test-Llama", + 13305, + host="example.com", + ) + + execute.assert_not_called() + + def test_pull_uses_legacy_sse_inside_absolute_deadline_worker(self) -> None: + deadline = time.monotonic() + 60 + response = VALIDATION._BufferedResponse( + 200, + {"Content-Type": "text/event-stream"}, + "utf-8", + b'{"terminal":"complete"}', + ) + with ( + mock.patch.object( + VALIDATION, + "_execute_http_worker", + return_value=response, + ) as execute, + mock.patch.object( + VALIDATION, + "_auth_headers", + return_value={"Authorization": "Bearer secret"}, + ), + ): + result = VALIDATION._pull_model_streaming( + "builtin.Test-Llama", + 13305, + host="127.0.0.1", + wall_clock_deadline=deadline, + ) + + self.assertEqual(result, (200, "")) + execute.assert_called_once_with( + "POST", + "http://127.0.0.1:13305/api/v1/pull", + VALIDATION.TIMEOUT_INFERENCE, + {"Authorization": "Bearer secret"}, + { + "model_name": "builtin.Test-Llama", + "stream": True, + "subscribe": True, + }, + "pull_sse", + deadline, + "model pull", + VALIDATION.MAX_PULL_SSE_WIRE_BYTES, + ) + + def test_pull_maps_only_bounded_worker_terminal_summaries(self) -> None: + cases = ( + ({"terminal": "complete"}, (200, "")), + ( + {"terminal": "error", "status_code": 500, "error": "disk full"}, + (500, "disk full"), + ), + ( + { + "terminal": "error", + "status_code": 400, + "error": "unknown_model", + }, + (400, "unknown_model"), + ), + ) + for summary, expected in cases: + with self.subTest(summary=summary): + response = VALIDATION._BufferedResponse( + 200, + {}, + "utf-8", + json.dumps(summary).encode("utf-8"), + ) + with mock.patch.object( + VALIDATION, + "_execute_http_worker", + return_value=response, + ): + result = VALIDATION._pull_model_streaming( + "builtin.Test-Llama", + 13305, + wall_clock_deadline=time.monotonic() + 60, + ) + self.assertEqual(result, expected) + + malformed = VALIDATION._BufferedResponse(200, {}, "utf-8", b"{}") + with ( + mock.patch.object( + VALIDATION, + "_execute_http_worker", + return_value=malformed, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "terminal summary", + ), + ): + VALIDATION._pull_model_streaming( + "builtin.Test-Llama", + 13305, + wall_clock_deadline=time.monotonic() + 60, + ) + + def test_pull_sse_reducer_handles_crlf_and_chunk_boundaries(self) -> None: + body = ( + b'event: progress\r\ndata: {"percent":50}\r\n\r\n' + b'event: complete\r\ndata: {"status":"ok"}\r\n\r\n' + ) + response = ChunkedResponse( + (body[:1], body[1:17], body[17:41], body[41:]), + content_type="text/event-stream; charset=utf-8", + ) + + summary = VALIDATION._reduce_pull_sse_response(response, len(body)) + + self.assertEqual(json.loads(summary), {"terminal": "complete"}) + self.assertLessEqual( + len(summary), + VALIDATION.MAX_PULL_TERMINAL_SUMMARY_BYTES, + ) + + def test_pull_sse_reducer_discards_large_progress_history(self) -> None: + progress = b'event: progress\ndata: {"percent":50}\n\n' + complete = b'event: complete\ndata: {"status":"ok"}\n\n' + event_total = 20_000 + + def chunks(): + for _index in range(event_total): + yield progress + yield complete + + response = ChunkedResponse(chunks()) + summary = VALIDATION._reduce_pull_sse_response( + response, + len(progress) * event_total + len(complete), + ) + + self.assertEqual(summary, b'{"terminal":"complete"}') + self.assertLess(len(summary), 64) + + def test_pull_sse_reducer_accepts_exact_wire_bound(self) -> None: + body = b'event: complete\ndata: {"status":"ok"}\n\n' + + summary = VALIDATION._reduce_pull_sse_response( + ChunkedResponse((body,)), + len(body), + ) + + self.assertEqual(summary, b'{"terminal":"complete"}') + + def test_pull_sse_reducer_enforces_wire_line_data_and_event_bounds(self) -> None: + complete = b'event: complete\ndata: {"status":"ok"}\n\n' + cases = ( + ( + ChunkedResponse((complete,)), + len(complete) - 1, + contextlib.nullcontext(), + "wire byte count", + ), + ( + ChunkedResponse((b"x" * 9 + b"\n",)), + 100, + mock.patch.object(VALIDATION, "MAX_PULL_SSE_LINE_BYTES", 8), + "line exceeds bounds", + ), + ( + ChunkedResponse((b'event: progress\ndata: {"x":1}\n\n',)), + 100, + mock.patch.object(VALIDATION, "MAX_PULL_SSE_EVENT_DATA_BYTES", 2), + "event data exceeds bounds", + ), + ( + ChunkedResponse( + (b'event: progress\ndata: {"percent":1}\n\n' + complete,) + ), + 1000, + mock.patch.object(VALIDATION, "MAX_PULL_SSE_EVENTS", 1), + "event count exceeds bounds", + ), + ) + for response, wire_bound, patched_bound, expected_error in cases: + with ( + self.subTest(expected_error=expected_error), + patched_bound, + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + expected_error, + ), + ): + VALIDATION._reduce_pull_sse_response(response, wire_bound) + + def test_pull_sse_reducer_requires_one_terminal_and_clean_eof(self) -> None: + progress = b'event: progress\ndata: {"percent":1}\n\n' + complete = b'event: complete\ndata: {"status":"ok"}\n\n' + cases = ( + (progress, "without a terminal"), + (complete + complete, "after its terminal"), + (complete + progress, "after its terminal"), + (complete[:-1], "incomplete event"), + (complete.rstrip(b"\n"), "incomplete line"), + (complete + b"data: {}\n\n", "after its terminal"), + ) + for body, expected_error in cases: + with ( + self.subTest(expected_error=expected_error), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + expected_error, + ), + ): + VALIDATION._reduce_pull_sse_response( + ChunkedResponse((body,)), + len(body), + ) + + def test_pull_sse_reducer_rejects_invalid_content_and_json(self) -> None: + cases = ( + ( + ChunkedResponse( + (b'event: complete\ndata: {"status":"ok"}\n\n',), + content_type="application/json", + ), + "content type", + ), + ( + ChunkedResponse((b'event: progress\ndata: {"x":1,"x":2}\n\n',)), + "event data is invalid", + ), + ( + ChunkedResponse((b'event: progress\ndata: {"x":NaN}\n\n',)), + "event data is invalid", + ), + ( + ChunkedResponse((b"event: complete\ndata: []\n\n",)), + "event data is invalid", + ), + ) + for response, expected_error in cases: + body_size = sum(len(chunk) for chunk in response.chunks) + with ( + self.subTest(expected_error=expected_error), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + expected_error, + ), + ): + VALIDATION._reduce_pull_sse_response(response, body_size) + + def test_pull_worker_always_closes_response_on_parser_failure(self) -> None: + response = ChunkedResponse((b'event: progress\ndata: {"percent":1}\n\n',)) + with ( + mock.patch.object( + VALIDATION.requests, + "request", + return_value=response, + create=True, + ), + mock.patch.object( + VALIDATION, + "_load_stream_worker_request", + return_value=( + "pull_sse", + "POST", + "http://127.0.0.1:13305/api/v1/pull", + 60, + {}, + {"model_name": "builtin.Test-Llama", "stream": True}, + VALIDATION.MAX_PULL_SSE_WIRE_BYTES, + ), + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "without a terminal event", + ), + ): + VALIDATION._run_stream_worker() + + self.assertTrue(response.closed) + + def test_pull_worker_bounds_and_fully_consumes_non_200_body(self) -> None: + response = ChunkedResponse( + (b"temporarily ", b"unavailable"), + status_code=503, + content_type="application/json", + ) + stdout = types.SimpleNamespace(buffer=io.BytesIO()) + with ( + mock.patch.object( + VALIDATION.requests, + "request", + return_value=response, + create=True, + ), + mock.patch.object( + VALIDATION, + "_load_stream_worker_request", + return_value=( + "pull_sse", + "POST", + "http://127.0.0.1:13305/api/v1/pull", + 60, + {}, + {"model_name": "builtin.Test-Llama", "stream": True}, + VALIDATION.MAX_PULL_SSE_WIRE_BYTES, + ), + ), + mock.patch.object(VALIDATION.sys, "stdout", stdout), + ): + VALIDATION._run_stream_worker() + + returned = VALIDATION._decode_http_worker_output(stdout.buffer.getvalue()) + self.assertEqual(returned.status_code, 503) + self.assertEqual(returned.text, "temporarily unavailable") + self.assertTrue(response.closed) + + oversized = ChunkedResponse( + (b"x" * VALIDATION.MAX_PULL_RESPONSE_BYTES, b"x"), + status_code=503, + content_type="application/json", + ) + with ( + mock.patch.object( + VALIDATION.requests, + "request", + return_value=oversized, + create=True, + ), + mock.patch.object( + VALIDATION, + "_load_stream_worker_request", + return_value=( + "pull_sse", + "POST", + "http://127.0.0.1:13305/api/v1/pull", + 60, + {}, + {"model_name": "builtin.Test-Llama", "stream": True}, + VALIDATION.MAX_PULL_SSE_WIRE_BYTES, + ), + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "HTTP response byte count exceeds bounds", + ), + ): + VALIDATION._run_stream_worker() + self.assertTrue(oversized.closed) + + def test_pull_deadline_kills_and_reaps_trickling_worker(self) -> None: + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with trickling_http_response() as (url, body_started, trickle_elapsed): + host, port = url.split("//", maxsplit=1)[1].split(":") + port = int(port.split("/", maxsplit=1)[0]) + with ( + mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ), + self.assertRaisesRegex( + capabilities.CapabilityValidationError, + "model pull exceeded its wall-clock deadline", + ), + ): + VALIDATION._pull_model_streaming( + "builtin.Test-Llama", + port, + host=host, + wall_clock_deadline=time.monotonic() + 5, + ) + + self.assertTrue(body_started.is_set()) + self.assertFalse(trickle_elapsed.is_set()) + + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_pull_success_worker_is_reaped_after_clean_terminal_eof(self) -> None: + prefix = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" + b"Connection: close\r\n\r\n" + ) + body = b'event: complete\ndata: {"status":"ok"}\n\n' + workers = [] + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + with stalled_http_response(prefix, body, stall_seconds=0) as ( + url, + response_started, + _stall_elapsed, + ): + host, port = url.split("//", maxsplit=1)[1].split(":") + port = int(port.split("/", maxsplit=1)[0]) + with mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ): + result = VALIDATION._pull_model_streaming( + "builtin.Test-Llama", + port, + host=host, + wall_clock_deadline=time.monotonic() + 10, + ) + self.assertTrue(response_started.is_set()) + + self.assertEqual(result, (200, "")) + self.assertEqual(len(workers), 1) + self.assertIsNotNone(workers[0].poll()) + + def test_pull_waits_for_clean_eof_after_terminal_event(self) -> None: + body = b'event: complete\ndata: {"status":"ok"}\n\n' + workers = [] + result = [] + errors = [] + finished = threading.Event() + real_popen = subprocess.Popen + + def record_worker(*args, **kwargs): + worker = real_popen(*args, **kwargs) + workers.append(worker) + return worker + + def pull(host, port): + try: + result.append( + VALIDATION._pull_model_streaming( + "builtin.Test-Llama", + port, + host=host, + wall_clock_deadline=time.monotonic() + 20, + ) + ) + except BaseException as exc: # pylint: disable=broad-exception-caught + errors.append(exc) + finally: + finished.set() + + with held_open_pull_response(body) as (url, terminal_sent, release): + host, port = url.split("//", maxsplit=1)[1].split(":") + port = int(port.split("/", maxsplit=1)[0]) + with mock.patch.object( + VALIDATION.subprocess, + "Popen", + side_effect=record_worker, + ): + caller = threading.Thread(target=pull, args=(host, port)) + caller.start() + self.assertTrue(terminal_sent.wait(10)) + self.assertFalse(finished.wait(0.25)) + self.assertEqual(len(workers), 1) + self.assertIsNone(workers[0].poll()) + release.set() + caller.join(timeout=10) + + self.assertFalse(caller.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(result, [(200, "")]) + self.assertIsNotNone(workers[0].poll()) + + def test_pull_ambiguous_failures_and_control_exceptions_are_not_retried( + self, + ) -> None: + failures = ( + capabilities.CapabilityValidationError("malformed SSE"), + KeyboardInterrupt(), + ) + for failure in failures: + with self.subTest(failure=failure.__class__.__name__): + with ( + mock.patch.object( + VALIDATION, + "_pull_model_streaming", + side_effect=failure, + ) as pull, + mock.patch.object(VALIDATION.time, "sleep") as sleep, + self.assertRaises(failure.__class__) as raised, + ): + VALIDATION.pull_model_with_retry( + "builtin.Test-Llama", + port=13305, + host="127.0.0.1", + ) + + self.assertIs(raised.exception, failure) + pull.assert_called_once() + sleep.assert_not_called() + + def test_pull_retries_only_returned_transient_terminal_status(self) -> None: + with ( + mock.patch.object( + VALIDATION, + "_pull_model_streaming", + side_effect=[(500, "download failed"), (200, "")], + ) as pull, + mock.patch.object(VALIDATION.time, "sleep") as sleep, + contextlib.redirect_stdout(io.StringIO()), + ): + VALIDATION.pull_model_with_retry( + "builtin.Test-Llama", + port=13305, + host="127.0.0.1", + ) + + self.assertEqual(pull.call_count, 2) + self.assertEqual( + pull.call_args_list[0].kwargs["wall_clock_deadline"], + pull.call_args_list[1].kwargs["wall_clock_deadline"], + ) + sleep.assert_called_once_with(2) + def test_model_rejects_raw_ifm_markers_in_response_fields(self) -> None: cases = { "content": ( @@ -655,7 +2342,12 @@ def test_model_runs_requested_capability_profile_and_records_evidence(self) -> N mock.patch.object( VALIDATION, "validate_capabilities", - return_value=(matrix, True, "12/12 capability cases passed"), + return_value=( + matrix, + True, + f"{len(capabilities.K2_HORIZON_CASE_IDS)}/" + f"{len(capabilities.K2_HORIZON_CASE_IDS)} capability cases passed", + ), ) as validate, contextlib.redirect_stdout(io.StringIO()), ): @@ -759,6 +2451,22 @@ def test_model_requires_health_to_confirm_the_requested_backend(self) -> None: }, "backend 'cpu' instead of 'vulkan'", ), + ( + 200, + { + "all_models_loaded": [ + { + "model_name": "builtin.Test-Llama", + "recipe_options": {"llamacpp_backend": "vulkan"}, + }, + { + "model_name": "builtin.Stale-Llama", + "recipe_options": {"llamacpp_backend": "vulkan"}, + }, + ] + }, + "exactly the requested model", + ), ) for status_code, health, expected_error in cases: @@ -792,6 +2500,68 @@ def request_json(method, url, timeout, **_kwargs): self.assertFalse(success) self.assertIn(expected_error, error) + def test_model_binds_top_level_response_model_to_the_request(self) -> None: + for response_model in (None, "builtin.Other-Llama"): + with self.subTest(response_model=response_model): + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat( + {"content": "The answer is 4."}, + response_model=response_model, + ), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, error, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + + self.assertFalse(success) + self.assertIn("response model", error) + + def test_model_accepts_canonically_equivalent_bare_response_model(self) -> None: + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat( + {"content": "The answer is 4."}, + response_model="Test-Llama", + ), + ), + contextlib.redirect_stdout(io.StringIO()), + ): + success, response, _stats = VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + + self.assertTrue(success, response) + + def test_model_treats_post_inference_unload_failure_as_fatal(self) -> None: + with ( + mock.patch.object( + VALIDATION, + "request_json", + side_effect=self.request_json_for_chat( + {"content": "The answer is 4."}, + unload_status_code=500, + ), + ), + contextlib.redirect_stdout(io.StringIO()), + self.assertRaisesRegex(RuntimeError, "unload.*HTTP 500"), + ): + VALIDATION.test_model( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + ) + def run_validation(self, explicit): operations = [] model = { @@ -818,7 +2588,11 @@ def run_validation(self, explicit): VALIDATION, "get_model_catalog", return_value=[model] ), mock.patch.object(VALIDATION, "get_builtin_model", return_value=model), - mock.patch.object(VALIDATION, "unload_all_models"), + mock.patch.object( + VALIDATION, + "unload_all_models", + return_value=self.response(), + ), mock.patch.object( VALIDATION, "request_json", @@ -873,6 +2647,33 @@ def test_reload_fails_when_the_first_model_remains_loaded(self) -> None: self.assertRegex(error, "still loaded") test_model.assert_called_once() + def test_reload_rejects_any_model_remaining_after_unload(self) -> None: + health = { + "all_models_loaded": [{"model_name": "builtin.Stale-Llama"}], + } + with ( + mock.patch.object( + VALIDATION, + "test_model", + return_value=(True, "answer", {}), + ) as test_model, + mock.patch.object( + VALIDATION, + "request_json", + return_value=(self.response(), health), + ), + ): + success, error, _stats = VALIDATION.validate_model_lifecycle( + "http://localhost:13305/api/v1", + "builtin.Test-Llama", + "vulkan", + reload_after_unload=True, + ) + + self.assertFalse(success) + self.assertIn("still loaded", error) + test_model.assert_called_once() + def test_reload_requires_a_valid_loaded_model_inventory(self) -> None: for health in ({}, {"all_models_loaded": "invalid"}): with self.subTest(health=health): @@ -938,6 +2739,139 @@ def test_reload_smoke_does_not_repeat_the_capability_profile(self) -> None: class LlamaCppValidationSideEffectTests(unittest.TestCase): + @staticmethod + def response(status_code=200): + return types.SimpleNamespace(status_code=status_code) + + def test_prepare_only_installs_and_pulls_without_running_inference(self) -> None: + model = { + "id": "Test-Llama", + "load_id": "builtin.Test-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/Test-Llama:model.gguf", + } + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--prepare-only", + "--model", + "Test-Llama", + ] + + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object(VALIDATION, "get_builtin_model", return_value=model), + mock.patch.object( + VALIDATION, + "unload_all_models", + return_value=self.response(), + ), + mock.patch.object(VALIDATION, "install_backend") as install, + mock.patch.object(VALIDATION, "pull_model_with_retry") as pull, + mock.patch.object(VALIDATION, "validate_model_lifecycle") as validate, + contextlib.redirect_stdout(io.StringIO()), + ): + try: + VALIDATION.main() + except SystemExit as exc: + self.fail(f"prepare-only exited unexpectedly: {exc}") + + install.assert_called_once() + pull.assert_called_once_with("builtin.Test-Llama", port=13305, host="127.0.0.1") + validate.assert_not_called() + + def test_result_file_creation_is_exclusive_and_rejects_non_finite_values( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + output_path = Path(directory) / "result.json" + output_path.write_text("preserved", encoding="utf-8") + with self.assertRaises(FileExistsError): + VALIDATION.write_results_file(output_path, []) + self.assertEqual(output_path.read_text(encoding="utf-8"), "preserved") + + non_finite_path = Path(directory) / "non-finite.json" + with self.assertRaises(ValueError): + VALIDATION.write_results_file( + non_finite_path, + [{"tokens_per_second": float("inf")}], + ) + + def test_prepare_only_stops_when_a_model_pull_fails(self) -> None: + model = { + "id": "Test-Llama", + "load_id": "builtin.Test-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/Test-Llama:model.gguf", + } + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--prepare-only", + "--model", + "Test-Llama", + ] + + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object(VALIDATION, "get_builtin_model", return_value=model), + mock.patch.object( + VALIDATION, + "unload_all_models", + return_value=self.response(), + ), + mock.patch.object(VALIDATION, "install_backend"), + mock.patch.object( + VALIDATION, + "pull_model_with_retry", + side_effect=AssertionError("pull failed"), + ), + mock.patch.object(VALIDATION, "validate_model_lifecycle") as validate, + contextlib.redirect_stdout(io.StringIO()), + ): + try: + with self.assertRaisesRegex(AssertionError, "pull failed"): + VALIDATION.main() + except SystemExit as exc: + self.fail(f"prepare-only exited before the pull: {exc}") + + validate.assert_not_called() + + def test_prepare_only_rejects_invalid_selection_before_server_mutation( + self, + ) -> None: + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--prepare-only", + "--model", + "Missing-Llama", + ] + + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object(VALIDATION, "get_builtin_model", return_value=None), + mock.patch.object(VALIDATION, "install_backend") as install, + mock.patch.object(VALIDATION, "pull_model_with_retry") as pull, + mock.patch.object(VALIDATION, "unload_all_models") as unload_models, + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit) as raised, + ): + VALIDATION.main() + + self.assertEqual(raised.exception.code, 1) + install.assert_not_called() + pull.assert_not_called() + unload_models.assert_not_called() + def test_invalid_explicit_model_is_rejected_before_server_mutation(self) -> None: argv = [ "validate_llamacpp.py", @@ -964,6 +2898,171 @@ def test_invalid_explicit_model_is_rejected_before_server_mutation(self) -> None set_channel.assert_not_called() unload_models.assert_not_called() + def test_wrong_builtin_resolver_id_is_rejected_before_server_mutation(self) -> None: + wrong_model = { + "id": "Different-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/different:model.gguf", + } + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--model", + "Test-Llama", + ] + + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object( + VALIDATION, + "get_builtin_model", + return_value=wrong_model, + ), + mock.patch.object(VALIDATION, "install_backend") as install, + mock.patch.object(VALIDATION, "unload_all_models") as unload_models, + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit) as raised, + ): + VALIDATION.main() + + self.assertEqual(raised.exception.code, 1) + install.assert_not_called() + unload_models.assert_not_called() + + def test_initial_clean_state_unload_http_failure_is_fatal(self) -> None: + model = { + "id": "Test-Llama", + "load_id": "builtin.Test-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/Test-Llama:model.gguf", + } + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--skip-install", + "--model", + "Test-Llama", + ] + + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object( + VALIDATION, "select_llamacpp_models", return_value=[model] + ), + mock.patch.object( + VALIDATION, + "unload_all_models", + return_value=self.response(500), + ), + mock.patch.object( + VALIDATION, + "validate_model_lifecycle", + return_value=(True, "Four.", {}), + ) as lifecycle, + contextlib.redirect_stdout(io.StringIO()), + self.assertRaisesRegex(RuntimeError, "clean state.*HTTP 500"), + ): + VALIDATION.main() + + lifecycle.assert_not_called() + + def test_final_cleanup_unload_request_failure_is_fatal(self) -> None: + model = { + "id": "Test-Llama", + "load_id": "builtin.Test-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/Test-Llama:model.gguf", + } + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--skip-install", + "--model", + "Test-Llama", + ] + + with tempfile.TemporaryDirectory() as temp_dir: + argv.extend(["--output", str(Path(temp_dir) / "results.json")]) + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object( + VALIDATION, "select_llamacpp_models", return_value=[model] + ), + mock.patch.object( + VALIDATION, + "unload_all_models", + side_effect=[ + self.response(), + VALIDATION.requests.RequestException("cleanup failed"), + ], + ), + mock.patch.object( + VALIDATION, + "validate_model_lifecycle", + return_value=(True, "Four.", {}), + ), + contextlib.redirect_stdout(io.StringIO()), + self.assertRaisesRegex(RuntimeError, "cleanup failed"), + ): + VALIDATION.main() + + def test_capability_execution_uses_the_requested_canonical_load_id(self) -> None: + model = { + "id": "Untrusted-Display-Id", + "load_id": "builtin.Test-Llama", + "recipe": "llamacpp", + "labels": ["chat"], + "checkpoint": "example/Test-Llama:model.gguf", + } + argv = [ + "validate_llamacpp.py", + "--backend", + "vulkan", + "--skip-install", + "--model", + "Test-Llama", + "--capability-profile", + capabilities.K2_HORIZON_PROFILE, + "--capability-model", + "Test-Llama", + ] + + with tempfile.TemporaryDirectory() as temp_dir: + argv.extend(["--output", str(Path(temp_dir) / "results.json")]) + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(VALIDATION, "require_running_server"), + mock.patch.object( + VALIDATION, "select_llamacpp_models", return_value=[model] + ), + mock.patch.object( + VALIDATION, + "unload_all_models", + return_value=self.response(), + ), + mock.patch.object( + VALIDATION, + "validate_model_lifecycle", + return_value=(True, "Four.", {}), + ) as lifecycle, + contextlib.redirect_stdout(io.StringIO()), + ): + VALIDATION.main() + + self.assertEqual( + lifecycle.call_args.kwargs["capability_profile"], + capabilities.K2_HORIZON_PROFILE, + ) + def test_capability_options_must_be_paired(self) -> None: cases = ( ["--capability-profile", capabilities.K2_HORIZON_PROFILE], @@ -1041,7 +3140,11 @@ def validate_lifecycle( mock.patch.object( VALIDATION, "get_model_catalog", return_value=[model] ), - mock.patch.object(VALIDATION, "unload_all_models"), + mock.patch.object( + VALIDATION, + "unload_all_models", + return_value=self.response(), + ), mock.patch.object( VALIDATION, "validate_model_lifecycle", diff --git a/test/test_validate_llamacpp_trusted_execution.py b/test/test_validate_llamacpp_trusted_execution.py new file mode 100644 index 0000000000..7aadadfc49 --- /dev/null +++ b/test/test_validate_llamacpp_trusted_execution.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Security-contract tests for protected llama.cpp validation execution.""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" +TESTING_GUIDE = ROOT / "docs" / "dev" / "testing.md" +PROTECTED_EVENT = ( + "github.event_name == 'pull_request_target' || " + "github.event_name == 'merge_group'" +) +UNPROTECTED_EVENT = ( + "github.event_name != 'pull_request_target' && " + "github.event_name != 'merge_group'" +) +CHECKOUT_ACTION = "actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0" + + +def job(workflow: str, name: str, following_name: str) -> str: + return workflow.split(f" {name}:\n", 1)[1].split(f" {following_name}:\n", 1)[0] + + +def step(job_text: str, name: str, following_name: str) -> str: + return job_text.split(f" - name: {name}\n", 1)[1].split( + f" - name: {following_name}\n", 1 + )[0] + + +class TrustedExecutionTests(unittest.TestCase): + def assert_trusted_validation_entrypoint(self, validation: str) -> None: + run_script = validation.split(" run: >-\n", 1)[1] + normalized = " ".join(run_script.split()) + trusted_entrypoint = ( + 'python "$VALIDATION_ROOT/.github/scripts/' 'run_llamacpp_validation.py"' + ) + + self.assertEqual(normalized.count(trusted_entrypoint), 1) + self.assertEqual(normalized.count("run_llamacpp_validation.py"), 1) + self.assertNotIn("test/validate_llamacpp.py", normalized) + self.assertNotIn("VALIDATION_CANDIDATE_ROOT", normalized) + self.assertNotIn("candidate/", normalized) + self.assertNotIn("cd candidate", normalized) + + def test_protected_validation_ignores_candidate_control_plane_files(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + build = job(workflow, "build", "validate") + validate = job(workflow, "validate", "reverify-protected-release-manifest") + + for job_text in (build, validate): + with self.subTest(job=job_text.splitlines()[0]): + control_checkout = step( + job_text, + "Check out validation control plane", + "Check out the validation candidate", + ) + candidate_checkout = step( + job_text, + "Check out the validation candidate", + "Verify authorized merge commit", + ) + self.assertIn(CHECKOUT_ACTION, control_checkout) + self.assertIn("github.event.pull_request.base.sha", control_checkout) + self.assertIn("github.event.merge_group.base_sha", control_checkout) + self.assertIn("github.sha", control_checkout) + self.assertIn("persist-credentials: false", control_checkout) + self.assertNotIn("allow-unsafe-pr-checkout", control_checkout) + self.assertIn("path: candidate", candidate_checkout) + self.assertIn("persist-credentials: false", candidate_checkout) + self.assertIn(f"if: {PROTECTED_EVENT}", candidate_checkout) + self.assertIn( + "github.event.pull_request.merge_commit_sha", candidate_checkout + ) + self.assertIn("github.event.merge_group.head_sha", candidate_checkout) + + protected_catalog = step( + build, + "Stage protected validation-only model catalog", + "Stage branch validation-only model catalog", + ) + self.assertIn(f"if: {PROTECTED_EVENT}", protected_catalog) + self.assertIn( + 'python "$GITHUB_WORKSPACE/test/utils/validation_model_catalog.py"', + protected_catalog, + ) + self.assertIn( + 'candidate_catalog="$GITHUB_WORKSPACE/candidate/src/cpp/resources/' + 'server_models.json"', + protected_catalog, + ) + self.assertIn( + '--overlay "$GITHUB_WORKSPACE/test/fixtures/' + 'llamacpp_validation_models.json"', + protected_catalog, + ) + self.assertIn( + "resolved_catalog.relative_to(candidate_root)", + protected_catalog, + ) + self.assertIn('--base "$candidate_catalog"', protected_catalog) + self.assertIn('--output "$candidate_catalog"', protected_catalog) + + branch_catalog = step( + build, + "Stage branch validation-only model catalog", + "Install Linux build dependencies", + ) + self.assertIn(f"if: {UNPROTECTED_EVENT}", branch_catalog) + self.assertIn( + "working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }}", + branch_catalog, + ) + self.assertIn("python -m test.utils.validation_model_catalog", branch_catalog) + + protected_setup = step( + validate, + "Set up trusted or branch validation environment", + "Run validation with lemond", + ) + self.assertIn("uses: ./.github/actions/setup-venv", protected_setup) + self.assertIn( + "requirements-file: '${{ github.workspace }}/test/requirements.txt'", + protected_setup, + ) + + before_trusted_runner = validate.split( + " - name: Run validation with lemond\n", 1 + )[0] + self.assertNotIn("lemond --version", before_trusted_runner) + self.assertNotIn("& $lemondExe --version", before_trusted_runner) + self.assertNotIn('"$lemond" --version', before_trusted_runner) + + validation = step(validate, "Run validation with lemond", "Upload results") + self.assertIn( + "VALIDATION_ROOT: ${{ github.workspace }}", + validation, + ) + self.assertIn("PYTHONPATH: ${{ github.workspace }}", validation) + self.assertIn("working-directory: ${{ github.workspace }}", validation) + self.assertIn( + 'python "$VALIDATION_ROOT/.github/scripts/' 'run_llamacpp_validation.py"', + validation, + ) + self.assert_trusted_validation_entrypoint(validation) + self.assertIn( + "CANDIDATE_LEMOND: ${{ runner.os == 'Windows' && " + "format('{0}/build/Release/lemond.exe', " + "env.VALIDATION_CANDIDATE_ROOT) || " + "format('{0}/build/lemond', env.VALIDATION_CANDIDATE_ROOT) }}", + validation, + ) + self.assertIn('--lemond "$CANDIDATE_LEMOND"', validation) + self.assertNotIn("uses: ./candidate/", validate) + self.assertNotIn("$GITHUB_WORKSPACE/candidate/.github", validation) + self.assertNotIn("$GITHUB_WORKSPACE/candidate/test", validation) + + controlled_files = ( + ".github/actions/cleanup-processes-linux/action.yml", + ".github/actions/cleanup-processes-windows/action.yml", + ".github/actions/setup-python/action.yml", + ".github/actions/setup-venv/action.yml", + ".github/scripts/run_llamacpp_validation.py", + "test/fixtures/llamacpp_validation_artifacts.json", + "test/fixtures/llamacpp_validation_models.json", + "test/requirements.txt", + "test/validate_llamacpp.py", + "test/utils/capabilities.py", + "test/utils/llamacpp_capability_validation.py", + "test/utils/llamacpp_validation_artifacts.py", + "test/utils/llamacpp_validation_evidence.py", + "test/utils/server_base.py", + "test/utils/test_models.py", + "test/utils/validation_model_catalog.py", + "test/utils/validation_model_selection.py", + ) + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + for relative_path in controlled_files: + trusted_file = workspace / relative_path + candidate_file = workspace / "candidate" / relative_path + trusted_file.parent.mkdir(parents=True, exist_ok=True) + candidate_file.parent.mkdir(parents=True, exist_ok=True) + trusted_file.write_text("TRUSTED\n", encoding="utf-8") + candidate_file.write_text("MALICIOUS\n", encoding="utf-8") + + execution_root = workspace + for relative_path in controlled_files: + with self.subTest(relative_path=relative_path): + selected = execution_root / relative_path + self.assertEqual(selected.read_text(encoding="utf-8"), "TRUSTED\n") + + def test_relative_candidate_validator_commands_fail_the_contract(self) -> None: + trusted = ( + " run: >-\n" + ' python "$VALIDATION_ROOT/.github/scripts/' + 'run_llamacpp_validation.py"\n' + ' --lemond "$CANDIDATE_LEMOND"\n' + ) + substitutions = ( + trusted.replace( + 'python "$VALIDATION_ROOT/.github/scripts/' + 'run_llamacpp_validation.py"', + "python candidate/test/validate_llamacpp.py", + ), + trusted + " && python candidate/test/validate_llamacpp.py\n", + trusted.replace( + "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py", + "$VALIDATION_CANDIDATE_ROOT/.github/scripts/" + "run_llamacpp_validation.py", + ), + trusted.replace("python ", "cd candidate && python ", 1), + ) + + for substitution in substitutions: + with self.subTest(substitution=substitution): + with self.assertRaises(AssertionError): + self.assert_trusted_validation_entrypoint(substitution) + + def test_protected_ephemeral_mac_has_no_token_bearing_post_run_step( + self, + ) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + validate = job(workflow, "validate", "reverify-protected-release-manifest") + validate_header = workflow.split(" validate:\n", 1)[1].split( + " steps:\n", 1 + )[0] + runtime_and_later = validate.split( + " - name: Run validation with lemond\n", 1 + )[1] + after_runtime = runtime_and_later.split(" - name: Upload results\n", 1)[1] + protected_mac_exclusion = ( + "!((github.event_name == 'pull_request_target' || " + "github.event_name == 'merge_group') && matrix.target == " + "'macos-metal' && runner.environment == 'github-hosted' && " + "join(matrix.runner, ',') == 'macos-latest')" + ) + + self.assertIn("permissions:\n contents: read\n", validate_header) + self.assertNotIn("GH_TOKEN:", validate) + for upload_name in ( + "Upload results", + "Upload restart results", + "Upload server logs", + ): + upload = validate.split(f" - name: {upload_name}\n", 1)[1] + upload = upload.split(" - name:", 1)[0] + with self.subTest(upload=upload_name): + self.assertIn(protected_mac_exclusion, " ".join(upload.split())) + self.assertNotIn("secrets.", after_runtime) + self.assertNotIn("actions/cache", after_runtime) + + def test_trust_boundary_does_not_claim_hostile_native_isolation(self) -> None: + guide = TESTING_GUIDE.read_text(encoding="utf-8") + + self.assertIn( + "base-branch checkout supplies the setup and cleanup actions", guide + ) + self.assertIn("every imported `test.utils` helper", guide) + self.assertIn("built `lemond` and adjacent build resources", guide) + self.assertIn( + "does **not** make candidate native code an adversarial sandbox", guide + ) + self.assertIn("execute as the runner's OS user", guide) + self.assertIn("single-job ephemeral protected self-hosted workers", guide) + self.assertIn("expose no repository secrets", guide) + self.assertIn("grant only a read-only repository token", guide) + self.assertIn("no artifact upload, cache upload", guide) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validate_llamacpp_workflow_plan.py b/test/test_validate_llamacpp_workflow_plan.py index de300d68a7..94ecf67256 100644 --- a/test/test_validate_llamacpp_workflow_plan.py +++ b/test/test_validate_llamacpp_workflow_plan.py @@ -3,35 +3,3585 @@ from __future__ import annotations +import importlib.util import json +import os +import subprocess +import sys import tempfile +import textwrap import unittest from pathlib import Path +import yaml + +from test.utils import llamacpp_release_assets as release_assets from test.utils import llamacpp_validation_plan as planning -K2_SMALL = "K2-Horizon-0.9B-GGUF" -K2_MEDIUM = "K2-Horizon-3.7B-GGUF" -K2_LARGE = "K2-Horizon-7B-GGUF" -ROOT = Path(__file__).resolve().parents[1] -LEGACY_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp.yml" -VALIDATION_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" -PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" -MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" -SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" -DOCS_AND_STYLE_WORKFLOW = ROOT / ".github" / "workflows" / "docs_and_style.yml" +K2_SMALL = "K2-Horizon-0.9B-GGUF" +K2_MEDIUM = "K2-Horizon-3.7B-GGUF" +K2_LARGE = "K2-Horizon-7B-GGUF" +CHECKOUT_SHA = "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09" +DOWNLOAD_ARTIFACT_SHA = "37930b1c2abaa49bbe596cd826c3c89aef350131" +ROOT = Path(__file__).resolve().parents[1] +LEGACY_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp.yml" +VALIDATION_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" +PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" +MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" +SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" +DOCS_AND_STYLE_WORKFLOW = ROOT / ".github" / "workflows" / "docs_and_style.yml" +VALIDATION_RUNNER = ROOT / ".github" / "scripts" / "run_llamacpp_validation.py" +REQUIRED_CHECK_POLICY = ( + ROOT / ".github" / "scripts" / "llamacpp_required_check_policy.py" +) +TESTING_GUIDE = ROOT / "docs" / "dev" / "testing.md" + + +def load_validation_runner(): + spec = importlib.util.spec_from_file_location( + "llamacpp_validation_runner", VALIDATION_RUNNER + ) + if spec is None or spec.loader is None: + raise RuntimeError("could not load llama.cpp validation runner") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_required_check_policy(): + spec = importlib.util.spec_from_file_location( + "llamacpp_required_check_policy", REQUIRED_CHECK_POLICY + ) + if spec is None or spec.loader is None: + raise RuntimeError("could not load llama.cpp required-check policy") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_workflow(path: Path) -> dict: + workflow = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + if not isinstance(workflow, dict): + raise ValueError(f"workflow must be a mapping: {path}") + return workflow + + +def workflow_step(path: Path, job_name: str, step_name: str) -> dict: + workflow = load_workflow(path) + steps = workflow["jobs"][job_name]["steps"] + matches = [step for step in steps if step.get("name") == step_name] + if len(matches) != 1: + raise ValueError(f"workflow step must be unique: {job_name}/{step_name}") + return matches[0] + + +def extract_workflow_step_script(step_name: str) -> str: + workflow = PR_WORKFLOW.read_text(encoding="utf-8") + step = workflow.split(f" - name: {step_name}\n", 1)[1].split( + " - name:", 1 + )[0] + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +def run_identity_step( + snapshots: list[dict[str, object]], + *, + event_merge_sha: str, + event_base_ref: str = "main", +) -> tuple[subprocess.CompletedProcess[str], dict[str, str], int]: + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + snapshots_path = temporary / "snapshots.json" + call_count_path = temporary / "call-count.txt" + output_path = temporary / "github-output.txt" + normalized_snapshots = json.loads(json.dumps(snapshots)) + for snapshot in normalized_snapshots: + snapshot["base"].setdefault("ref", "main") + snapshots_path.write_text(json.dumps(normalized_snapshots), encoding="utf-8") + + fake_gh = temporary / "gh" + fake_gh.write_text( + """#!/usr/bin/env python3 +import json +import os +from pathlib import Path + +snapshots = json.loads(Path(os.environ["FAKE_GH_SNAPSHOTS"]).read_text()) +count_path = Path(os.environ["FAKE_GH_CALL_COUNT"]) +count = int(count_path.read_text()) if count_path.exists() else 0 +count_path.write_text(str(count + 1)) +print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + fake_sleep = temporary / "sleep" + fake_sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake_sleep.chmod(0o755) + + environment = os.environ.copy() + base = snapshots[0]["base"] + head = snapshots[0]["head"] + if not isinstance(base, dict) or not isinstance(head, dict): + raise ValueError("fake pull request base and head must be objects") + environment.update( + { + "EVENT_BASE_SHA": str(base["sha"]), + "EVENT_BASE_REF": event_base_ref, + "EVENT_HEAD_SHA": str(head["sha"]), + "EVENT_MERGE_SHA": event_merge_sha, + "EVENT_NAME": "pull_request_target", + "FAKE_GH_CALL_COUNT": str(call_count_path), + "FAKE_GH_SNAPSHOTS": str(snapshots_path), + "GH_TOKEN": "test-token", + "GITHUB_OUTPUT": str(output_path), + "GITHUB_REPOSITORY": "LLM360/lemonade", + "MERGE_GROUP_BASE_SHA": "", + "MERGE_GROUP_HEAD_SHA": "", + "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", + "PR_NUMBER": "7", + } + ) + completed = subprocess.run( + [ + "bash", + "-c", + extract_workflow_step_script("Resolve the live candidate identity"), + ], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + outputs = {} + if output_path.exists(): + outputs = dict( + line.split("=", 1) + for line in output_path.read_text(encoding="utf-8").splitlines() + ) + calls = int(call_count_path.read_text()) if call_count_path.exists() else 0 + return completed, outputs, calls + + +def issue_event( + event_id: int, + event: str, + *, + actor: str = "maintainer", + actor_id: int = 100, + label: str | None = None, +) -> dict[str, object]: + record: dict[str, object] = { + "actor": { + "id": actor_id, + "login": actor, + "node_id": f"USER_{actor_id}", + "type": "User", + }, + "created_at": f"2026-01-01T00:00:{event_id:02d}Z", + "event": event, + "id": event_id, + "node_id": f"EVENT_{event_id}", + "performed_via_github_app": None, + } + if label is not None: + record["label"] = {"name": label} + return record + + +def run_output_step( + step_name: str, environment_values: dict[str, str] +) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: + with tempfile.TemporaryDirectory() as directory: + output_path = Path(directory) / "github-output.txt" + environment = os.environ.copy() + environment.update(environment_values) + environment["GITHUB_OUTPUT"] = str(output_path) + completed = subprocess.run( + ["bash", "-c", extract_workflow_step_script(step_name)], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + outputs = {} + if output_path.exists(): + outputs = dict( + line.split("=", 1) + for line in output_path.read_text(encoding="utf-8").splitlines() + ) + return completed, outputs + + +def run_failed_closed_assessment( + pull_request: dict[str, object], +) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + output_path = temporary / "github-output.txt" + fake_gh = temporary / "gh" + fake_gh.write_text( + f"#!/bin/sh\nprintf '%s\\n' '{json.dumps(pull_request)}'\n", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "EVENT_ACTION": "closed", + "EVENT_BASE_REF": "main", + "EVENT_BASE_SHA": "a" * 40, + "EVENT_HEAD_SHA": "b" * 40, + "EVENT_MERGE_SHA": "c" * 40, + "GITHUB_OUTPUT": str(output_path), + "GITHUB_REPOSITORY": "LLM360/lemonade", + "MODE": "", + "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", + "POLICY_CONCLUSION": "", + "POLICY_RESULT": "failure", + "POLICY_SUMMARY": "", + "PR_NUMBER": "7", + "VALIDATION_RESULT": "skipped", + } + ) + completed = subprocess.run( + [ + "bash", + "-c", + extract_workflow_step_script("Assess policy and validation"), + ], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + outputs = {} + if output_path.exists(): + outputs = dict( + line.split("=", 1) + for line in output_path.read_text(encoding="utf-8").splitlines() + ) + return completed, outputs + + +def run_policy_decision_step( + *, + action: str, + event_label: str, + event_actor: str, + event_actor_id: int, + event_updated_at: str, + issue_events: list[list[dict[str, object]]], + check_pages: list[dict[str, object]], + authorization_label_present: bool = True, + dependabot_pr: bool = False, + fail_check_api: bool = False, + permission: str = "write", + permission_actor_id: int = 100, + run_attempt: int = 1, + run_conclusion: str = "failure", + run_created_at: str = "2026-01-01T00:00:00Z", + run_event: str = "pull_request_target", + run_id: int = 500, + run_number: int = 50, + run_path: str = ".github/workflows/validate_llamacpp_pr.yml@main", + run_repository: str = "LLM360/lemonade", + run_response_attempt: int = 1, + run_response_head_branch: str = "feature/k2-horizon", + run_response_head_repository: str = "LLM360/lemonade", + run_response_head_sha: str = "b" * 40, + run_response_id: int | None = None, + run_response_number: int | None = None, + run_response_pr_base_ref: str = "main", + run_response_pr_base_sha: str = "a" * 40, + run_response_pr_head_ref: str = "feature/k2-horizon", + run_response_pr_head_repository: str = "LLM360/lemonade", + run_response_pr_head_sha: str = "b" * 40, + run_response_pr_number: int = 7, + run_response_pull_request_count: int = 1, + run_response_pull_requests_json: str | None = None, + run_started_at: str | None = None, + run_status: str = "completed", + triggering_actor: str = "maintainer", + identity_matches: bool = True, + relevant: bool = True, +) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + events_path = temporary / "events.json" + checks_path = temporary / "checks.json" + output_path = temporary / "github-output.txt" + events_path.write_text(json.dumps(issue_events), encoding="utf-8") + checks_path.write_text(json.dumps(check_pages), encoding="utf-8") + fake_gh = temporary / "gh" + fake_gh.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +endpoint = next( + (argument for argument in sys.argv[1:] if argument.startswith("repos/")), "" +) +if "/issues/" in endpoint: + print(Path(os.environ["FAKE_GH_EVENTS"]).read_text(encoding="utf-8")) +elif "/actions/runs/" in endpoint: + pull_request = { + "base": { + "ref": os.environ["FAKE_RUN_PR_BASE_REF"], + "repo": {"url": os.environ["FAKE_RUN_REPOSITORY_URL"]}, + "sha": os.environ["FAKE_RUN_PR_BASE_SHA"], + }, + "head": { + "ref": os.environ["FAKE_RUN_PR_HEAD_REF"], + "repo": {"url": os.environ["FAKE_RUN_PR_HEAD_REPOSITORY_URL"]}, + "sha": os.environ["FAKE_RUN_PR_HEAD_SHA"], + }, + "id": 4453422506, + "number": int(os.environ["FAKE_RUN_PR_NUMBER"]), + "url": os.environ["FAKE_RUN_PR_URL"], + } + pull_requests = [pull_request] * int(os.environ["FAKE_RUN_PR_COUNT"]) + if os.environ["FAKE_RUN_PULL_REQUESTS_JSON"]: + pull_requests = json.loads(os.environ["FAKE_RUN_PULL_REQUESTS_JSON"]) + print(json.dumps({ + "conclusion": os.environ["FAKE_RUN_CONCLUSION"], + "created_at": os.environ["FAKE_RUN_CREATED_AT"], + "event": os.environ["FAKE_RUN_EVENT"], + "head_branch": os.environ["FAKE_RUN_RESPONSE_HEAD_BRANCH"], + "head_repository": {"full_name": os.environ["FAKE_RUN_RESPONSE_HEAD_REPOSITORY"]}, + "head_sha": os.environ["FAKE_RUN_RESPONSE_HEAD_SHA"], + "id": int(os.environ["FAKE_RUN_RESPONSE_ID"]), + "path": os.environ["FAKE_RUN_PATH"], + "pull_requests": pull_requests, + "repository": {"full_name": os.environ["FAKE_RUN_REPOSITORY"], "id": 1356276914}, + "run_attempt": int(os.environ["FAKE_RUN_RESPONSE_ATTEMPT"]), + "run_number": int(os.environ["FAKE_RUN_RESPONSE_NUMBER"]), + "run_started_at": os.environ["FAKE_RUN_STARTED_AT"], + "status": os.environ["FAKE_RUN_STATUS"], + })) +elif "/collaborators/" in endpoint: + print(json.dumps({ + "permission": os.environ["FAKE_GH_PERMISSION"], + "user": {"id": int(os.environ["FAKE_GH_PERMISSION_ACTOR_ID"])}, + })) +elif "/check-runs" in endpoint: + if os.environ["FAKE_FAIL_CHECK_API"] == "true": + raise SystemExit("Checks API is unavailable") + print(Path(os.environ["FAKE_GH_CHECKS"]).read_text(encoding="utf-8")) +else: + raise SystemExit(f"unexpected endpoint: {endpoint}") +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "AUTHORIZATION_LABEL_PRESENT": str(authorization_label_present).lower(), + "BASE_REF_CHANGED": "false", + "CHECK_NAME": "llama.cpp authorized validation v2", + "CURRENT_RUN_ATTEMPT": str(run_attempt), + "CURRENT_RUN_ID": str(run_id), + "CURRENT_RUN_NUMBER": str(run_number), + "DEPENDABOT_PR": str(dependabot_pr).lower(), + "EVENT_ACTION": action, + "EVENT_ACTOR": event_actor, + "EVENT_ACTOR_ID": str(event_actor_id), + "EVENT_BASE_REF": "main", + "EVENT_BASE_SHA": "a" * 40, + "EVENT_LABEL": event_label, + "EVENT_HEAD_SHA": "b" * 40, + "EVENT_NAME": "pull_request_target", + "EVENT_UPDATED_AT": event_updated_at, + "FAKE_GH_CHECKS": str(checks_path), + "FAKE_GH_EVENTS": str(events_path), + "FAKE_GH_PERMISSION": permission, + "FAKE_GH_PERMISSION_ACTOR_ID": str(permission_actor_id), + "FAKE_FAIL_CHECK_API": str(fail_check_api).lower(), + "FAKE_RUN_CONCLUSION": run_conclusion, + "FAKE_RUN_CREATED_AT": run_created_at, + "FAKE_RUN_EVENT": run_event, + "FAKE_RUN_PATH": run_path, + "FAKE_RUN_REPOSITORY": run_repository, + "FAKE_RUN_REPOSITORY_URL": ( + f"https://api.github.com/repos/{run_repository}" + ), + "FAKE_RUN_RESPONSE_ATTEMPT": str(run_response_attempt), + "FAKE_RUN_RESPONSE_HEAD_BRANCH": run_response_head_branch, + "FAKE_RUN_RESPONSE_HEAD_REPOSITORY": run_response_head_repository, + "FAKE_RUN_RESPONSE_HEAD_SHA": run_response_head_sha, + "FAKE_RUN_RESPONSE_ID": str( + run_id if run_response_id is None else run_response_id + ), + "FAKE_RUN_RESPONSE_NUMBER": str( + run_number if run_response_number is None else run_response_number + ), + "FAKE_RUN_PR_BASE_REF": run_response_pr_base_ref, + "FAKE_RUN_PR_BASE_SHA": run_response_pr_base_sha, + "FAKE_RUN_PR_COUNT": str(run_response_pull_request_count), + "FAKE_RUN_PR_HEAD_REF": run_response_pr_head_ref, + "FAKE_RUN_PR_HEAD_REPOSITORY_URL": ( + "https://api.github.com/repos/" f"{run_response_pr_head_repository}" + ), + "FAKE_RUN_PR_HEAD_SHA": run_response_pr_head_sha, + "FAKE_RUN_PR_NUMBER": str(run_response_pr_number), + "FAKE_RUN_PULL_REQUESTS_JSON": (run_response_pull_requests_json or ""), + "FAKE_RUN_PR_URL": ( + f"https://api.github.com/repos/{run_repository}/pulls/" + f"{run_response_pr_number}" + ), + "FAKE_RUN_STARTED_AT": ( + run_created_at if run_started_at is None else run_started_at + ), + "FAKE_RUN_STATUS": run_status, + "GH_TOKEN": "test-token", + "GITHUB_OUTPUT": str(output_path), + "GITHUB_API_URL": "https://api.github.com", + "GITHUB_REPOSITORY": "LLM360/lemonade", + "IDENTITY_MATCHES": str(identity_matches).lower(), + "MERGED": "false", + "MERGE_CANDIDATE_AVAILABLE": "true", + "MERGE_SHA": "c" * 40, + "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", + "PR_NUMBER": "7", + "RELEVANT": str(relevant).lower(), + "RUNNER_TEMP": str(temporary), + "STATE": "open", + "TRIGGERING_ACTOR": triggering_actor, + } + ) + script = extract_workflow_step_script( + "Decide whether protected validation may run" + ).split("\nvalidate:\n", 1)[0] + script = script.replace( + "trusted/.github/scripts/llamacpp_required_check_policy.py", + str(REQUIRED_CHECK_POLICY), + ) + completed = subprocess.run( + ["bash", "-c", script], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + outputs = {} + if output_path.exists(): + outputs = dict( + line.split("=", 1) + for line in output_path.read_text(encoding="utf-8").splitlines() + ) + return completed, outputs + + +def run_observed_merge_invalidation( + check_pages: list[dict[str, object]], + *, + observed_merge_sha: str, + secondary_observed_merge_sha: str = "", + authorization_epoch: int = 1, +) -> tuple[subprocess.CompletedProcess[str], list[list[str]]]: + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + pages_path = temporary / "check-pages.json" + calls_path = temporary / "gh-calls.jsonl" + pages_path.write_text(json.dumps(check_pages), encoding="utf-8") + + fake_gh = temporary / "gh" + fake_gh.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +arguments = sys.argv[1:] +with Path(os.environ["FAKE_GH_CALLS"]).open("a", encoding="utf-8") as stream: + stream.write(json.dumps(arguments) + "\\n") +method = arguments[arguments.index("--method") + 1] if "--method" in arguments else "GET" +if method == "GET": + print(Path(os.environ["FAKE_GH_CHECK_PAGES"]).read_text(encoding="utf-8")) +elif method == "POST": + print('{"id": 80}') +else: + print('{}') +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "CHECK_NAME": "llama.cpp authorization diagnostics v2", + "AUTHORIZATION_EPOCH": str(authorization_epoch), + "FAKE_GH_CALLS": str(calls_path), + "FAKE_GH_CHECK_PAGES": str(pages_path), + "GH_TOKEN": "test-token", + "GITHUB_REPOSITORY": "LLM360/lemonade", + "GITHUB_RUN_ID": "500", + "GITHUB_SERVER_URL": "https://github.com", + "OBSERVED_MERGE_SHA": observed_merge_sha, + "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", + "PR_NUMBER": "7", + "RUN_ATTEMPT": "1", + "RUN_ID": "500", + "RUN_NUMBER": "50", + "SECONDARY_OBSERVED_MERGE_SHA": secondary_observed_merge_sha, + } + ) + completed = subprocess.run( + [ + "bash", + "-c", + extract_workflow_step_script( + "Invalidate an unconfirmed observed merge commit" + ), + ], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + calls = [] + if calls_path.exists(): + calls = [ + json.loads(line) + for line in calls_path.read_text(encoding="utf-8").splitlines() + ] + return completed, calls + + +def run_preserve_step( + *, + pull_request: dict[str, object], + issue_events: list[list[dict[str, object]]], + check_pages: list[dict[str, object]], + run_attempt: int = 1, + run_number: int = 50, +) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + output_path = temporary / "github-output.txt" + inputs_path = temporary / "inputs.json" + inputs_path.write_text( + json.dumps( + { + "checks": check_pages, + "events": issue_events, + "pull_request": pull_request, + } + ), + encoding="utf-8", + ) + fake_gh = temporary / "gh" + fake_gh.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +inputs = json.loads(Path(os.environ["FAKE_GH_INPUTS"]).read_text()) +endpoint = next( + (argument for argument in sys.argv[1:] if argument.startswith("repos/")), "" +) +if "/pulls/" in endpoint: + print(json.dumps(inputs["pull_request"])) +elif "/issues/" in endpoint: + print(json.dumps(inputs["events"])) +elif "/collaborators/" in endpoint: + print('{"permission":"write","user":{"id":100}}') +elif "/check-runs" in endpoint: + print(json.dumps(inputs["checks"])) +else: + raise SystemExit(f"unexpected endpoint: {endpoint}") +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "AUTHORIZATION_ACTOR": "maintainer", + "AUTHORIZATION_ACTOR_ID": "100", + "AUTHORIZATION_BOUNDARY": "2025-12-31T23:59:59Z", + "AUTHORIZATION_CREATED_AT": "2026-01-01T00:00:01Z", + "AUTHORIZATION_EPOCH": "1", + "BASE_REF": "main", + "BASE_SHA": "a" * 40, + "CHECK_NAME": "llama.cpp authorized validation v2", + "FAKE_GH_INPUTS": str(inputs_path), + "GH_TOKEN": "test-token", + "GITHUB_OUTPUT": str(output_path), + "GITHUB_REPOSITORY": "LLM360/lemonade", + "HEAD_SHA": "b" * 40, + "MERGE_SHA": "c" * 40, + "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", + "POLICY_HELPER": str(REQUIRED_CHECK_POLICY), + "PR_NUMBER": "7", + "RUN_ATTEMPT": str(run_attempt), + "RUN_NUMBER": str(run_number), + "RUNNER_TEMP": str(temporary), + } + ) + completed = subprocess.run( + [ + "bash", + "-c", + extract_workflow_step_script( + "Preserve the exact-commit gate for unrelated metadata" + ), + ], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + outputs = {} + if output_path.exists(): + outputs = dict( + line.split("=", 1) + for line in output_path.read_text(encoding="utf-8").splitlines() + ) + return completed, outputs + + +def run_report_step( + check_page_snapshots: list[list[dict[str, object]]], + *, + pull_request_snapshots: list[dict[str, object]] | None = None, + issue_event_snapshots: list[list[list[dict[str, object]]]] | None = None, + permission: str = "write", + permission_snapshots: list[str] | None = None, + run_attempt: int = 1, + run_id: int = 500, + run_number: int = 50, + mode: str = "validate", + conclusion: str = "success", + preserve_valid: bool = False, + event_action: str = "synchronize", + event_name: str | None = None, + authorization_created_at: str = "2026-01-01T00:00:01Z", + authorization_epoch: int = 1, + step_name: str = "Report result on the exact candidate commit", +) -> tuple[subprocess.CompletedProcess[str], list[list[str]]]: + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + snapshots_path = temporary / "check-snapshots.json" + check_count_path = temporary / "check-count.txt" + event_count_path = temporary / "event-count.txt" + pr_count_path = temporary / "pr-count.txt" + permission_count_path = temporary / "permission-count.txt" + events_path = temporary / "event-snapshots.json" + pull_requests_path = temporary / "pr-snapshots.json" + calls_path = temporary / "gh-calls.jsonl" + snapshots_path.write_text(json.dumps(check_page_snapshots), encoding="utf-8") + events_path.write_text( + json.dumps(issue_event_snapshots or []), encoding="utf-8" + ) + permission_path = temporary / "permission-snapshots.json" + permission_path.write_text( + json.dumps(permission_snapshots or [permission]), encoding="utf-8" + ) + normalized_pull_requests = json.loads(json.dumps(pull_request_snapshots or [])) + for pull_request in normalized_pull_requests: + pull_request["base"].setdefault("ref", "main") + pull_requests_path.write_text( + json.dumps(normalized_pull_requests), encoding="utf-8" + ) + + fake_gh = temporary / "gh" + fake_gh.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +arguments = sys.argv[1:] +with Path(os.environ["FAKE_GH_CALLS"]).open("a", encoding="utf-8") as stream: + stream.write(json.dumps(arguments) + "\\n") +method = arguments[arguments.index("--method") + 1] if "--method" in arguments else "GET" +endpoint = next((argument for argument in arguments if argument.startswith("repos/")), "") +if method == "GET" and "/check-runs" in endpoint and "/commits/" in endpoint: + count_path = Path(os.environ["FAKE_GH_CHECK_COUNT"]) + count = int(count_path.read_text()) if count_path.exists() else 0 + count_path.write_text(str(count + 1)) + snapshots = json.loads(Path(os.environ["FAKE_GH_SNAPSHOTS"]).read_text()) + print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) +elif method == "GET" and "/pulls/" in endpoint: + count_path = Path(os.environ["FAKE_GH_PR_COUNT"]) + count = int(count_path.read_text()) if count_path.exists() else 0 + count_path.write_text(str(count + 1)) + snapshots = json.loads(Path(os.environ["FAKE_GH_PULL_REQUESTS"]).read_text()) + print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) +elif method == "GET" and "/issues/" in endpoint and endpoint.endswith("/events?per_page=100"): + count_path = Path(os.environ["FAKE_GH_EVENT_COUNT"]) + count = int(count_path.read_text()) if count_path.exists() else 0 + count_path.write_text(str(count + 1)) + snapshots = json.loads(Path(os.environ["FAKE_GH_EVENTS"]).read_text()) + print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) +elif method == "GET" and "/collaborators/" in endpoint: + count_path = Path(os.environ["FAKE_GH_PERMISSION_COUNT"]) + count = int(count_path.read_text()) if count_path.exists() else 0 + count_path.write_text(str(count + 1)) + permissions = json.loads(Path(os.environ["FAKE_GH_PERMISSIONS"]).read_text()) + permission = permissions[min(count, len(permissions) - 1)] + if "--jq" in arguments: + print(permission) + else: + print(json.dumps({ + "permission": permission, + "user": {"id": 100}, + })) +elif method == "POST": + print('{"id": 80}') +else: + print('{}') +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "BASE_SHA": "a" * 40, + "BASE_REF": "main", + "CHECK_NAME": "llama.cpp authorized validation v2", + "CHECK_RUN_ID": "", + "CONCLUSION": conclusion, + "EVENT_ACTION": event_action, + "EVENT_NAME": ( + event_name + if event_name is not None + else ( + "pull_request_target" + if normalized_pull_requests + else "merge_group" + ) + ), + "FAKE_GH_CALLS": str(calls_path), + "FAKE_GH_CHECK_COUNT": str(check_count_path), + "FAKE_GH_EVENT_COUNT": str(event_count_path), + "FAKE_GH_EVENTS": str(events_path), + "FAKE_GH_PERMISSION_COUNT": str(permission_count_path), + "FAKE_GH_PERMISSIONS": str(permission_path), + "FAKE_GH_PR_COUNT": str(pr_count_path), + "FAKE_GH_PULL_REQUESTS": str(pull_requests_path), + "FAKE_GH_SNAPSHOTS": str(snapshots_path), + "GH_TOKEN": "test-token", + "GITHUB_REPOSITORY": "LLM360/lemonade", + "GITHUB_RUN_ID": str(run_id), + "GITHUB_SERVER_URL": "https://github.com", + "HEAD_SHA": "b" * 40, + "MERGE_SHA": "c" * 40, + "METADATA_CHECK_NAME": "llama.cpp authorization diagnostics v2", + "MODE": mode, + "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", + "POLICY_HELPER": str(REQUIRED_CHECK_POLICY), + "PR_NUMBER": "7", + "PRESERVE_VALID": str(preserve_valid).lower(), + "RUN_ATTEMPT": str(run_attempt), + "RUN_ID": str(run_id), + "RUN_NUMBER": str(run_number), + "SUMMARY": "Authorized validation passed.", + "TARGET_KEY": "pr-7", + "AUTHORIZATION_ACTOR": "maintainer", + "AUTHORIZATION_ACTOR_ID": "100", + "AUTHORIZATION_BOUNDARY": "2025-12-31T23:59:59Z", + "AUTHORIZATION_CREATED_AT": authorization_created_at, + "AUTHORIZATION_EPOCH": str(authorization_epoch), + } + ) + completed = subprocess.run( + [ + "bash", + "-c", + extract_workflow_step_script(step_name), + ], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + calls = [] + if calls_path.exists(): + calls = [ + json.loads(line) + for line in calls_path.read_text(encoding="utf-8").splitlines() + ] + return completed, calls + + +class LlamaCppValidationPlanTests(unittest.TestCase): + def test_authorization_provenance_is_a_fully_validated_epoch(self) -> None: + policy = load_required_check_policy() + valid = policy.classify_authorization_event_pages( + [ + [ + issue_event(1, "reopened"), + issue_event(2, "labeled", label="ci:upgrades"), + ] + ] + ) + self.assertTrue(valid.epoch_valid) + self.assertEqual(valid.authorization_event_id, 2) + self.assertEqual(valid.authorization_actor, "maintainer") + self.assertEqual(valid.authorization_actor_id, 100) + self.assertEqual(valid.authorization_created_at, "2026-01-01T00:00:02Z") + self.assertTrue( + policy.classify_authorization_event_pages( + [[issue_event(2, "labeled", label="ci:upgrades")]], + authorization_after="2026-01-01T00:00:01Z", + ).epoch_valid + ) + for boundary in ( + "2026-01-01T00:00:02Z", + "2026-01-01T00:00:03Z", + ): + with self.subTest(boundary=boundary): + stale_for_run = policy.classify_authorization_event_pages( + [[issue_event(2, "labeled", label="ci:upgrades")]], + authorization_after=boundary, + ) + self.assertFalse(stale_for_run.epoch_valid) + for invalid_boundary in ("not-a-time", "2026-01-01T00:00:00"): + with self.subTest(invalid_boundary=invalid_boundary): + with self.assertRaises(ValueError): + policy.classify_authorization_event_pages( + [[issue_event(2, "labeled", label="ci:upgrades")]], + authorization_after=invalid_boundary, + ) + + for invalidator in ("base_ref_changed", "reopened", "head_ref_force_pushed"): + with self.subTest(invalidator=invalidator): + invalid = policy.classify_authorization_event_pages( + [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, invalidator), + ] + ] + ) + self.assertFalse(invalid.epoch_valid) + + untrusted_readd = policy.classify_authorization_event_pages( + [ + [ + issue_event(1, "labeled", actor="trusted", label="ci:upgrades"), + issue_event(2, "unlabeled", label="ci:upgrades"), + issue_event(3, "labeled", actor="triage", label="ci:upgrades"), + ] + ] + ) + self.assertTrue(untrusted_readd.epoch_valid) + self.assertEqual(untrusted_readd.authorization_actor, "triage") + unrelated_bot_label = issue_event( + 1, + "labeled", + actor="github-actions[bot]", + actor_id=41898282, + label="automation:classified", + ) + unrelated_bot_label["actor"]["type"] = "Bot" + unrelated_bot_label["performed_via_github_app"] = {"id": 15368} + bot_label_before_authorization = policy.classify_authorization_event_pages( + [ + [ + unrelated_bot_label, + issue_event(2, "labeled", label="ci:upgrades"), + ] + ] + ) + self.assertTrue(bot_label_before_authorization.epoch_valid) + self.assertEqual( + bot_label_before_authorization.authorization_actor, + "maintainer", + ) + stale_epoch = policy.decide_pull_request_policy( + action="labeled", + event_label="ci:upgrades", + base_ref_changed=False, + state="open", + merged=False, + merge_candidate_available=True, + relevant=True, + identity_matches=True, + authorization_label_present=True, + authorization_epoch_valid=False, + actor_permission="write", + has_existing_check=True, + has_newer_check=False, + ) + self.assertEqual(stale_epoch.mode, "awaiting_rerun") + self.assertEqual(stale_epoch.conclusion, "failure") + delayed_delivery = policy.decide_pull_request_policy( + action="labeled", + event_label="ci:upgrades", + base_ref_changed=False, + state="open", + merged=False, + merge_candidate_available=True, + relevant=True, + identity_matches=True, + authorization_label_present=True, + authorization_epoch_valid=True, + authorization_rerun_matches=False, + actor_permission="write", + has_existing_check=False, + has_newer_check=False, + ) + self.assertEqual(delayed_delivery.mode, "awaiting_rerun") + with self.assertRaises(ValueError): + policy.classify_authorization_event_pages( + [ + [ + issue_event(1, "reopened"), + issue_event(1, "labeled", label="ci:upgrades"), + ] + ] + ) + malformed_actor = issue_event(1, "labeled", label="ci:upgrades") + malformed_actor["actor"] = { + "id": 100, + "login": "maintainer", + "node_id": "USER_100", + "type": "Bot", + } + duplicate_node = issue_event(2, "reopened") + duplicate_node["node_id"] = "EVENT_1" + for pages in ( + [[], [issue_event(1, "reopened")]], + [[issue_event(2, "reopened"), issue_event(1, "closed")]], + [[issue_event(1, "reopened"), duplicate_node]], + [[malformed_actor]], + ): + with self.subTest(invalid_pages=pages): + with self.assertRaises(ValueError): + policy.classify_authorization_event_pages(pages) + + def test_live_authorization_epoch_binding_is_enforced(self) -> None: + events = [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, "unlabeled", label="ci:upgrades"), + issue_event(3, "labeled", label="ci:upgrades"), + ] + ] + current_epoch_check = { + "app": {"slug": "github-actions"}, + "conclusion": "success", + "external_id": "lemonade-llamacpp-required-pr-7-3-40-1-400", + "id": 40, + "name": "llama.cpp authorized validation v2", + "status": "completed", + } + old_epoch_check = { + **current_epoch_check, + "external_id": "lemonade-llamacpp-required-pr-7-1-40-1-400", + } + + authorized, authorized_outputs = run_policy_decision_step( + action="synchronize", + event_label="", + event_actor="contributor", + event_actor_id=200, + event_updated_at="2026-01-01T00:00:00Z", + issue_events=events, + check_pages=[{"check_runs": []}], + run_attempt=2, + ) + self.assertEqual(authorized.returncode, 0, authorized.stderr) + self.assertEqual(authorized_outputs["mode"], "validate") + self.assertEqual(authorized_outputs["authorization_event_id"], "3") + + for run_path in ( + ".github/workflows/validate_llamacpp_pr.yml", + ".github/workflows/validate_llamacpp_pr.yml@refs/heads/main", + ): + with self.subTest(allowed_workflow_path=run_path): + allowed, allowed_outputs = run_policy_decision_step( + action="synchronize", + event_label="", + event_actor="contributor", + event_actor_id=200, + event_updated_at="2026-01-01T00:00:00Z", + issue_events=events, + check_pages=[{"check_runs": []}], + run_attempt=2, + run_path=run_path, + ) + self.assertEqual(allowed.returncode, 0, allowed.stderr) + self.assertEqual(allowed_outputs["mode"], "validate") + + fork, fork_outputs = run_policy_decision_step( + action="synchronize", + event_label="", + event_actor="contributor", + event_actor_id=200, + event_updated_at="2026-01-01T00:00:00Z", + issue_events=events, + check_pages=[{"check_runs": []}], + run_attempt=2, + run_response_head_branch="fork/k2-horizon", + run_response_head_repository="contributor/lemonade", + run_response_pr_head_ref="fork/k2-horizon", + run_response_pr_head_repository="contributor/lemonade", + run_response_pull_request_count=0, + ) + self.assertEqual(fork.returncode, 0, fork.stderr) + self.assertEqual(fork_outputs["mode"], "validate") + + multiple, multiple_outputs = run_policy_decision_step( + action="synchronize", + event_label="", + event_actor="contributor", + event_actor_id=200, + event_updated_at="2026-01-01T00:00:00Z", + issue_events=events, + check_pages=[{"check_runs": []}], + run_attempt=2, + run_response_pr_base_ref="release", + run_response_pr_base_sha="d" * 40, + run_response_pr_head_sha="e" * 40, + run_response_pr_number=99, + run_response_pull_request_count=3, + ) + self.assertEqual(multiple.returncode, 0, multiple.stderr) + self.assertEqual(multiple_outputs["mode"], "validate") + + for name, overrides, expected_mode in ( + ("first-attempt", {"run_attempt": 1}, "authorization_required"), + ( + "label-not-after-run", + {"run_created_at": "2026-01-01T00:00:03Z"}, + "authorization_required", + ), + ( + "label-after-queue-but-before-start", + { + "run_created_at": "2025-12-31T23:59:59Z", + "run_started_at": "2026-01-01T00:00:04Z", + }, + "authorization_required", + ), + ( + "missing-run-start", + {"run_started_at": ""}, + "authorization_required", + ), + ( + "wrong-permission-identity", + {"permission_actor_id": 999}, + "authorization_required", + ), + ( + "untrusted-rerun-actor", + {"triggering_actor": "triage"}, + "authorization_required", + ), + ( + "first-attempt-succeeded", + {"run_conclusion": "success"}, + "authorization_required", + ), + ( + "wrong-run-event", + {"run_event": "pull_request"}, + "authorization_required", + ), + ( + "wrong-workflow", + {"run_path": ".github/workflows/other.yml"}, + "authorization_required", + ), + ( + "wrong-workflow-ref", + {"run_path": (".github/workflows/validate_llamacpp_pr.yml@release")}, + "authorization_required", + ), + ( + "malformed-workflow-ref", + { + "run_path": ( + ".github/workflows/validate_llamacpp_pr.yml@main@release" + ) + }, + "authorization_required", + ), + ( + "wrong-repository", + {"run_repository": "attacker/lemonade"}, + "authorization_required", + ), + ( + "wrong-top-level-head-sha", + {"run_response_head_sha": "d" * 40}, + "authorization_required", + ), + ( + "malformed-pull-request-associations", + {"run_response_pull_requests_json": '"not-an-array"'}, + "authorization_required", + ), + ( + "live-identity-drift", + {"identity_matches": False}, + "stale_required", + ), + ( + "wrong-run-id", + {"run_response_id": 501}, + "authorization_required", + ), + ( + "wrong-run-number", + {"run_response_number": 51}, + "authorization_required", + ), + ( + "wrong-attempt-record", + {"run_response_attempt": 2}, + "authorization_required", + ), + ( + "incomplete-first-attempt", + {"run_status": "in_progress"}, + "authorization_required", + ), + ( + "label-event-rerun", + {"action": "labeled", "event_label": "ci:upgrades"}, + "awaiting_rerun", + ), + ): + arguments = { + "action": "synchronize", + "event_label": "", + "event_actor": "contributor", + "event_actor_id": 200, + "event_updated_at": "2026-01-01T00:00:00Z", + "issue_events": events, + "check_pages": [{"check_runs": []}], + "run_attempt": 2, + } + arguments.update(overrides) + with self.subTest(name=name): + denied, denied_outputs = run_policy_decision_step(**arguments) + self.assertEqual(denied.returncode, 0, denied.stderr) + self.assertEqual(denied_outputs["mode"], expected_mode) + self.assertEqual(denied_outputs["conclusion"], "failure") + + for name, checks, expected_mode in ( + ("same-epoch", [current_epoch_check], "preserve"), + ("old-epoch", [old_epoch_check], "authorization_required"), + ( + "in-progress", + [{**current_epoch_check, "conclusion": None, "status": "in_progress"}], + "authorization_required", + ), + ( + "failed", + [{**current_epoch_check, "conclusion": "failure"}], + "authorization_required", + ), + ( + "later-failed-decision", + [ + current_epoch_check, + { + **current_epoch_check, + "conclusion": "failure", + "external_id": ("lemonade-llamacpp-required-pr-7-3-41-1-401"), + "id": 41, + }, + ], + "authorization_required", + ), + ("missing", [], "authorization_required"), + ): + with self.subTest(name=name): + metadata, metadata_outputs = run_policy_decision_step( + action="edited", + event_label="", + event_actor="maintainer", + event_actor_id=100, + event_updated_at="2026-01-01T00:00:04Z", + issue_events=events, + check_pages=[{"check_runs": checks}], + ) + self.assertEqual(metadata.returncode, 0, metadata.stderr) + self.assertEqual(metadata_outputs["mode"], expected_mode) + if expected_mode != "preserve": + self.assertEqual(metadata_outputs["conclusion"], "failure") + + def test_dependabot_policy_does_not_require_the_checks_api(self) -> None: + authorization = [[issue_event(3, "labeled", label="ci:upgrades")]] + cases = ( + ( + "neutral", + { + "action": "synchronize", + "authorization_label_present": False, + "issue_events": [[]], + "relevant": False, + }, + "neutral", + "success", + ), + ( + "unauthorized", + { + "action": "synchronize", + "authorization_label_present": False, + "issue_events": [[]], + }, + "authorization_required", + "failure", + ), + ( + "awaiting-rerun", + { + "action": "labeled", + "event_label": "ci:upgrades", + "issue_events": authorization, + }, + "awaiting_rerun", + "failure", + ), + ( + "metadata-preserve", + {"action": "edited", "issue_events": authorization}, + "preserve", + "", + ), + ( + "authorized-rerun", + { + "action": "synchronize", + "issue_events": authorization, + "run_attempt": 2, + }, + "validate", + "", + ), + ) + for name, overrides, expected_mode, expected_conclusion in cases: + arguments = { + "action": "synchronize", + "event_label": "", + "event_actor": "dependabot[bot]", + "event_actor_id": 49699333, + "event_updated_at": "2026-01-01T00:00:00Z", + "issue_events": authorization, + "check_pages": [{"check_runs": []}], + "dependabot_pr": True, + "fail_check_api": True, + } + arguments.update(overrides) + with self.subTest(name=name): + completed, outputs = run_policy_decision_step(**arguments) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["mode"], expected_mode) + self.assertEqual(outputs["conclusion"], expected_conclusion) + + def test_dependabot_pr_uses_only_the_required_workflow_conclusion(self) -> None: + expected_identity = ( + "${{ github.event_name == 'pull_request_target' && " + "github.event.pull_request.user.login == 'dependabot[bot]' && " + "github.event.pull_request.user.id == 49699333 && " + "github.event.pull_request.user.type == 'Bot' }}" + ) + self.assertEqual( + load_workflow(PR_WORKFLOW)["env"]["DEPENDABOT_PR"], expected_identity + ) + self.assertEqual( + load_workflow(VALIDATION_WORKFLOW)["env"]["DEPENDABOT_PR"], + expected_identity, + ) + + decision = workflow_step( + PR_WORKFLOW, "policy", "Decide whether protected validation may run" + ) + self.assertEqual(decision["env"]["DEPENDABOT_PR"], "${{ env.DEPENDABOT_PR }}") + self.assertIn('if [ "$DEPENDABOT_PR" != "true" ]', decision["run"]) + self.assertIn('--required-workflow-only "$DEPENDABOT_PR"', decision["run"]) + + for step_name in ( + "Preserve the exact-commit gate for unrelated metadata", + "Invalidate an unconfirmed observed merge commit", + "Report result on the exact candidate commit", + ): + with self.subTest(step=step_name): + condition = workflow_step(PR_WORKFLOW, "reconcile", step_name)["if"] + self.assertIn("env.DEPENDABOT_PR != 'true'", condition) + + start = workflow_step( + VALIDATION_WORKFLOW, "validate-invocation", "Start authorized merge check" + ) + self.assertIn("env.DEPENDABOT_PR != 'true'", start["if"]) + enforce = workflow_step(PR_WORKFLOW, "reconcile", "Enforce policy result")["if"] + self.assertIn("env.DEPENDABOT_PR != 'true'", enforce) + + guide = TESTING_GUIDE.read_text(encoding="utf-8") + self.assertIn("Dependabot", guide) + self.assertIn("read-only `GITHUB_TOKEN`", guide) + self.assertIn("required-workflow conclusion", guide) + self.assertIn("omit custom diagnostic Checks API runs", guide) + + def test_required_workflow_only_metadata_preserves_without_a_custom_check( + self, + ) -> None: + policy = load_required_check_policy() + common = { + "action": "edited", + "event_label": "", + "base_ref_changed": False, + "state": "open", + "merged": False, + "merge_candidate_available": True, + "relevant": True, + "identity_matches": True, + "authorization_label_present": True, + "actor_permission": "write", + "has_existing_check": False, + "has_newer_check": False, + "authorization_epoch_valid": True, + "authorization_rerun_matches": False, + "required_workflow_only": True, + } + preserved = policy.decide_pull_request_policy(**common) + self.assertEqual(preserved.mode, "preserve") + self.assertFalse(preserved.run_validation) + self.assertEqual(preserved.conclusion, "") + + for name, overrides, expected_mode in ( + ( + "missing-authorization", + {"authorization_label_present": False}, + "authorization_required", + ), + ( + "untrusted-authorization", + {"actor_permission": "triage"}, + "authorization_required", + ), + ( + "stale-identity", + {"identity_matches": False}, + "stale", + ), + ( + "base-ref-change", + {"base_ref_changed": True}, + "authorization_required", + ), + ( + "protected-synchronize", + {"action": "synchronize"}, + "authorization_required", + ), + ): + arguments = dict(common) + arguments.update(overrides) + with self.subTest(name=name): + self.assertEqual( + policy.decide_pull_request_policy(**arguments).mode, + expected_mode, + ) + + def test_dependabot_authorized_result_is_revalidated_without_writes(self) -> None: + pull_request = { + "base": {"ref": "main", "sha": "a" * 40}, + "head": {"sha": "b" * 40}, + "labels": [{"name": "ci:upgrades"}], + "merge_commit_sha": "c" * 40, + "mergeable": True, + "merged": False, + "number": 7, + "state": "open", + } + events = [[issue_event(1, "labeled", label="ci:upgrades")]] + + valid, valid_calls = run_report_step( + [[]], + pull_request_snapshots=[pull_request], + issue_event_snapshots=[events], + step_name="Revalidate an authorized Dependabot result", + ) + self.assertEqual(valid.returncode, 0, valid.stderr) + self.assertFalse( + any( + "--method" in call + and call[call.index("--method") + 1] in {"POST", "PATCH"} + for call in valid_calls + ) + ) + + invalid_cases = ( + ( + "unlabeled-live-pr", + {**pull_request, "labels": []}, + events, + "write", + ), + ( + "invalidated-epoch", + pull_request, + [ + [ + *events[0], + issue_event(2, "unlabeled", label="ci:upgrades"), + ] + ], + "write", + ), + ("permission-loss", pull_request, events, "read"), + ( + "identity-drift", + {**pull_request, "head": {"sha": "d" * 40}}, + events, + "write", + ), + ) + for name, live_pull_request, live_events, permission in invalid_cases: + with self.subTest(name=name): + invalid, calls = run_report_step( + [[]], + pull_request_snapshots=[live_pull_request], + issue_event_snapshots=[live_events], + permission_snapshots=[permission], + step_name="Revalidate an authorized Dependabot result", + ) + self.assertNotEqual(invalid.returncode, 0) + self.assertFalse( + any( + "--method" in call + and call[call.index("--method") + 1] in {"POST", "PATCH"} + for call in calls + ) + ) + + def test_dependabot_unauthorized_result_still_fails_reconciliation(self) -> None: + assessed, outputs = run_output_step( + "Assess policy and validation", + { + "EVENT_ACTION": "synchronize", + "EVENT_BASE_REF": "main", + "EVENT_BASE_SHA": "a" * 40, + "EVENT_HEAD_SHA": "b" * 40, + "EVENT_MERGE_SHA": "c" * 40, + "GH_TOKEN": "test-token", + "MODE": "authorization_required", + "POLICY_CONCLUSION": "failure", + "POLICY_RESULT": "success", + "POLICY_SUMMARY": "Maintainer authorization is required.", + "PR_NUMBER": "7", + "VALIDATION_RESULT": "skipped", + }, + ) + self.assertEqual(assessed.returncode, 0, assessed.stderr) + self.assertEqual(outputs["mode"], "authorization_required") + self.assertEqual(outputs["passed"], "false") + + dependabot_pr = True + preserve_failed = ( + outputs["mode"] == "preserve" + and not dependabot_pr + and outputs.get("preserve_valid") != "true" + ) + self.assertTrue(outputs["passed"] != "true" or preserve_failed) + enforce_script = extract_workflow_step_script("Enforce policy result").replace( + "${{ steps.assess-policy.outputs.summary || " + "'Validation assessment failed.' }}", + "Maintainer authorization is required.", + ) + enforced = subprocess.run( + ["bash", "-c", enforce_script], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(enforced.returncode, 0) + + def test_required_workflow_rerun_consumes_only_a_fresh_label(self) -> None: + events = [[issue_event(1, "labeled", label="ci:upgrades")]] + + authorized, outputs = run_policy_decision_step( + action="synchronize", + event_label="", + event_actor="contributor", + event_actor_id=200, + event_updated_at="2026-01-01T00:00:00Z", + issue_events=events, + check_pages=[{"check_runs": []}], + run_attempt=2, + run_created_at="2025-12-31T23:59:59Z", + ) + + self.assertEqual(authorized.returncode, 0, authorized.stderr) + self.assertEqual(outputs["mode"], "validate") + self.assertEqual(outputs["run_validation"], "true") + self.assertEqual(outputs["authorization_event_id"], "1") + + later_attempt, later_outputs = run_policy_decision_step( + action="synchronize", + event_label="", + event_actor="contributor", + event_actor_id=200, + event_updated_at="2026-01-01T00:00:00Z", + issue_events=events, + check_pages=[{"check_runs": []}], + run_attempt=10, + run_created_at="2025-12-31T23:59:59Z", + ) + self.assertEqual(later_attempt.returncode, 0, later_attempt.stderr) + self.assertEqual(later_outputs["mode"], "validate") + + def test_diagnostic_check_cannot_make_a_required_workflow_pass(self) -> None: + policy = load_required_check_policy() + + decision = policy.decide_pull_request_policy( + action="synchronize", + event_label="", + base_ref_changed=False, + state="open", + merged=False, + merge_candidate_available=True, + relevant=True, + identity_matches=True, + authorization_label_present=True, + actor_permission="write", + has_existing_check=True, + has_newer_check=True, + authorization_epoch_valid=True, + authorization_rerun_matches=True, + ) + + self.assertEqual(decision.mode, "superseded") + self.assertFalse(decision.run_validation) + self.assertEqual(decision.conclusion, "failure") + + def test_required_workflow_rerun_rejects_identity_and_lifecycle_races( + self, + ) -> None: + newer_check = { + "app": {"slug": "github-actions"}, + "external_id": "lemonade-llamacpp-required-pr-7-2-51-1-501", + "id": 51, + "name": "llama.cpp authorized validation v2", + } + common = { + "action": "synchronize", + "event_label": "", + "event_actor": "contributor", + "event_actor_id": 200, + "event_updated_at": "2026-01-01T00:00:00Z", + "issue_events": [[issue_event(2, "labeled", label="ci:upgrades")]], + "check_pages": [{"check_runs": []}], + "run_attempt": 2, + "run_created_at": "2026-01-01T00:00:01Z", + } + cases = ( + ( + "label-at-run-boundary", + { + "issue_events": [[issue_event(1, "labeled", label="ci:upgrades")]], + }, + "authorization_required", + ), + ( + "base-retarget-after-label", + { + "issue_events": [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, "base_ref_changed"), + ] + ], + "run_created_at": "2026-01-01T00:00:00Z", + }, + "authorization_required", + ), + ( + "reopen-after-label", + { + "issue_events": [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, "reopened"), + ] + ], + "run_created_at": "2026-01-01T00:00:00Z", + }, + "authorization_required", + ), + ( + "remove-after-label", + { + "issue_events": [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, "unlabeled", label="ci:upgrades"), + ] + ], + "run_created_at": "2026-01-01T00:00:00Z", + }, + "authorization_required", + ), + ( + "push-after-original-run", + {"identity_matches": False}, + "stale_required", + ), + ( + "newer-default-event-run", + {"check_pages": [{"check_runs": [newer_check]}]}, + "superseded", + ), + ) + for name, overrides, expected_mode in cases: + arguments = dict(common) + arguments.update(overrides) + with self.subTest(name=name): + completed, outputs = run_policy_decision_step(**arguments) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["mode"], expected_mode) + self.assertEqual(outputs["run_validation"], "false") + self.assertEqual(outputs["conclusion"], "failure") + + for action in ("opened", "reopened", "synchronize"): + with self.subTest(valid_action=action): + completed, outputs = run_policy_decision_step( + **{ + **common, + "action": action, + "issue_events": [ + [ + issue_event(1, "reopened"), + issue_event(2, "labeled", label="ci:upgrades"), + ] + ], + } + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["mode"], "validate") + self.assertEqual(outputs["run_validation"], "true") + + def test_reauthorization_metadata_does_not_deadlock_the_required_rerun( + self, + ) -> None: + events = [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, "unlabeled", label="ci:upgrades"), + issue_event(3, "labeled", label="ci:upgrades"), + ] + ] + common = { + "action": "synchronize", + "event_label": "", + "event_actor": "contributor", + "event_actor_id": 200, + "event_updated_at": "2026-01-01T00:00:00Z", + "issue_events": events, + "run_attempt": 3, + "run_created_at": "2026-01-01T00:00:00Z", + } + prior_required = { + "app": {"slug": "github-actions"}, + "external_id": "lemonade-llamacpp-required-pr-7-1-50-2-500", + "id": 50, + "name": "llama.cpp authorized validation v2", + } + metadata_revocation = { + "app": {"slug": "github-actions"}, + "external_id": "lemonade-llamacpp-metadata-pr-7-2-51-1-501", + "id": 51, + "name": "llama.cpp authorization diagnostics v2", + } + legacy_unscoped_revocation = { + **metadata_revocation, + "external_id": "lemonade-llamacpp-pr-7-2-51-1-501", + "name": "llama.cpp authorized validation v2", + } + + initial_recovery, outputs = run_policy_decision_step( + check_pages=[{"check_runs": [metadata_revocation]}], + **{**common, "run_attempt": 2}, + ) + self.assertEqual(initial_recovery.returncode, 0, initial_recovery.stderr) + self.assertEqual(outputs["mode"], "validate") + self.assertEqual(outputs["run_validation"], "true") + + for name, checks in ( + ("scoped-metadata", [prior_required, metadata_revocation]), + ("legacy-unscoped-metadata", [legacy_unscoped_revocation]), + ): + with self.subTest(name=name): + completed, outputs = run_policy_decision_step( + check_pages=[{"check_runs": checks}], + **common, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["mode"], "validate") + self.assertEqual(outputs["run_validation"], "true") + + newer_required = { + **prior_required, + "external_id": "lemonade-llamacpp-required-pr-7-3-51-1-501", + "id": 52, + } + superseded, outputs = run_policy_decision_step( + check_pages=[{"check_runs": [newer_required]}], + **common, + ) + self.assertEqual(superseded.returncode, 0, superseded.stderr) + self.assertEqual(outputs["mode"], "superseded") + self.assertEqual(outputs["conclusion"], "failure") + + def test_recovered_required_result_is_distinct_from_metadata_diagnostics( + self, + ) -> None: + required_name = "llama.cpp authorized validation v2" + metadata_name = "llama.cpp authorization diagnostics v2" + metadata_report, metadata_calls = run_report_step( + [[]], + authorization_epoch=2, + conclusion="failure", + event_action="unlabeled", + event_name="pull_request_target", + mode="authorization_required", + run_id=501, + run_number=51, + ) + self.assertEqual(metadata_report.returncode, 0, metadata_report.stderr) + metadata_writes = [ + call + for call in metadata_calls + if "--method" in call + and call[call.index("--method") + 1] in {"POST", "PATCH"} + ] + self.assertEqual(len(metadata_writes), 1) + self.assertIn(f"name={metadata_name}", metadata_writes[0]) + self.assertNotIn(f"name={required_name}", metadata_writes[0]) + + metadata_check = { + "app": {"slug": "github-actions"}, + "external_id": "lemonade-llamacpp-metadata-pr-7-2-51-1-501", + "id": 90, + "name": metadata_name, + } + recovered_check = { + "app": {"slug": "github-actions"}, + "external_id": "lemonade-llamacpp-required-pr-7-3-50-3-500", + "id": 80, + "name": required_name, + } + live_pull_request = { + "base": {"ref": "main", "sha": "a" * 40}, + "head": {"sha": "b" * 40}, + "labels": [{"name": "ci:upgrades"}], + "merge_commit_sha": "c" * 40, + "mergeable": True, + "merged": False, + "number": 7, + "state": "open", + } + authorization_events = [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, "unlabeled", label="ci:upgrades"), + issue_event(3, "labeled", label="ci:upgrades"), + ] + ] + recovered, recovered_calls = run_report_step( + [ + [{"check_runs": [metadata_check]}], + [{"check_runs": [metadata_check, recovered_check]}], + ], + authorization_created_at="2026-01-01T00:00:03Z", + authorization_epoch=3, + issue_event_snapshots=[authorization_events, authorization_events], + pull_request_snapshots=[live_pull_request, live_pull_request], + run_attempt=3, + ) + self.assertEqual(recovered.returncode, 0, recovered.stderr) + recovered_writes = [ + call + for call in recovered_calls + if "--method" in call + and call[call.index("--method") + 1] in {"POST", "PATCH"} + ] + self.assertTrue( + any(f"name={required_name}" in call for call in recovered_writes) + ) + self.assertFalse( + any(f"name={metadata_name}" in call for call in recovered_writes) + ) + self.assertFalse( + any( + "repos/LLM360/lemonade/check-runs/90" in call + for call in recovered_writes + ) + ) + + def test_policy_failure_without_outputs_still_reports_and_enforces_failure( + self, + ) -> None: + completed, outputs = run_output_step( + "Assess policy and validation", + { + "EVENT_ACTION": "labeled", + "MODE": "", + "POLICY_CONCLUSION": "", + "POLICY_RESULT": "failure", + "POLICY_SUMMARY": "", + "VALIDATION_RESULT": "skipped", + }, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["mode"], "policy_error") + self.assertEqual(outputs["conclusion"], "failure") + self.assertEqual(outputs["passed"], "false") + + workflow = PR_WORKFLOW.read_text(encoding="utf-8") + assess = workflow.split(" - name: Assess policy and validation\n", 1)[ + 1 + ].split(" - name:", 1)[0] + report = workflow.split( + " - name: Report result on the exact candidate commit\n", 1 + )[1].split(" - name: Enforce policy result\n", 1)[0] + enforce = workflow.split(" - name: Enforce policy result\n", 1)[1] + self.assertIn(" if: always()\n", assess) + self.assertIn(" always() &&\n", report) + self.assertIn(" always() &&\n", enforce) + + def test_failed_closed_event_uses_live_lifecycle_before_skipping(self) -> None: + reopened = { + "base": {"ref": "main", "sha": "a" * 40}, + "head": {"sha": "b" * 40}, + "labels": [{"name": "ci:upgrades"}], + "merge_commit_sha": "c" * 40, + "mergeable": True, + "merged": False, + "number": 7, + "state": "open", + } + completed, outputs = run_failed_closed_assessment(reopened) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["mode"], "policy_error") + self.assertEqual(outputs["conclusion"], "failure") + self.assertEqual(outputs["passed"], "false") + self.assertEqual(outputs["fallback_merge_sha"], "c" * 40) + + def test_unconfirmed_observed_merge_gate_is_failed_with_run_ordering( + self, + ) -> None: + workflow = PR_WORKFLOW.read_text(encoding="utf-8") + step_name = "Invalidate an unconfirmed observed merge commit" + self.assertIn(f" - name: {step_name}\n", workflow) + observed_merge_sha = "c" * 40 + prefix = "lemonade-llamacpp-metadata-pr-7-" + old_check = { + "id": 11, + "name": "llama.cpp authorization diagnostics v2", + "app": {"slug": "github-actions"}, + "external_id": f"{prefix}1-10-1-9000", + } + foreign_check = { + "id": 12, + "name": "llama.cpp authorization diagnostics v2", + "app": {"slug": "foreign-app"}, + "external_id": f"{prefix}1-9-1-8000", + } + + completed, calls = run_observed_merge_invalidation( + [{"check_runs": [old_check, foreign_check]}], + observed_merge_sha=observed_merge_sha, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + mutations = [ + call for call in calls if call[call.index("--method") + 1] != "GET" + ] + posts = [ + call for call in mutations if call[call.index("--method") + 1] == "POST" + ] + self.assertEqual(len(posts), 1) + self.assertIn(f"head_sha={observed_merge_sha}", posts[0]) + self.assertIn("conclusion=failure", posts[0]) + patched_endpoints = { + next(argument for argument in call if argument.startswith("repos/")) + for call in mutations + if call[call.index("--method") + 1] == "PATCH" + } + self.assertIn("repos/LLM360/lemonade/check-runs/11", patched_endpoints) + self.assertNotIn("repos/LLM360/lemonade/check-runs/12", patched_endpoints) + + newer_check = dict(old_check, id=13, external_id=f"{prefix}1-60-1-10") + superseded, superseded_calls = run_observed_merge_invalidation( + [{"check_runs": [newer_check]}], + observed_merge_sha=observed_merge_sha, + ) + self.assertEqual(superseded.returncode, 0, superseded.stderr) + self.assertTrue( + all(call[call.index("--method") + 1] == "GET" for call in superseded_calls) + ) + + second_observed_merge_sha = "d" * 40 + both, both_calls = run_observed_merge_invalidation( + [{"check_runs": []}], + observed_merge_sha=observed_merge_sha, + secondary_observed_merge_sha=second_observed_merge_sha, + authorization_epoch=0, + ) + self.assertEqual(both.returncode, 0, both.stderr) + observed_posts = [ + call + for call in both_calls + if "--method" in call and call[call.index("--method") + 1] == "POST" + ] + self.assertEqual(len(observed_posts), 2) + self.assertEqual( + { + next(value for value in call if value.startswith("head_sha=")) + for call in observed_posts + }, + {f"head_sha={observed_merge_sha}", f"head_sha={second_observed_merge_sha}"}, + ) + self.assertTrue( + all( + "external_id=lemonade-llamacpp-metadata-pr-7-0-50-1-500" in call + for call in observed_posts + ) + ) + + def test_post_write_recheck_closes_concurrent_check_ordering_races( + self, + ) -> None: + prefix = "lemonade-llamacpp-required-pr-7-" + current = { + "id": 80, + "name": "llama.cpp authorized validation v2", + "app": {"slug": "github-actions"}, + "external_id": f"{prefix}1-50-1-500", + } + newer_revocation = { + "id": 90, + "name": "llama.cpp authorized validation v2", + "app": {"slug": "github-actions"}, + "external_id": f"{prefix}0-51-1-10", + } + + completed, calls = run_report_step( + [[], [{"check_runs": [current, newer_revocation]}]] + ) + self.assertNotEqual(completed.returncode, 0) + mutations = [ + call for call in calls if call[call.index("--method") + 1] != "GET" + ] + self.assertIn("conclusion=success", mutations[0]) + self.assertTrue( + any( + "repos/LLM360/lemonade/check-runs/80" in call + and "conclusion=cancelled" in call + for call in mutations + ) + ) + self.assertFalse( + any("repos/LLM360/lemonade/check-runs/90" in call for call in mutations) + ) + + superseded_before_write, pre_write_calls = run_report_step( + [[{"check_runs": [newer_revocation]}]] + ) + self.assertNotEqual(superseded_before_write.returncode, 0) + self.assertTrue( + all(call[call.index("--method") + 1] == "GET" for call in pre_write_calls) + ) + + prior_attempt = dict(current, id=70, external_id=f"{prefix}9-50-1-9000") + retry = dict(current, external_id=f"{prefix}1-50-2-500") + retried, retry_calls = run_report_step( + [ + [{"check_runs": [prior_attempt]}], + [{"check_runs": [prior_attempt, retry]}], + ], + run_attempt=2, + ) + self.assertEqual(retried.returncode, 0, retried.stderr) + self.assertTrue( + any( + "repos/LLM360/lemonade/check-runs/70" in call + and "conclusion=cancelled" in call + for call in retry_calls + ) + ) + + def test_success_is_demoted_when_live_identity_changes_after_write(self) -> None: + valid_pull_request = { + "base": {"ref": "main", "sha": "a" * 40}, + "head": {"sha": "b" * 40}, + "labels": [{"name": "ci:upgrades"}], + "merge_commit_sha": "c" * 40, + "mergeable": True, + "merged": False, + "number": 7, + "state": "open", + } + current_check = { + "app": {"slug": "github-actions"}, + "external_id": "lemonade-llamacpp-required-pr-7-1-50-1-500", + "id": 80, + "name": "llama.cpp authorized validation v2", + } + valid_events = [[issue_event(1, "labeled", label="ci:upgrades")]] + changed_snapshots = { + "authorization": { + **valid_pull_request, + "labels": [], + }, + "base-ref": { + **valid_pull_request, + "base": {"ref": "release", "sha": "a" * 40}, + }, + "head": { + **valid_pull_request, + "head": {"sha": "d" * 40}, + }, + "merge": { + **valid_pull_request, + "merge_commit_sha": "d" * 40, + }, + "base-epoch": valid_pull_request, + "reopened-epoch": valid_pull_request, + "permission": valid_pull_request, + } + for name, changed_pull_request in changed_snapshots.items(): + with self.subTest(name=name): + post_events = ( + [ + [ + *valid_events[0], + issue_event(2, "unlabeled", label="ci:upgrades"), + ] + ] + if name == "authorization" + else ( + [[*valid_events[0], issue_event(2, "base_ref_changed")]] + if name == "base-epoch" + else ( + [ + [ + *valid_events[0], + issue_event(2, "closed"), + issue_event(3, "reopened"), + ] + ] + if name == "reopened-epoch" + else valid_events + ) + ) + ) + completed, calls = run_report_step( + [[], [{"check_runs": [current_check]}]], + pull_request_snapshots=[valid_pull_request, changed_pull_request], + issue_event_snapshots=[valid_events, post_events], + permission_snapshots=( + ["write", "read"] if name == "permission" else None + ), + ) + self.assertNotEqual(completed.returncode, 0) + pull_reads = [ + call + for call in calls + if any("/pulls/7" in argument for argument in call) + ] + self.assertEqual(len(pull_reads), 2) + self.assertTrue( + any( + "repos/LLM360/lemonade/check-runs/80" in call + and "conclusion=failure" in call + for call in calls + ) + ) + + def test_missing_live_authorization_invalidates_a_prior_protected_gate( + self, + ) -> None: + policy = load_required_check_policy() + common = { + "base_ref_changed": False, + "state": "open", + "merged": False, + "merge_candidate_available": True, + "identity_matches": True, + "actor_permission": "write", + "has_newer_check": False, + } + + authorized = policy.decide_pull_request_policy( + action="synchronize", + event_label="", + relevant=True, + authorization_label_present=True, + authorization_rerun_matches=True, + has_existing_check=False, + **common, + ) + self.assertEqual(authorized.mode, "validate") + self.assertTrue(authorized.run_validation) + + for action, event_label in (("edited", ""), ("labeled", "ci:macos")): + with self.subTest(action=action, event_label=event_label): + missing_live_label = policy.decide_pull_request_policy( + action=action, + event_label=event_label, + relevant=True, + authorization_label_present=False, + has_existing_check=True, + **common, + ) + self.assertEqual(missing_live_label.mode, "authorization_required") + self.assertFalse(missing_live_label.run_validation) + self.assertEqual(missing_live_label.conclusion, "failure") + + neutral = policy.decide_pull_request_policy( + action="edited", + event_label="", + relevant=False, + authorization_label_present=False, + has_existing_check=True, + **common, + ) + self.assertEqual(neutral.mode, "neutral") + self.assertEqual(neutral.conclusion, "success") + + def test_authorization_lifecycle_events_cannot_be_replaced_or_bypassed( + self, + ) -> None: + policy = load_required_check_policy() + common = { + "base_ref_changed": False, + "state": "open", + "merged": False, + "merge_candidate_available": True, + "relevant": True, + "identity_matches": True, + "authorization_label_present": True, + "has_existing_check": True, + "has_newer_check": False, + } + + removed_before_fast_readd = policy.decide_pull_request_policy( + action="unlabeled", + event_label="ci:upgrades", + actor_permission="", + **common, + ) + self.assertEqual(removed_before_fast_readd.mode, "authorization_required") + self.assertEqual(removed_before_fast_readd.conclusion, "failure") + + triage_readd = policy.decide_pull_request_policy( + action="labeled", + event_label="ci:upgrades", + actor_permission="triage", + **common, + ) + self.assertEqual(triage_readd.mode, "awaiting_rerun") + self.assertFalse(triage_readd.run_validation) + + metadata_replacement = policy.decide_pull_request_policy( + action="labeled", + event_label="ci:macos", + actor_permission="write", + **common, + ) + self.assertEqual(metadata_replacement.mode, "preserve") + self.assertFalse(metadata_replacement.run_validation) + + metadata_after_untrusted_readd = policy.decide_pull_request_policy( + action="edited", + event_label="", + actor_permission="triage", + authorization_epoch_valid=True, + **common, + ) + self.assertEqual(metadata_after_untrusted_readd.mode, "authorization_required") + self.assertEqual(metadata_after_untrusted_readd.conclusion, "failure") + + metadata_before_readd = policy.decide_pull_request_policy( + action="edited", + event_label="", + actor_permission="", + has_existing_check=False, + **{ + key: value + for key, value in common.items() + if key != "has_existing_check" + }, + ) + self.assertEqual(metadata_before_readd.mode, "authorization_required") + self.assertEqual(metadata_before_readd.conclusion, "failure") + self.assertFalse(metadata_before_readd.run_validation) + + trusted_readd = policy.decide_pull_request_policy( + action="labeled", + event_label="ci:upgrades", + actor_permission="write", + **common, + ) + self.assertEqual(trusted_readd.mode, "awaiting_rerun") + self.assertFalse(trusted_readd.run_validation) + + trusted_rerun = policy.decide_pull_request_policy( + action="synchronize", + event_label="", + actor_permission="write", + authorization_rerun_matches=True, + **common, + ) + self.assertEqual(trusted_rerun.mode, "validate") + self.assertTrue(trusted_rerun.run_validation) + + delayed_removal = policy.decide_pull_request_policy( + action="unlabeled", + event_label="ci:upgrades", + actor_permission="", + has_newer_check=True, + **{key: value for key, value in common.items() if key != "has_newer_check"}, + ) + self.assertEqual(delayed_removal.mode, "preserve") + + workflow = PR_WORKFLOW.read_text(encoding="utf-8") + decision_environment = workflow.split( + " - name: Decide whether protected validation may run\n", 1 + )[1].split(" run: |\n", 1)[0] + concurrency = workflow.split("concurrency:\n", 1)[1].split("\njobs:\n", 1)[0] + self.assertIn("github.event.action == 'labeled'", concurrency) + self.assertIn("github.event.action == 'unlabeled'", concurrency) + self.assertIn("github.run_number", concurrency) + self.assertIn("github.run_attempt", concurrency) + self.assertNotIn("github.run_id", concurrency) + for action in ( + "opened", + "reopened", + "synchronize", + "edited", + "labeled", + "unlabeled", + "closed", + ): + self.assertIn(action, concurrency) + self.assertNotIn("github.event.pull_request.merge_commit_sha", concurrency) + self.assertIn("issues: read", workflow) + self.assertIn("issues/${PR_NUMBER}/events?per_page=100", workflow) + self.assertIn("--paginate --slurp", workflow) + self.assertIn("authorization_event_id", workflow) + self.assertIn("authorization_created_at", workflow) + self.assertIn("actions: read", workflow) + self.assertIn("CURRENT_RUN_ID: ${{ github.run_id }}", workflow) + self.assertIn( + "EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }}", + decision_environment, + ) + self.assertIn( + "EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }}", + decision_environment, + ) + self.assertIn( + "EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }}", + decision_environment, + ) + self.assertIn( + "TRIGGERING_ACTOR: ${{ github.triggering_actor || '' }}", workflow + ) + self.assertIn("actions/runs/${CURRENT_RUN_ID}/attempts/1", workflow) + self.assertIn(".github/workflows/validate_llamacpp_pr.yml", workflow) + self.assertNotIn(".head_branch == $base_ref", workflow) + self.assertNotIn(".head_repository.full_name == $repository", workflow) + self.assertNotIn(".head_sha == $base_sha", workflow) + self.assertIn('$path + "@" + $base_ref', workflow) + self.assertIn('$path + "@refs/heads/" + $base_ref', workflow) + self.assertIn('((.pull_requests | type) == "array")', workflow) + self.assertNotIn(".pull_requests | length", workflow) + self.assertNotIn(".pull_requests[0]", workflow) + self.assertIn(".head_sha == $head_sha", workflow) + self.assertIn('.conclusion == "failure"', workflow) + self.assertIn(".run_started_at", workflow) + self.assertIn("authorization_boundary", workflow) + self.assertIn("authorization_rerun_matches", workflow) + self.assertIn( + '--authorization-rerun-matches "$authorization_rerun_matches"', + workflow, + ) + self.assertIn('[[ "$CURRENT_RUN_ATTEMPT" =~ ^([2-9]|[1-9][0-9]+)$ ]]', workflow) + self.assertIn(".user.id == $actor_id", workflow) + report = workflow.split( + " - name: Report result on the exact candidate commit\n", 1 + )[1].split(" - name: Enforce policy result\n", 1)[0] + self.assertLess( + report.index('newest_number" -gt "$RUN_NUMBER'), + report.index("--method PATCH"), + ) + self.assertIn(".number < $current_number", report) + self.assertIn("RUN_NUMBER: ${{ github.run_number }}", report) + + def test_preserve_rechecks_the_live_epoch_or_reports_terminal_failure( + self, + ) -> None: + pull_request = { + "base": {"ref": "main", "sha": "a" * 40}, + "head": {"sha": "b" * 40}, + "labels": [{"name": "ci:upgrades"}], + "merge_commit_sha": "c" * 40, + "mergeable": True, + "merged": False, + "number": 7, + "state": "open", + } + current_epoch_check = { + "app": {"slug": "github-actions"}, + "conclusion": "success", + "external_id": "lemonade-llamacpp-required-pr-7-1-40-1-400", + "id": 40, + "name": "llama.cpp authorized validation v2", + "status": "completed", + } + valid, valid_outputs = run_preserve_step( + pull_request=pull_request, + issue_events=[[issue_event(1, "labeled", label="ci:upgrades")]], + check_pages=[{"check_runs": [current_epoch_check]}], + ) + self.assertEqual(valid.returncode, 0, valid.stderr) + self.assertEqual(valid_outputs["valid"], "true") + + for name, events, checks in ( + ( + "unlabeled-after-policy", + [ + [ + issue_event(1, "labeled", label="ci:upgrades"), + issue_event(2, "unlabeled", label="ci:upgrades"), + ] + ], + [current_epoch_check], + ), + ( + "old-epoch-only", + [[issue_event(1, "labeled", label="ci:upgrades")]], + [ + { + **current_epoch_check, + "external_id": "lemonade-llamacpp-required-pr-7-9-40-1-400", + } + ], + ), + ( + "malformed-final-provenance", + [ + [ + { + "actor": {"id": 100, "login": "maintainer"}, + "created_at": "2026-01-01T00:00:01Z", + "event": "labeled", + "id": 1, + "label": {"name": "ci:upgrades"}, + } + ] + ], + [current_epoch_check], + ), + ( + "in-progress-prior-validation", + [[issue_event(1, "labeled", label="ci:upgrades")]], + [{**current_epoch_check, "conclusion": None, "status": "in_progress"}], + ), + ( + "failed-prior-validation", + [[issue_event(1, "labeled", label="ci:upgrades")]], + [{**current_epoch_check, "conclusion": "failure"}], + ), + ( + "newer-required-decision", + [[issue_event(1, "labeled", label="ci:upgrades")]], + [ + current_epoch_check, + { + **current_epoch_check, + "conclusion": None, + "external_id": ("lemonade-llamacpp-required-pr-7-1-51-1-600"), + "id": 51, + "status": "in_progress", + }, + ], + ), + ( + "later-failed-prior-decision", + [[issue_event(1, "labeled", label="ci:upgrades")]], + [ + current_epoch_check, + { + **current_epoch_check, + "conclusion": "failure", + "external_id": ("lemonade-llamacpp-required-pr-7-1-49-1-499"), + "id": 49, + }, + ], + ), + ): + with self.subTest(name=name): + invalid, invalid_outputs = run_preserve_step( + pull_request=pull_request, + issue_events=events, + check_pages=[{"check_runs": checks}], + ) + self.assertEqual(invalid.returncode, 0, invalid.stderr) + self.assertEqual(invalid_outputs["valid"], "false") + + reported_check = { + **current_epoch_check, + "external_id": "lemonade-llamacpp-required-pr-7-1-50-1-500", + "id": 80, + } + reported, calls = run_report_step( + [[], [{"check_runs": [reported_check]}]], + mode="preserve", + conclusion="failure", + preserve_valid=invalid_outputs["valid"] == "true", + ) + self.assertEqual(reported.returncode, 0, reported.stderr) + self.assertTrue( + any( + "--method" in call + and call[call.index("--method") + 1] in {"POST", "PATCH"} + and "conclusion=failure" in call + for call in calls + ) + ) + + def test_live_identity_rejects_stale_merge_sha_until_mergeability_is_known( + self, + ) -> None: + base_sha = "a" * 40 + head_sha = "b" * 40 + stale_merge_sha = "c" * 40 + + def snapshot(mergeable: object) -> dict[str, object]: + return { + "number": 7, + "state": "open", + "merged": False, + "mergeable": mergeable, + "merge_commit_sha": stale_merge_sha, + "base": {"sha": base_sha}, + "head": {"sha": head_sha}, + "changed_files": 1, + "labels": [{"name": "ci:upgrades"}], + } + + cases = ( + ("unknown-through-timeout", [snapshot(None)] * 3, 3), + ("known-unmergeable", [snapshot(False)], 1), + ) + policy = load_required_check_policy() + for name, snapshots, expected_calls in cases: + with self.subTest(name=name): + completed, outputs, calls = run_identity_step( + snapshots, + event_merge_sha=stale_merge_sha, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(calls, expected_calls) + self.assertEqual(outputs["merge_candidate_available"], "false") + self.assertEqual(outputs["merge_sha"], head_sha) + self.assertEqual(outputs["observed_merge_sha"], stale_merge_sha) + self.assertEqual(outputs["identity_matches"], "true") + + decision = policy.decide_pull_request_policy( + action="synchronize", + event_label="", + base_ref_changed=False, + state=outputs["state"], + merged=outputs["merged"] == "true", + merge_candidate_available=( + outputs["merge_candidate_available"] == "true" + ), + relevant=True, + identity_matches=outputs["identity_matches"] == "true", + authorization_label_present=True, + authorization_rerun_matches=True, + actor_permission="write", + has_existing_check=False, + has_newer_check=False, + ) + self.assertEqual(decision.mode, "authorization_required") + self.assertFalse(decision.run_validation) + self.assertEqual(decision.conclusion, "failure") + + def test_identity_tracks_event_and_live_unconfirmed_merge_commits(self) -> None: + base_sha = "a" * 40 + head_sha = "b" * 40 + event_merge_sha = "c" * 40 + live_merge_sha = "d" * 40 + + def snapshot(merge_commit_sha: str | None) -> dict[str, object]: + return { + "base": {"ref": "main", "sha": base_sha}, + "changed_files": 1, + "head": {"sha": head_sha}, + "labels": [{"name": "ci:upgrades"}], + "merge_commit_sha": merge_commit_sha, + "mergeable": None, + "merged": False, + "number": 7, + "state": "open", + } + + for name, live_merge, expected_primary, expected_secondary in ( + ("event-only", None, event_merge_sha, ""), + ("live-and-event", live_merge_sha, live_merge_sha, event_merge_sha), + ): + with self.subTest(name=name): + completed, outputs, _ = run_identity_step( + [snapshot(live_merge)] * 3, + event_merge_sha=event_merge_sha, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["merge_sha"], head_sha) + self.assertEqual(outputs["observed_merge_sha"], expected_primary) + self.assertEqual( + outputs["secondary_observed_merge_sha"], expected_secondary + ) + + retargeted = snapshot(live_merge_sha) + retargeted["base"] = {"ref": "release", "sha": base_sha} + completed, outputs, _ = run_identity_step( + [retargeted] * 3, + event_base_ref="main", + event_merge_sha=event_merge_sha, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(outputs["base_ref"], "release") + self.assertEqual(outputs["identity_matches"], "false") + + def test_required_check_classifier_covers_protected_inputs(self) -> None: + policy = load_required_check_policy() + + protected_paths = ( + ".github/llamacpp_release_manifest.json", + ".github/workflows/validate_llamacpp_pr.yml", + ".github/workflows/validate_llamacpp_core.yml", + ".github/scripts/run_llamacpp_validation.py", + ".github/scripts/llamacpp_required_check_policy.py", + ".github/actions/setup-venv/action.yml", + ".github/actions/setup-python/action.yml", + ".github/actions/cleanup-processes-linux/action.yml", + ".github/actions/cleanup-processes-windows/action.yml", + "CMakeLists.txt", + "CMakePresets.json", + "cmake/DetectSystemHttplib.cmake", + "setup.sh", + "src/cpp/resources/backend_versions.json", + "src/cpp/resources/server_models.json", + "src/app/src/renderer/utils/toolDefinitions.json", + "src/cpp/server/backends/llamacpp/llamacpp_server.cpp", + "src/cpp/server/backends/vllm/vllm_server.cpp", + "src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp", + "src/cpp/include/lemon/backend_manager.h", + "src/cpp/server/backend_manager.cpp", + "src/cpp/include/lemon/runtime_config.h", + "src/cpp/server/runtime_config.cpp", + "src/cpp/include/lemon/backends/llamacpp/llamacpp_server.h", + "src/cpp/server/backends/backend_utils.cpp", + "src/cpp/include/lemon/backends/backend_utils.h", + "src/cpp/server/streaming_proxy.cpp", + "src/cpp/include/lemon/streaming_proxy.h", + "src/cpp/server/wrapped_server.cpp", + "src/cpp/include/lemon/wrapped_server.h", + "src/cpp/server/system_info.cpp", + "src/cpp/include/lemon/system_info.h", + "src/cpp/server/utils/platform/process_linux.cpp", + "src/cpp/include/lemon/utils/process_manager.h", + "test/requirements.txt", + "test/validate_llamacpp.py", + "test/utils/llamacpp_capability_validation.py", + "test/utils/capabilities.py", + "test/utils/server_base.py", + "test/utils/test_models.py", + "test/utils/validation_model_catalog.py", + "test/utils/validation_model_selection.py", + "test/fixtures/llamacpp_validation_models.json", + "new-root-build-input.txt", + ".github/actions/future-validation-action/action.yml", + ".github/scripts/future-validation-helper.py", + ) + for path in protected_paths: + with self.subTest(path=path): + self.assertTrue(policy.is_protected_path(path)) + + unrelated_paths = ( + "docs/dev/testing.md", + "src/app/src/renderer/ChatWindow.tsx", + "src/web-app/src/index.ts", + ) + for path in unrelated_paths: + with self.subTest(path=path): + self.assertFalse(policy.is_protected_path(path)) + + def test_required_check_classifier_fails_closed_on_incomplete_file_lists( + self, + ) -> None: + policy = load_required_check_policy() + base_sha = "a" * 40 + stale_target_sha = "b" * 40 + candidate_sha = "c" * 40 + neutral_comparison = { + "ahead_by": 1, + "base_commit": {"sha": base_sha}, + "behind_by": 0, + "commits": [{"sha": candidate_sha}], + "files": [{"filename": "docs/dev/testing.md", "status": "modified"}], + "merge_base_commit": {"sha": base_sha}, + "status": "ahead", + "total_commits": 1, + } + + self.assertFalse( + policy.classify_compare_response( + neutral_comparison, + expected_base_sha=base_sha, + expected_target_sha=candidate_sha, + expected_count=1, + ) + ) + stale_comparison = { + **neutral_comparison, + "commits": [{"sha": stale_target_sha}], + } + with self.assertRaises(ValueError): + policy.classify_compare_response( + stale_comparison, + expected_base_sha=base_sha, + expected_target_sha=candidate_sha, + expected_count=1, + ) + self.assertTrue( + policy.classify_compare_response( + {}, + expected_base_sha=base_sha, + expected_target_sha=candidate_sha, + expected_count=policy.MAX_COMPARE_FILES + 1, + ) + ) + + self.assertTrue( + policy.classify_changed_file_pages( + [ + [ + { + "filename": "docs/renamed.md", + "previous_filename": "CMakeLists.txt", + "status": "renamed", + } + ] + ], + expected_count=1, + ) + ) + self.assertTrue( + policy.classify_changed_file_pages( + [ + [ + { + "filename": "src/cpp/resources/backend_versions.json", + "status": "modified", + } + ] + ], + expected_count=1, + ) + ) + self.assertFalse( + policy.classify_changed_file_pages( + [[{"filename": "docs/dev/testing.md", "status": "modified"}]], + expected_count=1, + ) + ) + + for status in ( + "added", + "removed", + "modified", + "copied", + "changed", + "unchanged", + ): + with self.subTest(valid_status=status): + self.assertFalse( + policy.classify_changed_file_pages( + [[{"filename": "docs/dev/testing.md", "status": status}]], + expected_count=1, + ) + ) + + with self.assertRaises(ValueError): + policy.classify_changed_file_pages( + [ + [ + { + "filename": "docs/dev/testing.md", + "previous_filename": "docs/dev/old.md", + "status": status, + } + ] + ], + expected_count=1, + ) + + for previous_filename in (None, "", 3, "/docs/old.md", "docs/../old.md"): + with self.subTest(invalid_previous_filename=previous_filename): + with self.assertRaises(ValueError): + policy.classify_changed_file_pages( + [ + [ + { + "filename": "docs/new.md", + "previous_filename": previous_filename, + "status": "renamed", + } + ] + ], + expected_count=1, + ) + + invalid_cases = ( + ([], 1), + ([[{"filename": "docs/one.md", "status": "modified"}]], 2), + ( + [ + [ + {"filename": "docs/one.md", "status": "modified"}, + {"filename": "docs/one.md", "status": "modified"}, + ] + ], + 2, + ), + ([[{"filename": "/absolute/path", "status": "modified"}]], 1), + ([[{"filename": "docs/one.md"}]], 1), + ([[{"filename": "docs/one.md", "status": "moved"}]], 1), + ([[{"filename": "docs/one.md", "status": 3}]], 1), + ([[{"filename": "docs/renamed.md", "status": "renamed"}]], 1), + ( + [ + [ + { + "filename": "docs/one.md", + "previous_filename": "docs/old.md", + "status": "modified", + } + ] + ], + 1, + ), + ( + [ + [ + { + "filename": "docs/one.md", + "previous_filename": None, + "status": "modified", + } + ] + ], + 1, + ), + ( + [[{"filename": "docs/one.md", "status": "modified"}]], + policy.MAX_CHANGED_FILES + 1, + ), + ) + for pages, expected_count in invalid_cases: + with self.subTest(pages=pages, expected_count=expected_count): + with self.assertRaises(ValueError): + policy.classify_changed_file_pages( + pages, + expected_count=expected_count, + ) + + def test_required_check_policy_requires_fresh_maintainer_authorization( + self, + ) -> None: + policy = load_required_check_policy() + + def decide(**overrides): + arguments = { + "action": "synchronize", + "event_label": "", + "base_ref_changed": False, + "state": "open", + "merged": False, + "merge_candidate_available": True, + "relevant": True, + "identity_matches": True, + "authorization_label_present": True, + "actor_permission": "write", + "authorization_rerun_matches": False, + "has_existing_check": True, + "has_newer_check": False, + } + arguments.update(overrides) + return policy.decide_pull_request_policy(**arguments) + + for action in ("opened", "reopened", "synchronize"): + with self.subTest(action=action): + decision = decide(action=action) + self.assertEqual(decision.mode, "authorization_required") + self.assertFalse(decision.run_validation) + self.assertIn("remove and reapply", decision.summary) + + base_edit = decide(action="edited", base_ref_changed=True) + self.assertEqual(base_edit.mode, "authorization_required") + + removed = decide( + action="unlabeled", + event_label="ci:upgrades", + authorization_label_present=False, + ) + self.assertEqual(removed.mode, "authorization_required") + + authorized = decide( + action="synchronize", + authorization_rerun_matches=True, + ) + self.assertEqual(authorized.mode, "validate") + self.assertTrue(authorized.run_validation) + + authorized_override = decide( + action="synchronize", + authorization_rerun_matches=True, + relevant=False, + has_existing_check=False, + ) + self.assertEqual(authorized_override.mode, "validate") + self.assertTrue(authorized_override.run_validation) + + unauthorized_override = decide( + action="synchronize", + authorization_rerun_matches=True, + relevant=False, + actor_permission="read", + has_existing_check=False, + ) + self.assertEqual(unauthorized_override.mode, "neutral") + self.assertFalse(unauthorized_override.run_validation) + + for permission in ("read", "none", ""): + with self.subTest(permission=permission): + denied = decide( + action="synchronize", + authorization_rerun_matches=True, + actor_permission=permission, + ) + self.assertEqual(denied.mode, "authorization_required") + self.assertFalse(denied.run_validation) + + stale = decide( + action="synchronize", + authorization_rerun_matches=True, + identity_matches=False, + ) + self.assertEqual(stale.mode, "stale_required") + + stale_without_gate = decide( + action="synchronize", + authorization_rerun_matches=True, + identity_matches=False, + has_existing_check=False, + ) + self.assertEqual(stale_without_gate.mode, "stale_required") + + neutral_without_merge = decide( + action="opened", + merge_candidate_available=False, + relevant=False, + has_existing_check=False, + ) + self.assertEqual(neutral_without_merge.mode, "neutral") + + protected_without_merge = decide( + action="synchronize", + authorization_rerun_matches=True, + merge_candidate_available=False, + has_existing_check=False, + ) + self.assertEqual(protected_without_merge.mode, "authorization_required") + self.assertFalse(protected_without_merge.run_validation) + + preserved = decide(action="labeled", event_label="ci:macos") + self.assertEqual(preserved.mode, "preserve") + + missing_prior = decide( + action="labeled", + event_label="ci:macos", + has_existing_check=False, + ) + self.assertEqual(missing_prior.mode, "authorization_required") + self.assertEqual(missing_prior.conclusion, "failure") + + neutral = decide(relevant=False, has_existing_check=False) + self.assertEqual(neutral.mode, "neutral") + self.assertEqual(neutral.conclusion, "success") + + closed = decide(action="closed", state="closed") + self.assertEqual(closed.mode, "closed") + self.assertEqual(closed.conclusion, "cancelled") + + merged = decide(action="closed", state="closed", merged=True) + self.assertEqual(merged.mode, "merged") + self.assertEqual(merged.conclusion, "") + + stale_after_merge = decide(action="labeled", state="closed", merged=True) + self.assertEqual(stale_after_merge.mode, "merged") + self.assertEqual(stale_after_merge.conclusion, "") + + stale_close_after_reopen = decide( + action="closed", + state="open", + merged=False, + ) + self.assertEqual(stale_close_after_reopen.mode, "stale") + + removal_before_reapply_is_processed = decide( + action="unlabeled", + event_label="ci:upgrades", + authorization_label_present=True, + ) + self.assertEqual( + removal_before_reapply_is_processed.mode, "authorization_required" + ) + self.assertEqual(removal_before_reapply_is_processed.conclusion, "failure") + + delayed_synchronize = decide( + action="synchronize", + has_newer_check=True, + ) + self.assertEqual(delayed_synchronize.mode, "superseded") + self.assertEqual(delayed_synchronize.conclusion, "failure") + + def test_protected_validation_bootstrap_requires_the_trusted_workflow( + self, + ) -> None: + guide = TESTING_GUIDE.read_text(encoding="utf-8") + pull_request = PR_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn( + "| llama.cpp, vLLM, stable-diffusion.cpp validation | " + "Required workflow `Validate llama.cpp protected change`;", + guide, + ) + self.assertNotIn( + "| llama.cpp, vLLM, stable-diffusion.cpp validation | " + "`llama.cpp authorized validation v2`,", + guide, + ) + self.assertIn("Do not require `llama.cpp validation`", guide) + self.assertIn("`ci:upgrades` label", guide) + self.assertIn("protected self-hosted runners", guide) + self.assertIn("CODEOWNERS", guide) + self.assertIn("separate bootstrap merge", guide) + self.assertIn( + "Enforcement therefore starts with later protected pull requests", guide + ) + self.assertIn("current `.github/CODEOWNERS` covers only", guide) + self.assertIn("explicit entries for every trusted validation", guide) + self.assertIn("enforce code-owner review", guide) + self.assertIn("Missing exact CODEOWNERS coverage", guide) + self.assertIn("external activation **NO-GO**", guide) + self.assertIn("merge queue", guide) + self.assertIn( + "Allow GitHub Actions to create and approve pull requests", + guide, + ) + self.assertIn("`LLAMACPP_UPDATE_TOKEN`", guide) + self.assertIn("`LLAMACPP_UPDATE_ACTOR`", guide) + self.assertIn("repository-scoped fine-grained PAT", guide) + self.assertNotIn("short-lived dedicated GitHub App installation token", guide) + self.assertIn("installation tokens expire after one hour", guide) + self.assertIn("mint a fresh token inside each publication job", guide) + self.assertIn("Contents: read and write", guide) + self.assertIn("Pull requests: read and write", guide) + self.assertIn("Never substitute the repository's `GITHUB_TOKEN`", guide) + self.assertIn("does not trigger the required workflow", guide) + self.assertIn("deliberately creates a draft first", guide) + self.assertIn("missing secret or variable configuration", guide) + self.assertIn("neutral success", guide) + self.assertIn("synthetic test-merge commit", guide) + self.assertIn("falls back to the immutable current head", guide) + self.assertIn( + "protected validation never executes without a test-merge commit", guide + ) + self.assertIn("GitHub ignores configured activity filters", guide) + self.assertIn("`opened`, `synchronize`, and `reopened`", guide) + self.assertIn("**Re-run all jobs**", guide) + self.assertIn("The same maintainer must start the rerun", guide) + self.assertIn("a label at or before the original run start time", guide) + self.assertIn("not its queued creation time", guide) + self.assertIn("while the run was queued", guide) + self.assertIn("base-branch retarget", guide) + self.assertIn("supported default-event run", guide) + self.assertIn("Metadata diagnostic runs do not supersede", guide) + self.assertIn("`llama.cpp authorization diagnostics v2`", guide) + self.assertIn("uses a distinct visible name", guide) + self.assertIn("does not conflict with a recovered required result", guide) + self.assertIn("newer `opened`, `synchronize`, or `reopened` run", guide) + self.assertIn("rerun the same selected default-event run", guide) + self.assertIn("Removal after a completed green rerun is diagnostic only", guide) + self.assertIn("one exact default-event run and candidate", guide) + self.assertNotIn("The label authorizes one exact run attempt", guide) + self.assertIn("Activation is **NO-GO** unless", guide) + self.assertIn("the default branch advances after a green rerun", guide) + self.assertIn( + "a merge queue, strict up-to-date enforcement, or a live-proven equivalent", + guide, + ) + self.assertIn("the exact validated base-plus-head candidate", guide) + self.assertIn("exclusive GitHub App", guide) + self.assertIn( + "Do **not** require `llama.cpp authorized validation v2` during bootstrap", + guide, + ) + self.assertIn("immutable K2-capable backend releases", guide) + self.assertIn("all three pinned release repositories", guide) + self.assertIn("controlled immutable mirror", guide) + self.assertIn("all three `sm_121` artifacts", guide) + self.assertIn("source sidecars", guide) + self.assertIn("Require workflows to pass before merging", guide) + self.assertIn("organization-level", guide) + self.assertIn("`LLM360/lemonade`", guide) + self.assertIn("`main`", guide) + self.assertIn("`.github/workflows/validate_llamacpp_pr.yml`", guide) + self.assertIn("organization owner", guide) + self.assertIn("After the bootstrap merge is on the default branch", guide) + self.assertIn("separate, unmerged live fork PRs", guide) + self.assertIn("Do not combine validation-infrastructure bootstrap", guide) + self.assertNotIn("Keep the integration PR in draft until", guide) + self.assertIn("shared `github-actions` App", guide) + self.assertIn("does not identify the workflow", guide) + self.assertIn("diagnostic only", guide) + self.assertIn( + "Never configure `llama.cpp authorized validation v2` as a required " + "status check", + guide, + ) + self.assertNotIn( + "select **GitHub Actions** as the expected check source", guide + ) + self.assertIn(" pull_request_target:\n", pull_request) + self.assertIn(" merge_group:\n", pull_request) + + reconcile = pull_request.split(" reconcile:\n", 1)[1] + enforce = reconcile.split(" - name: Enforce policy result\n", 1)[1] + self.assertIn(" if: always()", reconcile) + self.assertIn("if: >-\n always()", enforce) + self.assertIn("exit 1", enforce) + + def test_pr_required_check_wrapper_covers_every_policy_lifecycle(self) -> None: + pull_request = PR_WORKFLOW.read_text(encoding="utf-8") + classify_step = extract_workflow_step_script( + "Classify the complete pull request file set" + ) + triggers = pull_request.split("permissions:", 1)[0] + report = pull_request.split( + " - name: Report result on the exact candidate commit\n", 1 + )[1].split(" - name: Enforce policy result\n", 1)[0] + + self.assertIn( + "types: [opened, reopened, synchronize, edited, labeled, unlabeled, closed]", + triggers, + ) + self.assertIn(" merge_group:\n", triggers) + self.assertIn( + "compare/${BASE_SHA}...${MERGE_SHA}", + classify_step, + ) + self.assertNotIn("pulls/${PR_NUMBER}/files?per_page=100", classify_step) + self.assertNotIn("--paginate --slurp", classify_step) + self.assertIn("classify-compare", classify_step) + self.assertIn( + "MERGE_SHA: ${{ steps.identity.outputs.merge_sha }}", pull_request + ) + self.assertIn("current_changed_files", pull_request) + self.assertIn("for attempt in 1 2 3; do", pull_request) + self.assertIn("snapshot_mergeable", pull_request) + self.assertIn("sleep 2", pull_request) + self.assertIn("llamacpp_required_check_policy.py", pull_request) + self.assertIn( + "previous_filename", REQUIRED_CHECK_POLICY.read_text(encoding="utf-8") + ) + self.assertIn( + "trusted/.github/scripts/llamacpp_required_check_policy.py", pull_request + ) + self.assertIn("ref: ${{ steps.identity.outputs.base_sha }}", pull_request) + self.assertIn("persist-credentials: false", pull_request) + self.assertIn("collaborators/${authorization_actor}/permission", pull_request) + self.assertIn("EVENT_ACTION: ${{ github.event.action || '' }}", pull_request) + self.assertIn("github.event.changes.base.ref.from", pull_request) + self.assertIn("llama.cpp authorized validation v2", pull_request) + self.assertIn("needs.policy.outputs.run_validation == 'true'", pull_request) + self.assertIn("needs.policy.outputs.mode == 'preserve'", pull_request) + self.assertIn("needs.policy.outputs.conclusion", pull_request) + self.assertIn("github.event.pull_request.merge_commit_sha", pull_request) + self.assertIn("github.event.merge_group.head_sha", pull_request) + self.assertIn("current_merge", pull_request) + self.assertIn("current_mergeable", report) + self.assertIn( + '[ "$current_mergeable" = "true" ] && [[ "$current_merge" =~ $oid_pattern ]]', + report, + ) + self.assertIn("current_base", pull_request) + self.assertIn("current_head", pull_request) + self.assertIn( + "current_merge=$(jq -r '.merge_commit_sha // empty'", pull_request + ) + self.assertIn('candidate_sha="$current_head"', pull_request) + self.assertIn( + '--merge-candidate-available "$MERGE_CANDIDATE_AVAILABLE"', + pull_request, + ) + self.assertIn("steps.assess-policy.outputs.mode != 'stale'", pull_request) + self.assertIn('.app.slug == "github-actions"', pull_request) + self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", pull_request) + + def test_protected_candidate_checkouts_opt_in_without_weakening_trusted_checkouts( + self, + ) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + unsafe_opt_in = ( + "allow-unsafe-pr-checkout: " + "${{ github.event_name == 'pull_request_target' }}" + ) + checkout_v5_1 = ( + "actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 " "# v5.1.0" + ) + + self.assertEqual(validation.count(unsafe_opt_in), 4) + self.assertEqual( + validation.count(checkout_v5_1), + validation.count("uses: actions/checkout@"), + ) + for trusted_checkout in validation.split( + " - name: Check out trusted validation tools\n" + )[1:]: + trusted_block = trusted_checkout.split(" - name:", 1)[0] + self.assertNotIn("allow-unsafe-pr-checkout", trusted_block) + wrapper = PR_WORKFLOW.read_text(encoding="utf-8") + policy_checkout = wrapper.split( + " - name: Check out trusted policy code\n", 1 + )[1].split(" - name:", 1)[0] + self.assertNotIn("allow-unsafe-pr-checkout", policy_checkout) + + def test_required_check_lookups_are_complete_source_bound_and_race_safe( + self, + ) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + pull_request = PR_WORKFLOW.read_text(encoding="utf-8") + invocation = validation.split(" validate-invocation:\n", 1)[1].split( + " plan:\n", 1 + )[0] + report = pull_request.split( + " - name: Report result on the exact candidate commit\n", 1 + )[1].split(" - name: Enforce policy result\n", 1)[0] + + self.assertIn( + 'EXTERNAL_PREFIX="lemonade-llamacpp-required-${TARGET_KEY}-"', + invocation, + ) + self.assertIn("decision_scope=metadata", report) + self.assertIn("decision_scope=required", report) + self.assertIn( + 'external_prefix="lemonade-llamacpp-${decision_scope}-${TARGET_KEY}-"', + report, + ) + + for lookup in (invocation, report): + with self.subTest(workflow="core" if lookup is invocation else "wrapper"): + self.assertIn("--paginate --slurp", lookup) + self.assertIn('.app.slug == "github-actions"', lookup) + self.assertIn('newest_number" -gt "$RUN_NUMBER', lookup) + self.assertIn(".number < $current_number", lookup) + self.assertLess( + lookup.index('newest_number" -gt "$RUN_NUMBER'), + lookup.index("--method PATCH"), + ) + self.assertGreaterEqual(lookup.count("--paginate --slurp"), 2) + self.assertIn("RUN_NUMBER: ${{ github.run_number }}", lookup) + + def test_required_check_filters_ignore_foreign_and_newer_runs(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + pull_request = PR_WORKFLOW.read_text(encoding="utf-8") + lookups = ( + validation.split(" validate-invocation:\n", 1)[1].split(" plan:\n", 1)[0], + pull_request.split( + " - name: Report result on the exact candidate commit\n", 1 + )[1].split(" - name: Enforce policy result\n", 1)[0], + ) + check_name = "llama.cpp authorized validation v2" + external_prefix = "lemonade-llamacpp-required-pr-7-" + check_pages = [ + { + "check_runs": [ + { + "id": 11, + "name": check_name, + "app": {"slug": "github-actions"}, + "conclusion": "success", + "external_id": f"{external_prefix}1-10-1-9000", + "status": "completed", + }, + { + "id": 12, + "name": check_name, + "app": {"slug": "github-actions"}, + "conclusion": "success", + "external_id": f"{external_prefix}1-20-1-8000", + "status": "completed", + }, + { + "id": 13, + "name": check_name, + "app": {"slug": "github-actions"}, + "conclusion": "success", + "external_id": f"{external_prefix}1-30-2-7000", + "status": "completed", + }, + { + "id": 14, + "name": check_name, + "app": {"slug": "foreign-app"}, + "conclusion": "success", + "external_id": f"{external_prefix}1-99-1-6000", + "status": "completed", + }, + { + "id": 15, + "name": "foreign check", + "app": {"slug": "github-actions"}, + "conclusion": "success", + "external_id": f"{external_prefix}1-100-1-5000", + "status": "completed", + }, + ] + } + ] + + def jq_program(block: str, anchor: str) -> str: + tail = block.split(anchor, 1)[1] + starts = [ + position + for token in ("'[.[]", "'.[]") + if (position := tail.find(token)) >= 0 + ] + self.assertTrue(starts) + start = min(starts) + end = tail.index('\' <<<"$check_runs"', start) + return tail[start + 1 : end] + + policy_job = pull_request.split(" policy:\n", 1)[1].split(" validate:\n", 1)[ + 0 + ] + preserve_step = pull_request.split( + " - name: Preserve the exact-commit gate for unrelated metadata\n", + 1, + )[1].split(" - name:", 1)[0] + for lookup in (policy_job, preserve_step): + latest_program = jq_program( + lookup, "read -r newest_check_epoch newest_check_number" + ) + base_command = [ + "jq", + "-r", + "--arg", + "check_name", + check_name, + "--arg", + "external_prefix", + external_prefix, + latest_program, + ] + trusted = subprocess.run( + base_command, + input=json.dumps(check_pages), + text=True, + capture_output=True, + check=True, + ) + self.assertEqual( + trusted.stdout.rstrip("\n").split("\t"), + ["1", "30", "2", "completed", "success"], + ) + foreign_only = [{"check_runs": [check_pages[0]["check_runs"][3]]}] + foreign = subprocess.run( + base_command, + input=json.dumps(foreign_only), + text=True, + capture_output=True, + check=True, + ) + self.assertEqual( + foreign.stdout.rstrip("\n").split("\t"), + ["0", "0", "0", "", ""], + ) + old_epoch_only = [ + { + "check_runs": [ + { + **check_pages[0]["check_runs"][0], + "external_id": f"{external_prefix}9-10-1-9000", + } + ] + } + ] + old_epoch = subprocess.run( + base_command, + input=json.dumps(old_epoch_only), + text=True, + capture_output=True, + check=True, + ) + self.assertEqual( + old_epoch.stdout.rstrip("\n").split("\t"), + ["9", "10", "1", "completed", "success"], + ) + for status, conclusion in ( + ("in_progress", None), + ("completed", "failure"), + ): + unfinished = [ + { + "check_runs": [ + { + **check_pages[0]["check_runs"][0], + "conclusion": conclusion, + "status": status, + } + ] + } + ] + rejected = subprocess.run( + base_command, + input=json.dumps(unfinished), + text=True, + capture_output=True, + check=True, + ) + self.assertEqual( + rejected.stdout.rstrip("\n").split("\t"), + ["1", "10", "1", status, conclusion or ""], + ) + + for lookup in lookups: + with self.subTest(workflow="core" if lookup is lookups[0] else "wrapper"): + newest = subprocess.run( + [ + "jq", + "-r", + "--arg", + "check_name", + check_name, + "--arg", + "external_prefix", + external_prefix, + jq_program(lookup, "read -r newest_number newest_attempt"), + ], + input=json.dumps(check_pages), + text=True, + capture_output=True, + check=True, + ) + self.assertEqual(newest.stdout.strip(), "30\t2") + + older = subprocess.run( + [ + "jq", + "-r", + "--arg", + "check_name", + check_name, + "--arg", + "external_prefix", + external_prefix, + "--argjson", + "current_number", + "20", + "--argjson", + "current_attempt", + "1", + jq_program(lookup, "prior_check_ids=$(jq -r"), + ], + input=json.dumps(check_pages), + text=True, + capture_output=True, + check=True, + ) + self.assertEqual(older.stdout.splitlines(), ["11"]) + + def test_portable_runner_builds_platform_neutral_validation_arguments(self) -> None: + runner = load_validation_runner() + + command = runner.build_validation_command( + python_executable=Path("python"), + backend="rocm", + channel="nightly", + target="windows-rocm-nightly", + models_csv=f"{K2_SMALL},{K2_MEDIUM}", + lite=False, + capability_profile="k2-horizon-v1", + capability_models_csv=K2_SMALL, + port=13305, + ) + + self.assertEqual( + command[0:2], + ["python", str(ROOT / "test" / "validate_llamacpp.py")], + ) + self.assertIn("--channel", command) + self.assertIn("nightly", command) + self.assertEqual(command.count("--model"), 2) + self.assertIn("--capability-profile", command) + self.assertIn("llamacpp_validation_windows-rocm-nightly.json", command) + + with self.assertRaisesRegex(ValueError, "mutually exclusive"): + runner.build_validation_command( + python_executable=Path("python"), + backend="cpu", + channel="", + target="linux-cpu", + models_csv=K2_SMALL, + lite=True, + capability_profile="k2-horizon-v1", + capability_models_csv=K2_SMALL, + port=13305, + ) + + inherited = {"HF_HUB_CACHE": "shared-cache", "PRESERVED": "yes"} + environment = runner.build_server_environment( + Path("target-isolated-cache"), + inherited=inherited, + private_temp_directory=Path("target-private-temp"), + candidate_defaults_path=Path("candidate-defaults.json"), + ) + self.assertEqual( + environment["HF_HUB_CACHE"], + str(Path("target-isolated-cache").resolve()), + ) + self.assertNotIn("PRESERVED", environment) + self.assertEqual(inherited["HF_HUB_CACHE"], "shared-cache") + + builtin_small_command = runner.build_validation_command( + python_executable=Path("python"), + backend="metal", + channel="", + target="macos-metal", + models_csv=f"builtin.{K2_SMALL}", + lite=False, + capability_profile="k2-horizon-v1", + capability_models_csv=K2_SMALL, + port=13305, + ) + self.assertEqual(builtin_small_command.count("--model"), 1) + + def test_validation_plan_supports_direct_script_invocation(self) -> None: + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "test/utils/llamacpp_validation_plan.py"), + "--help", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + + def test_validation_runner_supports_direct_script_invocation(self) -> None: + completed = subprocess.run( + [sys.executable, str(VALIDATION_RUNNER), "--help"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) -class LlamaCppValidationPlanTests(unittest.TestCase): def assert_active_lanes(self, rows): self.assertEqual( - [(row["backend"], row["channel"]) for row in rows], - [("vulkan", ""), ("rocm", "stable"), ("rocm", "nightly")], + [ + ( + row["target"], + row["backend"], + row["channel"], + row["managed_pin"], + row["build_platform"], + ) + for row in rows + ], + [ + ("windows-vulkan", "vulkan", "", "vulkan", "windows"), + ("linux-vulkan", "vulkan", "", "vulkan", "linux"), + ("linux-cpu", "cpu", "", "cpu", "linux"), + ("macos-metal", "metal", "", "metal", "macos"), + ("windows-cuda", "cuda", "", "cuda", "windows"), + ( + "windows-rocm-stable", + "rocm", + "stable", + "rocm-stable", + "windows", + ), + ( + "windows-rocm-nightly", + "rocm", + "nightly", + "rocm-nightly", + "windows", + ), + ], ) self.assertEqual( - [row["target"] for row in rows], - ["windows-vulkan", "rocm-stable", "rocm-nightly"], + {row["managed_pin"] for row in rows}, + {"cpu", "cuda", "metal", "rocm-nightly", "rocm-stable", "vulkan"}, + ) + vulkan_platforms = { + row["build_platform"] for row in rows if row["backend"] == "vulkan" + } + self.assertEqual(vulkan_platforms, {"linux", "windows"}) + self.assertTrue( + all(row["capability_profile"] == "k2-horizon-v1" for row in rows) ) + self.assertTrue(all(row["capability_models"] == [K2_SMALL] for row in rows)) + self.assertTrue(all(K2_SMALL in row["models"] for row in rows if row["models"])) + + for row in rows: + if K2_MEDIUM in row["models"] or K2_LARGE in row["models"]: + self.assertEqual(row["target"], "windows-vulkan") + self.assertIn("128gb", row["runner"]) def test_pull_request_and_merge_group_require_k2(self) -> None: for event_name in ("pull_request", "merge_group"): @@ -40,51 +3590,91 @@ def test_pull_request_and_merge_group_require_k2(self) -> None: self.assert_active_lanes(rows) self.assertEqual(rows[0]["models"], [K2_SMALL, K2_MEDIUM, K2_LARGE]) - self.assertEqual(rows[1]["models"], [K2_SMALL]) - self.assertEqual(rows[2]["models"], [K2_SMALL]) + self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) self.assertTrue(all(not row["lite"] for row in rows)) self.assertIn("128gb", rows[0]["runner"]) - self.assertTrue( - all(row["capability_profile"] == "k2-horizon-v1" for row in rows) - ) - self.assertTrue( - all(row["capability_models"] == [K2_SMALL] for row in rows) - ) - def test_schedule_captures_exact_hot_model_evidence_contract(self) -> None: + def test_schedule_captures_exact_validation_candidate_contract(self) -> None: rows = planning.create_validation_plan("schedule")["include"] self.assert_active_lanes(rows) - expected = planning.load_hot_llamacpp_model_ids() + expected = planning.load_validation_candidate_model_ids() self.assertIn(K2_SMALL, expected) - self.assertTrue(all(row["models"] == [] for row in rows)) - self.assertTrue(all(row["expected_models"] == expected for row in rows)) + self.assertIn(K2_MEDIUM, expected) + self.assertIn(K2_LARGE, expected) + self.assertEqual(rows[0]["models"], expected) + self.assertEqual(rows[0]["expected_models"], expected) + self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) + self.assertTrue(all(row["expected_models"] == [K2_SMALL] for row in rows[1:])) self.assertTrue(all(not row["lite"] for row in rows)) - self.assertTrue(all("128gb" in row["runner"] for row in rows)) - self.assertTrue( - all(row["capability_profile"] == "k2-horizon-v1" for row in rows) + self.assertIn("128gb", rows[0]["runner"]) + + def test_validation_candidate_set_reads_overlay_outside_product_catalog( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + registry = Path(directory) / "server_models.json" + overlay = Path(directory) / "validation_models.json" + registry.write_text( + json.dumps( + { + "Existing-Hot-Model": { + "recipe": "llamacpp", + "labels": ["chat", "hot"], + } + } + ), + encoding="utf-8", + ) + overlay.write_text( + json.dumps( + { + "Candidate-Not-Hot": { + "recipe": "llamacpp", + "labels": ["chat"], + } + } + ), + encoding="utf-8", + ) + + models = planning.load_validation_candidate_model_ids(registry, overlay) + + self.assertEqual( + models, + ["Candidate-Not-Hot", "Existing-Hot-Model"], ) - self.assertTrue(all(row["capability_models"] == [K2_SMALL] for row in rows)) - def test_default_dispatch_keeps_runtime_hot_selection(self) -> None: + def test_default_dispatch_selects_hot_models_and_its_capability_model( + self, + ) -> None: rows = planning.create_validation_plan("workflow_dispatch")["include"] self.assert_active_lanes(rows) - self.assertTrue(all(row["models"] == [] for row in rows)) + self.assertEqual( + set(rows[0]["models"]), + {K2_SMALL, *planning.load_hot_llamacpp_model_ids()}, + ) + self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) + self.assertTrue( + all( + set(row["capability_models"]).issubset( + {model.removeprefix("builtin.") for model in row["models"]} + ) + for row in rows + ) + ) self.assertTrue(all(not row["lite"] for row in rows)) - self.assertTrue(all("128gb" in row["runner"] for row in rows)) - self.assertTrue(all("capability_profile" not in row for row in rows)) - self.assertTrue(all("capability_models" not in row for row in rows)) + self.assertIn("128gb", rows[0]["runner"]) - def test_manual_lite_selection_uses_small_runners(self) -> None: + def test_manual_lite_selection_runs_only_k2_small(self) -> None: rows = planning.create_validation_plan("workflow_dispatch", lite=True)[ "include" ] self.assert_active_lanes(rows) - self.assertTrue(all(row["models"] == [] for row in rows)) - self.assertTrue(all(row["lite"] for row in rows)) - self.assertTrue(all("stx-halo" in row["runner"] for row in rows)) + self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows)) + self.assertTrue(all(not row["lite"] for row in rows)) def test_manual_models_are_trimmed_and_keep_order_and_duplicates(self) -> None: rows = planning.create_validation_plan( @@ -93,9 +3683,39 @@ def test_manual_models_are_trimmed_and_keep_order_and_duplicates(self) -> None: )["include"] self.assert_active_lanes(rows) - self.assertTrue(all(row["models"] == ["Zulu", "Alpha", "Zulu"] for row in rows)) + self.assertEqual( + rows[0]["models"], + [K2_SMALL, "Zulu", "Alpha", "Zulu"], + ) + self.assertTrue( + all( + row["models"] == [K2_SMALL, "Zulu", "Alpha", "Zulu"] for row in rows[1:] + ) + ) self.assertTrue(all(not row["lite"] for row in rows)) - self.assertTrue(all("128gb" in row["runner"] for row in rows)) + self.assertIn("128gb", rows[0]["runner"]) + + def test_manual_large_k2_models_are_confined_to_large_vulkan_lane(self) -> None: + rows = planning.create_validation_plan( + "workflow_dispatch", + models_csv=f"{K2_MEDIUM},{K2_LARGE}", + )["include"] + + self.assert_active_lanes(rows) + self.assertEqual(rows[0]["models"], [K2_SMALL, K2_MEDIUM, K2_LARGE]) + self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) + + def test_manual_builtin_large_k2_aliases_are_confined_to_large_lane(self) -> None: + builtin_medium = f"builtin.{K2_MEDIUM}" + builtin_large = f"builtin.{K2_LARGE}" + rows = planning.create_validation_plan( + "workflow_dispatch", + models_csv=f"{builtin_medium},{builtin_large}", + )["include"] + + self.assert_active_lanes(rows) + self.assertEqual(rows[0]["models"], [K2_SMALL, builtin_medium, builtin_large]) + self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) def test_manual_models_and_lite_are_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "mutually exclusive"): @@ -134,6 +3754,47 @@ def test_only_scheduled_runs_are_promotion_eligible(self) -> None: expected, ) + def test_promotion_coverage_rejects_missing_or_mapped_wrong_lanes(self) -> None: + complete = planning.create_validation_plan("schedule") + planning.require_complete_promotion_coverage(complete) + + missing = json.loads(json.dumps(complete)) + missing["include"].pop() + with self.assertRaisesRegex(ValueError, "exactly"): + planning.require_complete_promotion_coverage(missing) + + wrong_mapping = json.loads(json.dumps(complete)) + wrong_mapping["include"][2]["managed_pin"] = "cuda" + with self.assertRaisesRegex(ValueError, "mapping"): + planning.require_complete_promotion_coverage(wrong_mapping) + + wrong_vulkan_platform = json.loads(json.dumps(complete)) + wrong_vulkan_platform["include"][1]["build_platform"] = "windows" + with self.assertRaisesRegex(ValueError, "mapping"): + planning.require_complete_promotion_coverage(wrong_vulkan_platform) + + unexecuted_models = json.loads(json.dumps(complete)) + unexecuted_models["include"][0]["models"] = [] + with self.assertRaisesRegex(ValueError, "selected models"): + planning.require_complete_promotion_coverage(unexecuted_models) + + lite_lane = json.loads(json.dumps(complete)) + lite_lane["include"][0]["lite"] = True + with self.assertRaisesRegex(ValueError, "lite"): + planning.require_complete_promotion_coverage(lite_lane) + + missing_candidate = json.loads(json.dumps(complete)) + for field in ("models", "expected_models"): + missing_candidate["include"][0][field].remove(K2_MEDIUM) + with self.assertRaisesRegex(ValueError, "exact candidate model set"): + planning.require_complete_promotion_coverage(missing_candidate) + + unexpected_small_lane_model = json.loads(json.dumps(complete)) + for field in ("models", "expected_models"): + unexpected_small_lane_model["include"][1][field].append("Other-Model") + with self.assertRaisesRegex(ValueError, "exact K2 small model set"): + planning.require_complete_promotion_coverage(unexpected_small_lane_model) + def test_github_output_contains_compact_matrix_json(self) -> None: plan = planning.create_validation_plan("merge_group") with tempfile.TemporaryDirectory() as directory: @@ -153,12 +3814,17 @@ def test_github_output_contains_compact_matrix_json(self) -> None: self.assertNotIn("\n", outputs["matrix"]) self.assertEqual(outputs["promotion_eligible"], "false") - def test_validation_core_is_read_only_and_has_no_legacy_entrypoint(self) -> None: + def test_validation_core_has_constrained_permissions_and_no_legacy_entrypoint( + self, + ) -> None: self.assertFalse(LEGACY_WORKFLOW.exists()) validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") validation_triggers = validation.split("permissions:", 1)[0] workflow_scope = validation.split("jobs:", 1)[0] + invocation_job = validation.split(" validate-invocation:\n", 1)[1].split( + " plan:\n", 1 + )[0] plan_job = validation.split(" plan:\n", 1)[1].split( " get-latest-releases:\n", 1 )[0] @@ -171,7 +3837,13 @@ def test_validation_core_is_read_only_and_has_no_legacy_entrypoint(self) -> None )[1].split(" - name: Upload results\n", 1)[0] result_upload = validate_job.split(" - name: Upload results\n", 1)[ 1 - ].split(" - name: Upload server logs\n", 1)[0] + ].split(" - name: Upload restart results\n", 1)[0] + restart_result_upload = validate_job.split( + " - name: Upload restart results\n", 1 + )[1].split(" - name: Upload server logs\n", 1)[0] + server_log_upload = validate_job.split(" - name: Upload server logs\n", 1)[ + 1 + ].split(" - name: Cleanup\n", 1)[0] self.assertIn(" workflow_call:\n", validation_triggers) self.assertNotIn(" pull_request_target:\n", validation_triggers) @@ -182,7 +3854,8 @@ def test_validation_core_is_read_only_and_has_no_legacy_entrypoint(self) -> None self.assertIn("permissions:\n contents: read\n", validation) self.assertNotIn("contents: write", validation) self.assertNotIn("pull-requests: write", validation) - self.assertNotIn("checks: write", validation) + self.assertEqual(validation.count("checks: write"), 1) + self.assertIn("checks: write", invocation_job) self.assertNotIn("concurrency:", workflow_scope) self.assertNotIn(" create-pr:\n", validation) self.assertIn(" HUGGINGFACE_ACCESS_TOKEN:\n", validation_triggers) @@ -192,26 +3865,166 @@ def test_validation_core_is_read_only_and_has_no_legacy_entrypoint(self) -> None self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", build_job) self.assertEqual(validate_job.count("HF_TOKEN:"), 1) self.assertIn( - " HF_TOKEN: ${{ github.event_name != 'pull_request_target' && github.event_name != 'merge_group' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }}\n", + " HF_TOKEN: ${{ github.event_name == 'schedule' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }}\n", validation_step, ) self.assertNotIn( " HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}\n", validation_step, ) - self.assertIn(" $cleanupExitCode = 0\n", validation_step) - self.assertIn(" $cleanupExitCode = 1\n", validation_step) - self.assertIn(" if ($cleanupExitCode -ne 0) {\n", validation_step) + self.assertIn(".github/scripts/run_llamacpp_validation.py", validation_step) + self.assertIn(" id: run-validation\n", validation_step) + self.assertIn("steps.validation-venv.outputs.python-path", validation_step) + self.assertIn("--lemond", validation_step) + self.assertIn( + "${{ github.event_name == 'schedule' && '--allow-huggingface-download-credentials' || '' }}", + validation_step, + ) + self.assertEqual(validation_step.count("LEMONADE_PROTECTED_RUNNER_CONTEXT:"), 1) + self.assertEqual( + validation_step.count("--allow-darwin-github-hosted-ephemeral-runner"), + 1, + ) + self.assertIn("matrix.target == 'macos-metal'", validation_step) + self.assertIn("runner.environment == 'github-hosted'", validation_step) + self.assertIn("join(matrix.runner, ',') == 'macos-latest'", validation_step) + self.assertIn("github-hosted:macos-latest:macos-metal", validation_step) + self.assertNotIn("shell: PowerShell", validation_step) + protected_hosted_mac_exclusion = ( + "!((github.event_name == 'pull_request_target' || " + "github.event_name == 'merge_group') &&" + ) + for upload in (result_upload, restart_result_upload, server_log_upload): + self.assertIn(" if: >-\n", upload) + self.assertIn( + " steps.run-validation.outcome == 'success' &&\n", upload + ) + self.assertIn(protected_hosted_mac_exclusion, upload) + self.assertIn("matrix.target == 'macos-metal'", upload) + self.assertIn("runner.environment == 'github-hosted'", upload) + self.assertIn("join(matrix.runner, ',') == 'macos-latest'", upload) + self.assertIn( + "path: ${{ steps.run-validation.outputs.validation_result }}", + result_upload, + ) self.assertIn(" if-no-files-found: error\n", result_upload) + self.assertNotIn("llamacpp_restart_validation_", restart_result_upload) + self.assertIn(" if-no-files-found: error\n", restart_result_upload) + self.assertIn( + "path: ${{ steps.run-validation.outputs.restart_result }}", + restart_result_upload, + ) + self.assertNotIn("server-restart-logs", server_log_upload) + for output_name in ( + "prepare_stdout_log", + "prepare_stderr_log", + "validation_stdout_log", + "validation_stderr_log", + "restart_stdout_log", + "restart_stderr_log", + ): + self.assertIn( + f"steps.run-validation.outputs.{output_name}", + server_log_upload, + ) + self.assertIn(" if-no-files-found: error\n", server_log_upload) + self.assertNotIn("if: always()", result_upload) + self.assertNotIn("if: always()", restart_result_upload) + self.assertNotIn("if: always()", server_log_upload) + + runner = VALIDATION_RUNNER.read_text(encoding="utf-8") + self.assertIn("verify_validation_artifacts", runner) + self.assertIn('cache_directory / "hf-hub"', runner) + self.assertIn('environment["HF_HUB_CACHE"]', runner) + self.assertIn("VALIDATION_ARTIFACT_LOCKS", runner) + + def test_build_and_validation_orchestration_are_native_per_platform(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + build_job = validation.split(" build:\n", 1)[1].split(" validate:\n", 1)[0] + validate_job = validation.split(" validate:\n", 1)[1].split( + " reverify-protected-release-manifest:\n", 1 + )[0] + + for platform, runner in ( + ("windows", "windows-latest"), + ("linux", "ubuntu-latest"), + ("macos", "macos-latest"), + ): + with self.subTest(platform=platform): + self.assertIn(f"platform: {platform}", build_job) + self.assertIn(f"runner: {runner}", build_job) + self.assertIn("runs-on: ${{ matrix.runner }}", build_job) + self.assertIn("llamacpp-build-${{ matrix.platform }}", build_job) + self.assertIn("if: runner.os == 'Windows'", build_job) + self.assertIn("if: runner.os != 'Windows'", build_job) + self.assertIn("llamacpp-build-${{ matrix.build_platform }}", validate_job) + self.assertIn("if: runner.os == 'Windows'", validate_job) + self.assertIn("if: runner.os == 'Linux'", validate_job) + self.assertIn("if: runner.os != 'Windows'", validate_job) + + protected_overlay_step = build_job.split( + " - name: Stage protected validation-only model catalog\n", 1 + )[1].split(" - name: Stage branch validation-only model catalog\n", 1)[0] + branch_overlay_step = build_job.split( + " - name: Stage branch validation-only model catalog\n", 1 + )[1].split(" - name: Install Linux build dependencies\n", 1)[0] + self.assertIn( + "$GITHUB_WORKSPACE/test/utils/validation_model_catalog.py", + protected_overlay_step, + ) + self.assertIn("$GITHUB_WORKSPACE/candidate", protected_overlay_step) + self.assertIn( + "$GITHUB_WORKSPACE/test/fixtures/llamacpp_validation_models.json", + protected_overlay_step, + ) + self.assertIn( + "python -m test.utils.validation_model_catalog", branch_overlay_step + ) + self.assertIn( + "--base src/cpp/resources/server_models.json", branch_overlay_step + ) + self.assertIn( + "--overlay test/fixtures/llamacpp_validation_models.json", + branch_overlay_step, + ) + self.assertIn( + "--output src/cpp/resources/server_models.json", branch_overlay_step + ) + self.assertLess( + build_job.index( + " - name: Stage protected validation-only model catalog\n" + ), + build_job.index(" cmake --preset vs18"), + ) + self.assertLess( + build_job.index( + " - name: Stage branch validation-only model catalog\n" + ), + build_job.index(" cmake --preset default"), + ) def test_focused_ci_runs_the_capability_contract_tests(self) -> None: workflow = DOCS_AND_STYLE_WORKFLOW.read_text(encoding="utf-8") self.assertIn("test.test_llamacpp_capabilities", workflow) + self.assertIn("test.test_llamacpp_validation_artifacts", workflow) self.assertIn("test.test_llamacpp_validation_evidence", workflow) + self.assertIn("test.test_llamacpp_pr_file_classification", workflow) + self.assertIn("test.test_run_llamacpp_validation", workflow) + self.assertIn("test.test_validation_model_catalog", workflow) + self.assertIn("test.test_validate_llamacpp_action_pins", workflow) self.assertIn("test.test_validate_llamacpp_selection", workflow) + self.assertIn("test.test_validate_llamacpp_trusted_execution", workflow) self.assertIn("test.test_verify_llamacpp_pin_manifest", workflow) + def test_focused_ci_installs_process_tracking_dependency(self) -> None: + workflow = DOCS_AND_STYLE_WORKFLOW.read_text(encoding="utf-8") + focused_tests = workflow.split( + " - name: Run focused Python unit tests\n", 1 + )[0] + + self.assertIn("pip install pre-commit psutil", focused_tests) + def test_release_asset_manifest_is_captured_and_exported(self) -> None: validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") release_job = validation.split(" get-latest-releases:\n", 1)[1].split( @@ -233,6 +4046,8 @@ def test_release_asset_manifest_is_captured_and_exported(self) -> None: self.assertIn("attach_release_claims", asset_job) self.assertIn("--source-manifest-asset-id", asset_job) self.assertIn("--validate-source-manifest", asset_job) + self.assertIn("--build-target-attestation-output", asset_job) + self.assertIn(".build_target_attestations", asset_job) self.assertIn("Accept: application/octet-stream", asset_job) self.assertIn("source_manifest_max_bytes=65536", asset_job) self.assertIn("source_manifest_directory=$(mktemp -d)", asset_job) @@ -350,6 +4165,92 @@ def test_manifest_recheck_anchors_to_live_trusted_upstream(self) -> None: ) self.assertIn("release_pattern='^b[0-9]+$'", capture) + def test_release_commit_resolution_uses_explicit_ref_namespaces(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + asset_job = validation.split(" verify-release-assets:\n", 1)[1].split( + " build:\n", 1 + )[0] + capture = ( + ROOT / ".github" / "scripts" / "capture_llamacpp_release_manifest.sh" + ).read_text(encoding="utf-8") + + for source in (asset_job, capture): + with self.subTest(source="workflow" if source is asset_job else "capture"): + self.assertIn( + "commits/heads/${source_default_branch}", + source, + ) + self.assertNotIn( + "commits/${source_default_branch}", + source, + ) + self.assertIn( + '"repos/${repository}/commits/tags/${tag}"', + source, + ) + self.assertNotIn( + '"repos/${repository}/commits/${tag}"', + source, + ) + + def test_release_asset_eligibility_uses_size_and_rocm_target_metadata(self) -> None: + validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") + asset_job = validation.split(" verify-release-assets:\n", 1)[1].split( + " build:\n", 1 + )[0] + + self.assertIn("def read_asset_metadata(path):", asset_job) + self.assertIn('asset_sizes[asset["name"]] = asset["size"]', asset_job) + self.assertIn('read_asset_metadata("ggml_release.json")', asset_job) + self.assertIn('read_asset_metadata("rocm_release.json")', asset_job) + self.assertIn('read_asset_metadata("lemonade_release.json")', asset_job) + self.assertIn("build_rocm_asset_target_requirements", asset_job) + self.assertIn("required_asset_targets=rocm_target_requirements", asset_job) + self.assertIn("attested_asset_targets=rocm_attested_targets", asset_job) + + versions = { + "therock": {"version": "7.14.0"}, + "rocm_asset_families": { + "gfx1033": "gfx103X", + "gfx1035": "gfx103X", + "gfx1036": "gfx103X", + }, + } + requirements = release_assets.build_asset_requirements( + ggml_release="b100", + rocm_release="b300", + lemonade_release="b200", + backend_versions=versions, + ) + required_targets = release_assets.build_rocm_asset_target_requirements( + rocm_release="b300", + backend_versions=versions, + ) + available = { + asset_name: 1 for asset_name in requirements["rocm"]["rocm-nightly"] + } + for platform in ("windows", "ubuntu"): + target_asset = f"llama-b300-{platform}-rocm-gfx103X-x64.zip" + for missing_target in ("gfx1033", "gfx1035", "gfx1036"): + with self.subTest( + platform=platform, + missing_target=missing_target, + ): + attested_targets = dict(required_targets) + attested_targets[target_asset] = tuple( + target + for target in required_targets[target_asset] + if target != missing_target + ) + eligible, missing = release_assets.evaluate_asset_group( + available, + requirements["rocm"], + required_asset_targets=required_targets, + attested_asset_targets=attested_targets, + ) + self.assertEqual(eligible, []) + self.assertEqual(missing["rocm-nightly"], [target_asset]) + def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: pull_request = PR_WORKFLOW.read_text(encoding="utf-8") validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") @@ -357,90 +4258,92 @@ def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: concurrency = pull_request.split("concurrency:\n", 1)[1].split("\njobs:\n", 1)[ 0 ] - authorization_job = pull_request.split(" authorize-invocation:\n", 1)[1].split( - " start-check:\n", 1 - )[0] - start_check_job = pull_request.split(" start-check:\n", 1)[1].split( - " validate:\n", 1 + policy_job = pull_request.split(" policy:\n", 1)[1].split(" validate:\n", 1)[ + 0 + ] + invocation_job = validation.split(" validate-invocation:\n", 1)[1].split( + " plan:\n", 1 )[0] validation_job = pull_request.split(" validate:\n", 1)[1].split( - " report:\n", 1 + " reconcile:\n", 1 )[0] - report_job = pull_request.split(" report:\n", 1)[1] + report_job = pull_request.split(" reconcile:\n", 1)[1] report_step = report_job.split( - " - name: Report result on the authorized merge commit\n", 1 - )[1].split(" - name: Enforce result\n", 1)[0] + " - name: Report result on the exact candidate commit\n", 1 + )[1].split(" - name: Enforce policy result\n", 1)[0] self.assertIn(" pull_request_target:\n", pull_request_triggers) self.assertIn(" merge_group:\n", pull_request_triggers) self.assertNotIn(" pull_request:\n", pull_request_triggers) self.assertNotIn(" workflow_call:\n", pull_request_triggers) self.assertIn( - " types: [synchronize, edited, labeled, unlabeled, closed]\n", + "types: [opened, reopened, synchronize, edited, labeled, unlabeled, closed]", pull_request_triggers, ) self.assertIn("github.event.pull_request.number", concurrency) - self.assertNotIn("github.ref", concurrency) - self.assertIn("github.run_id", concurrency) - self.assertIn("github.event.action == 'synchronize'", concurrency) - self.assertIn("github.event.action == 'edited'", concurrency) - self.assertIn("github.event.changes.base.ref.from", concurrency) - self.assertIn("github.event.action == 'closed'", concurrency) - self.assertIn("github.event.action == 'unlabeled'", concurrency) - self.assertIn("github.event.label.name == 'ci:upgrades'", concurrency) + self.assertIn("github.event.merge_group.head_sha", concurrency) + self.assertIn("github.run_number", concurrency) + self.assertIn("github.run_attempt", concurrency) + self.assertNotIn("github.run_id", concurrency) self.assertNotIn("cancel-in-progress: true", concurrency) - self.assertIn("github.event.action", authorization_job) - self.assertIn('if [ "$EVENT_NAME" = "merge_group" ]', authorization_job) - self.assertIn("github.event.label.name", authorization_job) - self.assertIn("github.actor", authorization_job) - self.assertIn("collaborators/${ACTOR}/permission", authorization_job) - self.assertIn("admin|maintain|write|push", authorization_job) - self.assertIn( - 'echo "run_validation=true" >> "$GITHUB_OUTPUT"', authorization_job - ) - self.assertNotIn("actions/checkout", authorization_job) - - self.assertIn("needs: authorize-invocation", start_check_job) - self.assertIn("checks: write", start_check_job) - self.assertIn("github.event.pull_request.merge_commit_sha", start_check_job) - self.assertIn("github.event.merge_group.head_sha", start_check_job) - self.assertNotIn("github.event.pull_request.head.sha", start_check_job) - self.assertIn('head_sha="$MERGE_SHA"', start_check_job) - self.assertIn("CHECK_NAME: llama.cpp authorized validation v2", start_check_job) - self.assertIn('name="$CHECK_NAME"', start_check_job) - self.assertIn('external_id="$EXTERNAL_ID"', start_check_job) - self.assertIn("github.run_id", start_check_job) - self.assertIn("github.run_attempt", start_check_job) - self.assertIn("status=in_progress", start_check_job) - self.assertIn("check_run_id", start_check_job) - - self.assertIn("needs: [authorize-invocation, start-check]", validation_job) - self.assertIn( - "needs.authorize-invocation.outputs.run_validation == 'true'", + self.assertIn('if [ "$EVENT_NAME" = "merge_group" ]', policy_job) + self.assertIn("collaborators/${authorization_actor}/permission", policy_job) + self.assertIn("merge_candidate_available", policy_job) + self.assertIn('candidate_sha="$current_head"', policy_job) + self.assertIn("--paginate --slurp", policy_job) + self.assertIn('.app.slug == "github-actions"', policy_job) + + self.assertNotIn(" start-check:\n", pull_request) + self.assertIn("checks: write", invocation_job) + self.assertIn("github.event.pull_request.merge_commit_sha", invocation_job) + self.assertIn("github.event.merge_group.head_sha", invocation_job) + self.assertNotIn("github.event.pull_request.head.sha", invocation_job) + self.assertIn('head_sha="$MERGE_SHA"', invocation_job) + self.assertIn("CHECK_NAME: llama.cpp authorized validation v2", invocation_job) + self.assertIn('name="$CHECK_NAME"', invocation_job) + self.assertIn('external_id="$EXTERNAL_ID"', invocation_job) + self.assertIn("github.run_id", invocation_job) + self.assertIn("github.run_number", invocation_job) + self.assertIn("github.run_attempt", invocation_job) + self.assertIn("status=in_progress", invocation_job) + self.assertIn("check_run_id", invocation_job) + self.assertIn("--paginate --slurp", invocation_job) + self.assertIn('.app.slug == "github-actions"', invocation_job) + self.assertIn("newest_number", invocation_job) + self.assertIn(".number < $current_number", invocation_job) + self.assertIn( + "value: ${{ jobs.validate-invocation.outputs.check_run_id }}", validation + ) + + self.assertIn("needs: policy", validation_job) + self.assertNotIn("start-check", validation_job) + self.assertIn( + "needs.policy.outputs.run_validation == 'true'", validation_job, ) self.assertIn( "uses: ./.github/workflows/validate_llamacpp_core.yml", validation_job ) - self.assertIn("permissions:\n contents: read", validation_job) + self.assertIn( + "permissions:\n checks: write\n contents: read", validation_job + ) self.assertNotIn("secrets:", validation_job) self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", pull_request) - self.assertIn( - "needs: [authorize-invocation, start-check, validate]", report_job - ) + self.assertIn("needs: [policy, validate]", report_job) + self.assertIn("if: always()", report_job) self.assertIn("checks: write", report_job) self.assertIn("repos/${GITHUB_REPOSITORY}/check-runs", report_job) self.assertIn("github.event.pull_request.head.sha", report_job) - self.assertIn("needs.start-check.outputs.check_run_id", report_job) + self.assertIn("needs.validate.outputs.check_run_id", report_job) self.assertIn("commits/${MERGE_SHA}/check-runs", report_step) self.assertIn(".external_id", report_step) self.assertIn("--arg external_id", report_step) self.assertIn('head_sha="$MERGE_SHA"', report_step) self.assertIn("CHECK_NAME: llama.cpp authorized validation v2", report_job) self.assertIn('name="$CHECK_NAME"', report_step) - self.assertIn('external_id="$EXTERNAL_ID"', report_step) + self.assertIn('external_id="$external_id"', report_step) self.assertNotIn('head_sha="$HEAD_SHA"', report_step) self.assertNotIn('-f name="llama.cpp validation"', pull_request) self.assertNotIn(" name: llama.cpp validation\n", pull_request) @@ -454,20 +4357,17 @@ def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: self.assertIn(".base.sha", report_job) self.assertIn(".head.sha", report_job) self.assertIn(".merge_commit_sha", report_job) + self.assertIn('current_candidate="$current_head"', report_step) + self.assertIn('.app.slug == "github-actions"', report_step) + self.assertIn("newest_number", report_step) + self.assertIn(".number < $current_number", report_step) self.assertLess( report_step.index("repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}"), report_step.index("--method PATCH"), ) - self.assertIn("github.event.action == 'labeled'", report_job) - self.assertIn("github.event.label.name == 'ci:upgrades'", report_job) self.assertIn("needs.validate.result", report_job) - self.assertIn("github.event.action == 'labeled'", report_step) - self.assertIn("github.event.label.name == 'ci:upgrades'", report_step) self.assertIn(" validate-invocation:\n", validation) - invocation_job = validation.split(" validate-invocation:\n", 1)[1].split( - " plan:\n", 1 - )[0] plan_job = validation.split(" plan:\n", 1)[1].split( " get-latest-releases:\n", 1 )[0] @@ -501,51 +4401,36 @@ def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: ) self.assertIn("github.event.pull_request.base.sha", validation) self.assertIn("github.event.pull_request.head.sha", validation) - self.assertEqual(validation.count("persist-credentials: false"), 8) + self.assertEqual(validation.count("persist-credentials: false"), 10) - def test_authorization_removal_revokes_the_merge_check(self) -> None: + def test_authorization_removal_reconciles_from_fresh_live_state(self) -> None: pull_request = PR_WORKFLOW.read_text(encoding="utf-8") + policy_job = pull_request.split(" policy:\n", 1)[1].split(" validate:\n", 1)[ + 0 + ] + helper = REQUIRED_CHECK_POLICY.read_text(encoding="utf-8") - self.assertIn(" revoke-check:\n", pull_request) - revoke_job = pull_request.split(" revoke-check:\n", 1)[1].split( - " start-check:\n", 1 - )[0] + self.assertIn("current_authorized", policy_job) + self.assertIn("authorization_label_present=$current_authorized", policy_job) + self.assertIn("EVENT_ACTION", policy_job) + self.assertIn("EVENT_LABEL", policy_job) + self.assertIn("AUTHORIZATION_LABEL_PRESENT", policy_job) + self.assertIn("not authorization_label_present", helper) + self.assertIn("conclusion=failure", pull_request) + self.assertNotIn(" revoke-check:\n", pull_request) - self.assertIn("github.event.action == 'unlabeled'", revoke_job) - self.assertIn("github.event.label.name == 'ci:upgrades'", revoke_job) - self.assertIn("github.event.pull_request.state == 'open'", revoke_job) - self.assertIn("checks: write", revoke_job) - self.assertIn("pull-requests: read", revoke_job) - self.assertIn("repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", revoke_job) - self.assertIn("current_state", revoke_job) - self.assertIn("current_authorized", revoke_job) - self.assertIn( - 'if [ "$current_state" != "open" ] || ' - '[ "$current_authorized" = "true" ]', - revoke_job, - ) - self.assertIn("github.event.pull_request.merge_commit_sha", revoke_job) - self.assertIn("commits/${MERGE_SHA}/check-runs", revoke_job) - self.assertIn("llama.cpp authorized validation v2", revoke_job) - self.assertIn("conclusion=cancelled", revoke_job) - self.assertIn("--method PATCH", revoke_job) - self.assertIn("--method POST", revoke_job) - self.assertNotIn("actions/checkout", revoke_job) - - def test_closed_pull_requests_cannot_start_protected_validation(self) -> None: + def test_closed_pull_requests_use_live_merged_state_before_mutating(self) -> None: pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - authorization_job = pull_request.split(" authorize-invocation:\n", 1)[1].split( - " revoke-check:\n", 1 - )[0] + helper = REQUIRED_CHECK_POLICY.read_text(encoding="utf-8") - self.assertIn("current_state=$(jq -r '.state'", authorization_job) - self.assertIn('if [ "$current_state" != "open" ]', authorization_job) - state_guard = authorization_job.index('if [ "$current_state" != "open" ]') - authorization_output = authorization_job.index( - 'echo "run_validation=true" >> "$GITHUB_OUTPUT"', - state_guard, - ) - self.assertLess(state_guard, authorization_output) + self.assertIn("current_state", pull_request) + self.assertIn("current_merged", pull_request) + self.assertIn("if merged:", helper) + self.assertIn('if state == "closed":', helper) + self.assertIn("mode != 'merged'", pull_request) + self.assertIn("mode != 'skip_closed'", pull_request) + self.assertNotIn(" authorize-invocation:\n", pull_request) + self.assertNotIn(" revoke-check:\n", pull_request) def test_protected_change_runner_matrix_is_defined_by_trusted_yaml(self) -> None: validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") @@ -568,11 +4453,35 @@ def test_protected_change_runner_matrix_is_defined_by_trusted_yaml(self) -> None self.assertIn('"K2-Horizon-3.7B-GGUF"', trusted_step) self.assertIn('"K2-Horizon-7B-GGUF"', trusted_step) self.assertIn('target: "windows-vulkan"', trusted_step) - self.assertEqual(trusted_step.count('capability_profile: "k2-horizon-v1"'), 3) + for target in ( + "windows-vulkan", + "linux-vulkan", + "linux-cpu", + "macos-metal", + "windows-cuda", + "windows-rocm-stable", + "windows-rocm-nightly", + ): + with self.subTest(target=target): + self.assertIn(f'target: "{target}"', trusted_step) + self.assertEqual(trusted_step.count('capability_profile: "k2-horizon-v1"'), 7) self.assertEqual( trusted_step.count('capability_models: ["K2-Horizon-0.9B-GGUF"]'), - 3, + 7, ) + self.assertEqual(trusted_step.count('managed_pin: "vulkan"'), 2) + for managed_pin in ( + "cpu", + "cuda", + "metal", + "rocm-stable", + "rocm-nightly", + ): + self.assertEqual(trusted_step.count(f'managed_pin: "{managed_pin}"'), 1) + self.assertEqual(trusted_step.count('build_platform: "windows"'), 4) + self.assertEqual(trusted_step.count('build_platform: "linux"'), 2) + self.assertEqual(trusted_step.count('build_platform: "macos"'), 1) + self.assertEqual(trusted_step.count('runner: ["macos-latest"]'), 1) self.assertNotIn("python -m test.utils", trusted_step) self.assertIn("github.event_name != 'pull_request_target'", branch_step) self.assertIn("github.event_name != 'merge_group'", branch_step) @@ -583,16 +4492,17 @@ def test_protected_change_runner_matrix_is_defined_by_trusted_yaml(self) -> None validate_job = validation.split(" validate:\n", 1)[1].split( " validation-gate:\n", 1 )[0] - self.assertIn(' $label = "${{ matrix.target }}"\n', validate_job) + self.assertIn(".github/scripts/run_llamacpp_validation.py", validate_job) + self.assertIn(' TARGET: "${{ matrix.target }}"\n', validate_job) self.assertIn( " CAPABILITY_PROFILE: ${{ matrix.capability_profile || '' }}\n", validate_job, ) self.assertIn(" CAPABILITY_MODELS:", validate_job) - self.assertIn("-CapabilityProfile $env:CAPABILITY_PROFILE", validate_job) - self.assertIn("-CapabilityModels $env:CAPABILITY_MODELS", validate_job) + self.assertIn(" LLAMACPP_BACKEND: ${{ matrix.backend }}", validate_job) + self.assertIn(" LLAMACPP_CHANNEL: ${{ matrix.channel }}", validate_job) self.assertIn("validation-results-${{ matrix.target }}", validate_job) - self.assertIn("llamacpp_validation_${{ matrix.target }}.json", validate_job) + self.assertIn("steps.run-validation.outputs.validation_result", validate_job) def test_manual_validation_calls_the_read_only_core(self) -> None: manual = MANUAL_WORKFLOW.read_text(encoding="utf-8") @@ -602,16 +4512,58 @@ def test_manual_validation_calls_the_read_only_core(self) -> None: self.assertNotIn(" schedule:\n", manual_triggers) self.assertIn("permissions:\n contents: read\n", manual) self.assertNotIn("contents: write", manual) - self.assertNotIn("checks: write", manual) + self.assertIn("permissions:\n checks: write\n contents: read", manual) self.assertNotIn("pull-requests: write", manual) self.assertNotIn("secrets: inherit", manual) + self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", manual) self.assertIn( - "HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}", + "uses: ./.github/workflows/validate_llamacpp_core.yml", manual, ) + + def test_validation_credential_aliases_are_scrubbed_by_trusted_control_plane( + self, + ) -> None: + step = workflow_step( + VALIDATION_WORKFLOW, + "validate", + "Run validation with lemond", + ) + environment = step["env"] + run = step["run"] + credential_aliases = { + "HF_TOKEN", + "HUGGINGFACE_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + } + + self.assertEqual( + {key for key in environment if key.upper() in credential_aliases}, + credential_aliases, + ) + self.assertEqual( + environment["HF_TOKEN"], + "${{ github.event_name == 'schedule' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }}", + ) + self.assertEqual(environment["HUGGINGFACE_TOKEN"], "") + self.assertEqual(environment["HUGGING_FACE_HUB_TOKEN"], "") + self.assertEqual(run.count("--allow-huggingface-download-credentials"), 1) self.assertIn( - "uses: ./.github/workflows/validate_llamacpp_core.yml", - manual, + "${{ github.event_name == 'schedule' && '--allow-huggingface-download-credentials' || '' }}", + run, + ) + self.assertIn( + 'python "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py"', + run, + ) + self.assertNotIn("VALIDATION_CANDIDATE_ROOT/.github/scripts", run) + + manual_job = load_workflow(MANUAL_WORKFLOW)["jobs"]["validate"] + schedule_job = load_workflow(SCHEDULE_WORKFLOW)["jobs"]["validate"] + self.assertNotIn("secrets", manual_job) + self.assertEqual( + schedule_job["secrets"], + {"HUGGINGFACE_ACCESS_TOKEN": ("${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}")}, ) def test_scheduled_publication_is_isolated_and_serialized(self) -> None: @@ -621,6 +4573,13 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: 0 ] publication = schedule.split(" publish:\n", 1)[1] + checkout_marker = f" - uses: actions/checkout@{CHECKOUT_SHA} # v5.1.0\n" + publication_identity_step = publication.split( + " - name: Require dedicated publication identity\n", 1 + )[1].split(checkout_marker, 1)[0] + checkout_step = publication.split(checkout_marker, 1)[1].split( + "\n - name:", 1 + )[0] self.assertIn(" schedule:\n", schedule_triggers) self.assertNotIn(" workflow_dispatch:\n", schedule_triggers) @@ -632,9 +4591,13 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: "uses: ./.github/workflows/validate_llamacpp_core.yml", validation_job, ) - self.assertIn("permissions:\n contents: read", validation_job) + self.assertIn( + "permissions:\n checks: write\n contents: read", validation_job + ) self.assertNotIn("contents: write", validation_job) - self.assertNotIn("checks: write", validation_job) + self.assertIn( + "permissions:\n checks: write\n contents: read", validation_job + ) self.assertNotIn("secrets: inherit", schedule) self.assertIn( "HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}", @@ -645,18 +4608,43 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: self.assertIn( "needs.validate.outputs.promotion_eligible == 'true'", publication ) - self.assertIn("contents: write", publication) - self.assertIn("pull-requests: write", publication) + self.assertIn("permissions:\n contents: read", publication) + self.assertNotIn("contents: write", publication) + self.assertNotIn("pull-requests: write", publication) + self.assertNotIn("secrets.GITHUB_TOKEN", publication) self.assertIn("publish_llamacpp_update.sh", publication) + self.assertIn("fetch-depth: 0", checkout_step) + self.assertIn("persist-credentials: false", checkout_step) + self.assertIn("token: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", checkout_step) + self.assertEqual( + publication.count( + f"uses: actions/download-artifact@{DOWNLOAD_ARTIFACT_SHA} # v7.0.0" + ), + 2, + ) + self.assertNotIn("actions/checkout@v5", publication) + self.assertNotIn("actions/download-artifact@v7", publication) + self.assertIn("gh api graphql", publication_identity_step) + self.assertIn(".data.viewer.login", publication_identity_step) self.assertIn( - " - uses: actions/checkout@v5\n with:\n fetch-depth: 0", - publication, + "GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", + publication_identity_step, + ) + self.assertIn( + "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", + publication_identity_step, + ) + self.assertLess( + publication.index(" - name: Require dedicated publication identity"), + publication.index(f" - uses: actions/checkout@{CHECKOUT_SHA}"), ) self.assertIn("VALIDATED_BASE_SHA: ${{ github.sha }}", publication) self.assertIn( "BASE_BRANCH: ${{ github.event.repository.default_branch }}", publication, ) + self.assertIn("pattern: validation-results-*", publication) + self.assertIn("pattern: restart-validation-results-*", publication) evidence_step = publication.split( " - name: Validate complete validation evidence\n", 1 )[1].split( @@ -664,6 +4652,9 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: )[ 0 ] + update_step = publication.split( + " - name: Update backend_versions.json from release manifest\n", 1 + )[1].split(" - name: Generate PR body\n", 1)[0] manifest_step = publication.split( " - name: Verify release asset manifest\n", 1 )[1].split(" - name: Publish update pull request\n", 1)[0] @@ -673,6 +4664,7 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: "needs.validate.outputs.validation_matrix", evidence_step, ) + self.assertEqual(update_step.count("set -euo pipefail"), 1) self.assertIn("llamacpp_release_manifest", manifest_step) self.assertIn( "needs.validate.outputs.release_asset_manifest", @@ -680,6 +4672,7 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: ) self.assertIn("capture_llamacpp_release_manifest.sh", manifest_step) self.assertIn("--materialize-manifest", manifest_step) + self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", manifest_step) self.assertIn("EXPECTED_RELEASE_ASSET_MANIFEST", publication) self.assertNotIn("\n - name:", manifest_step) self.assertLess( @@ -690,6 +4683,14 @@ def test_scheduled_publication_is_isolated_and_serialized(self) -> None: publication.index(" - name: Verify release asset manifest\n"), publication.index(" - name: Publish update pull request\n"), ) + publisher_step = publication.split( + " - name: Publish update pull request\n", 1 + )[1] + self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", publisher_step) + self.assertIn( + "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", + publisher_step, + ) validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") output_names = ( diff --git a/test/test_validation_model_catalog.py b/test/test_validation_model_catalog.py new file mode 100644 index 0000000000..40c084e123 --- /dev/null +++ b/test/test_validation_model_catalog.py @@ -0,0 +1,178 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from test.utils.validation_model_catalog import ( + ModelCatalogOverlayError, + merge_model_catalog_files, + merge_model_catalogs, +) + + +class ValidationModelCatalogTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.root = Path(self.temp_dir.name) + + def _write(self, name, payload): + path = self.root / name + path.write_text(payload, encoding="utf-8") + return path + + def _merge(self, base, overlay): + output = self.root / "merged.json" + merge_model_catalog_files(base, overlay, output) + return output.read_text(encoding="utf-8") + + def test_merges_disjoint_models_deterministically(self): + base = self._write( + "base.json", + json.dumps({"Existing": {"recipe": "llamacpp", "size": 1.0}}), + ) + overlay = self._write( + "overlay.json", + json.dumps({"Candidate": {"recipe": "llamacpp", "size": 2.0}}), + ) + + first = self._merge(base, overlay) + second = self._merge(base, overlay) + + self.assertEqual(first, second) + self.assertEqual( + json.loads(first), + { + "Candidate": {"recipe": "llamacpp", "size": 2.0}, + "Existing": {"recipe": "llamacpp", "size": 1.0}, + }, + ) + self.assertTrue(first.endswith("\n")) + + def test_accepts_an_identical_promoted_model(self): + model = {"recipe": "llamacpp", "size": 2.0} + base = self._write("base.json", json.dumps({"Candidate": model})) + overlay = self._write("overlay.json", json.dumps({"Candidate": model})) + + merged = json.loads(self._merge(base, overlay)) + + self.assertEqual(merged, {"Candidate": model}) + + def test_returns_the_merged_catalog_without_writing_a_file(self): + base = self._write( + "base.json", + json.dumps({"Existing": {"recipe": "llamacpp"}}), + ) + overlay = self._write( + "overlay.json", + json.dumps({"Candidate": {"recipe": "llamacpp"}}), + ) + + self.assertEqual( + merge_model_catalogs(base, overlay), + { + "Existing": {"recipe": "llamacpp"}, + "Candidate": {"recipe": "llamacpp"}, + }, + ) + self.assertFalse((self.root / "merged.json").exists()) + + def test_rejects_a_conflicting_promoted_model(self): + base = self._write( + "base.json", json.dumps({"Candidate": {"recipe": "llamacpp"}}) + ) + overlay = self._write( + "overlay.json", json.dumps({"Candidate": {"recipe": "vllm"}}) + ) + + with self.assertRaisesRegex( + ModelCatalogOverlayError, "conflicting model definition: Candidate" + ): + self._merge(base, overlay) + + def test_rejects_duplicate_json_keys(self): + base = self._write( + "base.json", + '{"Duplicate":{"recipe":"llamacpp"},' '"Duplicate":{"recipe":"llamacpp"}}', + ) + overlay = self._write("overlay.json", "{}") + + with self.assertRaisesRegex( + ModelCatalogOverlayError, "duplicate JSON member: Duplicate" + ): + self._merge(base, overlay) + + def test_rejects_non_finite_json_numbers(self): + valid = self._write("valid.json", "{}") + for literal in ("NaN", "Infinity", "-Infinity"): + with self.subTest(literal=literal): + invalid = self._write( + "invalid.json", + '{"Candidate":{"recipe":"llamacpp","size":' + literal + "}}", + ) + + with self.assertRaisesRegex( + ModelCatalogOverlayError, "non-finite JSON number" + ): + self._merge(valid, invalid) + + def test_rejects_floating_point_overflow_in_both_catalogs(self): + for invalid_side in ("base", "overlay"): + with self.subTest(invalid_side=invalid_side): + invalid = self._write( + f"{invalid_side}-overflow.json", + '{"Candidate":{"recipe":"llamacpp","size":1e400}}', + ) + valid = self._write(f"{invalid_side}-valid.json", "{}") + base, overlay = ( + (invalid, valid) if invalid_side == "base" else (valid, invalid) + ) + + with self.assertRaisesRegex( + ModelCatalogOverlayError, "non-finite JSON number" + ): + self._merge(base, overlay) + + def test_promoted_model_comparison_is_type_sensitive(self): + base = self._write( + "base.json", + '{"Candidate":{"recipe":"llamacpp","value":1}}', + ) + overlay = self._write( + "overlay.json", + '{"Candidate":{"recipe":"llamacpp","value":true}}', + ) + + with self.assertRaisesRegex( + ModelCatalogOverlayError, "conflicting model definition: Candidate" + ): + self._merge(base, overlay) + + def test_rejects_non_object_catalogs_and_models(self): + valid = self._write("valid.json", "{}") + catalog_list = self._write("list.json", "[]") + invalid_model = self._write("model.json", '{"Candidate":[]}') + + with self.assertRaisesRegex( + ModelCatalogOverlayError, "catalog must be a JSON object" + ): + self._merge(catalog_list, valid) + with self.assertRaisesRegex( + ModelCatalogOverlayError, "Candidate must be a JSON object" + ): + self._merge(valid, invalid_model) + + def test_does_not_replace_output_after_a_validation_failure(self): + output = self.root / "merged.json" + output.write_text('{"preserved":true}\n', encoding="utf-8") + base = self._write("base.json", "{}") + overlay = self._write("overlay.json", '{"Candidate":[]}') + + with self.assertRaises(ModelCatalogOverlayError): + merge_model_catalog_files(base, overlay, output) + + self.assertEqual(output.read_text(encoding="utf-8"), '{"preserved":true}\n') + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_verify_llamacpp_pin_manifest.py b/test/test_verify_llamacpp_pin_manifest.py index f1ef59a0a3..ecf0a1dbe7 100644 --- a/test/test_verify_llamacpp_pin_manifest.py +++ b/test/test_verify_llamacpp_pin_manifest.py @@ -12,38 +12,47 @@ import unittest from pathlib import Path +from test.utils import llamacpp_release_assets as release_assets from test.utils import llamacpp_release_manifest as manifest ROOT = Path(__file__).resolve().parents[1] VERIFIER = ROOT / ".github" / "scripts" / "verify_llamacpp_pin_manifest.sh" MANIFEST_TOOL = ROOT / "test" / "utils" / "llamacpp_release_manifest.py" +ASSET_TOOL = ROOT / "test" / "utils" / "llamacpp_release_assets.py" VERSIONS_PATH = Path("src/cpp/resources/backend_versions.json") RELEASE_MANIFEST_PATH = Path(".github/llamacpp_release_manifest.json") MISSING = object() -def release_payload(repository: str, tag: str, release_id: int) -> dict: +def release_payload( + repository: str, + tag: str, + release_id: int, + asset_names: list[str] | None = None, +) -> dict: release_tag_commit = f"{release_id:040x}" claim_type = ( "source-release-tag" if repository == manifest.TRUSTED_SOURCE_REPOSITORY else "immutable-source-manifest" ) + names = asset_names or [f"llama-{release_id}.zip"] assets = [ { - "digest": "sha256:" + f"{release_id:064x}"[-64:], - "id": release_id * 10, - "name": f"llama-{release_id}.zip", + "digest": "sha256:" + f"{release_id * 1000 + index:064x}"[-64:], + "id": release_id * 1000 + index, + "name": name, "size": release_id, "state": "uploaded", "updated_at": "2026-09-05T12:30:00Z", } + for index, name in enumerate(names, start=1) ] if claim_type == "immutable-source-manifest": assets.append( { "digest": "sha256:" + "a" * 64, - "id": release_id * 10 + 1, + "id": release_id * 1000 + len(assets) + 1, "name": manifest.SOURCE_MANIFEST_ASSET_NAME, "size": 512, "state": "uploaded", @@ -76,6 +85,7 @@ def setUp(self) -> None: utilities.mkdir(parents=True) shutil.copy2(VERIFIER, scripts / VERIFIER.name) shutil.copy2(MANIFEST_TOOL, utilities / MANIFEST_TOOL.name) + shutil.copy2(ASSET_TOOL, utilities / ASSET_TOOL.name) capture = scripts / "capture_llamacpp_release_manifest.sh" capture.write_text( "#!/usr/bin/env bash\n" @@ -95,7 +105,9 @@ def setUp(self) -> None: "rocm-nightly": "b3", "rocm-stable": "b2", "vulkan": "b1", - } + }, + "rocm_asset_families": {}, + "therock": {"version": "7.14.0"}, } def tearDown(self) -> None: @@ -139,26 +151,73 @@ def initialize_repository( text=True, ).stdout.strip() - def matching_manifest(self, ggml_tag: str = "b9") -> str: - return manifest.build_release_asset_manifest( - [ - ( - "ggml-org/llama.cpp", - ggml_tag, - release_payload("ggml-org/llama.cpp", ggml_tag, 9), - ), - ( - "lemonade-sdk/llama.cpp", - "b2", - release_payload("lemonade-sdk/llama.cpp", "b2", 2), - ), - ( - "lemonade-sdk/llamacpp-rocm", - "b3", - release_payload("lemonade-sdk/llamacpp-rocm", "b3", 3), - ), - ] + def matching_manifest( + self, + ggml_tag: str = "b9", + rocm_tag: str = "b3", + lemonade_tag: str = "b2", + *, + missing_asset: str | None = None, + missing_attested_target: tuple[str, str] | None = None, + zero_size_asset: str | None = None, + versions: dict | None = None, + ) -> str: + selected_versions = versions or self.base_versions + requirements = release_assets.build_asset_requirements( + ggml_release=ggml_tag, + rocm_release=rocm_tag, + lemonade_release=lemonade_tag, + backend_versions=selected_versions, + ) + rocm_target_requirements = release_assets.build_rocm_asset_target_requirements( + rocm_release=rocm_tag, + backend_versions=selected_versions, + ) + release_inputs = ( + ("ggml-org/llama.cpp", ggml_tag, 9, "ggml"), + ("lemonade-sdk/llama.cpp", lemonade_tag, 2, "lemonade"), + ("lemonade-sdk/llamacpp-rocm", rocm_tag, 3, "rocm"), ) + releases = [] + for repository, tag, release_id, requirement_group in release_inputs: + names = [ + name + for family in requirements[requirement_group].values() + for name in family + ] + payload = release_payload(repository, tag, release_id, names) + if missing_asset is not None: + payload["assets"] = [ + asset + for asset in payload["assets"] + if asset["name"] != missing_asset + ] + if zero_size_asset is not None: + for asset in payload["assets"]: + if asset["name"] == zero_size_asset: + asset["size"] = 0 + if repository == "lemonade-sdk/llamacpp-rocm": + payload["build_target_attestations"] = [] + for asset in payload["assets"]: + required_targets = rocm_target_requirements.get(asset["name"]) + if required_targets is None: + continue + build_targets = list(required_targets) + if ( + missing_attested_target is not None + and asset["name"] == missing_attested_target[0] + ): + build_targets.remove(missing_attested_target[1]) + payload["build_target_attestations"].append( + { + "build_targets": build_targets, + "digest": asset["digest"], + "name": asset["name"], + "size": asset["size"], + } + ) + releases.append((repository, tag, payload)) + return manifest.build_release_asset_manifest(releases) def run_verifier( self, @@ -250,6 +309,128 @@ def test_capture_failure_is_propagated(self) -> None: self.assertEqual(result.returncode, 37, result.stderr) self.assertTrue(self.capture_log.exists()) + def test_changed_pins_require_complete_positive_cross_platform_assets( + self, + ) -> None: + cases = ( + ( + "cpu", + {"ggml_tag": "b9"}, + "llama-b9-bin-win-cpu-x64.zip", + ), + ( + "vulkan", + {"ggml_tag": "b9"}, + "llama-b9-bin-ubuntu-vulkan-arm64.tar.gz", + ), + ( + "metal", + {"ggml_tag": "b9"}, + "llama-b9-bin-macos-arm64.tar.gz", + ), + ( + "cuda", + {"ggml_tag": "b1", "lemonade_tag": "b9"}, + "llama-b9-ubuntu-cuda-sm_121-arm64.tar.xz", + ), + ( + "rocm-stable", + {"ggml_tag": "b1", "lemonade_tag": "b9"}, + "llama-b9-bin-win-rocm-7.14-x64.zip", + ), + ( + "rocm-nightly", + {"ggml_tag": "b1", "rocm_tag": "b9"}, + "llama-b9-ubuntu-rocm-gfx942-x64.zip", + ), + ) + for backend, tags, required_asset in cases: + candidate = copy.deepcopy(self.base_versions) + candidate["llamacpp"][backend] = "b9" + for defect in ("missing", "zero-size"): + with self.subTest(backend=backend, defect=defect): + manifest_json = self.matching_manifest( + **tags, + missing_asset=required_asset if defect == "missing" else None, + zero_size_asset=( + required_asset if defect == "zero-size" else None + ), + ) + result = self.run_verifier( + candidate, + candidate_manifest=manifest_json, + capture_exit=0, + ) + + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn(f"llamacpp.{backend}", result.stderr) + self.assertFalse(self.capture_log.exists()) + + def test_rocm_nightly_pin_rejects_incomplete_concrete_target_attestation( + self, + ) -> None: + candidate = copy.deepcopy(self.base_versions) + candidate["llamacpp"]["rocm-nightly"] = "b9" + candidate["rocm_asset_families"] = { + "gfx1033": "gfx103X", + "gfx1035": "gfx103X", + "gfx1036": "gfx103X", + } + target_asset = "llama-b9-windows-rocm-gfx103X-x64.zip" + + result = self.run_verifier( + candidate, + candidate_manifest=self.matching_manifest( + ggml_tag="b1", + rocm_tag="b9", + versions=candidate, + missing_attested_target=(target_asset, "gfx1035"), + ), + capture_exit=0, + ) + + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("incomplete build-target attestation", result.stderr) + self.assertIn(target_asset, result.stderr) + self.assertFalse(self.capture_log.exists()) + + def test_selector_only_changes_require_live_manifests(self) -> None: + mapping_candidate = copy.deepcopy(self.base_versions) + mapping_candidate["rocm_asset_families"] = {"gfx1033": "gfx103X"} + missing_mapping_manifest = self.run_verifier(mapping_candidate) + self.assertNotEqual(missing_mapping_manifest.returncode, 0) + self.assertIn( + "require a committed release manifest", missing_mapping_manifest.stderr + ) + self.assertFalse(self.capture_log.exists()) + + therock_candidates = [] + for field, value in ( + ("version", "7.14.1"), + ("architectures", ["gfx103X", "gfx110X"]), + ("url_mapping", {"windows": "runtime.zip"}), + ): + candidate = copy.deepcopy(self.base_versions) + candidate["therock"][field] = value + therock_candidates.append(candidate) + with self.subTest(therock_selector=field): + missing_therock_manifest = self.run_verifier(candidate) + self.assertNotEqual(missing_therock_manifest.returncode, 0) + self.assertIn( + "require a committed release manifest", + missing_therock_manifest.stderr, + ) + self.assertFalse(self.capture_log.exists()) + + therock_candidate = therock_candidates[-1] + accepted = self.run_verifier( + therock_candidate, + candidate_manifest=self.matching_manifest(versions=therock_candidate), + capture_exit=0, + ) + self.assertEqual(accepted.returncode, 0, accepted.stderr) + self.assertTrue(self.capture_log.exists()) + if __name__ == "__main__": unittest.main() diff --git a/test/utils/llamacpp_capability_validation.py b/test/utils/llamacpp_capability_validation.py index 29f2601e70..15619f9315 100644 --- a/test/utils/llamacpp_capability_validation.py +++ b/test/utils/llamacpp_capability_validation.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import time from collections.abc import Callable, Iterable from typing import Any @@ -37,7 +38,7 @@ None, ), "openai_reasoning_medium": ("openai", False, "stop", True, True, 0, None), - "openai_reasoning_low": ("openai", False, "stop", True, True, 0, None), + "openai_reasoning_low": ("openai", False, "stop", True, False, 0, None), "openai_tool_json": ("openai", False, "tool_calls", False, False, 1, None), "openai_tool_xml": ("openai", False, "tool_calls", False, False, 1, None), "openai_tool_xml_typed": ( @@ -95,6 +96,11 @@ "include_hourly": False, } IFM_CONTROL_PREFIXES = ("", start) if end < 0: end = min(start + 79, len(candidate) - 1) - return candidate[start : end + 1], path - elif isinstance(candidate, dict): - for key, nested_value in reversed(list(candidate.items())): - pending.append((f"{path}.{key}", nested_value)) - elif isinstance(candidate, (list, tuple)): - for index in range(len(candidate) - 1, -1, -1): - pending.append((f"{path}[{index}]", candidate[index])) + return candidate[start : end + 1], "".join(path_components) + elif isinstance(candidate, (dict, list, tuple)): + pending.append( + ( + iter(_marker_children(candidate)), + len(path_components), + ) + ) return None @@ -219,6 +244,23 @@ def reject_duplicate_json_object(pairs: list[tuple[str, object]]) -> dict: return result +def _stream_event_size(event: object) -> int: + if isinstance(event, bytes): + return len(event) + 1 + if isinstance(event, str): + return len(event.encode("utf-8")) + 1 + try: + encoded = json.dumps( + event, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise CapabilityValidationError("stream event cannot be encoded") from exc + return len(encoded) + 1 + + def _validate_tool_arguments(arguments: object) -> dict: _require(isinstance(arguments, dict), "tool arguments must be an object") _require( @@ -378,7 +420,11 @@ def _tool_payload(model: str, tool_format: str | None, *, stream: bool) -> dict: return payload -def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: +def _decode_openai_stream( + events: Iterable[object], *, deadline: float | None = None +) -> tuple[dict, str, bool, str]: + if deadline is None: + deadline = time.monotonic() + STREAM_WALL_CLOCK_TIMEOUT_SECONDS content_parts: list[str] = [] reasoning_parts: list[str] = [] tools: dict[int, dict[str, Any]] = {} @@ -386,8 +432,21 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: stream_model = None saw_choice = False saw_done = False + total_sse_bytes = 0 + semantic_event_count = 0 + total_text_bytes = 0 + total_tool_argument_bytes = 0 for event_index, event in enumerate(events): + _require( + time.monotonic() < deadline, + "stream exceeded its wall-clock deadline", + ) + total_sse_bytes += _stream_event_size(event) + _require( + total_sse_bytes <= MAX_STREAM_SSE_BYTES, + "stream SSE byte count exceeds bounds", + ) if isinstance(event, bytes): try: event = event.decode("utf-8") @@ -399,6 +458,11 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: line = event.strip() if not line or line.startswith(":"): continue + semantic_event_count += 1 + _require( + semantic_event_count <= MAX_STREAM_EVENTS, + "stream event count exceeds bounds", + ) _require( not saw_done, "stream contains data after the terminal [DONE] frame" ) @@ -423,6 +487,11 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: f"stream event {event_index} contains invalid JSON: {exc}" ) from exc else: + semantic_event_count += 1 + _require( + semantic_event_count <= MAX_STREAM_EVENTS, + "stream event count exceeds bounds", + ) _require( not saw_done, "stream contains data after the terminal [DONE] frame" ) @@ -464,6 +533,11 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: value = delta.get(key) if value is not None: _require(isinstance(value, str), f"stream {key} must be a string") + total_text_bytes += len(value.encode("utf-8")) + _require( + total_text_bytes <= MAX_STREAM_TEXT_BYTES, + "stream content and reasoning exceed bounds", + ) destination.append(value) tool_deltas = delta.get("tool_calls", []) @@ -479,7 +553,12 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: ) target = tools.setdefault( index, - {"id": None, "type": None, "name": "", "arguments": ""}, + { + "id": None, + "type": None, + "name_parts": [], + "argument_parts": [], + }, ) for key in ("id", "type"): value = tool_delta.get(key) @@ -499,14 +578,19 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: name = function.get("name") if name is not None: _require(isinstance(name, str), "stream tool name must be a string") - target["name"] += name + target["name_parts"].append(name) arguments = function.get("arguments") if arguments is not None: _require( isinstance(arguments, str), "stream tool arguments must be a string", ) - target["arguments"] += arguments + total_tool_argument_bytes += len(arguments.encode("utf-8")) + _require( + total_tool_argument_bytes <= MAX_STREAM_TOOL_ARGUMENT_BYTES, + "stream tool arguments exceed bounds", + ) + target["argument_parts"].append(arguments) current_finish = choice.get("finish_reason") if current_finish is not None: @@ -520,6 +604,7 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: _require(saw_choice, "stream contains no choice chunks") _require(saw_done, "stream is missing the terminal [DONE] frame") _require(isinstance(finish_reason, str), "stream has no terminal finish_reason") + _require(isinstance(stream_model, str), "stream model is missing") tool_calls = [] for index in sorted(tools): tool = tools[index] @@ -528,8 +613,8 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: "id": tool["id"], "type": tool["type"], "function": { - "name": tool["name"], - "arguments": tool["arguments"], + "name": "".join(tool["name_parts"]), + "arguments": "".join(tool["argument_parts"]), }, } ) @@ -540,7 +625,7 @@ def _decode_openai_stream(events: Iterable[object]) -> tuple[dict, str, bool]: "tool_calls": tool_calls, } _marker_free(message, "assembled_message") - return message, finish_reason, saw_done + return message, finish_reason, saw_done, stream_model def _post_openai_stream( @@ -549,14 +634,29 @@ def _post_openai_stream( timeout: int, payload: dict, ) -> tuple[dict, str, int, bool]: - response, events = request_stream("POST", url, timeout=timeout, json=payload) + deadline = time.monotonic() + min(timeout, STREAM_WALL_CLOCK_TIMEOUT_SECONDS) + response, events = request_stream( + "POST", + url, + timeout=timeout, + wall_clock_deadline=deadline, + json=payload, + ) status_code = _status_code(response) _require(status_code == 200, f"stream returned HTTP {status_code}") expected_model = payload.get("model") _require( isinstance(expected_model, str) and expected_model, "request model is missing" ) - message, finish_reason, terminal = _decode_openai_stream(events) + message, finish_reason, terminal, response_model = _decode_openai_stream( + events, + deadline=deadline, + ) + _require_response_model( + {"model": response_model}, + expected_model, + "stream response", + ) return message, finish_reason, status_code, terminal @@ -634,11 +734,22 @@ def _openai_plain_stream_case(request_stream, url, timeout, payload): }, message -def _openai_reasoning_case(request_json, url, timeout, payload): +def _openai_reasoning_case( + request_json, + url, + timeout, + payload, + *, + require_reasoning: bool, +): body, status = _post_json(request_json, url, timeout, payload) message = _openai_message(body, "stop") content = _visible_content(message, "42") - reasoning = _reasoning_content(message) + if require_reasoning: + reasoning = _reasoning_content(message) + else: + _require_no_reasoning(message) + reasoning = "" _require_no_tool_calls(message) return { "status_code": status, @@ -914,8 +1025,12 @@ def validate_capabilities( f"openai_reasoning_{effort}", "openai", False, - lambda payload=payload: _openai_reasoning_case( - request_json, openai_url, timeout, payload + lambda payload=payload, effort=effort: _openai_reasoning_case( + request_json, + openai_url, + timeout, + payload, + require_reasoning=effort != "low", ), ) if effort == "high": diff --git a/test/utils/llamacpp_release_assets.py b/test/utils/llamacpp_release_assets.py index d090b430c4..29e2feea4d 100644 --- a/test/utils/llamacpp_release_assets.py +++ b/test/utils/llamacpp_release_assets.py @@ -6,6 +6,7 @@ from collections.abc import Collection, Iterable, Mapping _RELEASE_RE = re.compile(r"^b[0-9]+$") +_ROCM_CONCRETE_ISA_RE = re.compile(r"^gfx[0-9a-f]{3,4}$") _CUDA_COMPUTE_CAPABILITIES = ( "sm_75", "sm_80", @@ -16,28 +17,27 @@ "sm_120", "sm_121", ) -_ROCM_NIGHTLY_TARGETS_BY_PLATFORM = { - "windows": ( - "gfx1151", - "gfx1150", - "gfx120X", - "gfx110X", - "gfx103X", - "gfx90a", - "gfx908", - "gfx1152", - ), - "ubuntu": ( - "gfx1151", - "gfx1150", - "gfx120X", - "gfx110X", - "gfx103X", - "gfx90a", - "gfx908", - "gfx1152", - "gfx942", - ), +_ROCM_RDNA_NIGHTLY_ISAS = ( + "gfx1030", + "gfx1031", + "gfx1032", + "gfx1033", + "gfx1034", + "gfx1035", + "gfx1036", + "gfx1100", + "gfx1101", + "gfx1102", + "gfx1103", + "gfx1150", + "gfx1151", + "gfx1152", + "gfx1200", + "gfx1201", +) +_ROCM_NIGHTLY_ISAS_BY_PLATFORM = { + "windows": _ROCM_RDNA_NIGHTLY_ISAS, + "ubuntu": (*_ROCM_RDNA_NIGHTLY_ISAS, "gfx908", "gfx90a", "gfx942"), } @@ -59,17 +59,50 @@ def _therock_major_minor(backend_versions: Mapping[str, object]) -> str: return major_minor -def _mapped_rocm_targets( - targets: Iterable[str], asset_families: Mapping[str, object] -) -> list[str]: - mapped = [] - for target in targets: - family = asset_families.get(target, target) +def _validate_rocm_asset_families(asset_families: Mapping[str, object]) -> None: + for isa, family in asset_families.items(): + if isa == "comment": + continue + if not isinstance(isa, str) or _ROCM_CONCRETE_ISA_RE.fullmatch(isa) is None: + raise ValueError( + f"ROCm asset family key must be a concrete ROCm ISA: {isa!r}" + ) if not isinstance(family, str) or not family: - raise ValueError(f"Invalid ROCm asset family for {target}: {family!r}") - if family not in mapped: - mapped.append(family) - return mapped + raise ValueError(f"Invalid ROCm asset family for {isa}: {family!r}") + + +def _rocm_asset_families( + backend_versions: Mapping[str, object], +) -> Mapping[str, object]: + asset_families = backend_versions.get("rocm_asset_families", {}) + if not isinstance(asset_families, Mapping): + raise ValueError("backend_versions.json rocm_asset_families must be an object") + _validate_rocm_asset_families(asset_families) + return asset_families + + +def build_rocm_asset_target_requirements( + *, + rocm_release: str, + backend_versions: Mapping[str, object], +) -> dict[str, tuple[str, ...]]: + """Bind every required ROCm nightly asset to its concrete build targets.""" + release = _validated_release(rocm_release, "ROCM_RELEASE") + asset_families = _rocm_asset_families(backend_versions) + grouped_targets: dict[str, list[str]] = {} + for platform, concrete_isas in _ROCM_NIGHTLY_ISAS_BY_PLATFORM.items(): + for concrete_isa in concrete_isas: + family = asset_families.get(concrete_isa, concrete_isa) + if not isinstance(family, str) or not family: + raise ValueError( + f"Invalid ROCm asset family for {concrete_isa}: {family!r}" + ) + asset_name = f"llama-{release}-{platform}-rocm-{family}-x64.zip" + grouped_targets.setdefault(asset_name, []).append(concrete_isa) + return { + asset_name: tuple(concrete_isas) + for asset_name, concrete_isas in grouped_targets.items() + } def build_asset_requirements( @@ -85,16 +118,12 @@ def build_asset_requirements( lemonade_release = _validated_release(lemonade_release, "LEMONADE_RELEASE") therock = _therock_major_minor(backend_versions) - asset_families = backend_versions.get("rocm_asset_families", {}) - if not isinstance(asset_families, Mapping): - raise ValueError("backend_versions.json rocm_asset_families must be an object") - - rocm_nightly = [] - for platform, targets in _ROCM_NIGHTLY_TARGETS_BY_PLATFORM.items(): - for target in _mapped_rocm_targets(targets, asset_families): - rocm_nightly.append( - f"llama-{rocm_release}-{platform}-rocm-{target}-x64.zip" - ) + rocm_nightly = list( + build_rocm_asset_target_requirements( + rocm_release=rocm_release, + backend_versions=backend_versions, + ) + ) cuda = [] for compute_capability in _CUDA_COMPUTE_CAPABILITIES: @@ -134,14 +163,41 @@ def build_asset_requirements( def evaluate_asset_group( - available_assets: Collection[str], + available_assets: Collection[str] | Mapping[str, object], backend_requirements: Mapping[str, Iterable[str]], + *, + required_asset_targets: Mapping[str, Iterable[str]] | None = None, + attested_asset_targets: Mapping[str, Iterable[str]] | None = None, ) -> tuple[list[str], dict[str, list[str]]]: """Return eligible backends and their missing required assets.""" + + def has_required_targets(asset: str) -> bool: + if required_asset_targets is None: + return True + required = required_asset_targets.get(asset) + if required is None or isinstance(required, str): + return False + if attested_asset_targets is None: + return False + attested = attested_asset_targets.get(asset) + if attested is None or isinstance(attested, str): + return False + return set(required).issubset(attested) + + def is_available(asset: str) -> bool: + if not isinstance(available_assets, Mapping): + has_asset = asset in available_assets + else: + size = available_assets.get(asset) + has_asset = ( + isinstance(size, int) and not isinstance(size, bool) and size > 0 + ) + return has_asset and has_required_targets(asset) + eligible = [] missing_by_backend = {} for backend, required_assets in backend_requirements.items(): - missing = [asset for asset in required_assets if asset not in available_assets] + missing = [asset for asset in required_assets if not is_available(asset)] missing_by_backend[backend] = missing if not missing: eligible.append(backend) diff --git a/test/utils/llamacpp_release_manifest.py b/test/utils/llamacpp_release_manifest.py index 3b7be3f57b..ab26fb7244 100644 --- a/test/utils/llamacpp_release_manifest.py +++ b/test/utils/llamacpp_release_manifest.py @@ -11,13 +11,28 @@ from typing import Iterable from urllib.parse import urlparse +if __package__: + from .llamacpp_release_assets import ( + build_asset_requirements, + build_rocm_asset_target_requirements, + evaluate_asset_group, + ) +else: + from llamacpp_release_assets import ( # type: ignore[import-not-found] + build_asset_requirements, + build_rocm_asset_target_requirements, + evaluate_asset_group, + ) + SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-fA-F]{64}$") LOWER_SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") GIT_OID_RE = re.compile(r"^[0-9a-fA-F]{40}$") RELEASE_TAG_RE = re.compile(r"^b[0-9]+$") +ROCM_CONCRETE_ISA_RE = re.compile(r"^gfx[0-9a-f]{3,4}$") SOURCE_MANIFEST_ASSET_NAME = ".llamacpp-source.json" SOURCE_MANIFEST_MAX_BYTES = 65_536 TRUSTED_SOURCE_REPOSITORY = "ggml-org/llama.cpp" +ROCM_RELEASE_REPOSITORY = "lemonade-sdk/llamacpp-rocm" PUBLISHER_CLAIM_TYPES = frozenset({"immutable-source-manifest", "source-release-tag"}) FORK_RELEASE_REPOSITORIES = frozenset( {"lemonade-sdk/llama.cpp", "lemonade-sdk/llamacpp-rocm"} @@ -30,6 +45,11 @@ "rocm-stable": "lemonade-sdk/llama.cpp", "vulkan": "ggml-org/llama.cpp", } +ASSET_REQUIREMENT_GROUPS = { + "ggml-org/llama.cpp": "ggml", + "lemonade-sdk/llama.cpp": "lemonade", + "lemonade-sdk/llamacpp-rocm": "rocm", +} class ReleaseManifestError(ValueError): @@ -196,14 +216,36 @@ def _source_manifest_asset_bindings( return sorted(bindings, key=lambda asset: asset["name"]) -def validate_immutable_source_manifest( +def _normalize_build_targets(value: object, prefix: str) -> list[str]: + if not isinstance(value, list): + raise ReleaseManifestError(f"{prefix} build_targets must be an array") + targets = [] + seen: set[str] = set() + for target in value: + if ( + not isinstance(target, str) + or ROCM_CONCRETE_ISA_RE.fullmatch(target) is None + ): + raise ReleaseManifestError( + f"{prefix} build target must be a concrete ROCm ISA" + ) + if target in seen: + raise ReleaseManifestError( + f"{prefix} has duplicate build target {target!r}" + ) + seen.add(target) + targets.append(target) + return sorted(targets) + + +def validate_immutable_source_manifest_with_attestation( evidence: bytes, release_payload: object, release_repository: str, release_tag: str, release_tag_commit: str, -) -> str: - """Validate one immutable publisher claim and return its upstream source OID.""" +) -> tuple[str, list[dict]]: + """Validate one immutable publisher claim and normalize its build targets.""" source_asset = locate_immutable_source_manifest_asset( release_payload, release_repository, @@ -249,13 +291,19 @@ def validate_immutable_source_manifest( raise ReleaseManifestError( f"{release_repository}: source manifest has unexpected fields" ) - if ( - isinstance(document["schema_version"], bool) - or not isinstance(document["schema_version"], int) - or document["schema_version"] != 1 - ): + schema_version = document["schema_version"] + if isinstance(schema_version, bool) or not isinstance(schema_version, int): + raise ReleaseManifestError( + f"{release_repository}: source manifest schema_version must be 1 or 2" + ) + if schema_version == 2 and release_repository != ROCM_RELEASE_REPOSITORY: + raise ReleaseManifestError( + f"{release_repository}: source manifest schema_version 2 is only valid " + f"for {ROCM_RELEASE_REPOSITORY}" + ) + if schema_version not in {1, 2}: raise ReleaseManifestError( - f"{release_repository}: source manifest schema_version must be 1" + f"{release_repository}: source manifest schema_version must be 1 or 2" ) if document["release_repository"] != release_repository: raise ReleaseManifestError( @@ -296,7 +344,10 @@ def validate_immutable_source_manifest( names: set[str] = set() for index, binding in enumerate(raw_bindings): prefix = f"{release_repository}: source manifest asset {index}" - if not isinstance(binding, dict) or set(binding) != {"digest", "name", "size"}: + expected_binding_keys = {"digest", "name", "size"} + if schema_version == 2: + expected_binding_keys.add("build_targets") + if not isinstance(binding, dict) or set(binding) != expected_binding_keys: raise ReleaseManifestError(f"{prefix} has unexpected fields") name = binding["name"] if not isinstance(name, str) or not name: @@ -319,17 +370,115 @@ def validate_immutable_source_manifest( or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None ): raise ReleaseManifestError(f"{prefix} must have a lowercase sha256 digest") - bindings.append({"digest": digest, "name": name, "size": size}) + normalized_binding = {"digest": digest, "name": name, "size": size} + if schema_version == 2: + normalized_binding["build_targets"] = _normalize_build_targets( + binding["build_targets"], + prefix, + ) + bindings.append(normalized_binding) expected_bindings = _source_manifest_asset_bindings( release_payload, release_repository, ) - if sorted(bindings, key=lambda asset: asset["name"]) != expected_bindings: + bound_asset_metadata = [ + {"digest": binding["digest"], "name": binding["name"], "size": binding["size"]} + for binding in bindings + ] + if ( + sorted(bound_asset_metadata, key=lambda asset: asset["name"]) + != expected_bindings + ): raise ReleaseManifestError( f"{release_repository}: source manifest asset metadata does not match " "the immutable release" ) - return source_commit.lower() + attestations = ( + sorted(bindings, key=lambda asset: asset["name"]) if schema_version == 2 else [] + ) + return source_commit.lower(), attestations + + +def validate_immutable_source_manifest( + evidence: bytes, + release_payload: object, + release_repository: str, + release_tag: str, + release_tag_commit: str, +) -> str: + """Validate one immutable publisher claim and return its upstream source OID.""" + source_commit, _ = validate_immutable_source_manifest_with_attestation( + evidence, + release_payload, + release_repository, + release_tag, + release_tag_commit, + ) + return source_commit + + +def _canonical_build_target_attestations( + repository: str, + value: object, + release_assets: list[dict], +) -> list[dict]: + if not isinstance(value, list): + raise ReleaseManifestError( + f"{repository}: build_target_attestations must be an array" + ) + if repository != ROCM_RELEASE_REPOSITORY and value: + raise ReleaseManifestError( + f"{repository}: build target attestations are only valid for " + f"{ROCM_RELEASE_REPOSITORY}" + ) + release_bindings = { + (asset["name"], asset["size"], asset["digest"]) + for asset in release_assets + if asset["name"] != SOURCE_MANIFEST_ASSET_NAME + } + attestations = [] + names: set[str] = set() + for index, attestation in enumerate(value): + prefix = f"{repository}: build target attestation {index}" + if not isinstance(attestation, dict) or set(attestation) != { + "build_targets", + "digest", + "name", + "size", + }: + raise ReleaseManifestError(f"{prefix} has unexpected fields") + name = attestation["name"] + if not isinstance(name, str) or not name: + raise ReleaseManifestError(f"{prefix} name must be nonempty") + if name in names: + raise ReleaseManifestError( + f"{repository}: duplicate build target attestation for {name!r}" + ) + names.add(name) + size = attestation["size"] + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise ReleaseManifestError(f"{prefix} size must be nonnegative") + digest = attestation["digest"] + if ( + not isinstance(digest, str) + or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None + ): + raise ReleaseManifestError(f"{prefix} must have a lowercase sha256 digest") + if (name, size, digest) not in release_bindings: + raise ReleaseManifestError( + f"{prefix} does not match release asset name, size, and digest" + ) + attestations.append( + { + "build_targets": _normalize_build_targets( + attestation["build_targets"], prefix + ), + "digest": digest, + "name": name, + "size": size, + } + ) + return sorted(attestations, key=lambda item: item["name"]) def _canonical_release( @@ -461,6 +610,11 @@ def _canonical_release( ) assets.sort(key=lambda asset: (asset["name"], asset["id"])) + build_target_attestations = _canonical_build_target_attestations( + repository, + payload.get("build_target_attestations", []), + assets, + ) if publisher_claim_type == "immutable-source-manifest": source_assets = [ asset for asset in assets if asset["name"] == SOURCE_MANIFEST_ASSET_NAME @@ -480,6 +634,7 @@ def _canonical_release( ) return { "assets": assets, + "build_target_attestations": build_target_attestations, "publisher_claim_type": publisher_claim_type, "publisher_claimed_source_commit": publisher_claimed_source_commit.lower(), "release_draft": release_draft, @@ -520,7 +675,7 @@ def build_release_asset_manifest( ) canonical_releases.sort(key=lambda release: release["repository"]) return json.dumps( - {"releases": canonical_releases, "schema_version": 4}, + {"releases": canonical_releases, "schema_version": 5}, separators=(",", ":"), sort_keys=True, ) @@ -532,9 +687,9 @@ def canonicalize_release_asset_manifest(manifest_json: str) -> str: not isinstance(document, dict) or isinstance(document.get("schema_version"), bool) or not isinstance(document.get("schema_version"), int) - or document.get("schema_version") != 4 + or document.get("schema_version") != 5 ): - raise ReleaseManifestError("release manifest schema_version must be 4") + raise ReleaseManifestError("release manifest schema_version must be 5") releases = document.get("releases") if not isinstance(releases, list): raise ReleaseManifestError("release manifest releases must be an array") @@ -545,6 +700,7 @@ def canonicalize_release_asset_manifest(manifest_json: str) -> str: raise ReleaseManifestError("release manifest entry must be an object") expected_keys = { "assets", + "build_target_attestations", "publisher_claim_type", "publisher_claimed_source_commit", "release_draft", @@ -565,6 +721,7 @@ def canonicalize_release_asset_manifest(manifest_json: str) -> str: release["tag_name"], { "assets": release["assets"], + "build_target_attestations": release["build_target_attestations"], "draft": release["release_draft"], "id": release["release_id"], "immutable": release["release_immutable"], @@ -621,12 +778,40 @@ def managed_llamacpp_pin_changes( sections.append(section) base_section, candidate_section = sections - return { + changes = { backend: (base_section[backend], candidate_section[backend]) for backend in MANAGED_PIN_REPOSITORIES if base_section[backend] != candidate_section[backend] } + def therock_selector(document: object) -> object: + if not isinstance(document, dict): + return None + therock = document.get("therock") + if not isinstance(therock, dict): + return therock + return {key: value for key, value in therock.items() if key != "comment"} + + def rocm_asset_families(document: object) -> object: + if not isinstance(document, dict): + return None + families = document.get("rocm_asset_families") + if not isinstance(families, dict): + return families + return {key: value for key, value in families.items() if key != "comment"} + + if therock_selector(base_versions) != therock_selector(candidate_versions): + changes.setdefault( + "rocm-stable", + (base_section["rocm-stable"], candidate_section["rocm-stable"]), + ) + if rocm_asset_families(base_versions) != rocm_asset_families(candidate_versions): + changes.setdefault( + "rocm-nightly", + (base_section["rocm-nightly"], candidate_section["rocm-nightly"]), + ) + return changes + def require_release_manifest_for_managed_pin_changes( base_versions: object, @@ -657,6 +842,59 @@ def require_release_manifest_for_managed_pin_changes( f"release manifest does not match changed managed llama.cpp pin " f"llamacpp.{backend}={candidate_tag}" ) + + try: + requirements = build_asset_requirements( + ggml_release=releases["ggml-org/llama.cpp"]["tag_name"], + rocm_release=releases["lemonade-sdk/llamacpp-rocm"]["tag_name"], + lemonade_release=releases["lemonade-sdk/llama.cpp"]["tag_name"], + backend_versions=candidate_versions, + ) + rocm_target_requirements = build_rocm_asset_target_requirements( + rocm_release=releases[ROCM_RELEASE_REPOSITORY]["tag_name"], + backend_versions=candidate_versions, + ) + except ValueError as exc: + raise ReleaseManifestError( + f"candidate backend versions cannot define release assets: {exc}" + ) from exc + + for backend in changes: + repository = MANAGED_PIN_REPOSITORIES[backend] + group = ASSET_REQUIREMENT_GROUPS[repository] + required_assets = requirements[group][backend] + asset_sizes = { + asset["name"]: asset["size"] for asset in releases[repository]["assets"] + } + evaluation_options = {} + if backend == "rocm-nightly": + evaluation_options = { + "required_asset_targets": rocm_target_requirements, + "attested_asset_targets": { + attestation["name"]: attestation["build_targets"] + for attestation in releases[repository]["build_target_attestations"] + }, + } + _, missing_by_backend = evaluate_asset_group( + asset_sizes, + {backend: required_assets}, + **evaluation_options, + ) + for asset_name in missing_by_backend[backend]: + if backend == "rocm-nightly" and asset_name in asset_sizes: + raise ReleaseManifestError( + "release manifest has incomplete build-target attestation for " + f"llamacpp.{backend}: {asset_name}" + ) + if asset_name in asset_sizes: + raise ReleaseManifestError( + f"release manifest required asset for llamacpp.{backend} must " + f"have positive size: {asset_name}" + ) + raise ReleaseManifestError( + f"release manifest is missing required asset for " + f"llamacpp.{backend}: {asset_name}" + ) return True @@ -737,6 +975,7 @@ def main() -> None: "RELEASE_TAG_COMMIT", ), ) + parser.add_argument("--build-target-attestation-output", type=Path) parser.add_argument( "--require-upstream-ancestry", nargs=4, @@ -765,18 +1004,32 @@ def main() -> None: mode is not None for mode in (args.source_manifest_asset_id, args.validate_source_manifest) ) + if ( + args.build_target_attestation_output is not None + and args.validate_source_manifest is None + ): + parser.error( + "--build-target-attestation-output requires --validate-source-manifest" + ) if source_modes: - if source_modes != 1 or any( - ( - args.release, - args.require_upstream_ancestry, - args.expected_json is not None, - args.github_output, - args.manifest_json is not None, - args.materialize_manifest, - args.append_summary_to, - args.managed_pin_changes, - args.require_managed_pin_manifest, + if ( + source_modes != 1 + or ( + args.source_manifest_asset_id is not None + and args.build_target_attestation_output is not None + ) + or any( + ( + args.release, + args.require_upstream_ancestry, + args.expected_json is not None, + args.github_output, + args.manifest_json is not None, + args.materialize_manifest, + args.append_summary_to, + args.managed_pin_changes, + args.require_managed_pin_manifest, + ) ) ): parser.error("source-manifest modes cannot be combined with other modes") @@ -796,8 +1049,8 @@ def main() -> None: release_tag, release_tag_commit, ) = args.validate_source_manifest - print( - validate_immutable_source_manifest( + source_commit, build_target_attestations = ( + validate_immutable_source_manifest_with_attestation( Path(source_path).read_bytes(), _load_json_file(Path(release_path)), release_repository, @@ -805,6 +1058,17 @@ def main() -> None: release_tag_commit, ) ) + if args.build_target_attestation_output is not None: + args.build_target_attestation_output.write_text( + json.dumps( + build_target_attestations, + separators=(",", ":"), + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + print(source_commit) except (OSError, ReleaseManifestError) as exc: parser.exit(1, f"ERROR: {exc}\n") return diff --git a/test/utils/llamacpp_validation_artifacts.py b/test/utils/llamacpp_validation_artifacts.py new file mode 100644 index 0000000000..66b8efcff2 --- /dev/null +++ b/test/utils/llamacpp_validation_artifacts.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""Verify validation-only K2-Horizon artifacts in an isolated HF cache.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import stat +import struct +import sys +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from test.utils.validation_model_catalog import load_model_catalog + + +class ValidationArtifactError(ValueError): + """Raised when a validation artifact does not match its immutable lock.""" + + +@dataclass(frozen=True) +class VerifiedValidationArtifact: + model_id: str + resolved_path: Path + size_bytes: int + sha256: str + + +LOCK_FIELDS = { + "filename", + "gguf_architecture", + "max_context_window", + "revision", + "sha256", + "size_bytes", +} +LOWER_HEX_40 = re.compile(r"^[0-9a-f]{40}$") +LOWER_HEX_64 = re.compile(r"^[0-9a-f]{64}$") +REPOSITORY_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +ARCHITECTURE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") +MAX_METADATA_ITEMS = 1_000_000 +MAX_STRING_BYTES = 64 * 1024 * 1024 +SCALAR_FORMATS = { + 0: " bytes: + if count < 0 or count > self.size - self.stream.tell(): + raise ValidationArtifactError( + f"{self.path}: GGUF metadata exceeds file bounds" + ) + value = self.stream.read(count) + if len(value) != count: + raise ValidationArtifactError( + f"{self.path}: GGUF metadata exceeds file bounds" + ) + return value + + def unpack(self, format_string: str): + size = struct.calcsize(format_string) + return struct.unpack(format_string, self.read(size))[0] + + def string(self, *, decode: bool) -> str | None: + length = self.unpack(" MAX_STRING_BYTES: + raise ValidationArtifactError( + f"{self.path}: GGUF string length exceeds bounds" + ) + value = self.read(length) + if not decode: + return None + try: + return value.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValidationArtifactError( + f"{self.path}: GGUF metadata is not UTF-8" + ) from exc + + +def _read_scalar(reader: _Reader, value_type: int): + format_string = SCALAR_FORMATS.get(value_type) + if format_string is None: + raise ValidationArtifactError( + f"{reader.path}: unsupported GGUF metadata type {value_type}" + ) + return reader.unpack(format_string) + + +def _skip_value(reader: _Reader, value_type: int) -> None: + if value_type == 8: + reader.string(decode=False) + return + if value_type != 9: + _read_scalar(reader, value_type) + return + + element_type = reader.unpack(" MAX_METADATA_ITEMS or element_type == 9: + raise ValidationArtifactError( + f"{reader.path}: GGUF array count or type exceeds bounds" + ) + if element_type == 8: + for _ in range(count): + reader.string(decode=False) + return + format_string = SCALAR_FORMATS.get(element_type) + if format_string is None: + raise ValidationArtifactError( + f"{reader.path}: unsupported GGUF array type {element_type}" + ) + reader.read(struct.calcsize(format_string) * count) + + +def _read_gguf_identity(open_artifact, path: Path) -> tuple[str, int]: + open_artifact.seek(0) + reader = _Reader(open_artifact, path) + if reader.read(4) != b"GGUF": + raise ValidationArtifactError(f"{path}: invalid GGUF magic") + version = reader.unpack(" MAX_METADATA_ITEMS: + raise ValidationArtifactError(f"{path}: GGUF metadata count exceeds bounds") + + metadata = {} + for _ in range(metadata_count): + key = reader.string(decode=True) + if key in metadata: + raise ValidationArtifactError(f"{path}: duplicate GGUF key {key}") + value_type = reader.unpack(" tuple[str, int]: + with path.open("rb") as artifact: + return _read_gguf_identity(artifact, path) + + +def _parse_checkpoint(model_id: str, checkpoint: object) -> tuple[str, str]: + if not isinstance(checkpoint, str) or checkpoint.count(":") != 1: + raise ValidationArtifactError( + f"{model_id}: validation checkpoint must be repository:filename" + ) + repository, filename = checkpoint.split(":", 1) + components = repository.split("/") + if ( + len(components) != 2 + or any(not REPOSITORY_COMPONENT.fullmatch(part) for part in components) + or any(part in {".", ".."} or "--" in part for part in components) + ): + raise ValidationArtifactError(f"{model_id}: unsafe HF repository name") + if ( + not filename + or Path(filename).name != filename + or "/" in filename + or "\\" in filename + or ":" in filename + or "\0" in filename + or filename in {".", ".."} + ): + raise ValidationArtifactError(f"{model_id}: filename must be a safe basename") + return repository, filename + + +def _validate_lock(model_id: str, lock: dict) -> None: + if set(lock) != LOCK_FIELDS: + raise ValidationArtifactError(f"{model_id}: artifact lock fields are invalid") + if not isinstance(lock["revision"], str) or not LOWER_HEX_40.fullmatch( + lock["revision"] + ): + raise ValidationArtifactError(f"{model_id}: revision must be lowercase hex") + if not isinstance(lock["sha256"], str) or not LOWER_HEX_64.fullmatch( + lock["sha256"] + ): + raise ValidationArtifactError(f"{model_id}: sha256 must be lowercase hex") + for field in ("size_bytes", "max_context_window"): + if type(lock[field]) is not int or not 0 < lock[field] <= 2**63 - 1: + raise ValidationArtifactError(f"{model_id}: {field} is out of bounds") + if not isinstance(lock["gguf_architecture"], str) or not ARCHITECTURE.fullmatch( + lock["gguf_architecture"] + ): + raise ValidationArtifactError(f"{model_id}: invalid GGUF architecture") + if not isinstance(lock["filename"], str): + raise ValidationArtifactError(f"{model_id}: filename must be a safe basename") + _parse_checkpoint(model_id, f"x/y:{lock['filename']}") + + +def _sha256(open_artifact) -> str: + digest = hashlib.sha256() + open_artifact.seek(0) + for chunk in iter(lambda: open_artifact.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _artifact_state(file_status: os.stat_result) -> tuple[int, ...]: + return ( + file_status.st_dev, + file_status.st_ino, + file_status.st_mode, + file_status.st_size, + file_status.st_mtime_ns, + file_status.st_ctime_ns, + ) + + +def _verify_locked_artifact( + cache_root: Path, + repository_path: Path, + model_id: str, + lock: dict, +) -> VerifiedValidationArtifact: + artifact_path = repository_path / "snapshots" / lock["revision"] / lock["filename"] + try: + resolved_artifact = artifact_path.resolve(strict=True) + resolved_artifact.relative_to(cache_root) + except (OSError, ValueError) as exc: + raise ValidationArtifactError( + f"{model_id}: artifact path escapes or is absent from the HF cache" + ) from exc + try: + with resolved_artifact.open("rb") as artifact: + status_before = os.fstat(artifact.fileno()) + if not stat.S_ISREG(status_before.st_mode): + raise ValidationArtifactError(f"{model_id}: artifact is not a file") + if status_before.st_size != lock["size_bytes"]: + raise ValidationArtifactError( + f"{model_id}: artifact size {status_before.st_size} != " + f"{lock['size_bytes']}" + ) + artifact_sha256 = _sha256(artifact) + if artifact_sha256 != lock["sha256"]: + raise ValidationArtifactError(f"{model_id}: artifact SHA-256 mismatch") + architecture, context = _read_gguf_identity(artifact, resolved_artifact) + status_after = os.fstat(artifact.fileno()) + + path_after = artifact_path.resolve(strict=True) + path_after.relative_to(cache_root) + path_status = path_after.stat() + except ValidationArtifactError: + raise + except (OSError, ValueError) as exc: + raise ValidationArtifactError( + f"{model_id}: artifact changed during verification" + ) from exc + if ( + path_after != resolved_artifact + or _artifact_state(status_before) != _artifact_state(status_after) + or _artifact_state(status_after) != _artifact_state(path_status) + ): + raise ValidationArtifactError( + f"{model_id}: artifact changed during verification" + ) + if architecture != lock["gguf_architecture"]: + raise ValidationArtifactError( + f"{model_id}: GGUF architecture {architecture} != " + f"{lock['gguf_architecture']}" + ) + if context != lock["max_context_window"]: + raise ValidationArtifactError( + f"{model_id}: GGUF context {context} != {lock['max_context_window']}" + ) + return VerifiedValidationArtifact( + model_id=model_id, + resolved_path=resolved_artifact, + size_bytes=status_after.st_size, + sha256=artifact_sha256, + ) + + +def _verify_reference_and_artifact( + cache_root: Path, + repository_path: Path, + model_id: str, + lock: dict, +) -> VerifiedValidationArtifact: + reference_path = repository_path / "refs" / "main" + try: + resolved_reference = reference_path.resolve(strict=True) + resolved_reference.relative_to(cache_root) + except OSError as exc: + raise ValidationArtifactError( + f"{model_id}: could not read HF refs/main" + ) from exc + except ValueError as exc: + raise ValidationArtifactError( + f"{model_id}: HF refs/main escapes the cache" + ) from exc + + expected_reference = lock["revision"].encode("ascii") + canonical_references = {expected_reference, expected_reference + b"\n"} + try: + with resolved_reference.open("rb") as reference_file: + status_before = os.fstat(reference_file.fileno()) + if not stat.S_ISREG(status_before.st_mode): + raise ValidationArtifactError(f"{model_id}: HF refs/main is not a file") + reference_bytes = reference_file.read(42) + if reference_bytes not in canonical_references: + raise ValidationArtifactError( + f"{model_id}: HF refs/main does not match locked revision" + ) + + verified_artifact = _verify_locked_artifact( + cache_root, + repository_path, + model_id, + lock, + ) + + reference_file.seek(0) + reference_bytes_after = reference_file.read(42) + status_after = os.fstat(reference_file.fileno()) + + path_after = reference_path.resolve(strict=True) + path_after.relative_to(cache_root) + path_status = path_after.stat() + except ValidationArtifactError: + raise + except (OSError, ValueError) as exc: + raise ValidationArtifactError( + f"{model_id}: HF refs/main changed during verification" + ) from exc + if ( + reference_bytes_after not in canonical_references + or reference_bytes_after != reference_bytes + or path_after != resolved_reference + or _artifact_state(status_before) != _artifact_state(status_after) + or _artifact_state(status_after) != _artifact_state(path_status) + ): + raise ValidationArtifactError( + f"{model_id}: HF refs/main changed during verification" + ) + return verified_artifact + + +def verify_validation_artifacts( + cache_path: Path | str, + catalog_path: Path | str, + lock_path: Path | str, + model_ids: list[str], +) -> dict[str, VerifiedValidationArtifact]: + cache_path = Path(cache_path) + catalog = load_model_catalog(catalog_path) + locks = load_model_catalog(lock_path) + if set(catalog) != set(locks): + raise ValidationArtifactError( + "validation catalog and artifact lock IDs must match exactly" + ) + + requested = list( + dict.fromkeys(value.removeprefix("builtin.") for value in model_ids) + ) + locked_requested = [model_id for model_id in requested if model_id in locks] + if not locked_requested: + return {} + + try: + cache_root = cache_path.resolve(strict=True) + except OSError as exc: + raise ValidationArtifactError("HF cache does not exist") from exc + verified_artifacts = {} + for model_id in locked_requested: + model = catalog[model_id] + lock = locks[model_id] + _validate_lock(model_id, lock) + repository, checkpoint_filename = _parse_checkpoint( + model_id, model.get("checkpoint") + ) + if checkpoint_filename != lock["filename"]: + raise ValidationArtifactError( + f"{model_id}: checkpoint and artifact filenames differ" + ) + + repository_path = cache_path / ("models--" + repository.replace("/", "--")) + verified_artifacts[model_id] = _verify_reference_and_artifact( + cache_root, + repository_path, + model_id, + lock, + ) + + return verified_artifacts + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--cache", required=True, type=Path) + parser.add_argument("--catalog", required=True, type=Path) + parser.add_argument("--locks", required=True, type=Path) + parser.add_argument("--model", action="append", required=True) + args = parser.parse_args() + verify_validation_artifacts( + args.cache, + args.catalog, + args.locks, + args.model, + ) + + +if __name__ == "__main__": + main() diff --git a/test/utils/llamacpp_validation_evidence.py b/test/utils/llamacpp_validation_evidence.py index 24e52184bb..9cd21a54aa 100644 --- a/test/utils/llamacpp_validation_evidence.py +++ b/test/utils/llamacpp_validation_evidence.py @@ -5,7 +5,10 @@ import argparse import json +import math +import os import re +import stat from pathlib import Path if __package__: @@ -14,19 +17,33 @@ CapabilityValidationError, validate_capability_matrix_evidence, ) + from .llamacpp_validation_plan import ( + K2_LARGE, + K2_MEDIUM, + K2_SMALL, + VALIDATION_LANES, + require_complete_promotion_coverage, + ) else: from llamacpp_capability_validation import ( # type: ignore[import-not-found] K2_HORIZON_PROFILE, CapabilityValidationError, validate_capability_matrix_evidence, ) + from llamacpp_validation_plan import ( # type: ignore[import-not-found] + K2_LARGE, + K2_MEDIUM, + K2_SMALL, + VALIDATION_LANES, + require_complete_promotion_coverage, + ) -EXPECTED_RESULT_FILES = ( - "llamacpp_validation_windows-vulkan.json", - "llamacpp_validation_rocm-stable.json", - "llamacpp_validation_rocm-nightly.json", +EXPECTED_RESULT_FILES = tuple( + f"llamacpp_validation_{lane['target']}.json" for lane in VALIDATION_LANES +) +EXPECTED_RESTART_RESULT_FILES = tuple( + f"llamacpp_restart_validation_{lane['target']}.json" for lane in VALIDATION_LANES ) -K2_SMALL = "K2-Horizon-0.9B-GGUF" REQUIRED_RESULT_FIELDS = { "model", "pass", @@ -36,6 +53,8 @@ "time_to_first_token", "tokens_per_second", } +MAX_VALIDATION_EVIDENCE_BYTES = 4 * 1024 * 1024 +_EVIDENCE_READ_CHUNK_BYTES = 64 * 1024 class ValidationEvidenceError(ValueError): @@ -51,9 +70,26 @@ def _unique_json_object(pairs: list[tuple[str, object]]) -> dict: return result +def _reject_json_constant(value: str) -> None: + raise ValidationEvidenceError(f"nonstandard JSON numeric constant: {value}") + + +def _parse_finite_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + raise ValidationEvidenceError(f"non-finite JSON number: {value}") + return parsed + + def _validate_record(filename: str, index: int, record: object) -> dict: if not isinstance(record, dict): raise ValidationEvidenceError(f"{filename}: result {index} must be an object") + unexpected_fields = set(record) - REQUIRED_RESULT_FIELDS - {"capability_matrix"} + if unexpected_fields: + raise ValidationEvidenceError( + f"{filename}: result {index} has unexpected fields: " + + ", ".join(sorted(unexpected_fields)) + ) missing_fields = REQUIRED_RESULT_FIELDS - set(record) if missing_fields: raise ValidationEvidenceError( @@ -94,6 +130,7 @@ def _validate_record(filename: str, index: int, record: object) -> dict: isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0 + and (isinstance(value, int) or math.isfinite(value)) ) ): raise ValidationEvidenceError( @@ -106,9 +143,205 @@ def _canonical_model_id(model_id: str) -> str: return model_id.removeprefix("builtin.") +def _stat_snapshot(status: os.stat_result) -> tuple[int, int, int, int, int, int]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _read_bounded_result_file(path: Path) -> str: + filename = path.name + try: + path_status = os.stat(path, follow_symlinks=False) + except FileNotFoundError as exc: + raise ValidationEvidenceError(f"missing validation result: {filename}") from exc + except OSError as exc: + raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc + if not stat.S_ISREG(path_status.st_mode): + raise ValidationEvidenceError( + f"validation result must be a regular file: {filename}" + ) + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc + + try: + opened_status = os.fstat(descriptor) + if not stat.S_ISREG(opened_status.st_mode): + raise ValidationEvidenceError( + f"validation result must be a regular file: {filename}" + ) + if (opened_status.st_dev, opened_status.st_ino) != ( + path_status.st_dev, + path_status.st_ino, + ): + raise ValidationEvidenceError( + f"validation result changed before reading: {filename}" + ) + if opened_status.st_size > MAX_VALIDATION_EVIDENCE_BYTES: + raise ValidationEvidenceError( + f"validation result exceeds byte limit: {filename}" + ) + + chunks = [] + total_bytes = 0 + while total_bytes <= MAX_VALIDATION_EVIDENCE_BYTES: + chunk = os.read( + descriptor, + min( + _EVIDENCE_READ_CHUNK_BYTES, + MAX_VALIDATION_EVIDENCE_BYTES + 1 - total_bytes, + ), + ) + if not chunk: + break + chunks.append(chunk) + total_bytes += len(chunk) + final_status = os.fstat(descriptor) + except ValidationEvidenceError: + raise + except OSError as exc: + raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc + finally: + os.close(descriptor) + + if total_bytes > MAX_VALIDATION_EVIDENCE_BYTES: + raise ValidationEvidenceError( + f"validation result exceeds byte limit: {filename}" + ) + if _stat_snapshot(opened_status) != _stat_snapshot(final_status): + raise ValidationEvidenceError( + f"validation result changed while reading: {filename}" + ) + if total_bytes != final_status.st_size: + raise ValidationEvidenceError( + f"validation result size changed while reading: {filename}" + ) + try: + final_path_status = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise ValidationEvidenceError( + f"validation result changed while reading: {filename}" + ) from exc + if not stat.S_ISREG(final_path_status.st_mode) or ( + final_path_status.st_dev, + final_path_status.st_ino, + ) != (final_status.st_dev, final_status.st_ino): + raise ValidationEvidenceError( + f"validation result changed while reading: {filename}" + ) + try: + return b"".join(chunks).decode("utf-8") + except UnicodeError as exc: + raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc + + +def load_and_validate_result_file( + path: Path, + expected_models: list[str], + *, + capability_profile: str = "", + capability_models: list[str] | None = None, +) -> list[dict]: + filename = path.name + if ( + not expected_models + or not all(isinstance(model, str) and model for model in expected_models) + or len({_canonical_model_id(model) for model in expected_models}) + != len(expected_models) + ): + raise ValidationEvidenceError( + f"{filename}: expected models must be unique nonempty model identifiers" + ) + expected_capability_models = capability_models or [] + if bool(capability_profile) != bool(expected_capability_models): + raise ValidationEvidenceError( + f"{filename}: capability profile and models must be paired" + ) + canonical_expected_models = [ + _canonical_model_id(model) for model in expected_models + ] + canonical_capability_models = { + _canonical_model_id(model) for model in expected_capability_models + } + if len(canonical_capability_models) != len(expected_capability_models): + raise ValidationEvidenceError(f"{filename}: capability models must be unique") + if not canonical_capability_models.issubset(canonical_expected_models): + raise ValidationEvidenceError(f"{filename}: capability models must be selected") + raw = _read_bounded_result_file(path) + if not raw.strip(): + raise ValidationEvidenceError(f"validation result is empty: {filename}") + try: + records = json.loads( + raw, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_json_constant, + parse_float=_parse_finite_float, + ) + except json.JSONDecodeError as exc: + raise ValidationEvidenceError( + f"validation result is invalid JSON: {filename}: {exc}" + ) from exc + if not isinstance(records, list) or not records: + raise ValidationEvidenceError( + f"validation result must be a nonempty array: {filename}" + ) + checked = [ + _validate_record(filename, index, record) + for index, record in enumerate(records) + ] + models = [record["model"] for record in checked] + canonical_models = [_canonical_model_id(model) for model in models] + if len(canonical_models) != len(set(canonical_models)): + raise ValidationEvidenceError( + f"validation result contains duplicate models: {filename}" + ) + if canonical_models != canonical_expected_models: + raise ValidationEvidenceError( + f"{filename}: result models do not match planned models" + ) + for record, model_id in zip(checked, canonical_models): + if model_id not in canonical_capability_models: + if "capability_matrix" in record: + raise ValidationEvidenceError( + f"{filename}: {record['model']} has an unexpected capability matrix" + ) + continue + if "capability_matrix" not in record: + raise ValidationEvidenceError( + f"{filename}: {record['model']} is missing capability evidence" + ) + try: + validate_capability_matrix_evidence( + record["capability_matrix"], + capability_profile, + record["model"], + ) + except CapabilityValidationError as exc: + raise ValidationEvidenceError( + f"{filename}: {record['model']} capability evidence is invalid: {exc}" + ) from exc + return checked + + def _expected_models_by_file(expected_matrix: object) -> dict[str, dict]: if not isinstance(expected_matrix, dict): raise ValidationEvidenceError("validation matrix must be an object") + try: + require_complete_promotion_coverage(expected_matrix) + except ValueError as exc: + raise ValidationEvidenceError( + f"invalid scheduled validation lanes: {exc}" + ) from exc rows = expected_matrix.get("include") if not isinstance(rows, list): raise ValidationEvidenceError("validation matrix include must be an array") @@ -133,9 +366,14 @@ def _expected_models_by_file(expected_matrix: object) -> dict[str, dict]: filename = f"llamacpp_validation_{target}.json" if filename in expected: raise ValidationEvidenceError(f"duplicate validation lane: {target}") - if row.get("models") != []: + selected_models = row.get("models") + if ( + not isinstance(selected_models, list) + or not all(isinstance(model, str) and model for model in selected_models) + or len(selected_models) != len(set(selected_models)) + ): raise ValidationEvidenceError( - f"scheduled validation lane {target} must keep runtime hot selection" + f"validation matrix lane {target} needs unique selected models" ) models = row.get("expected_models") if ( @@ -147,7 +385,11 @@ def _expected_models_by_file(expected_matrix: object) -> dict[str, dict]: raise ValidationEvidenceError( f"validation matrix lane {target} needs unique expected models" ) - if K2_SMALL not in models: + if selected_models != models: + raise ValidationEvidenceError( + f"validation matrix lane {target} selected and expected models differ" + ) + if not any(_canonical_model_id(model) == K2_SMALL for model in models): raise ValidationEvidenceError( f"validation matrix lane {target} must include {K2_SMALL}" ) @@ -176,7 +418,11 @@ def _expected_models_by_file(expected_matrix: object) -> dict[str, dict]: raise ValidationEvidenceError( f"validation matrix lane {target} needs unique capability models" ) - if any(model not in models for model in capability_models): + canonical_models = {_canonical_model_id(model) for model in models} + if any( + _canonical_model_id(model) not in canonical_models + for model in capability_models + ): raise ValidationEvidenceError( f"validation matrix lane {target} has an unselected capability model" ) @@ -184,31 +430,24 @@ def _expected_models_by_file(expected_matrix: object) -> dict[str, dict]: raise ValidationEvidenceError( f"validation matrix lane {target} must validate {K2_SMALL} capabilities" ) + if any( + _canonical_model_id(model) in {K2_MEDIUM, K2_LARGE} for model in models + ) and (target != "windows-vulkan" or "128gb" not in row["runner"]): + raise ValidationEvidenceError( + f"validation matrix lane {target} needs the 128gb Vulkan runner " + "for K2-Horizon-3.7B/7B" + ) expected[filename] = { "models": models, "capability_profile": capability_profile, "capability_models": capability_models, + "restart_model": capability_models[0], } if set(expected) != set(EXPECTED_RESULT_FILES): raise ValidationEvidenceError( "validation matrix must contain exactly the scheduled validation lanes" ) - planned_model_lists = [lane["models"] for lane in expected.values()] - if any(models != planned_model_lists[0] for models in planned_model_lists[1:]): - raise ValidationEvidenceError( - "scheduled validation lanes must use the same planned model set" - ) - planned_capability_models = [ - lane["capability_models"] for lane in expected.values() - ] - if any( - models != planned_capability_models[0] - for models in planned_capability_models[1:] - ): - raise ValidationEvidenceError( - "scheduled validation lanes must use the same capability model set" - ) return expected @@ -219,66 +458,20 @@ def load_and_validate_results( expected_lanes = _expected_models_by_file(expected_matrix) validated = {} for filename in EXPECTED_RESULT_FILES: - path = root / filename - if not path.is_file(): - raise ValidationEvidenceError(f"missing validation result: {filename}") - try: - raw = path.read_text(encoding="utf-8") - except OSError as exc: - raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc - if not raw.strip(): - raise ValidationEvidenceError(f"validation result is empty: {filename}") - try: - records = json.loads(raw, object_pairs_hook=_unique_json_object) - except json.JSONDecodeError as exc: - raise ValidationEvidenceError( - f"validation result is invalid JSON: {filename}: {exc}" - ) from exc - if not isinstance(records, list) or not records: - raise ValidationEvidenceError( - f"validation result must be a nonempty array: {filename}" - ) - checked = [ - _validate_record(filename, index, record) - for index, record in enumerate(records) - ] - models = [record["model"] for record in checked] - if len(models) != len(set(models)): - raise ValidationEvidenceError( - f"validation result contains duplicate models: {filename}" - ) expected_lane = expected_lanes[filename] - if models != expected_lane["models"]: - raise ValidationEvidenceError( - f"{filename}: result models do not match planned models" - ) - capability_models = { - _canonical_model_id(model) for model in expected_lane["capability_models"] - } - for record in checked: - model_id = _canonical_model_id(record["model"]) - if model_id not in capability_models: - if "capability_matrix" in record: - raise ValidationEvidenceError( - f"{filename}: {record['model']} has an unexpected capability matrix" - ) - continue - if "capability_matrix" not in record: - raise ValidationEvidenceError( - f"{filename}: {record['model']} is missing capability evidence" - ) - capability_matrix = record["capability_matrix"] - try: - validate_capability_matrix_evidence( - capability_matrix, - expected_lane["capability_profile"], - record["model"], - ) - except CapabilityValidationError as exc: - raise ValidationEvidenceError( - f"{filename}: {record['model']} capability evidence is invalid: {exc}" - ) from exc - validated[filename] = checked + validated[filename] = load_and_validate_result_file( + root / filename, + expected_lane["models"], + capability_profile=expected_lane["capability_profile"], + capability_models=expected_lane["capability_models"], + ) + restart_filename = filename.replace( + "llamacpp_validation_", "llamacpp_restart_validation_", 1 + ) + validated[restart_filename] = load_and_validate_result_file( + root / restart_filename, + [expected_lane["restart_model"]], + ) return validated @@ -291,6 +484,8 @@ def main() -> None: expected_matrix = json.loads( args.expected_matrix_json, object_pairs_hook=_unique_json_object, + parse_constant=_reject_json_constant, + parse_float=_parse_finite_float, ) results = load_and_validate_results(args.directory, expected_matrix) except (json.JSONDecodeError, ValidationEvidenceError) as exc: diff --git a/test/utils/llamacpp_validation_plan.py b/test/utils/llamacpp_validation_plan.py index da0d5c5eb1..2f922a9dbf 100644 --- a/test/utils/llamacpp_validation_plan.py +++ b/test/utils/llamacpp_validation_plan.py @@ -7,10 +7,25 @@ import os from pathlib import Path +if __package__: + from .validation_model_catalog import ( + load_model_catalog, + merge_model_catalogs, + ) +else: + from validation_model_catalog import ( # type: ignore[import-not-found] + load_model_catalog, + merge_model_catalogs, + ) + K2_SMALL = "K2-Horizon-0.9B-GGUF" K2_MEDIUM = "K2-Horizon-3.7B-GGUF" K2_LARGE = "K2-Horizon-7B-GGUF" SUPPORTED_EVENTS = {"merge_group", "pull_request", "schedule", "workflow_dispatch"} +K2_CAPABILITY_PROFILE = "k2-horizon-v1" +MANAGED_PINS = frozenset( + {"cpu", "cuda", "metal", "rocm-nightly", "rocm-stable", "vulkan"} +) MODEL_REGISTRY = ( Path(__file__).resolve().parents[2] / "src" @@ -18,6 +33,9 @@ / "resources" / "server_models.json" ) +VALIDATION_MODEL_OVERLAY = ( + Path(__file__).resolve().parents[1] / "fixtures" / "llamacpp_validation_models.json" +) LARGE_VULKAN_RUNNER = [ "self-hosted", @@ -26,21 +44,21 @@ "vulkan", "lemon-prod", ] -LARGE_ROCM_RUNNER = [ +LINUX_VULKAN_RUNNER = [ "self-hosted", - "Windows", - "128gb", - "rocm", + "Linux", + "vulkan", "lemon-prod", ] -SMALL_VULKAN_RUNNER = [ +LINUX_CPU_RUNNER = ["self-hosted", "Linux", "lemon-prod"] +MACOS_METAL_RUNNER = ["macos-latest"] +WINDOWS_CUDA_RUNNER = [ "self-hosted", "Windows", - "stx-halo", - "vulkan", + "cuda", "lemon-prod", ] -SMALL_ROCM_RUNNER = [ +WINDOWS_ROCM_RUNNER = [ "self-hosted", "Windows", "stx-halo", @@ -48,18 +66,75 @@ "lemon-prod", ] +VALIDATION_LANES = ( + { + "target": "windows-vulkan", + "backend": "vulkan", + "channel": "", + "managed_pin": "vulkan", + "build_platform": "windows", + "runner": LARGE_VULKAN_RUNNER, + }, + { + "target": "linux-vulkan", + "backend": "vulkan", + "channel": "", + "managed_pin": "vulkan", + "build_platform": "linux", + "runner": LINUX_VULKAN_RUNNER, + }, + { + "target": "linux-cpu", + "backend": "cpu", + "channel": "", + "managed_pin": "cpu", + "build_platform": "linux", + "runner": LINUX_CPU_RUNNER, + }, + { + "target": "macos-metal", + "backend": "metal", + "channel": "", + "managed_pin": "metal", + "build_platform": "macos", + "runner": MACOS_METAL_RUNNER, + }, + { + "target": "windows-cuda", + "backend": "cuda", + "channel": "", + "managed_pin": "cuda", + "build_platform": "windows", + "runner": WINDOWS_CUDA_RUNNER, + }, + { + "target": "windows-rocm-stable", + "backend": "rocm", + "channel": "stable", + "managed_pin": "rocm-stable", + "build_platform": "windows", + "runner": WINDOWS_ROCM_RUNNER, + }, + { + "target": "windows-rocm-nightly", + "backend": "rocm", + "channel": "nightly", + "managed_pin": "rocm-nightly", + "build_platform": "windows", + "runner": WINDOWS_ROCM_RUNNER, + }, +) + def _parse_models(models_csv: str) -> list[str]: return [model.strip() for model in models_csv.split(",") if model.strip()] -def load_hot_llamacpp_model_ids(path: Path = MODEL_REGISTRY) -> list[str]: - try: - registry = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise ValueError(f"Could not read model registry {path}: {exc}") from exc - if not isinstance(registry, dict): - raise ValueError(f"Model registry {path} must be an object") +def _canonical_model_id(model_id: str) -> str: + return model_id.removeprefix("builtin.") + + +def _hot_llamacpp_model_ids(registry: dict, source: Path) -> list[str]: models = sorted( model_id for model_id, model in registry.items() @@ -69,37 +144,160 @@ def load_hot_llamacpp_model_ids(path: Path = MODEL_REGISTRY) -> list[str]: and "hot" in model["labels"] ) if not models: - raise ValueError(f"Model registry {path} has no hot llama.cpp models") + raise ValueError(f"Model registry {source} has no hot llama.cpp models") return models +def load_hot_llamacpp_model_ids(path: Path = MODEL_REGISTRY) -> list[str]: + return _hot_llamacpp_model_ids(load_model_catalog(path), path) + + +def load_validation_candidate_model_ids( + path: Path = MODEL_REGISTRY, + overlay_path: Path = VALIDATION_MODEL_OVERLAY, +) -> list[str]: + product_hot_models = _hot_llamacpp_model_ids(load_model_catalog(path), path) + overlay = load_model_catalog(overlay_path) + merged = merge_model_catalogs(path, overlay_path) + + for model_id in overlay: + model = merged[model_id] + if model.get("recipe") != "llamacpp": + raise ValueError( + f"Validation model {model_id} must use the llamacpp recipe" + ) + labels = model.get("labels") + if not isinstance(labels, list) or "chat" not in labels: + raise ValueError(f"Validation model {model_id} must support chat") + + return sorted(set(product_hot_models) | set(overlay)) + + def _row( - target: str, - backend: str, - channel: str, - runner: list[str], + lane: dict[str, object], models: list[str], - lite: bool, expected_models: list[str] | None = None, - capability_profile: str = "", - capability_models: list[str] | None = None, ) -> dict[str, object]: row = { - "target": target, - "backend": backend, - "channel": channel, - "runner": list(runner), + "target": lane["target"], + "backend": lane["backend"], + "channel": lane["channel"], + "managed_pin": lane["managed_pin"], + "build_platform": lane["build_platform"], + "runner": list(lane["runner"]), "models": list(models), - "lite": lite, + "lite": False, + "capability_profile": K2_CAPABILITY_PROFILE, + "capability_models": [K2_SMALL], } if expected_models is not None: row["expected_models"] = list(expected_models) - if capability_profile: - row["capability_profile"] = capability_profile - row["capability_models"] = list(capability_models or []) return row +def _with_required_k2_small(models: list[str]) -> list[str]: + if any(_canonical_model_id(model) == K2_SMALL for model in models): + return list(models) + return [K2_SMALL, *models] + + +def _small_lane_models(models: list[str]) -> list[str]: + return [ + model + for model in _with_required_k2_small(models) + if _canonical_model_id(model) not in {K2_MEDIUM, K2_LARGE} + ] + + +def require_complete_promotion_coverage( + plan: dict[str, list[dict[str, object]]], +) -> None: + rows = plan.get("include") if isinstance(plan, dict) else None + if not isinstance(rows, list) or len(rows) != len(VALIDATION_LANES): + raise ValueError("Promotion requires exactly the seven validation lanes") + + candidate_models = load_validation_candidate_model_ids() + actual_targets = [] + for index, (row, lane) in enumerate(zip(rows, VALIDATION_LANES)): + if not isinstance(row, dict): + raise ValueError(f"Validation lane {index} must be an object") + actual_targets.append(row.get("target")) + for field in ( + "target", + "backend", + "channel", + "managed_pin", + "build_platform", + ): + if row.get(field) != lane[field]: + raise ValueError( + f"Validation lane {index} has an invalid {field} mapping" + ) + if row.get("runner") != lane["runner"]: + raise ValueError(f"Validation lane {index} has an invalid runner mapping") + if row.get("capability_profile") != K2_CAPABILITY_PROFILE: + raise ValueError(f"Validation lane {index} lacks the K2 capability profile") + if row.get("capability_models") != [K2_SMALL]: + raise ValueError( + f"Validation lane {index} lacks the K2 small capability model" + ) + + if row.get("lite") is not False: + raise ValueError(f"Validation lane {index} cannot use lite mode") + + selected_models = row.get("models") + expected_models = row.get("expected_models") + for field_name, model_ids in ( + ("selected", selected_models), + ("expected", expected_models), + ): + if ( + not isinstance(model_ids, list) + or not model_ids + or not all( + isinstance(model_id, str) and model_id for model_id in model_ids + ) + or len(model_ids) != len(set(model_ids)) + ): + raise ValueError( + f"Validation lane {index} {field_name} models must be " + "unique nonempty strings" + ) + if selected_models != expected_models: + raise ValueError( + f"Validation lane {index} selected models do not match expected models" + ) + + planned_models = expected_models + if not any( + _canonical_model_id(model_id) == K2_SMALL for model_id in planned_models + ): + raise ValueError(f"Validation lane {index} does not include {K2_SMALL}") + if any( + _canonical_model_id(model_id) in {K2_MEDIUM, K2_LARGE} + for model_id in planned_models + ): + if row["target"] != "windows-vulkan" or "128gb" not in row["runner"]: + raise ValueError( + "K2-Horizon-3.7B/7B validation requires the 128gb Vulkan lane" + ) + + if row["target"] == "windows-vulkan": + if planned_models != candidate_models: + raise ValueError( + "The 128gb Vulkan lane must use the exact candidate model set" + ) + elif planned_models != [K2_SMALL]: + raise ValueError( + f"Validation lane {index} must use the exact K2 small model set" + ) + + if actual_targets != [lane["target"] for lane in VALIDATION_LANES]: + raise ValueError("Promotion validation lanes must use the exact target order") + if {row["managed_pin"] for row in rows} != MANAGED_PINS: + raise ValueError("Promotion validation does not cover every managed pin") + + def create_validation_plan( event_name: str, models_csv: str = "", @@ -118,80 +316,60 @@ def create_validation_plan( return { "include": [ _row( - "windows-vulkan", - "vulkan", - "", - LARGE_VULKAN_RUNNER, - [K2_SMALL, K2_MEDIUM, K2_LARGE], - False, - capability_profile="k2-horizon-v1", - capability_models=[K2_SMALL], - ), - _row( - "rocm-stable", - "rocm", - "stable", - SMALL_ROCM_RUNNER, - [K2_SMALL], - False, - capability_profile="k2-horizon-v1", - capability_models=[K2_SMALL], - ), + lane, + ( + [K2_SMALL, K2_MEDIUM, K2_LARGE] + if lane["target"] == "windows-vulkan" + else [K2_SMALL] + ), + ) + for lane in VALIDATION_LANES + ] + } + + if event_name == "schedule": + candidate_models = load_validation_candidate_model_ids() + plan = { + "include": [ _row( - "rocm-nightly", - "rocm", - "nightly", - SMALL_ROCM_RUNNER, - [K2_SMALL], - False, - capability_profile="k2-horizon-v1", - capability_models=[K2_SMALL], - ), + lane, + ( + candidate_models + if lane["target"] == "windows-vulkan" + else [K2_SMALL] + ), + ( + candidate_models + if lane["target"] == "windows-vulkan" + else [K2_SMALL] + ), + ) + for lane in VALIDATION_LANES ] } + require_complete_promotion_coverage(plan) + return plan - runner_vulkan = SMALL_VULKAN_RUNNER if lite else LARGE_VULKAN_RUNNER - runner_rocm = SMALL_ROCM_RUNNER if lite else LARGE_ROCM_RUNNER - expected_models = ( - load_hot_llamacpp_model_ids() if event_name == "schedule" else None - ) - capability_profile = "k2-horizon-v1" if event_name == "schedule" else "" - capability_models = [K2_SMALL] if event_name == "schedule" else None + if lite: + large_lane_models = [K2_SMALL] + small_lane_models = [K2_SMALL] + elif models: + large_lane_models = _with_required_k2_small(models) + small_lane_models = _small_lane_models(models) + else: + large_lane_models = _with_required_k2_small(load_hot_llamacpp_model_ids()) + small_lane_models = [K2_SMALL] return { "include": [ _row( - "windows-vulkan", - "vulkan", - "", - runner_vulkan, - models, - lite, - expected_models, - capability_profile, - capability_models, - ), - _row( - "rocm-stable", - "rocm", - "stable", - runner_rocm, - models, - lite, - expected_models, - capability_profile, - capability_models, - ), - _row( - "rocm-nightly", - "rocm", - "nightly", - runner_rocm, - models, - lite, - expected_models, - capability_profile, - capability_models, - ), + lane, + ( + large_lane_models + if lane["target"] == "windows-vulkan" + else small_lane_models + ), + ) + for lane in VALIDATION_LANES ] } @@ -201,8 +379,11 @@ def is_promotion_eligible( models_csv: str = "", lite: bool = False, ) -> bool: - create_validation_plan(event_name, models_csv, lite) - return event_name == "schedule" + plan = create_validation_plan(event_name, models_csv, lite) + if event_name != "schedule": + return False + require_complete_promotion_coverage(plan) + return True def write_github_output( diff --git a/test/utils/server_base.py b/test/utils/server_base.py index 9e22b60743..bfa095bea0 100644 --- a/test/utils/server_base.py +++ b/test/utils/server_base.py @@ -131,7 +131,7 @@ def get_cli_binary(): return _config["cli_binary"] -def wait_for_server(port=PORT, timeout=60): +def wait_for_server(port=PORT, timeout=60, host="localhost"): """ Wait for the server to start by checking if the port is available. @@ -147,7 +147,7 @@ def wait_for_server(port=PORT, timeout=60): if time.time() - start_time > timeout: raise TimeoutError(f"Server failed to start within {timeout} seconds") try: - conn = socket.create_connection(("localhost", port)) + conn = socket.create_connection((host, port)) conn.close() return True except socket.error: @@ -181,13 +181,13 @@ def set_server_config(config: dict, port=PORT): return response.json() -def unload_all_models(port=PORT, attempts=3): +def unload_all_models(port=PORT, attempts=3, host="localhost"): """POST /api/v1/unload to unload all models for clean state.""" response = None for attempt in range(1, attempts + 1): try: response = requests.post( - f"http://localhost:{port}/api/v1/unload", + f"http://{host}:{port}/api/v1/unload", json={}, headers=_auth_headers(), timeout=30, @@ -280,7 +280,7 @@ def _is_transient_pull_status(status_code): return status_code in {408, 409, 429, 500, 502, 503, 504} -def _pull_model_streaming(model_name, port): +def _pull_model_streaming(model_name, port, host="localhost"): """Pull via the SSE streaming mode and block until the download completes. Large models (10+ GB) exceed any fixed read timeout on the synchronous @@ -288,7 +288,7 @@ def _pull_model_streaming(model_name, port): applies between events, not to the whole download. """ with requests.post( - f"http://localhost:{port}/api/v1/pull", + f"http://{host}:{port}/api/v1/pull", json={"model_name": model_name, "stream": True}, stream=True, timeout=TIMEOUT_MODEL_OPERATION, @@ -312,7 +312,7 @@ def _pull_model_streaming(model_name, port): return 500, "SSE stream ended without a 'complete' event" -def pull_model_with_retry(model_name, attempts=3, port=PORT): +def pull_model_with_retry(model_name, attempts=3, port=PORT, host="localhost"): """Pull a model with bounded retry for transient setup failures. Test setup should tolerate one-off transient pull failures, but persistent @@ -327,7 +327,7 @@ def pull_model_with_retry(model_name, attempts=3, port=PORT): time.sleep(min(30, 2 ** (attempt - 1))) try: - status, body = _pull_model_streaming(model_name, port) + status, body = _pull_model_streaming(model_name, port, host=host) except requests.RequestException as exc: last_error = exc if attempt < attempts: diff --git a/test/utils/validation_model_catalog.py b/test/utils/validation_model_catalog.py new file mode 100644 index 0000000000..34745a5f49 --- /dev/null +++ b/test/utils/validation_model_catalog.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Merge validation-only model records into a disposable Lemonade catalog.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import tempfile +from pathlib import Path + + +class ModelCatalogOverlayError(ValueError): + """Raised when a validation catalog overlay cannot be merged safely.""" + + +def _reject_duplicate_members(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ModelCatalogOverlayError(f"duplicate JSON member: {key}") + result[key] = value + return result + + +def _reject_non_finite_number(value: str): + raise ModelCatalogOverlayError(f"non-finite JSON number: {value}") + + +def _parse_finite_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + _reject_non_finite_number(value) + return parsed + + +def _json_values_equal(left: object, right: object) -> bool: + if type(left) is not type(right): + return False + if isinstance(left, dict): + return left.keys() == right.keys() and all( + _json_values_equal(left[key], right[key]) for key in left + ) + if isinstance(left, list): + return len(left) == len(right) and all( + _json_values_equal(left_value, right_value) + for left_value, right_value in zip(left, right) + ) + return left == right + + +def load_model_catalog(path: Path | str) -> dict: + path = Path(path) + try: + catalog = json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=_reject_duplicate_members, + parse_constant=_reject_non_finite_number, + parse_float=_parse_finite_float, + ) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ModelCatalogOverlayError(f"{path}: invalid JSON: {exc}") from exc + if not isinstance(catalog, dict): + raise ModelCatalogOverlayError(f"{path}: catalog must be a JSON object") + for model_id, model in catalog.items(): + if not isinstance(model, dict): + raise ModelCatalogOverlayError( + f"{path}: model {model_id} must be a JSON object" + ) + return catalog + + +def merge_model_catalogs( + base_path: Path | str, + overlay_path: Path | str, +) -> dict: + base_path = Path(base_path) + overlay_path = Path(overlay_path) + merged = load_model_catalog(base_path) + overlay = load_model_catalog(overlay_path) + + for model_id, candidate in overlay.items(): + if model_id in merged and not _json_values_equal(merged[model_id], candidate): + raise ModelCatalogOverlayError(f"conflicting model definition: {model_id}") + merged[model_id] = candidate + return merged + + +def merge_model_catalog_files( + base_path: Path | str, + overlay_path: Path | str, + output_path: Path | str, +) -> None: + output_path = Path(output_path) + merged = merge_model_catalogs(base_path, overlay_path) + + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + "w", + dir=output_path.parent, + encoding="utf-8", + delete=False, + ) as temporary: + temporary_path = Path(temporary.name) + json.dump(merged, temporary, indent=4, sort_keys=True, allow_nan=False) + temporary.write("\n") + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, output_path) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Merge a validation-only model overlay into a catalog" + ) + parser.add_argument("--base", required=True, type=Path) + parser.add_argument("--overlay", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + merge_model_catalog_files(args.base, args.overlay, args.output) + + +if __name__ == "__main__": + main() diff --git a/test/utils/validation_model_selection.py b/test/utils/validation_model_selection.py index 28e1c462be..276279084a 100644 --- a/test/utils/validation_model_selection.py +++ b/test/utils/validation_model_selection.py @@ -35,6 +35,10 @@ def _model_size(model: dict[str, Any]) -> float: return math.inf +def _canonical_builtin_model_id(model_id: str) -> str: + return model_id if model_id.startswith("builtin.") else f"builtin.{model_id}" + + def _require_pullable_checkpoint(model: dict[str, Any]) -> None: checkpoint = model.get("checkpoint") if isinstance(checkpoint, str) and checkpoint.strip(): @@ -72,11 +76,7 @@ def select_llamacpp_models( ) selected = [] for requested_model_id in requested_model_ids: - canonical_model_id = ( - requested_model_id - if requested_model_id.startswith("builtin.") - else f"builtin.{requested_model_id}" - ) + canonical_model_id = _canonical_builtin_model_id(requested_model_id) model = builtin_model_resolver(canonical_model_id) if model is None: raise ModelSelectionError( @@ -84,6 +84,14 @@ def select_llamacpp_models( "model catalog" ) model = dict(model) + resolved_model_id = model.get("id") + if not isinstance(resolved_model_id, str) or ( + _canonical_builtin_model_id(resolved_model_id) != canonical_model_id + ): + raise ModelSelectionError( + f"requested built-in model '{canonical_model_id}' resolved to " + f"unrelated id '{resolved_model_id}'" + ) model["load_id"] = canonical_model_id recipe = model.get("recipe") if recipe != "llamacpp": diff --git a/test/validate_llamacpp.py b/test/validate_llamacpp.py index f5b56e2367..8e07b3ef74 100644 --- a/test/validate_llamacpp.py +++ b/test/validate_llamacpp.py @@ -24,19 +24,26 @@ import json import os import shutil +import socket +import subprocess import sys import tempfile +import time from urllib.parse import quote import requests from utils.llamacpp_capability_validation import ( + CapabilityValidationError, K2_HORIZON_PROFILE, + MAX_STREAM_EVENTS, + MAX_STREAM_SSE_BYTES, + STREAM_WALL_CLOCK_TIMEOUT_SECONDS, find_raw_ifm_control_marker, reject_duplicate_json_object, validate_capabilities, ) -from utils.server_base import _auth_headers, unload_all_models, wait_for_server +from utils.server_base import _auth_headers from utils.test_models import PORT, TIMEOUT_DEFAULT from utils.validation_model_selection import ( ModelSelectionError, @@ -46,7 +53,25 @@ TIMEOUT_HEALTH = 60 TIMEOUT_INFERENCE = 1800 # 30 minutes; large models may need 60+ GB download +PULL_WALL_CLOCK_TIMEOUT_SECONDS = 3 * 60 * 60 +MAX_PULL_RESPONSE_BYTES = 64 * 1024 +MAX_PULL_SSE_WIRE_BYTES = 1024 * 1024 * 1024 +MAX_PULL_SSE_EVENTS = 4 * 1024 * 1024 +MAX_PULL_SSE_LINE_BYTES = 64 * 1024 +MAX_PULL_SSE_EVENT_DATA_BYTES = 64 * 1024 +MAX_PULL_TERMINAL_SUMMARY_BYTES = 4096 +SERVER_CONNECT_ATTEMPT_TIMEOUT_SECONDS = 1 +SERVER_CONNECT_RETRY_DELAY_SECONDS = 1 VALIDATION_CTX_SIZE = 8192 +VALIDATION_HOST = "127.0.0.1" +INTERNAL_STREAM_WORKER_ARGUMENT = "--internal-stream-worker" +STREAM_WORKER_PROTOCOL_VERSION = 2 +STREAM_WORKER_MAGIC = b"LEMONADE_STREAM_WORKER_V2\n" +MAX_STREAM_WORKER_INPUT_BYTES = 1024 * 1024 +MAX_STREAM_WORKER_HEADER_BYTES = 64 * 1024 +MAX_STREAM_WORKER_ERROR_BYTES = 4096 +STREAM_WORKER_CHUNK_BYTES = 64 * 1024 +MAX_STREAM_SSE_LINE_BYTES = 64 * 1024 CHAT_PROMPT = [ {"role": "user", "content": "What is 2+2? Reply in one sentence."}, ] @@ -75,39 +100,759 @@ def collect_server_logs(output_dir): def request_json(method, url, timeout, **kwargs): - """Perform an HTTP request and parse the JSON response when present.""" + """Perform a bounded HTTP request and parse the JSON response when present.""" + deadline = kwargs.pop("wall_clock_deadline", None) + if deadline is None: + deadline = time.monotonic() + timeout + unsupported_options = set(kwargs) - {"headers", "json"} + if unsupported_options: + raise CapabilityValidationError("JSON request options are invalid") auth_headers = _auth_headers() headers = {**(kwargs.get("headers") or {}), **auth_headers} - request_kwargs = {**kwargs, "headers": headers} - response = requests.request(method, url, timeout=timeout, **request_kwargs) + response = _execute_http_worker( + method, + url, + timeout, + headers, + kwargs.get("json"), + "json", + deadline, + "HTTP response", + ) body = {} if response.content: try: body = response.json(object_pairs_hook=reject_duplicate_json_object) except ValueError: body = {"raw_text": response.text} + if time.monotonic() >= deadline: + raise CapabilityValidationError( + "HTTP response exceeded its wall-clock deadline" + ) return response, body -def request_stream(method, url, timeout, **kwargs): - """Perform a streaming HTTP request and return decoded response lines.""" - auth_headers = _auth_headers() - headers = {**(kwargs.get("headers") or {}), **auth_headers} - request_kwargs = {**kwargs, "headers": headers, "stream": True} - response = requests.request(method, url, timeout=timeout, **request_kwargs) +class _BufferedResponse: + def __init__(self, status_code, headers, encoding, content): + self.status_code = status_code + self.headers = headers + self.encoding = encoding + self.content = content + + @property + def text(self): + encoding = self.encoding or "utf-8" + try: + return self.content.decode(encoding, errors="replace") + except LookupError: + return self.content.decode("utf-8", errors="replace") + + def json(self, **kwargs): + return json.loads(self.text, **kwargs) + + def close(self): + return None + + +def _encode_stream_worker_output( + status_code, + body, + headers=None, + encoding=None, +): + header = json.dumps( + { + "version": STREAM_WORKER_PROTOCOL_VERSION, + "status_code": status_code, + "body_bytes": len(body), + "headers": headers or {}, + "encoding": encoding, + }, + allow_nan=False, + separators=(",", ":"), + ).encode("ascii") + if len(header) > MAX_STREAM_WORKER_HEADER_BYTES: + raise CapabilityValidationError("HTTP worker header exceeds bounds") + return STREAM_WORKER_MAGIC + header + b"\n" + body + + +def _decode_http_worker_output(output): + maximum_output_bytes = ( + len(STREAM_WORKER_MAGIC) + + MAX_STREAM_WORKER_HEADER_BYTES + + 1 + + MAX_STREAM_SSE_BYTES + ) + if not isinstance(output, bytes) or len(output) > maximum_output_bytes: + raise CapabilityValidationError("HTTP worker output exceeds bounds") + if not output.startswith(STREAM_WORKER_MAGIC): + raise CapabilityValidationError("HTTP worker output has invalid framing") + raw_header, separator, body = output[len(STREAM_WORKER_MAGIC) :].partition(b"\n") + if not separator or len(raw_header) > MAX_STREAM_WORKER_HEADER_BYTES: + raise CapabilityValidationError("HTTP worker output has invalid framing") + try: + header = json.loads( + raw_header.decode("ascii"), + object_pairs_hook=reject_duplicate_json_object, + ) + except (UnicodeError, json.JSONDecodeError) as exc: + raise CapabilityValidationError( + "HTTP worker output has an invalid header" + ) from exc + if not isinstance(header, dict) or set(header) != { + "version", + "status_code", + "body_bytes", + "headers", + "encoding", + }: + raise CapabilityValidationError("HTTP worker output has an invalid header") + status_code = header["status_code"] + body_bytes = header["body_bytes"] + headers = header["headers"] + encoding = header["encoding"] + if ( + header["version"] != STREAM_WORKER_PROTOCOL_VERSION + or not isinstance(status_code, int) + or isinstance(status_code, bool) + or not 100 <= status_code <= 599 + or not isinstance(body_bytes, int) + or isinstance(body_bytes, bool) + or body_bytes != len(body) + or body_bytes > MAX_STREAM_SSE_BYTES + or not isinstance(headers, dict) + or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in headers.items() + ) + or not (encoding is None or isinstance(encoding, str)) + ): + raise CapabilityValidationError("HTTP worker output has an invalid header") + return _BufferedResponse(status_code, headers, encoding, body) + + +def _decode_stream_worker_output(output): + response = _decode_http_worker_output(output) + return response, _split_stream_lines(response.content) + + +def _split_stream_lines(body): + lines = [] + event_count = 0 + line_start = 0 + while line_start < len(body): + newline = body.find(b"\n", line_start) + if newline < 0: + line = body[line_start:] + line_start = len(body) + else: + line = body[line_start:newline] + line_start = newline + 1 + if len(line) > MAX_STREAM_SSE_LINE_BYTES: + raise CapabilityValidationError("stream SSE line exceeds bounds") + stripped = line.strip() + if not stripped or stripped.startswith(b":"): + continue + if not stripped.startswith(b"data:"): + raise CapabilityValidationError("stream line is not an SSE data frame") + event_count += 1 + if event_count > MAX_STREAM_EVENTS: + raise CapabilityValidationError("stream event count exceeds bounds") + lines.append(line) + return tuple(lines) + + +def _iter_bounded_response_lines(response, max_response_bytes): + total_bytes = 0 + pending = bytearray() + for chunk in response.iter_content( + chunk_size=STREAM_WORKER_CHUNK_BYTES, + decode_unicode=False, + ): + if not isinstance(chunk, bytes): + raise CapabilityValidationError("HTTP response chunk must be bytes") + total_bytes += len(chunk) + if total_bytes > max_response_bytes: + raise CapabilityValidationError( + "model pull SSE wire byte count exceeds bounds" + ) + start = 0 + while start < len(chunk): + newline = chunk.find(b"\n", start) + end = len(chunk) if newline < 0 else newline + segment = chunk[start:end] + if len(pending) + len(segment) > MAX_PULL_SSE_LINE_BYTES: + raise CapabilityValidationError("model pull SSE line exceeds bounds") + pending.extend(segment) + if newline < 0: + break + line = bytes(pending) + pending.clear() + if line.endswith(b"\r"): + line = line[:-1] + yield line + start = newline + 1 + if pending: + raise CapabilityValidationError("model pull SSE ended with an incomplete line") + + +def _reject_non_finite_json(value): + raise ValueError(f"non-finite JSON value: {value}") + + +def _parse_pull_event_data(data): + try: + payload = json.loads( + data.decode("utf-8"), + object_pairs_hook=reject_duplicate_json_object, + parse_constant=_reject_non_finite_json, + ) + except (UnicodeError, ValueError) as exc: + raise CapabilityValidationError("model pull SSE event data is invalid") from exc + if not isinstance(payload, dict): + raise CapabilityValidationError("model pull SSE event data is invalid") + return payload + + +def _pull_content_type(response): + for key, value in response.headers.items(): + if str(key).lower() == "content-type": + return str(value).split(";", maxsplit=1)[0].strip().lower() + return "" + + +def _bounded_pull_error(error): + encoded = str(error).encode("utf-8", errors="replace")[:1000] + return encoded.decode("utf-8", errors="ignore") + + +def _reduce_pull_sse_response(response, max_response_bytes): + if _pull_content_type(response) != "text/event-stream": + raise CapabilityValidationError( + "model pull returned an invalid SSE content type" + ) + + event_name = None + event_data = bytearray() + data_seen = False + event_count = 0 + terminal = None + + for line in _iter_bounded_response_lines(response, max_response_bytes): + if not line: + if event_name is None and not data_seen: + continue + event_count += 1 + if event_count > MAX_PULL_SSE_EVENTS: + raise CapabilityValidationError( + "model pull SSE event count exceeds bounds" + ) + if terminal is not None: + raise CapabilityValidationError( + "model pull SSE contained an event after its terminal event" + ) + if event_name not in {"progress", "complete", "error"} or not data_seen: + raise CapabilityValidationError("model pull SSE event is invalid") + payload = _parse_pull_event_data(bytes(event_data)) + if event_name == "complete": + terminal = {"terminal": "complete"} + elif event_name == "error": + error = payload.get("error") + code = payload.get("code") + if ( + not isinstance(error, str) + or not error + or not (code is None or isinstance(code, str)) + ): + raise CapabilityValidationError( + "model pull SSE error event is invalid" + ) + message = _bounded_pull_error(error) + terminal = { + "terminal": "error", + "status_code": ( + 400 + if code == "unknown_model" or "unknown_model" in message + else 500 + ), + "error": message, + } + event_name = None + event_data.clear() + data_seen = False + continue + + if terminal is not None: + raise CapabilityValidationError( + "model pull SSE contained data after its terminal event" + ) + if line.startswith(b"event:"): + if event_name is not None: + raise CapabilityValidationError("model pull SSE event is invalid") + raw_name = line[len(b"event:") :] + if raw_name.startswith(b" "): + raw_name = raw_name[1:] + try: + event_name = raw_name.decode("ascii") + except UnicodeError as exc: + raise CapabilityValidationError( + "model pull SSE event is invalid" + ) from exc + elif line.startswith(b"data:"): + raw_data = line[len(b"data:") :] + if raw_data.startswith(b" "): + raw_data = raw_data[1:] + additional_bytes = len(raw_data) + (1 if data_seen else 0) + if len(event_data) + additional_bytes > MAX_PULL_SSE_EVENT_DATA_BYTES: + raise CapabilityValidationError( + "model pull SSE event data exceeds bounds" + ) + if data_seen: + event_data.extend(b"\n") + event_data.extend(raw_data) + data_seen = True + else: + raise CapabilityValidationError("model pull SSE line is invalid") + + if event_name is not None or data_seen: + raise CapabilityValidationError("model pull SSE ended with an incomplete event") + if terminal is None: + raise CapabilityValidationError("model pull SSE ended without a terminal event") + summary = json.dumps( + terminal, + allow_nan=False, + separators=(",", ":"), + ).encode("utf-8") + if len(summary) > MAX_PULL_TERMINAL_SUMMARY_BYTES: + raise CapabilityValidationError("model pull terminal summary exceeds bounds") + return summary + + +def _buffer_response_body(response, max_response_bytes, failure_message): + buffered_body = bytearray() + for chunk in response.iter_content( + chunk_size=STREAM_WORKER_CHUNK_BYTES, + decode_unicode=False, + ): + if not isinstance(chunk, bytes): + raise CapabilityValidationError("HTTP response chunk must be bytes") + if len(buffered_body) + len(chunk) > max_response_bytes: + raise CapabilityValidationError(failure_message) + buffered_body.extend(chunk) + return bytes(buffered_body) + + +def _load_stream_worker_request(): + raw_request = sys.stdin.buffer.read(MAX_STREAM_WORKER_INPUT_BYTES + 1) + if len(raw_request) > MAX_STREAM_WORKER_INPUT_BYTES: + raise CapabilityValidationError("HTTP worker input exceeds bounds") + try: + request = json.loads( + raw_request.decode("utf-8"), + object_pairs_hook=reject_duplicate_json_object, + ) + except (UnicodeError, json.JSONDecodeError) as exc: + raise CapabilityValidationError("HTTP worker input is invalid") from exc + if not isinstance(request, dict) or set(request) != { + "version", + "response_kind", + "max_response_bytes", + "method", + "url", + "timeout", + "headers", + "json", + }: + raise CapabilityValidationError("HTTP worker input is invalid") + response_kind = request["response_kind"] + max_response_bytes = request["max_response_bytes"] + method = request["method"] + url = request["url"] + timeout = request["timeout"] + headers = request["headers"] + payload = request["json"] + if ( + request["version"] != STREAM_WORKER_PROTOCOL_VERSION + or response_kind not in ("json", "stream", "pull_sse") + or not isinstance(max_response_bytes, int) + or isinstance(max_response_bytes, bool) + or not 1 + <= max_response_bytes + <= ( + MAX_PULL_SSE_WIRE_BYTES + if response_kind == "pull_sse" + else MAX_STREAM_SSE_BYTES + ) + or not isinstance(method, str) + or not method + or not isinstance(url, str) + or not url + or not isinstance(timeout, (int, float)) + or isinstance(timeout, bool) + or timeout <= 0 + or not isinstance(headers, dict) + or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in headers.items() + ) + or not (payload is None or isinstance(payload, dict)) + ): + raise CapabilityValidationError("HTTP worker input is invalid") + if response_kind in {"stream", "pull_sse"} and not isinstance(payload, dict): + raise CapabilityValidationError("HTTP worker input is invalid") + return ( + response_kind, + method, + url, + timeout, + headers, + payload, + max_response_bytes, + ) - def lines(): + +def _run_stream_worker(): + response_kind, method, url, timeout, headers, payload, max_response_bytes = ( + _load_stream_worker_request() + ) + request_options = { + "timeout": timeout, + "headers": headers, + "stream": True, + } + if payload is not None: + request_options["json"] = payload + session = requests.Session() + session.trust_env = False + try: + response = session.request(method, url, **request_options) try: - yield from response.iter_lines(decode_unicode=True) + if response_kind == "pull_sse" and response.status_code == 200: + body = _reduce_pull_sse_response(response, max_response_bytes) + else: + response_bound = ( + min(max_response_bytes, MAX_PULL_RESPONSE_BYTES) + if response_kind == "pull_sse" + else max_response_bytes + ) + failure_message = ( + "stream SSE byte count exceeds bounds" + if response_kind == "stream" + else "HTTP response byte count exceeds bounds" + ) + body = _buffer_response_body( + response, + response_bound, + failure_message, + ) finally: response.close() + output = _encode_stream_worker_output( + response.status_code, + body, + dict(response.headers), + response.encoding, + ) + sys.stdout.buffer.write(output) + sys.stdout.buffer.flush() + finally: + session.close() + + +def _loopback_pull_url(host, port): + if host not in {"127.0.0.1", "localhost", "::1"}: + raise CapabilityValidationError("model pull host must be loopback") + if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535: + raise CapabilityValidationError("model pull port is invalid") + authority = f"[{host}]" if host == "::1" else host + return f"http://{authority}:{port}/api/v1/pull" + - return response, lines() +def _stream_worker_main(): + try: + _run_stream_worker() + except BaseException as exc: # pylint: disable=broad-exception-caught + error = f"{exc.__class__.__name__}: {exc}".encode("utf-8", errors="replace") + sys.stderr.buffer.write(error[:MAX_STREAM_WORKER_ERROR_BYTES]) + sys.stderr.buffer.flush() + return 1 + return 0 + + +def _kill_and_reap_stream_worker(process): + try: + if process.poll() is None: + process.kill() + finally: + process.communicate() + + +def _encode_stream_worker_request( + method, + url, + timeout, + headers, + payload, + response_kind="stream", + max_response_bytes=MAX_STREAM_SSE_BYTES, +): + try: + encoded = json.dumps( + { + "version": STREAM_WORKER_PROTOCOL_VERSION, + "response_kind": response_kind, + "max_response_bytes": max_response_bytes, + "method": method, + "url": url, + "timeout": timeout, + "headers": headers, + "json": payload, + }, + allow_nan=False, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise CapabilityValidationError("HTTP request cannot be encoded") from exc + if len(encoded) > MAX_STREAM_WORKER_INPUT_BYTES: + raise CapabilityValidationError("HTTP worker input exceeds bounds") + return encoded + + +def _execute_http_worker( + method, + url, + timeout, + headers, + payload, + response_kind, + deadline, + deadline_subject, + max_response_bytes=MAX_STREAM_SSE_BYTES, +): + worker_input = _encode_stream_worker_request( + method, + url, + timeout, + headers, + payload, + response_kind, + max_response_bytes, + ) + deadline_message = f"{deadline_subject} exceeded its wall-clock deadline" + remaining = deadline - time.monotonic() + if remaining <= 0: + raise CapabilityValidationError(deadline_message) + process = None + worker_reaped = False + try: + process = subprocess.Popen( + [ + sys.executable, + os.path.realpath(__file__), + INTERNAL_STREAM_WORKER_ARGUMENT, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise CapabilityValidationError(deadline_message) + try: + output, error = process.communicate(input=worker_input, timeout=remaining) + except subprocess.TimeoutExpired as exc: + raise CapabilityValidationError(deadline_message) from exc + worker_reaped = True + if time.monotonic() >= deadline: + raise CapabilityValidationError(deadline_message) + if process.returncode != 0: + detail = error[:MAX_STREAM_WORKER_ERROR_BYTES].decode( + "utf-8", errors="replace" + ) + raise CapabilityValidationError( + f"{deadline_subject} transport worker failed: " + f"{detail or 'unknown error'}" + ) + response = _decode_http_worker_output(output) + if time.monotonic() >= deadline: + raise CapabilityValidationError(deadline_message) + return response + except BaseException: # pylint: disable=broad-exception-caught + if process is not None and not worker_reaped: + _kill_and_reap_stream_worker(process) + raise + + +def request_stream(method, url, timeout, **kwargs): + """Perform a bounded streaming HTTP request and return raw response lines.""" + deadline = kwargs.pop("wall_clock_deadline", None) + if deadline is None: + deadline = time.monotonic() + min( + timeout, + STREAM_WALL_CLOCK_TIMEOUT_SECONDS, + ) + unsupported_options = set(kwargs) - {"headers", "json"} + if unsupported_options or not isinstance(kwargs.get("json"), dict): + raise CapabilityValidationError("stream request options are invalid") + auth_headers = _auth_headers() + headers = {**(kwargs.get("headers") or {}), **auth_headers} + response = _execute_http_worker( + method, + url, + timeout, + headers, + kwargs["json"], + "stream", + deadline, + "stream", + ) + response_and_lines = response, _split_stream_lines(response.content) + if time.monotonic() >= deadline: + raise CapabilityValidationError("stream exceeded its wall-clock deadline") + return response_and_lines + + +def wait_for_server(port=PORT, timeout=TIMEOUT_HEALTH, host="localhost"): + """Wait for a server while bounding every loopback connection attempt.""" + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Server failed to start within {timeout} seconds") + try: + connection = socket.create_connection( + (host, port), + timeout=min(SERVER_CONNECT_ATTEMPT_TIMEOUT_SECONDS, remaining), + ) + connection.close() + return True + except OSError: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + f"Server failed to start within {timeout} seconds" + ) from None + time.sleep(min(SERVER_CONNECT_RETRY_DELAY_SECONDS, remaining)) + + +def unload_all_models(port=PORT, attempts=3, host="localhost"): + """Unload all models through the bounded HTTP worker.""" + response = None + for attempt in range(1, attempts + 1): + try: + response, _body = request_json( + "POST", + f"http://{host}:{port}/api/v1/unload", + timeout=TIMEOUT_DEFAULT, + json={}, + ) + if response.status_code in (200, 404): + return response + except (requests.RequestException, CapabilityValidationError): + if attempt == attempts: + raise + if attempt < attempts: + time.sleep(1) + return response + + +def _is_transient_pull_status(status_code): + return status_code in {408, 409, 429, 500, 502, 503, 504} + + +def _pull_model_streaming( + model_name, + port, + host=VALIDATION_HOST, + *, + wall_clock_deadline=None, +): + if wall_clock_deadline is None: + wall_clock_deadline = time.monotonic() + PULL_WALL_CLOCK_TIMEOUT_SECONDS + pull_url = _loopback_pull_url(host, port) + response = _execute_http_worker( + "POST", + pull_url, + TIMEOUT_INFERENCE, + _auth_headers(), + {"model_name": model_name, "stream": True, "subscribe": True}, + "pull_sse", + wall_clock_deadline, + "model pull", + MAX_PULL_SSE_WIRE_BYTES, + ) + if response.status_code != 200: + return response.status_code, response.text[:1000] + try: + summary = response.json( + object_pairs_hook=reject_duplicate_json_object, + parse_constant=_reject_non_finite_json, + ) + except (UnicodeError, ValueError) as exc: + raise CapabilityValidationError( + "model pull worker returned an invalid terminal summary" + ) from exc + if summary == {"terminal": "complete"}: + return 200, "" + if ( + isinstance(summary, dict) + and set(summary) == {"terminal", "status_code", "error"} + and summary["terminal"] == "error" + and summary["status_code"] in {400, 500} + and not isinstance(summary["status_code"], bool) + and isinstance(summary["error"], str) + and summary["error"] + ): + return summary["status_code"], summary["error"] + raise CapabilityValidationError( + "model pull worker returned an invalid terminal summary" + ) + + +def pull_model_with_retry(model_name, attempts=3, port=PORT, host=VALIDATION_HOST): + """Pull a model with bounded retry for transient setup failures.""" + last_status = None + last_body = "" + wall_clock_deadline = time.monotonic() + PULL_WALL_CLOCK_TIMEOUT_SECONDS + + for attempt in range(1, attempts + 1): + if attempt > 1: + remaining = wall_clock_deadline - time.monotonic() + if remaining <= 0: + raise CapabilityValidationError( + "model pull exceeded its wall-clock deadline" + ) + time.sleep(min(30, 2 ** (attempt - 1), remaining)) + + status, body = _pull_model_streaming( + model_name, + port, + host=host, + wall_clock_deadline=wall_clock_deadline, + ) + + if status == 200: + return + + last_status = status + last_body = body + if _is_transient_pull_status(status) and attempt < attempts: + print( + f"Transient /pull setup failure for {model_name}: " + f"status={status}, attempt={attempt}/{attempts}. Retrying..." + ) + continue + break + + raise AssertionError( + f"Expected 200 from /api/v1/pull for {model_name} after " + f"{attempts} attempt(s), got {last_status}. Body: {last_body}" + ) def require_running_server(base_url, port): """Wait for a running server and confirm the health endpoint responds.""" - wait_for_server(port=port, timeout=TIMEOUT_HEALTH) + wait_for_server(port=port, timeout=TIMEOUT_HEALTH, host=VALIDATION_HOST) response, body = request_json( "GET", f"{base_url}/health", @@ -204,10 +949,20 @@ def unload_model(base_url, model_name=None): json=payload, ) if response.status_code not in (200, 404): - print( - f" Warning: unload returned HTTP {response.status_code}: {body}", - flush=True, - ) + raise RuntimeError(f"Model unload failed: HTTP {response.status_code} - {body}") + + +def require_all_models_unloaded(port, phase): + """Unload every model and reject an unsuccessful cleanup response.""" + response = unload_all_models(port=port, host=VALIDATION_HOST) + if response is None or response.status_code not in (200, 404): + status = "no response" if response is None else f"HTTP {response.status_code}" + raise RuntimeError(f"{phase} model unload failed: {status}") + + +def canonical_model_id(model_name): + """Return the canonical built-in identity used at API boundaries.""" + return model_name if model_name.startswith("builtin.") else f"builtin.{model_name}" def test_model( @@ -270,21 +1025,27 @@ def test_model( {}, ) - aliases = {model_name} - if model_name.startswith("builtin."): - aliases.add(model_name.removeprefix("builtin.")) - else: - aliases.add(f"builtin.{model_name}") - loaded_model = next( - (model for model in loaded_models if model["model_name"] in aliases), - None, - ) - if loaded_model is None: + requested_model_id = canonical_model_id(model_name) + requested_models = [ + model + for model in loaded_models + if canonical_model_id(model["model_name"]) == requested_model_id + ] + if not requested_models: return ( False, f"Model '{model_name}' is absent from the loaded inventory", {}, ) + if len(loaded_models) != 1: + inventory = ", ".join(model["model_name"] for model in loaded_models) + return ( + False, + f"Loaded inventory must contain exactly the requested model " + f"'{model_name}', found: {inventory or ''}", + {}, + ) + loaded_model = requested_models[0] if "recipe_options" not in loaded_model: return False, f"Loaded model '{model_name}' is missing recipe_options", {} @@ -316,6 +1077,17 @@ def test_model( if chat_resp.status_code != 200: return False, f"HTTP {chat_resp.status_code}: {chat_body}", {} + response_model = chat_body.get("model") + if not isinstance(response_model, str) or ( + canonical_model_id(response_model) != requested_model_id + ): + return ( + False, + f"Chat response model '{response_model}' does not match " + f"requested model '{model_name}'", + {}, + ) + message = chat_body["choices"][0]["message"] marker = find_raw_ifm_control_marker(message, "message") if marker: @@ -369,10 +1141,7 @@ def test_model( except requests.RequestException as exc: return False, f"Request failed: {exc}", {} finally: - try: - unload_model(base_url, model_name) - except requests.RequestException as exc: - print(f" Warning: failed to unload {model_name}: {exc}", flush=True) + unload_model(base_url, model_name) def validate_model_lifecycle( @@ -421,18 +1190,24 @@ def validate_model_lifecycle( ): return False, "Could not verify unload: invalid model inventory", result[2] - aliases = {model_name} - if model_name.startswith("builtin."): - aliases.add(model_name.removeprefix("builtin.")) - else: - aliases.add(f"builtin.{model_name}") - if any(model["model_name"] in aliases for model in loaded_models): - return False, f"Model '{model_name}' is still loaded after unload", result[2] + if loaded_models: + remaining_models = ", ".join(model["model_name"] for model in loaded_models) + return ( + False, + f"Models are still loaded after unloading '{model_name}': " + f"{remaining_models}", + result[2], + ) print(" Reloading model after unload...", flush=True) return test_model(base_url, model_name, backend) +def write_results_file(output_path, results): + with open(output_path, "x", encoding="utf-8") as output_file: + json.dump(results, output_file, indent=2, allow_nan=False) + + def main(): parser = argparse.ArgumentParser( description="Validate a llama.cpp backend against selected Lemonade models" @@ -472,6 +1247,11 @@ def main(): action="store_true", help="Skip backend installation step", ) + parser.add_argument( + "--prepare-only", + action="store_true", + help="Install the backend and pull selected models without inference", + ) parser.add_argument( "--logs-dir", default=None, @@ -508,7 +1288,7 @@ def main(): # Label used for output filenames and artifact names label = f"{args.backend}-{args.channel}" if args.channel else args.backend - base_url = f"http://localhost:{args.port}/api/v1" + base_url = f"http://{VALIDATION_HOST}:{args.port}/api/v1" output_path = args.output or f"llamacpp_validation_{label}.json" require_running_server(base_url, args.port) @@ -528,9 +1308,8 @@ def main(): sys.exit(1) selected_model_ids = { - candidate.removeprefix("builtin.") + canonical_model_id(model.get("load_id", model["id"])).removeprefix("builtin.") for model in selected_models - for candidate in (model["id"], model.get("load_id", model["id"])) } missing_capability_models = sorted( set(canonical_capability_models) - selected_model_ids @@ -549,10 +1328,7 @@ def main(): set_rocm_channel(base_url, args.channel) print("Unloading all models for clean state...", flush=True) - try: - unload_all_models(port=args.port) - except requests.RequestException as exc: - print(f"Warning: failed to unload pre-existing models: {exc}", flush=True) + require_all_models_unloaded(args.port, "Pre-validation clean state") selection_mode = "explicitly selected" if args.model else "hot" print( @@ -573,19 +1349,36 @@ def main(): if not args.skip_install: install_backend(base_url, args.backend) + if args.prepare_only: + try: + for model in selected_models: + model_name = model.get("load_id", model["id"]) + print(f"Pulling model: {model_name}", flush=True) + pull_model_with_retry( + model_name, + port=args.port, + host=VALIDATION_HOST, + ) + finally: + if args.logs_dir: + collect_server_logs(args.logs_dir) + return + results = [] all_passed = True try: for model in selected_models: model_name = model["id"] + load_id = model.get("load_id", model_name) print(f"\nTesting: {model_name}", flush=True) runs_capability_profile = ( - model_name.removeprefix("builtin.") in capability_model_ids + canonical_model_id(load_id).removeprefix("builtin.") + in capability_model_ids ) capability_evidence = {} if runs_capability_profile else None success, response_text, stats = validate_model_lifecycle( base_url, - model.get("load_id", model_name), + load_id, args.backend, reload_after_unload=bool(args.model), capability_profile=( @@ -612,14 +1405,12 @@ def main(): print(f" Error: {response_text}", flush=True) finally: try: - unload_all_models(port=args.port) - except requests.RequestException as exc: - print(f"Warning: failed to unload models during cleanup: {exc}", flush=True) - if args.logs_dir: - collect_server_logs(args.logs_dir) - - with open(output_path, "w", encoding="utf-8") as output_file: - json.dump(results, output_file, indent=2) + require_all_models_unloaded(args.port, "Final cleanup") + finally: + if args.logs_dir: + collect_server_logs(args.logs_dir) + + write_results_file(output_path, results) print(f"\nResults written to {output_path}", flush=True) print(f"\n{'=' * 60}", flush=True) @@ -635,4 +1426,6 @@ def main(): if __name__ == "__main__": + if sys.argv[1:] == [INTERNAL_STREAM_WORKER_ARGUMENT]: + sys.exit(_stream_worker_main()) main() From 09e83c9c0cb26a4cbd708b92cb0b74057be76820 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Mon, 7 Sep 2026 03:13:18 -0700 Subject: [PATCH 07/14] test: stabilize interrupted SSE fixture --- test/cpp/test_llamacpp_stream_model.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/test_llamacpp_stream_model.cpp b/test/cpp/test_llamacpp_stream_model.cpp index 8c7db746f0..6a76f7c07b 100644 --- a/test/cpp/test_llamacpp_stream_model.cpp +++ b/test/cpp/test_llamacpp_stream_model.cpp @@ -513,8 +513,9 @@ int main() { } if (!truncated.empty()) { + const std::size_t advertised_length = truncated.size() + 128; response.set_content_provider( - truncated.size() + 128, + advertised_length, "text/event-stream", [truncated = std::move(truncated)]( size_t offset, size_t, httplib::DataSink& sink) { From ac1655c4467e57d9509501df0dba32a9a0c5db92 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:32:28 -0700 Subject: [PATCH 08/14] docs: limit K2 integration to custom backend setup --- .github/actions/setup-venv/action.yml | 2 +- .../capture_llamacpp_release_manifest.sh | 161 - .../scripts/llamacpp_required_check_policy.py | 695 --- .github/scripts/publish_llamacpp_update.sh | 479 -- .github/scripts/run_llamacpp_validation.py | 2578 --------- .github/scripts/validate_backend_bench.py | 20 +- .../scripts/verify_llamacpp_pin_manifest.sh | 72 - .../cpp_server_build_test_release.yml | 6 +- .github/workflows/docs_and_style.yml | 28 +- .github/workflows/validate_llamacpp.yml | 1154 ++++ .github/workflows/validate_llamacpp_core.yml | 1498 ------ .../workflows/validate_llamacpp_manual.yml | 28 - .github/workflows/validate_llamacpp_pr.yml | 1303 ----- .../workflows/validate_llamacpp_schedule.yml | 280 - CMakeLists.txt | 67 +- docs/dev/testing.md | 47 +- docs/embeddable/backends.md | 62 +- docs/guide/configuration/README.md | 4 +- src/cpp/include/lemon/streaming_proxy.h | 36 +- src/cpp/include/lemon/utils/process_manager.h | 30 +- .../include/lemon/utils/process_platform.h | 6 +- src/cpp/include/lemon/wrapped_server.h | 13 +- .../backends/fastflowlm/fastflowlm_models.cpp | 49 +- .../backends/llamacpp/llamacpp_server.cpp | 305 +- .../backends/llamacpp/llamacpp_system_utils.h | 110 - src/cpp/server/mcp_client.cpp | 240 +- src/cpp/server/streaming_proxy.cpp | 367 +- src/cpp/server/system_info.cpp | 40 +- .../server/utils/platform/process_linux.cpp | 197 +- .../server/utils/platform/process_macos.cpp | 613 +-- .../server/utils/platform/process_windows.cpp | 313 +- src/cpp/server/utils/process_manager.cpp | 102 +- src/cpp/server/wrapped_server.cpp | 55 +- test/cpp/test_llamacpp_stream_model.cpp | 893 ---- test/cpp/test_llamacpp_system_backend.cpp | 89 - test/cpp/test_mcp_client_config.cpp | 124 +- test/cpp/test_mcp_client_process_contract.cpp | 92 - test/cpp/test_process_manager.cpp | 797 --- test/cpp/test_process_manager_windows.cpp | 549 -- test/cpp/test_streaming_heartbeat.cpp | 100 - .../llamacpp_validation_artifacts.json | 26 - test/fixtures/llamacpp_validation_models.json | 36 - test/requirements.txt | 3 - test/test_llamacpp_capabilities.py | 960 ---- test/test_llamacpp_check_reporting.py | 116 - test/test_llamacpp_pr_file_classification.py | 301 -- test/test_llamacpp_release_assets.py | 300 -- test/test_llamacpp_release_manifest.py | 1397 ----- test/test_llamacpp_system_backend.py | 287 +- test/test_llamacpp_validation_artifacts.py | 314 -- test/test_llamacpp_validation_evidence.py | 639 --- test/test_process_inheritance_contract.py | 299 -- test/test_publish_llamacpp_update.py | 1982 ------- test/test_run_llamacpp_validation.py | 4466 ---------------- test/test_validate_llamacpp_action_pins.py | 281 - test/test_validate_llamacpp_selection.py | 3211 ----------- ...est_validate_llamacpp_trusted_execution.py | 275 - test/test_validate_llamacpp_workflow_plan.py | 4724 ----------------- test/test_validation_model_catalog.py | 178 - test/test_verify_llamacpp_pin_manifest.py | 436 -- test/utils/llamacpp_capability_validation.py | 1238 ----- test/utils/llamacpp_release_assets.py | 204 - test/utils/llamacpp_release_manifest.py | 1205 ----- test/utils/llamacpp_validation_artifacts.py | 448 -- test/utils/llamacpp_validation_evidence.py | 498 -- test/utils/llamacpp_validation_plan.py | 432 -- test/utils/server_base.py | 16 +- test/utils/validation_model_catalog.py | 130 - test/utils/validation_model_selection.py | 123 - test/validate_llamacpp.py | 1202 +---- 70 files changed, 1839 insertions(+), 37492 deletions(-) delete mode 100755 .github/scripts/capture_llamacpp_release_manifest.sh delete mode 100644 .github/scripts/llamacpp_required_check_policy.py delete mode 100755 .github/scripts/publish_llamacpp_update.sh delete mode 100644 .github/scripts/run_llamacpp_validation.py delete mode 100755 .github/scripts/verify_llamacpp_pin_manifest.sh create mode 100644 .github/workflows/validate_llamacpp.yml delete mode 100644 .github/workflows/validate_llamacpp_core.yml delete mode 100644 .github/workflows/validate_llamacpp_manual.yml delete mode 100644 .github/workflows/validate_llamacpp_pr.yml delete mode 100644 .github/workflows/validate_llamacpp_schedule.yml delete mode 100644 test/cpp/test_llamacpp_stream_model.cpp delete mode 100644 test/cpp/test_mcp_client_process_contract.cpp delete mode 100644 test/cpp/test_process_manager_windows.cpp delete mode 100644 test/fixtures/llamacpp_validation_artifacts.json delete mode 100644 test/fixtures/llamacpp_validation_models.json delete mode 100644 test/test_llamacpp_capabilities.py delete mode 100644 test/test_llamacpp_check_reporting.py delete mode 100644 test/test_llamacpp_pr_file_classification.py delete mode 100644 test/test_llamacpp_release_assets.py delete mode 100644 test/test_llamacpp_release_manifest.py delete mode 100644 test/test_llamacpp_validation_artifacts.py delete mode 100644 test/test_llamacpp_validation_evidence.py delete mode 100644 test/test_process_inheritance_contract.py delete mode 100644 test/test_publish_llamacpp_update.py delete mode 100644 test/test_run_llamacpp_validation.py delete mode 100644 test/test_validate_llamacpp_action_pins.py delete mode 100644 test/test_validate_llamacpp_selection.py delete mode 100644 test/test_validate_llamacpp_trusted_execution.py delete mode 100644 test/test_validate_llamacpp_workflow_plan.py delete mode 100644 test/test_validation_model_catalog.py delete mode 100644 test/test_verify_llamacpp_pin_manifest.py delete mode 100644 test/utils/llamacpp_capability_validation.py delete mode 100644 test/utils/llamacpp_release_assets.py delete mode 100644 test/utils/llamacpp_release_manifest.py delete mode 100644 test/utils/llamacpp_validation_artifacts.py delete mode 100644 test/utils/llamacpp_validation_evidence.py delete mode 100644 test/utils/llamacpp_validation_plan.py delete mode 100644 test/utils/validation_model_catalog.py delete mode 100644 test/utils/validation_model_selection.py diff --git a/.github/actions/setup-venv/action.yml b/.github/actions/setup-venv/action.yml index ee595a0dda..abf72d075e 100644 --- a/.github/actions/setup-venv/action.yml +++ b/.github/actions/setup-venv/action.yml @@ -37,7 +37,7 @@ runs: # For GitHub-hosted runners, use actions/setup-python - name: Setup Python (GitHub-hosted) if: ${{ !contains(runner.name, 'stx') && !contains(runner.name, 'rai') && !contains(runner.name, 'selfhost') }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@v6 with: python-version: ${{ inputs.python-version }} diff --git a/.github/scripts/capture_llamacpp_release_manifest.sh b/.github/scripts/capture_llamacpp_release_manifest.sh deleted file mode 100755 index f1779ff9bb..0000000000 --- a/.github/scripts/capture_llamacpp_release_manifest.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 EXPECTED_MANIFEST_PATH SNAPSHOT_DIRECTORY" >&2 - exit 2 -fi - -expected_manifest_path=$1 -snapshot_directory=$2 -manifest_tool=${LLAMACPP_RELEASE_MANIFEST_TOOL:-test/utils/llamacpp_release_manifest.py} -trusted_source_repository=ggml-org/llama.cpp -source_manifest_max_bytes=65536 -source_manifest_directory=$(mktemp -d) - -mkdir -p "$snapshot_directory" - -canonical_expected_path="${snapshot_directory}/expected.json" -python "$manifest_tool" \ - --materialize-manifest "$canonical_expected_path" \ - --manifest-json "$(<"$expected_manifest_path")" >/dev/null - -source_repository=$(jq -er '.releases[0].source_repository' "$canonical_expected_path") -upstream_reference=$(jq -er '.releases[0].upstream_reference' "$canonical_expected_path") -upstream_reference_head=$(jq -er \ - '.releases[0].upstream_reference_head' "$canonical_expected_path") -if [[ "$source_repository" != "$trusted_source_repository" ]]; then - echo "Manifest source repository must be ${trusted_source_repository}." >&2 - exit 1 -fi - -tag_for_repository() { - jq -er --arg repository "$1" \ - '.releases | map(select(.repository == $repository)) - | if length == 1 then .[0].tag_name else error("missing release") end' \ - "$canonical_expected_path" -} - -ggml_release=$(tag_for_repository ggml-org/llama.cpp) -rocm_release=$(tag_for_repository lemonade-sdk/llamacpp-rocm) -lemonade_release=$(tag_for_repository lemonade-sdk/llama.cpp) - -release_pattern='^b[0-9]+$' -for release in "$ggml_release" "$rocm_release" "$lemonade_release"; do - if [[ ! "$release" =~ $release_pattern ]]; then - echo "Manifest release tag must match bNNNN: ${release}" >&2 - exit 1 - fi -done - -source_default_branch=$(gh api "repos/${trusted_source_repository}" \ - --jq '.default_branch') -if [[ -z "$source_default_branch" ]]; then - echo "Could not resolve the trusted source repository default branch." >&2 - exit 1 -fi -trusted_upstream_reference="refs/heads/${source_default_branch}" -if [[ "$upstream_reference" != "$trusted_upstream_reference" ]]; then - echo "Manifest upstream reference is not the trusted default branch." >&2 - exit 1 -fi -live_upstream_head=$(gh api \ - "repos/${trusted_source_repository}/commits/heads/${source_default_branch}" \ - --jq '.sha') -if ! [[ "$live_upstream_head" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "Could not resolve the trusted upstream reference to a full commit." >&2 - exit 1 -fi - -anchor_comparison_path="${snapshot_directory}/upstream-anchor-comparison.json" -gh api \ - "repos/${trusted_source_repository}/compare/${upstream_reference_head}...${live_upstream_head}" \ - >"$anchor_comparison_path" -python "$manifest_tool" \ - --require-upstream-ancestry "$anchor_comparison_path" \ - "$trusted_source_repository" "$upstream_reference_head" "$live_upstream_head" - -ggml_path="${snapshot_directory}/ggml.json" -rocm_path="${snapshot_directory}/rocm.json" -lemonade_path="${snapshot_directory}/lemonade.json" - -gh api "repos/ggml-org/llama.cpp/releases/tags/${ggml_release}" >"$ggml_path" -gh api "repos/lemonade-sdk/llamacpp-rocm/releases/tags/${rocm_release}" >"$rocm_path" -gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${lemonade_release}" \ - >"$lemonade_path" - -attach_release_claims() { - local repository=$1 - local tag=$2 - local path=$3 - local release_tag_commit publisher_claim_type - local publisher_claimed_source_commit comparison_path - local source_manifest_asset_id source_manifest_path - local build_target_attestation_path - - release_tag_commit=$(gh api \ - "repos/${repository}/commits/tags/${tag}" --jq '.sha') - if ! [[ "$release_tag_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "Could not resolve ${repository} ${tag} to a release tag commit." >&2 - exit 1 - fi - if [[ "$repository" == "$source_repository" ]]; then - publisher_claim_type="source-release-tag" - publisher_claimed_source_commit=$release_tag_commit - build_target_attestation_path="${source_manifest_directory}/source-release-targets.json" - printf '[]\n' >"$build_target_attestation_path" - else - publisher_claim_type="immutable-source-manifest" - source_manifest_asset_id=$(python "$manifest_tool" \ - --source-manifest-asset-id "$path" "$repository") - source_manifest_path="${source_manifest_directory}/source-${source_manifest_asset_id}.json" - build_target_attestation_path="${source_manifest_path}.build-targets" - gh api "repos/${repository}/releases/assets/${source_manifest_asset_id}" \ - -H "Accept: application/octet-stream" | - head -c "$((source_manifest_max_bytes + 1))" >"$source_manifest_path" - publisher_claimed_source_commit=$(python "$manifest_tool" \ - --validate-source-manifest "$source_manifest_path" "$path" \ - "$repository" "$tag" "$release_tag_commit" \ - --build-target-attestation-output "$build_target_attestation_path") - fi - if ! [[ "$publisher_claimed_source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "${repository} ${tag} did not bind one full upstream source commit." >&2 - exit 1 - fi - comparison_path="${path}.upstream-comparison.json" - gh api \ - "repos/${source_repository}/compare/${publisher_claimed_source_commit}...${upstream_reference_head}" \ - >"$comparison_path" - python "$manifest_tool" \ - --require-upstream-ancestry "$comparison_path" "$source_repository" \ - "$publisher_claimed_source_commit" "$upstream_reference_head" - jq \ - --slurpfile build_target_attestations "$build_target_attestation_path" \ - --arg publisher_claim_type "$publisher_claim_type" \ - --arg publisher_claimed_source_commit "$publisher_claimed_source_commit" \ - --arg release_tag_commit "$release_tag_commit" \ - --arg source_repository "$source_repository" \ - --arg upstream_reference "$upstream_reference" \ - --arg upstream_reference_head "$upstream_reference_head" \ - '.build_target_attestations = $build_target_attestations[0] - | .publisher_claim_type = $publisher_claim_type - | .publisher_claimed_source_commit = $publisher_claimed_source_commit - | .release_tag_commit = $release_tag_commit - | .source_repository = $source_repository - | .upstream_reference = $upstream_reference - | .upstream_reference_head = $upstream_reference_head - | del(.source_commit)' \ - "$path" >"${path}.with-source" - mv "${path}.with-source" "$path" -} - -attach_release_claims ggml-org/llama.cpp "$ggml_release" "$ggml_path" -attach_release_claims lemonade-sdk/llamacpp-rocm "$rocm_release" "$rocm_path" -attach_release_claims lemonade-sdk/llama.cpp "$lemonade_release" "$lemonade_path" - -python "$manifest_tool" \ - --release ggml-org/llama.cpp "$ggml_release" "$ggml_path" \ - --release lemonade-sdk/llamacpp-rocm "$rocm_release" "$rocm_path" \ - --release lemonade-sdk/llama.cpp "$lemonade_release" "$lemonade_path" \ - --expected-json "$(<"$canonical_expected_path")" diff --git a/.github/scripts/llamacpp_required_check_policy.py b/.github/scripts/llamacpp_required_check_policy.py deleted file mode 100644 index 8cfd1539ec..0000000000 --- a/.github/scripts/llamacpp_required_check_policy.py +++ /dev/null @@ -1,695 +0,0 @@ -#!/usr/bin/env python3 -"""Trusted path and lifecycle policy for the llama.cpp required check.""" - -from __future__ import annotations - -import argparse -from datetime import datetime -import json -from pathlib import Path, PurePosixPath -import re -from typing import NamedTuple - -MAX_CHANGED_FILES = 3000 -MAX_COMPARE_COMMITS = 250 -MAX_COMPARE_FILES = 300 -MAX_PAGE_SIZE = 100 -MAX_EVENT_PAGES = 100 -CHANGED_FILE_STATUSES = { - "added", - "changed", - "copied", - "modified", - "removed", - "renamed", - "unchanged", -} -AUTHORIZATION_LABEL = "ci:upgrades" -TRUSTED_PERMISSIONS = {"admin", "maintain", "push", "write"} -AUTHORIZATION_INVALIDATING_EVENTS = { - "automatic_base_change_succeeded", - "base_ref_changed", - "closed", - "head_ref_deleted", - "head_ref_force_pushed", - "head_ref_restored", - "merged", - "reopened", -} -IDENTITY_CHANGING_ACTIONS = {"opened", "reopened", "synchronize"} -REQUIRED_WORKFLOW_ACTIONS = {"opened", "reopened", "synchronize"} -SUPPORTED_ACTIONS = { - "closed", - "edited", - "labeled", - "opened", - "reopened", - "synchronize", - "unlabeled", -} -COMPARE_STATUSES = {"ahead", "behind", "diverged", "identical"} -FULL_OID_PATTERN = re.compile(r"[0-9a-f]{40}") - -NEUTRAL_PREFIXES = ( - "docs/", - "src/app/", - "src/web-app/", -) -PROTECTED_OVERRIDES = frozenset({"src/app/src/renderer/utils/toolDefinitions.json"}) - - -class PolicyDecision(NamedTuple): - mode: str - run_validation: bool - conclusion: str - summary: str - - -class AuthorizationProvenance(NamedTuple): - authorization_event_id: int - authorization_event: str - authorization_actor: str - authorization_actor_id: int - authorization_created_at: str - latest_invalidator_event_id: int - epoch_valid: bool - - -def _validated_path(path: object, field: str) -> str: - if not isinstance(path, str) or not path: - raise ValueError(f"{field} must be a non-empty string") - if "\\" in path or path.startswith("/"): - raise ValueError(f"{field} must be a repository-relative POSIX path") - parsed = PurePosixPath(path) - if any(part in {"", ".", ".."} for part in parsed.parts) or str(parsed) != path: - raise ValueError(f"{field} is not a normalized repository path") - return path - - -def is_protected_path(path: str) -> bool: - normalized = _validated_path(path, "filename") - return normalized in PROTECTED_OVERRIDES or not normalized.startswith( - NEUTRAL_PREFIXES - ) - - -def _classify_file_records(records: list[object], *, source: str) -> bool: - filenames: set[str] = set() - relevant = False - for index, record in enumerate(records): - if not isinstance(record, dict): - raise ValueError(f"{source} file record {index} must be an object") - filename = _validated_path(record.get("filename"), "filename") - if filename in filenames: - raise ValueError(f"duplicate {source} filename: {filename}") - filenames.add(filename) - relevant = relevant or is_protected_path(filename) - - status = record.get("status") - if not isinstance(status, str) or status not in CHANGED_FILE_STATUSES: - raise ValueError(f"{source} file has an invalid status: {filename}") - - has_previous_filename = "previous_filename" in record - if status == "renamed": - if not has_previous_filename: - raise ValueError( - f"renamed file is missing previous_filename: {filename}" - ) - previous = _validated_path(record["previous_filename"], "previous_filename") - relevant = relevant or is_protected_path(previous) - elif has_previous_filename: - raise ValueError( - f"previous_filename is only valid for renamed files: {filename}" - ) - - return relevant - - -def classify_changed_file_pages(pages: object, *, expected_count: int) -> bool: - if isinstance(expected_count, bool) or not isinstance(expected_count, int): - raise ValueError("expected_count must be an integer") - if expected_count < 0 or expected_count > MAX_CHANGED_FILES: - raise ValueError(f"changed_files must be between 0 and {MAX_CHANGED_FILES}") - if not isinstance(pages, list): - raise ValueError("paginated pull-request files must be a JSON array") - if len(pages) > MAX_CHANGED_FILES // MAX_PAGE_SIZE: - raise ValueError("pull-request files exceeded the API pagination limit") - - records: list[object] = [] - for page in pages: - if not isinstance(page, list): - raise ValueError("each pull-request files page must be a JSON array") - if len(page) > MAX_PAGE_SIZE: - raise ValueError("a pull-request files page exceeded per_page=100") - records.extend(page) - - if len(records) != expected_count: - raise ValueError( - "paginated pull-request file count does not match changed_files" - ) - - return _classify_file_records(records, source="pull-request") - - -def _validated_oid(value: object, field: str) -> str: - if not isinstance(value, str) or FULL_OID_PATTERN.fullmatch(value) is None: - raise ValueError(f"{field} must be a lowercase full Git OID") - return value - - -def _validated_count(value: object, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ValueError(f"{field} must be a nonnegative integer") - return value - - -def _nested_oid(response: dict[str, object], field: str) -> str: - value = response.get(field) - if not isinstance(value, dict): - raise ValueError(f"compare response {field} must be an object") - return _validated_oid(value.get("sha"), f"compare response {field}.sha") - - -def classify_compare_response( - response: object, - *, - expected_base_sha: str, - expected_target_sha: str, - expected_count: int, -) -> bool: - expected_base = _validated_oid(expected_base_sha, "expected base SHA") - expected_target = _validated_oid(expected_target_sha, "expected target SHA") - expected_files = _validated_count(expected_count, "expected file count") - if expected_files > MAX_COMPARE_FILES: - return True - if not isinstance(response, dict): - raise ValueError("compare response must be a JSON object") - - base_commit = _nested_oid(response, "base_commit") - merge_base_commit = _nested_oid(response, "merge_base_commit") - if base_commit != expected_base: - raise ValueError("compare response does not identify the expected base commit") - - status = response.get("status") - if not isinstance(status, str) or status not in COMPARE_STATUSES: - raise ValueError("compare response has an invalid status") - ahead_by = _validated_count(response.get("ahead_by"), "compare response ahead_by") - behind_by = _validated_count( - response.get("behind_by"), "compare response behind_by" - ) - total_commits = _validated_count( - response.get("total_commits"), "compare response total_commits" - ) - if total_commits != ahead_by: - raise ValueError("compare response commit counts are inconsistent") - - expected_relationships = { - "ahead": (True, False), - "behind": (False, True), - "diverged": (True, True), - "identical": (False, False), - } - expects_ahead, expects_behind = expected_relationships[status] - if (ahead_by > 0) != expects_ahead or (behind_by > 0) != expects_behind: - raise ValueError("compare response status and commit counts are inconsistent") - if status in {"ahead", "identical"} and merge_base_commit != expected_base: - raise ValueError("compare response has an unexpected merge-base commit") - if status == "behind" and merge_base_commit != expected_target: - raise ValueError( - "compare response does not identify the expected target commit" - ) - if status == "diverged" and merge_base_commit in {expected_base, expected_target}: - raise ValueError("compare response has an inconsistent divergent merge base") - if (status == "identical") != (expected_base == expected_target): - raise ValueError("compare response has an inconsistent identical target") - - commits = response.get("commits") - if not isinstance(commits, list): - raise ValueError("compare response commits must be a JSON array") - expected_commit_records = min(total_commits, MAX_COMPARE_COMMITS) - if len(commits) != expected_commit_records: - raise ValueError("compare response commit list is incomplete") - commit_oids: set[str] = set() - for index, commit in enumerate(commits): - if not isinstance(commit, dict): - raise ValueError(f"compare response commit {index} must be an object") - commit_oid = _validated_oid( - commit.get("sha"), f"compare response commit {index}.sha" - ) - if commit_oid in commit_oids: - raise ValueError(f"duplicate compare response commit: {commit_oid}") - commit_oids.add(commit_oid) - if status in {"ahead", "diverged"} and commits[-1]["sha"] != expected_target: - raise ValueError( - "compare response does not identify the expected target commit" - ) - - files = response.get("files") - if not isinstance(files, list): - raise ValueError("compare response files must be a JSON array") - if len(files) > MAX_COMPARE_FILES: - raise ValueError("compare response exceeded the file-list limit") - if status in {"behind", "identical"} and files: - raise ValueError("compare response status cannot contain changed files") - if len(files) != expected_files: - return True - return _classify_file_records(files, source="compare response") - - -def _parse_timestamp(value: str, field: str) -> datetime: - try: - timestamp = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as error: - raise ValueError(f"{field} has an invalid timestamp") from error - if timestamp.tzinfo is None: - raise ValueError(f"{field} has a naive timestamp") - return timestamp - - -def classify_authorization_event_pages( - pages: object, - *, - authorization_after: str | None = None, -) -> AuthorizationProvenance: - if not isinstance(pages, list): - raise ValueError("paginated issue events must be a JSON array") - if len(pages) > MAX_EVENT_PAGES: - raise ValueError("issue events exceeded the trusted pagination limit") - - boundary = ( - _parse_timestamp(authorization_after, "authorization boundary") - if authorization_after is not None - else None - ) - records: list[object] = [] - for page_index, page in enumerate(pages): - if not isinstance(page, list): - raise ValueError("each issue-events page must be a JSON array") - if len(page) > MAX_PAGE_SIZE: - raise ValueError("an issue-events page exceeded per_page=100") - if not page and page_index != len(pages) - 1: - raise ValueError("an intermediate issue-events page was empty") - records.extend(page) - if ( - len(pages) == MAX_EVENT_PAGES - and pages - and isinstance(pages[-1], list) - and len(pages[-1]) == MAX_PAGE_SIZE - ): - raise ValueError("issue events may have exceeded the trusted pagination limit") - - seen_ids: set[int] = set() - seen_node_ids: set[str] = set() - previous_event_id = 0 - previous_order: tuple[datetime, int] | None = None - latest_authorization: tuple[int, dict[str, object]] | None = None - latest_invalidator: tuple[int, int] | None = None - for index, record in enumerate(records): - if not isinstance(record, dict): - raise ValueError(f"issue event record {index} must be an object") - event_id = record.get("id") - node_id = record.get("node_id") - event = record.get("event") - created_at = record.get("created_at") - if ( - isinstance(event_id, bool) - or not isinstance(event_id, int) - or event_id <= 0 - or not isinstance(node_id, str) - or not node_id - or not isinstance(event, str) - or not event - or not isinstance(created_at, str) - ): - raise ValueError(f"issue event record {index} has invalid common fields") - if event_id in seen_ids: - raise ValueError(f"duplicate issue event ID: {event_id}") - if event_id <= previous_event_id: - raise ValueError("issue event IDs are not in strict API order") - if node_id in seen_node_ids: - raise ValueError(f"duplicate issue event node ID: {node_id}") - seen_ids.add(event_id) - previous_event_id = event_id - seen_node_ids.add(node_id) - timestamp = _parse_timestamp( - created_at, - f"issue event record {index} created_at", - ) - order = (timestamp, event_id) - if previous_order is not None and order <= previous_order: - raise ValueError("issue events are not in strict chronological order") - previous_order = order - - if event in {"labeled", "unlabeled"}: - label = record.get("label") - actor = record.get("actor") - if ( - not isinstance(label, dict) - or not isinstance(label.get("name"), str) - or not label["name"] - or not isinstance(actor, dict) - or isinstance(actor.get("id"), bool) - or not isinstance(actor.get("id"), int) - or actor["id"] <= 0 - or not isinstance(actor.get("login"), str) - or not actor["login"] - ): - raise ValueError(f"issue label event {event_id} is malformed") - if label["name"] == AUTHORIZATION_LABEL: - if ( - not isinstance(actor.get("node_id"), str) - or not actor["node_id"] - or actor.get("type") != "User" - or record.get("performed_via_github_app") is not None - or re.fullmatch( - r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})", actor["login"] - ) - is None - ): - raise ValueError( - f"authorization label event {event_id} has an invalid actor" - ) - latest_authorization = (index, record) - - if event in AUTHORIZATION_INVALIDATING_EVENTS: - latest_invalidator = (index, event_id) - - invalidator_id = latest_invalidator[1] if latest_invalidator is not None else 0 - if latest_authorization is None: - return AuthorizationProvenance(0, "", "", 0, "", invalidator_id, False) - - authorization_index, authorization = latest_authorization - actor = authorization["actor"] - assert isinstance(actor, dict) - invalidator_index = latest_invalidator[0] if latest_invalidator is not None else -1 - authorization_created_at = str(authorization["created_at"]) - authorization_timestamp = _parse_timestamp( - authorization_created_at, - "authorization event created_at", - ) - return AuthorizationProvenance( - authorization_event_id=int(authorization["id"]), - authorization_event=str(authorization["event"]), - authorization_actor=str(actor["login"]), - authorization_actor_id=int(actor["id"]), - authorization_created_at=authorization_created_at, - latest_invalidator_event_id=invalidator_id, - epoch_valid=( - authorization["event"] == "labeled" - and authorization_index > invalidator_index - and (boundary is None or authorization_timestamp > boundary) - ), - ) - - -def decide_pull_request_policy( - *, - action: str, - event_label: str, - base_ref_changed: bool, - state: str, - merged: bool, - merge_candidate_available: bool, - relevant: bool, - identity_matches: bool, - authorization_label_present: bool, - actor_permission: str, - has_existing_check: bool, - has_newer_check: bool, - authorization_epoch_valid: bool = True, - authorization_rerun_matches: bool = False, - required_workflow_only: bool = False, -) -> PolicyDecision: - if action not in SUPPORTED_ACTIONS: - raise ValueError(f"unsupported pull_request_target action: {action}") - - if state not in {"open", "closed"}: - raise ValueError(f"unsupported pull request state: {state}") - if merged and state != "closed": - raise ValueError("a merged pull request must be closed") - if merged: - return PolicyDecision( - "merged", - False, - "", - "The pull request merged; the pre-merge gate is left unchanged.", - ) - if state == "closed": - return PolicyDecision( - "closed", - False, - "cancelled", - "The pull request was closed; protected validation was cancelled.", - ) - if action == "labeled" and event_label == AUTHORIZATION_LABEL: - return PolicyDecision( - "awaiting_rerun", - False, - "failure", - "The authorization label was recorded. Re-run all jobs on the " - "matching failed required-workflow run.", - ) - if has_newer_check: - if action in REQUIRED_WORKFLOW_ACTIONS: - return PolicyDecision( - "superseded", - False, - "failure", - "A newer diagnostic decision exists for this candidate. Use the " - "newest failed default-event workflow run for authorization.", - ) - return PolicyDecision( - "preserve", - False, - "", - "A newer exact-commit policy decision superseded this event.", - ) - - authorization_summary = ( - "Protected llama.cpp inputs changed. After the latest required-workflow " - "run fails, a maintainer must apply ci:upgrades and then use Re-run all " - "jobs on that same workflow run. If the label is already present, remove " - "and reapply it after the latest candidate change." - ) - if not identity_matches: - if action in REQUIRED_WORKFLOW_ACTIONS: - return PolicyDecision( - "stale_required", - False, - "failure", - "The required-workflow run no longer identifies the live candidate. " - "Use the newest failed default-event workflow run.", - ) - return PolicyDecision( - "stale", - False, - "", - "A stale event cannot change the live candidate's exact-commit gate.", - ) - if action == "closed": - return PolicyDecision( - "stale", - False, - "", - "A stale lifecycle event cannot change the open pull request's gate.", - ) - - if ( - action in REQUIRED_WORKFLOW_ACTIONS - and identity_matches - and merge_candidate_available - and authorization_label_present - and authorization_epoch_valid - and authorization_rerun_matches - and actor_permission in TRUSTED_PERMISSIONS - ): - return PolicyDecision( - "validate", - True, - "", - "A trusted maintainer authorized protected llama.cpp validation.", - ) - - if not relevant: - return PolicyDecision( - "neutral", - False, - "success", - "No protected llama.cpp validation inputs changed; the gate is neutral.", - ) - - if action == "unlabeled" and event_label == AUTHORIZATION_LABEL: - return PolicyDecision( - "authorization_required", False, "failure", authorization_summary - ) - - if not authorization_label_present: - return PolicyDecision( - "authorization_required", False, "failure", authorization_summary - ) - - if not authorization_epoch_valid or actor_permission not in TRUSTED_PERMISSIONS: - return PolicyDecision( - "authorization_required", False, "failure", authorization_summary - ) - - if not merge_candidate_available: - return PolicyDecision( - "authorization_required", - False, - "failure", - "GitHub has not produced a synthetic test merge commit. Resolve " - "merge conflicts, establish a new default-event workflow run, then " - "apply ci:upgrades and re-run all jobs on that run.", - ) - - invalidated = action in IDENTITY_CHANGING_ACTIONS or ( - action == "edited" and base_ref_changed - ) - if invalidated: - return PolicyDecision( - "authorization_required", False, "failure", authorization_summary - ) - - if required_workflow_only: - return PolicyDecision( - "preserve", - False, - "", - "An unrelated metadata event left the required-workflow gate unchanged.", - ) - - if not has_existing_check: - return PolicyDecision( - "authorization_required", False, "failure", authorization_summary - ) - - return PolicyDecision( - "preserve", - False, - "", - "An unrelated metadata event preserved the exact-commit gate.", - ) - - -def _parse_bool(value: str) -> bool: - if value == "true": - return True - if value == "false": - return False - raise argparse.ArgumentTypeError("expected true or false") - - -def _write_outputs(path: Path, values: dict[str, object]) -> None: - with path.open("a", encoding="utf-8") as output: - for key, value in values.items(): - rendered = str(value).lower() if isinstance(value, bool) else str(value) - if "\n" in rendered or "\r" in rendered: - raise ValueError(f"output {key} must fit on one line") - output.write(f"{key}={rendered}\n") - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - - classify = subparsers.add_parser("classify") - classify.add_argument("--pages", type=Path, required=True) - classify.add_argument("--expected-count", type=int, required=True) - classify.add_argument("--github-output", type=Path, required=True) - - classify_compare = subparsers.add_parser("classify-compare") - classify_compare.add_argument("--response", type=Path, required=True) - classify_compare.add_argument("--expected-base-sha", required=True) - classify_compare.add_argument("--expected-target-sha", required=True) - classify_compare.add_argument("--expected-count", type=int, required=True) - classify_compare.add_argument("--github-output", type=Path, required=True) - - provenance = subparsers.add_parser("provenance") - provenance.add_argument("--pages", type=Path, required=True) - provenance.add_argument("--authorization-after") - - decide = subparsers.add_parser("decide") - decide.add_argument("--action", required=True) - decide.add_argument("--event-label", default="") - decide.add_argument("--base-ref-changed", type=_parse_bool, required=True) - decide.add_argument("--state", required=True) - decide.add_argument("--merged", type=_parse_bool, required=True) - decide.add_argument("--merge-candidate-available", type=_parse_bool, required=True) - decide.add_argument("--relevant", type=_parse_bool, required=True) - decide.add_argument("--identity-matches", type=_parse_bool, required=True) - decide.add_argument( - "--authorization-label-present", type=_parse_bool, required=True - ) - decide.add_argument("--authorization-epoch-valid", type=_parse_bool, required=True) - decide.add_argument( - "--authorization-rerun-matches", type=_parse_bool, required=True - ) - decide.add_argument("--actor-permission", default="") - decide.add_argument("--has-existing-check", type=_parse_bool, required=True) - decide.add_argument("--has-newer-check", type=_parse_bool, required=True) - decide.add_argument("--required-workflow-only", type=_parse_bool, required=True) - decide.add_argument("--github-output", type=Path, required=True) - return parser - - -def main() -> int: - args = _build_parser().parse_args() - if args.command == "classify": - pages = json.loads(args.pages.read_text(encoding="utf-8")) - relevant = classify_changed_file_pages( - pages, - expected_count=args.expected_count, - ) - _write_outputs(args.github_output, {"relevant": relevant}) - return 0 - if args.command == "classify-compare": - response = json.loads(args.response.read_text(encoding="utf-8")) - relevant = classify_compare_response( - response, - expected_base_sha=args.expected_base_sha, - expected_target_sha=args.expected_target_sha, - expected_count=args.expected_count, - ) - _write_outputs(args.github_output, {"relevant": relevant}) - return 0 - if args.command == "provenance": - pages = json.loads(args.pages.read_text(encoding="utf-8")) - provenance = classify_authorization_event_pages( - pages, - authorization_after=args.authorization_after, - ) - print(json.dumps(provenance._asdict(), separators=(",", ":"))) - return 0 - - decision = decide_pull_request_policy( - action=args.action, - event_label=args.event_label, - base_ref_changed=args.base_ref_changed, - state=args.state, - merged=args.merged, - merge_candidate_available=args.merge_candidate_available, - relevant=args.relevant, - identity_matches=args.identity_matches, - authorization_label_present=args.authorization_label_present, - authorization_epoch_valid=args.authorization_epoch_valid, - authorization_rerun_matches=args.authorization_rerun_matches, - actor_permission=args.actor_permission, - has_existing_check=args.has_existing_check, - has_newer_check=args.has_newer_check, - required_workflow_only=args.required_workflow_only, - ) - _write_outputs( - args.github_output, - { - "mode": decision.mode, - "run_validation": decision.run_validation, - "conclusion": decision.conclusion, - "summary": decision.summary, - }, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/scripts/publish_llamacpp_update.sh b/.github/scripts/publish_llamacpp_update.sh deleted file mode 100755 index f032fbed94..0000000000 --- a/.github/scripts/publish_llamacpp_update.sh +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -versions_path=${VERSIONS_PATH:-src/cpp/resources/backend_versions.json} -manifest_path=${RELEASE_MANIFEST_PATH:-.github/llamacpp_release_manifest.json} -body_file=${PR_BODY_FILE:-pr_body.md} -base_branch=${BASE_BRANCH:-main} -repository=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required} -repository_owner=${GITHUB_REPOSITORY_OWNER:?GITHUB_REPOSITORY_OWNER is required} -validated_base_sha=${VALIDATED_BASE_SHA:?VALIDATED_BASE_SHA is required} -llamacpp_release=${LLAMACPP_RELEASE:?LLAMACPP_RELEASE is required} -lemonade_release=${LLAMACPP_LEMONADE_RELEASE:?LLAMACPP_LEMONADE_RELEASE is required} -rocm_release=${LLAMACPP_ROCM_RELEASE:?LLAMACPP_ROCM_RELEASE is required} -expected_release_manifest=${EXPECTED_RELEASE_ASSET_MANIFEST:?EXPECTED_RELEASE_ASSET_MANIFEST is required} -: "${GH_TOKEN:?GH_TOKEN must contain a separately provisioned publication token}" -publication_actor=${EXPECTED_PUBLICATION_ACTOR:?EXPECTED_PUBLICATION_ACTOR is required} - -if ! authenticated_actor=$(gh api graphql \ - -f 'query=query { viewer { login } }' \ - --jq '.data.viewer.login'); then - echo "Could not authenticate the dedicated publication token." >&2 - exit 1 -fi -if [[ -z "$authenticated_actor" || - "$authenticated_actor" != "$publication_actor" ]]; then - echo "The authenticated publication actor does not match EXPECTED_PUBLICATION_ACTOR." >&2 - exit 1 -fi - -script_directory=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -repository_root=$(cd -- "${script_directory}/../.." && pwd) -manifest_tool="${repository_root}/test/utils/llamacpp_release_manifest.py" -capture_tool="${script_directory}/capture_llamacpp_release_manifest.sh" -snapshot_root=$(mktemp -d) -git_askpass_path="${snapshot_root}/git-askpass.sh" -# The generated helper expands these variables only when Git invokes it. -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'case "$1" in' \ - ' *Username*) printf "%s\n" x-access-token ;;' \ - ' *Password*) printf "%s\n" "$GH_TOKEN" ;;' \ - ' *) exit 1 ;;' \ - 'esac' > "$git_askpass_path" -chmod 0700 "$git_askpass_path" - -manifest_digest=$(python "$manifest_tool" \ - --materialize-manifest "$manifest_path" \ - --manifest-json "$expected_release_manifest" \ - --append-summary-to "$body_file") - -manifest_tag() { - jq -er --arg repository "$1" \ - '.releases | map(select(.repository == $repository)) - | if length == 1 then .[0].tag_name else error("missing release") end' \ - "$manifest_path" -} - -if [[ $(manifest_tag ggml-org/llama.cpp) != "$llamacpp_release" || - $(manifest_tag lemonade-sdk/llamacpp-rocm) != "$rocm_release" || - $(manifest_tag lemonade-sdk/llama.cpp) != "$lemonade_release" ]]; then - echo "The release manifest does not match the validated candidate tags." >&2 - exit 1 -fi - -snapshot_number=0 -recheck_release_manifest() { - snapshot_number=$((snapshot_number + 1)) - LLAMACPP_RELEASE_MANIFEST_TOOL="$manifest_tool" \ - bash "$capture_tool" "$manifest_path" \ - "${snapshot_root}/snapshot-${snapshot_number}" >/dev/null -} - -require_managed_pin_manifest() { - python "$manifest_tool" \ - --require-managed-pin-manifest \ - "$base_versions_path" "$versions_path" "$manifest_path" >/dev/null -} - -release_pattern='^b[0-9]+$' -for release in "$llamacpp_release" "$lemonade_release" "$rocm_release"; do - if [[ ! "$release" =~ $release_pattern ]]; then - echo "Invalid llama.cpp release tag: ${release}" >&2 - exit 1 - fi -done - -local_base_sha=$(git rev-parse HEAD) -if [[ "$local_base_sha" != "$validated_base_sha" ]]; then - echo "Local HEAD does not match the validated base SHA." >&2 - exit 1 -fi - -base_versions_path="${snapshot_root}/base-backend-versions.json" -git show "${validated_base_sha}:${versions_path}" > "$base_versions_path" -python - "$base_versions_path" "$versions_path" <<'PY' -import json -import re -import sys - -managed_backends = ("cpu", "cuda", "metal", "rocm-nightly", "rocm-stable", "vulkan") -release_pattern = re.compile(r"b[0-9]+") - -try: - with open(sys.argv[1], encoding="utf-8") as base_file: - base_versions = json.load(base_file)["llamacpp"] - with open(sys.argv[2], encoding="utf-8") as candidate_file: - candidate_versions = json.load(candidate_file)["llamacpp"] -except (KeyError, OSError, TypeError, ValueError) as error: - raise SystemExit( - "Scheduled llama.cpp update found invalid llamacpp version mappings." - ) from error - -if not isinstance(base_versions, dict) or not isinstance(candidate_versions, dict): - raise SystemExit( - "Scheduled llama.cpp update found invalid llamacpp version mappings." - ) - -for backend in managed_backends: - if backend not in base_versions or backend not in candidate_versions: - raise SystemExit( - f"Scheduled llama.cpp update changed the managed llamacpp.{backend} pin shape." - ) - base_release = base_versions[backend] - candidate_release = candidate_versions[backend] - if ( - not isinstance(base_release, str) - or release_pattern.fullmatch(base_release) is None - or not isinstance(candidate_release, str) - or release_pattern.fullmatch(candidate_release) is None - ): - raise SystemExit( - f"Scheduled llama.cpp update found an invalid llamacpp.{backend} release tag." - ) - if int(candidate_release[1:]) < int(base_release[1:]): - raise SystemExit( - f"Scheduled llama.cpp update would downgrade llamacpp.{backend} " - f"from {base_release} to {candidate_release}." - ) -PY - -if [[ $(git rev-parse --is-shallow-repository) == "true" ]]; then - git fetch --unshallow --no-tags origin "refs/heads/${base_branch}" -else - git fetch --no-tags origin "refs/heads/${base_branch}" -fi -fetched_base_sha=$(git rev-parse FETCH_HEAD) -if [[ "$fetched_base_sha" != "$validated_base_sha" ]]; then - echo "The ${base_branch} branch advanced after validation; refusing publication." >&2 - exit 1 -fi - -assert_candidate_releases_current() { - local fresh_llamacpp_release fresh_rocm_release fresh_lemonade_release - fresh_llamacpp_release=$(gh api repos/ggml-org/llama.cpp/releases \ - --jq '[.[] | select(.draft | not) | .tag_name | select(test("^b[0-9]+$"))][0] // empty') - fresh_rocm_release=$(gh api \ - repos/lemonade-sdk/llamacpp-rocm/releases/latest --jq '.tag_name') - fresh_lemonade_release=$(gh api \ - repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name') - - if [[ "$fresh_llamacpp_release" != "$llamacpp_release" || - "$fresh_rocm_release" != "$rocm_release" || - "$fresh_lemonade_release" != "$lemonade_release" ]]; then - echo "The candidate releases changed during validation; refusing publication." >&2 - exit 1 - fi -} - -assert_candidate_releases_current - -recheck_release_manifest - -has_update=true -if git diff --quiet HEAD -- "$versions_path"; then - has_update=false - branch="" -else - versions_blob=$(git hash-object "$versions_path") - branch="auto/llamacpp-update-${llamacpp_release}-${lemonade_release}-${rocm_release}-${validated_base_sha}-${versions_blob}-${manifest_digest}" -fi - -open_pr_records=$(gh api "repos/${repository}/pulls" \ - --method GET \ - -f state=open \ - -f per_page=100 \ - --paginate \ - --jq '.[] | select(.head.ref | startswith("auto/llamacpp-update-")) | [.number, .head.ref, (.head.repo.full_name // ""), .user.login, .base.ref] | @tsv') - -open_pr="" -stale_prs=() -while IFS=$'\t' read -r number head_ref head_repository author base_ref; do - if [[ -z "$number" ]]; then - continue - fi - - trusted=false - if [[ "$head_repository" == "$repository" && - "$author" == "$publication_actor" && - "$base_ref" == "$base_branch" ]]; then - trusted=true - fi - - if [[ "$has_update" == "true" && - "$head_ref" == "$branch" && - "$head_repository" == "$repository" ]]; then - if [[ "$trusted" != "true" ]]; then - echo "Found an untrusted open pull request for ${repository_owner}:${branch}." >&2 - exit 1 - fi - if [[ -n "$open_pr" ]]; then - echo "Multiple open pull requests use ${repository_owner}:${branch}; refusing publication." >&2 - exit 1 - fi - open_pr=$number - elif [[ "$trusted" == "true" ]]; then - stale_prs+=("${number}"$'\t'"${head_ref}") - fi -done <<< "$open_pr_records" - -assert_base_is_current() { - local remote_base_sha - remote_base_sha=$(git ls-remote --heads origin "refs/heads/${base_branch}" | - awk 'NR == 1 { print $1 }') - if [[ -z "$remote_base_sha" || "$remote_base_sha" != "$validated_base_sha" ]]; then - echo "The ${base_branch} branch advanced after validation; refusing publication." >&2 - exit 1 - fi -} - -assert_publication_branch() { - local expected_oid=$1 - local observed_oid - observed_oid=$(git ls-remote --heads origin "refs/heads/${branch}" | - awk 'NR == 1 { print $1 }') - if [[ -z "$observed_oid" || "$observed_oid" != "$expected_oid" ]]; then - echo "The publication branch changed after push." >&2 - exit 1 - fi -} - -require_pull_request_identity() { - local number=$1 - local expected_head_ref=$2 - local expected_head_sha=${3:-} - local expected_state=${4:-open} - local expected_base_sha=${5:-} - local pull_request_json - - if ! [[ "$number" =~ ^[1-9][0-9]*$ ]]; then - echo "Invalid pull request number: ${number}" >&2 - exit 1 - fi - pull_request_json=$(gh api "repos/${repository}/pulls/${number}") - if ! pull_request_is_draft=$(jq -r \ - 'if (.draft | type) == "boolean" then .draft else error("invalid draft") end' \ - <<< "$pull_request_json"); then - echo "Pull request #${number} has invalid draft state." >&2 - exit 1 - fi - if ! jq -e \ - --arg state "$expected_state" \ - --arg repository "$repository" \ - --arg head_ref "$expected_head_ref" \ - --arg head_sha "$expected_head_sha" \ - --arg actor "$publication_actor" \ - --arg base_ref "$base_branch" \ - --arg base_sha "$expected_base_sha" \ - '.state == $state and - .head.repo.full_name == $repository and - .head.ref == $head_ref and - ($head_sha == "" or .head.sha == $head_sha) and - .user.login == $actor and - .base.ref == $base_ref and - ($base_sha == "" or .base.sha == $base_sha)' \ - <<< "$pull_request_json" >/dev/null; then - echo "Pull request #${number} identity changed before publication." >&2 - exit 1 - fi -} - -report_stale_prs() { - local stale_record stale_pr stale_head_ref - for stale_record in "${stale_prs[@]}"; do - IFS=$'\t' read -r stale_pr stale_head_ref <<< "$stale_record" - echo "Superseded pull request #${stale_pr} (${stale_head_ref}) requires manual cleanup." - done -} - -reconcile_failed_pull_request_creation() { - local number head_ref head_repository author base_ref head_sha records - local matching_pr="" - local record_count=0 - - failed_create_reconciliation=unknown - failed_create_pr="" - if ! records=$(gh api "repos/${repository}/pulls" \ - --method GET \ - -f state=open \ - -f head="${repository_owner}:${branch}" \ - -f base="$base_branch" \ - --paginate \ - --jq '.[] | [.number, .head.ref, (.head.repo.full_name // ""), .user.login, .base.ref, .head.sha] | @tsv'); then - echo "Pull request creation failed, and its result could not be reconciled; preserving ${branch}." >&2 - return - fi - - while IFS=$'\t' read -r number head_ref head_repository author base_ref head_sha; do - if [[ -z "$number" ]]; then - continue - fi - record_count=$((record_count + 1)) - if ! [[ "$number" =~ ^[1-9][0-9]*$ ]] || - [[ "$head_ref" != "$branch" ]] || - [[ "$head_repository" != "$repository" ]] || - [[ "$author" != "$publication_actor" ]] || - [[ "$base_ref" != "$base_branch" ]] || - [[ "$head_sha" != "$published_oid" ]]; then - echo "Pull request creation failed, and its result was ambiguous; preserving ${branch}." >&2 - return - fi - matching_pr=$number - done <<< "$records" - - if [[ "$record_count" -eq 0 ]]; then - failed_create_reconciliation=absent - elif [[ "$record_count" -eq 1 ]]; then - failed_create_reconciliation=adopted - failed_create_pr=$matching_pr - else - echo "Pull request creation failed, and multiple matching pull requests were found; preserving ${branch}." >&2 - fi -} - -if [[ "$has_update" != "true" ]]; then - assert_base_is_current - recheck_release_manifest - assert_candidate_releases_current - report_stale_prs - echo "backend_versions.json is unchanged; nothing to update." - exit 0 -fi - -assert_base_is_current -title="Update llama.cpp to ${llamacpp_release}" -remote_oid=$(git ls-remote --heads origin "refs/heads/$branch" | awk 'NR == 1 { print $1 }') -remote_tree="" -if [[ -n "$remote_oid" ]]; then - git fetch --no-tags origin "refs/heads/$branch" - fetched_oid=$(git rev-parse FETCH_HEAD) - if [[ "$fetched_oid" != "$remote_oid" ]]; then - echo "The publication branch changed while it was inspected." >&2 - exit 1 - fi - remote_tree=$(git rev-parse "${remote_oid}^{tree}") - if ! merge_base=$(git merge-base "$validated_base_sha" "$remote_oid"); then - echo "Publication branch ${branch} has unrelated history." >&2 - exit 1 - fi - if [[ "$merge_base" != "$validated_base_sha" ]]; then - echo "Publication branch ${branch} is not based on the validated commit." >&2 - exit 1 - fi - unexpected_path=$(git diff --name-only "$merge_base" "$remote_oid" -- | - awk -v versions="$versions_path" -v manifest="$manifest_path" \ - '$0 != versions && $0 != manifest { print; exit }') - if [[ -n "$unexpected_path" ]]; then - echo "Publication branch ${branch} changes unexpected path ${unexpected_path}." >&2 - exit 1 - fi -elif [[ -n "$open_pr" ]]; then - echo "Trusted pull request #${open_pr} has no publication branch." >&2 - exit 1 -fi - -git config user.name "$publication_actor" -git config user.email "${publication_actor}@users.noreply.github.com" -git checkout -b "$branch" -recheck_release_manifest -git add "$versions_path" "$manifest_path" -git commit \ - -m "Update llama.cpp to ${llamacpp_release}, rocm-stable to ${lemonade_release}, rocm-nightly to ${rocm_release}" \ - -m "LlamaCpp-Release-Manifest-SHA256: ${manifest_digest}" -desired_tree=$(git rev-parse 'HEAD^{tree}') - -if [[ -n "$remote_oid" && -z "$open_pr" && "$remote_tree" != "$desired_tree" ]]; then - echo "Existing publication branch does not match the desired update." >&2 - exit 1 -fi -if [[ -n "$remote_oid" && -n "$open_pr" && "$remote_tree" != "$desired_tree" ]]; then - echo "Trusted pull request #${open_pr} does not match the desired update." >&2 - exit 1 -fi - -assert_base_is_current -recheck_release_manifest -require_managed_pin_manifest -if [[ -n "$remote_oid" ]]; then - lease="refs/heads/${branch}:${remote_oid}" - push_source="$remote_oid" - published_oid=$remote_oid -else - lease="refs/heads/${branch}:" - push_source=HEAD - published_oid=$(git rev-parse HEAD) -fi -assert_candidate_releases_current -GIT_ASKPASS="$git_askpass_path" \ - GIT_TERMINAL_PROMPT=0 \ - GIT_CONFIG_COUNT=1 \ - GIT_CONFIG_KEY_0=credential.helper \ - GIT_CONFIG_VALUE_0='' \ - git push --force-with-lease="$lease" origin \ - "${push_source}:refs/heads/$branch" - -assert_base_is_current -assert_publication_branch "$published_oid" -if [[ -n "$open_pr" ]]; then - require_pull_request_identity \ - "$open_pr" "$branch" "$published_oid" open "$validated_base_sha" - gh pr edit "$open_pr" \ - --repo "$repository" \ - --title "$title" \ - --body-file "$body_file" - current_pr=$open_pr - echo "Refreshed pull request #${open_pr}." -else - if ! created_pr_url=$(gh pr create \ - --repo "$repository" \ - --title "$title" \ - --body-file "$body_file" \ - --base "$base_branch" \ - --head "$branch" \ - --draft); then - assert_candidate_releases_current - reconcile_failed_pull_request_creation - if [[ "$failed_create_reconciliation" == "adopted" ]]; then - current_pr=$failed_create_pr - echo "Recovered pull request #${current_pr} after an ambiguous creation result." - else - if [[ "$failed_create_reconciliation" == "absent" ]]; then - echo "Pull request creation failed; preserving ${branch}." >&2 - fi - exit 1 - fi - else - created_pr_url=${created_pr_url%/} - current_pr=${created_pr_url##*/} - created_draft_pr=true - fi -fi - -require_pull_request_identity \ - "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" -if [[ "${created_draft_pr:-false}" == "true" && - "$pull_request_is_draft" != "true" ]]; then - echo "New pull request #${current_pr} was not created as a draft." >&2 - exit 1 -fi -assert_base_is_current -assert_publication_branch "$published_oid" -recheck_release_manifest -assert_candidate_releases_current -assert_base_is_current -assert_publication_branch "$published_oid" -require_pull_request_identity \ - "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" -if [[ "$pull_request_is_draft" == "true" ]]; then - gh pr ready "$current_pr" --repo "$repository" - echo "Marked pull request #${current_pr} ready for review." -fi -require_pull_request_identity \ - "$current_pr" "$branch" "$published_oid" open "$validated_base_sha" -if [[ "$pull_request_is_draft" != "false" ]]; then - echo "Pull request #${current_pr} remained a draft after promotion." >&2 - exit 1 -fi -report_stale_prs diff --git a/.github/scripts/run_llamacpp_validation.py b/.github/scripts/run_llamacpp_validation.py deleted file mode 100644 index 7f6d6154db..0000000000 --- a/.github/scripts/run_llamacpp_validation.py +++ /dev/null @@ -1,2578 +0,0 @@ -#!/usr/bin/env python3 -"""Run one llama.cpp validation lane on Windows, Linux, or macOS.""" - -from __future__ import annotations - -import argparse -import ctypes -import os -import re -import signal -import socket -import stat -import subprocess -import sys -import tempfile -import threading -import time -import urllib.request -from pathlib import Path -from typing import NamedTuple - -ROOT = Path(__file__).resolve().parents[2] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from test.utils.llamacpp_validation_artifacts import ( - VerifiedValidationArtifact, - verify_validation_artifacts, -) -from test.utils.llamacpp_validation_evidence import ( - ValidationEvidenceError, - load_and_validate_result_file, -) -from test.utils.validation_model_catalog import load_model_catalog - -K2_HORIZON_PROFILE = "k2-horizon-v1" -TARGET_PATTERN = re.compile(r"^[a-z0-9-]+$") -INFERENCE_PROCESS_NAMES = { - "flm", - "lemond", - "lemonade", - "lemonadeserver", - "llama-server", - "moonshine-server", - "ort-server", -} -VALIDATION_MODEL_CATALOG = ROOT / "test/fixtures/llamacpp_validation_models.json" -VALIDATION_ARTIFACT_LOCKS = ROOT / "test/fixtures/llamacpp_validation_artifacts.json" -INHERITED_RUNTIME_ENVIRONMENT_KEYS = { - "COMSPEC", - "CUDA_VISIBLE_DEVICES", - "CURL_CA_BUNDLE", - "GGML_METAL_NO_RESIDENCY", - "GPU_DEVICE_ORDINAL", - "HIP_VISIBLE_DEVICES", - "HSA_OVERRIDE_GFX_VERSION", - "LANG", - "LC_ALL", - "LC_CTYPE", - "NVIDIA_DRIVER_CAPABILITIES", - "NVIDIA_VISIBLE_DEVICES", - "NUMBER_OF_PROCESSORS", - "PATH", - "PATHEXT", - "PROCESSOR_ARCHITECTURE", - "REQUESTS_CA_BUNDLE", - "ROCR_VISIBLE_DEVICES", - "SSL_CERT_DIR", - "SSL_CERT_FILE", - "SYSTEMDRIVE", - "SYSTEMROOT", - "TZ", - "VK_ADD_DRIVER_FILES", - "VK_DRIVER_FILES", - "VK_ICD_FILENAMES", - "WINDIR", - "__NV_PRIME_RENDER_OFFLOAD", -} -LOG_PREFIX = r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \[[A-Za-z]+\]" -CHILD_LOG_PREFIX = r"(?:\[\d{1,10}\]\s+)?\d{1,6}\.\d{2}\.\d{3}\.\d{3}\s+[A-Z]\s+" -LOADING_MODEL_MARKER = re.compile( - LOG_PREFIX + r" \(LlamaCpp\) Loading model:\s+(?P\S+)\s*$" -) -BACKEND_MARKER = re.compile( - LOG_PREFIX + r" \(LlamaCpp\) Using LlamaCpp Backend:\s+(?P\S+)\s*$" -) -GGUF_MARKER = re.compile( - LOG_PREFIX + r" \(LlamaCpp\) Using GGUF:\s+(?P.+?\S)\s*$" -) -LOADED_MODEL_MARKER = re.compile( - LOG_PREFIX - + r" \(Process\) " - + CHILD_LOG_PREFIX - + r"srv\s+llama_server:\s+model loaded\s*$" -) -DEVICE_MARKER = re.compile( - LOG_PREFIX - + r" \(Process\) " - + CHILD_LOG_PREFIX - + r"llama_prepare_model_devices:\s+using device\s+(?P\S+).*$" -) -BACKEND_PREFIX = "Using LlamaCpp Backend:" -DEVICE_PREFIX = "llama_prepare_model_devices: using device" -OFFLOAD_MARKER = re.compile( - LOG_PREFIX - + r" \(Process\) " - + CHILD_LOG_PREFIX - + r"load_tensors:\s+offloaded\s+(\d+)/(\d+)\s+layers\s+to\s+GPU\s*$" -) -CPU_BUFFER_MARKER = re.compile( - LOG_PREFIX - + r" \(Process\) " - + CHILD_LOG_PREFIX - + r"load_tensors:\s+CPU(?:_Mapped)?\s+model buffer size\s*=\s*" - + r"(?P[0-9]+(?:\.[0-9]+)?)\s+MiB\s*$" -) -OFFLOAD_PREFIX = "load_tensors: offloaded" -MAX_ATTESTATION_LOG_BYTES = 512 * 1024 * 1024 -MAX_ATTESTATION_LINE_BYTES = 1024 * 1024 -VALIDATION_PHASE_TIMEOUT_SECONDS = 4 * 60 * 60 -PROCESS_WAIT_POLL_SECONDS = 0.2 -EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS = 4.0 -EXCEPTIONAL_TERM_GRACE_SECONDS = 2.0 -DARWIN_EPHEMERAL_RUNNER_CONTEXT_KEY = "LEMONADE_PROTECTED_RUNNER_CONTEXT" -DARWIN_EPHEMERAL_RUNNER_CONTEXT = "github-hosted:macos-latest:macos-metal" -_PR_SET_CHILD_SUBREAPER = 36 -_PR_GET_CHILD_SUBREAPER = 37 -_POSIX_GATE_PROGRAM = """ -import os -import sys - -gate = int(sys.argv[1]) -command = sys.argv[2:] -released = os.read(gate, 1) -os.close(gate) -if released != b"1" or not command: - raise SystemExit(125) -os.execvpe(command[0], command, os.environ) -""" -_WINDOWS_GATE_PROGRAM = """ -import ctypes -import subprocess -import sys - -gate = int(sys.argv[1]) -command = sys.argv[2:] -kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) -kernel32.WaitForSingleObject.argtypes = (ctypes.c_void_p, ctypes.c_uint32) -kernel32.WaitForSingleObject.restype = ctypes.c_uint32 -kernel32.CloseHandle.argtypes = (ctypes.c_void_p,) -kernel32.CloseHandle.restype = ctypes.c_int -wait_result = kernel32.WaitForSingleObject(gate, 0xFFFFFFFF) -closed = kernel32.CloseHandle(gate) -if wait_result != 0 or not closed or not command: - raise SystemExit(125) -raise SystemExit(subprocess.call(command)) -""" -ACCELERATOR_DEVICE_PREFIXES = { - "cuda": "CUDA", - "metal": "MTL", - "rocm": "ROCm", - "vulkan": "Vulkan", -} -ATTESTED_BACKENDS = { - "cpu", - "cuda", - "metal", - "rocm-nightly", - "rocm-stable", - "vulkan", -} -_JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 -_JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION = 1 -_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 -_HANDLE_FLAG_INHERIT = 0x00000001 -_LINUX_SUBREAPER_LOCK = threading.Lock() -_LINUX_SUBREAPER_LEASES = 0 -_LINUX_SUBREAPER_ORIGINAL_STATE: bool | None = None - - -class _JobObjectBasicLimitInformation(ctypes.Structure): - _fields_ = ( - ("per_process_user_time_limit", ctypes.c_longlong), - ("per_job_user_time_limit", ctypes.c_longlong), - ("limit_flags", ctypes.c_uint32), - ("minimum_working_set_size", ctypes.c_size_t), - ("maximum_working_set_size", ctypes.c_size_t), - ("active_process_limit", ctypes.c_uint32), - ("affinity", ctypes.c_size_t), - ("priority_class", ctypes.c_uint32), - ("scheduling_class", ctypes.c_uint32), - ) - - -class _JobObjectBasicAccountingInformation(ctypes.Structure): - _fields_ = ( - ("total_user_time", ctypes.c_longlong), - ("total_kernel_time", ctypes.c_longlong), - ("this_period_total_user_time", ctypes.c_longlong), - ("this_period_total_kernel_time", ctypes.c_longlong), - ("total_page_fault_count", ctypes.c_uint32), - ("total_processes", ctypes.c_uint32), - ("active_processes", ctypes.c_uint32), - ("total_terminated_processes", ctypes.c_uint32), - ) - - -class _IoCounters(ctypes.Structure): - _fields_ = ( - ("read_operation_count", ctypes.c_ulonglong), - ("write_operation_count", ctypes.c_ulonglong), - ("other_operation_count", ctypes.c_ulonglong), - ("read_transfer_count", ctypes.c_ulonglong), - ("write_transfer_count", ctypes.c_ulonglong), - ("other_transfer_count", ctypes.c_ulonglong), - ) - - -class _JobObjectExtendedLimitInformation(ctypes.Structure): - _fields_ = ( - ("basic_limit_information", _JobObjectBasicLimitInformation), - ("io_info", _IoCounters), - ("process_memory_limit", ctypes.c_size_t), - ("job_memory_limit", ctypes.c_size_t), - ("peak_process_memory_used", ctypes.c_size_t), - ("peak_job_memory_used", ctypes.c_size_t), - ) - - -class _DirectoryIdentity(NamedTuple): - path: Path - resolved_path: Path - device: int - inode: int - - -class _FileIdentity(NamedTuple): - path: Path - resolved_path: Path - device: int - inode: int - size: int - modified_ns: int - changed_ns: int - - -class ValidationCache(NamedTuple): - temporary_root: _DirectoryIdentity - validation_root: _DirectoryIdentity - cache_directory: _DirectoryIdentity - config_directory: _DirectoryIdentity - hf_cache_directory: _DirectoryIdentity - private_temp_directory: _DirectoryIdentity - results_directory: _DirectoryIdentity - logs_directory: _DirectoryIdentity - - -class _CacheProcessScan(NamedTuple): - processes: list[object] - complete: bool - - -class _ValidationCancelled(RuntimeError): - def __init__(self, signum: int, cleanup_deadline: float): - self.signum = signum - self.cleanup_deadline = cleanup_deadline - super().__init__(signum) - - -_ACTIVE_CANCELLATION_STATE = None - - -class _CancellationSignalGuard: - def __init__(self): - self.signum: int | None = None - self.cleanup_deadline: float | None = None - self._previous_handlers: dict[int, object] = {} - - def _handle_signal(self, signum: int, _frame) -> None: - if self.signum is None: - self.signum = signum - - def _restore_handlers(self) -> None: - for selected_signal, previous_handler in self._previous_handlers.items(): - signal.signal(selected_signal, previous_handler) - self._previous_handlers.clear() - - def __enter__(self): - global _ACTIVE_CANCELLATION_STATE - if _ACTIVE_CANCELLATION_STATE is not None: - raise RuntimeError("Cancellation signal guard is already active") - _ACTIVE_CANCELLATION_STATE = self - if os.name == "nt": - return self - try: - for selected_signal in (signal.SIGINT, signal.SIGTERM): - self._previous_handlers[selected_signal] = signal.getsignal( - selected_signal - ) - signal.signal(selected_signal, self._handle_signal) - except BaseException: - self._restore_handlers() - _ACTIVE_CANCELLATION_STATE = None - raise - return self - - def __exit__(self, _exc_type, _exc_value, _traceback) -> None: - global _ACTIVE_CANCELLATION_STATE - try: - if os.name != "nt": - self._restore_handlers() - finally: - _ACTIVE_CANCELLATION_STATE = None - - -def _raise_if_cancelled() -> None: - cancellation = _ACTIVE_CANCELLATION_STATE - if cancellation is not None and cancellation.signum is not None: - cleanup_deadline = getattr(cancellation, "cleanup_deadline", None) - if cleanup_deadline is None: - cleanup_deadline = time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS - cancellation.cleanup_deadline = cleanup_deadline - raise _ValidationCancelled( - cancellation.signum, - cleanup_deadline, - ) - - -def _cancellation_pending() -> bool: - cancellation = _ACTIVE_CANCELLATION_STATE - return cancellation is not None and cancellation.signum is not None - - -def _windows_last_error() -> int: - getter = getattr(ctypes, "get_last_error", None) - return getter() if getter is not None else 0 - - -def _load_windows_kernel32(): - loader = getattr(ctypes, "WinDLL", None) - if loader is None: - raise RuntimeError("Windows Job Objects are unavailable") - kernel32 = loader("kernel32", use_last_error=True) - handle_type = ctypes.c_void_p - kernel32.CreateJobObjectW.argtypes = (ctypes.c_void_p, ctypes.c_wchar_p) - kernel32.CreateJobObjectW.restype = handle_type - kernel32.SetInformationJobObject.argtypes = ( - handle_type, - ctypes.c_int, - ctypes.c_void_p, - ctypes.c_uint32, - ) - kernel32.SetInformationJobObject.restype = ctypes.c_int - kernel32.AssignProcessToJobObject.argtypes = (handle_type, handle_type) - kernel32.AssignProcessToJobObject.restype = ctypes.c_int - kernel32.QueryInformationJobObject.argtypes = ( - handle_type, - ctypes.c_int, - ctypes.c_void_p, - ctypes.c_uint32, - ctypes.c_void_p, - ) - kernel32.QueryInformationJobObject.restype = ctypes.c_int - kernel32.CreateEventW.argtypes = ( - ctypes.c_void_p, - ctypes.c_int, - ctypes.c_int, - ctypes.c_wchar_p, - ) - kernel32.CreateEventW.restype = handle_type - kernel32.SetEvent.argtypes = (handle_type,) - kernel32.SetEvent.restype = ctypes.c_int - kernel32.SetHandleInformation.argtypes = ( - handle_type, - ctypes.c_uint32, - ctypes.c_uint32, - ) - kernel32.SetHandleInformation.restype = ctypes.c_int - kernel32.CloseHandle.argtypes = (handle_type,) - kernel32.CloseHandle.restype = ctypes.c_int - return kernel32 - - -class _WindowsKillOnCloseJob: - def __init__(self, api, handle: int): - self._api = api - self._handle = handle - self._close_result: bool | None = None - - @classmethod - def create(cls, api=None): - selected_api = api or _load_windows_kernel32() - handle = selected_api.CreateJobObjectW(None, None) - if not handle: - error = _windows_last_error() - raise OSError(error, "CreateJobObjectW failed") - information = _JobObjectExtendedLimitInformation() - information.basic_limit_information.limit_flags = ( - _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE - ) - configured = selected_api.SetInformationJobObject( - handle, - _JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, - ctypes.byref(information), - ctypes.sizeof(information), - ) - if not configured: - error = _windows_last_error() - selected_api.CloseHandle(handle) - raise OSError(error, "SetInformationJobObject failed") - return cls(selected_api, handle) - - @staticmethod - def popen_kwargs() -> dict[str, object]: - return {} - - def start_process(self, command: list[str], **kwargs) -> subprocess.Popen: - if not command: - raise ValueError("Contained command must not be empty") - if self._handle is None: - raise RuntimeError("Windows Job Object is already closed") - if kwargs.get("startupinfo") is not None: - raise ValueError("Contained Windows commands cannot override startupinfo") - if kwargs.get("close_fds") is False: - raise ValueError("Contained Windows commands require close_fds=True") - kwargs.pop("startupinfo", None) - kwargs.pop("close_fds", None) - - gate_handle = self._api.CreateEventW(None, True, False, None) - if not gate_handle: - error = _windows_last_error() - raise OSError(error, "CreateEventW failed") - gate_inheritable = False - process = None - try: - if not self._api.SetHandleInformation( - gate_handle, - _HANDLE_FLAG_INHERIT, - _HANDLE_FLAG_INHERIT, - ): - error = _windows_last_error() - raise OSError(error, "SetHandleInformation failed") - gate_inheritable = True - startup_info = subprocess.STARTUPINFO() - startup_info.lpAttributeList = {"handle_list": [int(gate_handle)]} - process = subprocess.Popen( - [ - sys.executable, - "-I", - "-S", - "-c", - _WINDOWS_GATE_PROGRAM, - str(int(gate_handle)), - *command, - ], - startupinfo=startup_info, - close_fds=True, - **kwargs, - ) - self.assign(process) - if not self._api.SetHandleInformation( - gate_handle, - _HANDLE_FLAG_INHERIT, - 0, - ): - error = _windows_last_error() - raise OSError(error, "SetHandleInformation failed") - gate_inheritable = False - _raise_if_cancelled() - if not self._api.SetEvent(gate_handle): - error = _windows_last_error() - raise OSError(error, "SetEvent failed") - if not self._api.CloseHandle(gate_handle): - error = _windows_last_error() - raise OSError(error, "CloseHandle failed for Windows launch gate") - gate_handle = None - return process - except BaseException as exc: - cleanup_deadline = ( - exc.cleanup_deadline - if isinstance(exc, _ValidationCancelled) - else time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS - ) - if process is not None: - self.abort(cleanup_deadline) - _reap_direct_process(process, cleanup_deadline) - raise - finally: - if gate_handle is not None: - if gate_inheritable: - self._api.SetHandleInformation( - gate_handle, - _HANDLE_FLAG_INHERIT, - 0, - ) - self._api.CloseHandle(gate_handle) - - def assign(self, process: subprocess.Popen) -> None: - process_handle = getattr(process, "_handle", None) - if process_handle is None: - raise RuntimeError("Windows child process handle is unavailable") - if not self._api.AssignProcessToJobObject(self._handle, int(process_handle)): - error = _windows_last_error() - raise OSError(error, "AssignProcessToJobObject failed") - - def close(self) -> bool: - if self._close_result is not None: - return self._close_result - handle = self._handle - self._handle = None - accounting = _JobObjectBasicAccountingInformation() - queried = bool( - self._api.QueryInformationJobObject( - handle, - _JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION, - ctypes.byref(accounting), - ctypes.sizeof(accounting), - None, - ) - ) - closed = bool(self._api.CloseHandle(handle)) - self._close_result = queried and accounting.active_processes == 0 and closed - return self._close_result - - def abort(self, deadline: float) -> bool: - del deadline - return self.close() - - -def _linux_child_subreaper_state() -> bool: - libc = ctypes.CDLL(None, use_errno=True) - state = ctypes.c_int() - if libc.prctl(_PR_GET_CHILD_SUBREAPER, ctypes.byref(state), 0, 0, 0) != 0: - error = ctypes.get_errno() - raise OSError(error, "PR_GET_CHILD_SUBREAPER failed") - return bool(state.value) - - -def _set_linux_child_subreaper(enabled: bool) -> None: - libc = ctypes.CDLL(None, use_errno=True) - if libc.prctl(_PR_SET_CHILD_SUBREAPER, int(enabled), 0, 0, 0) != 0: - error = ctypes.get_errno() - raise OSError(error, "PR_SET_CHILD_SUBREAPER failed") - - -def _acquire_linux_child_subreaper() -> None: - global _LINUX_SUBREAPER_LEASES - global _LINUX_SUBREAPER_ORIGINAL_STATE - with _LINUX_SUBREAPER_LOCK: - if _LINUX_SUBREAPER_LEASES == 0: - original_state = _linux_child_subreaper_state() - if not original_state: - _set_linux_child_subreaper(True) - _LINUX_SUBREAPER_ORIGINAL_STATE = original_state - _LINUX_SUBREAPER_LEASES += 1 - - -def _release_linux_child_subreaper() -> None: - global _LINUX_SUBREAPER_LEASES - global _LINUX_SUBREAPER_ORIGINAL_STATE - with _LINUX_SUBREAPER_LOCK: - if _LINUX_SUBREAPER_LEASES <= 0: - raise RuntimeError("Linux child-subreaper lease was not acquired") - if _LINUX_SUBREAPER_LEASES == 1 and _LINUX_SUBREAPER_ORIGINAL_STATE is False: - _set_linux_child_subreaper(False) - _LINUX_SUBREAPER_LEASES -= 1 - if _LINUX_SUBREAPER_LEASES == 0: - _LINUX_SUBREAPER_ORIGINAL_STATE = None - - -class _PosixProcessGroupContainment: - def __init__(self, *, allow_untracked_descendants: bool = False): - self._process_groups: list[int] = [] - self._root_processes: dict[tuple[int, float], object] = {} - self._retained_descendants: dict[tuple[int, float], object] = {} - self._tracking_complete = True - self._cleanup_finished = False - self._close_result: bool | None = None - self._linux_runner_process = None - self._linux_subreaper_lease = False - if sys.platform.startswith("linux"): - self._initialize_linux_subreaper() - elif sys.platform == "darwin": - if not allow_untracked_descendants: - raise RuntimeError( - "Darwin cannot contain session-escaping descendants; protected " - "validation requires --allow-darwin-github-hosted-ephemeral-runner " - "on the trusted macos-latest job" - ) - elif not allow_untracked_descendants: - raise RuntimeError( - "Kernel-backed descendant containment is unavailable on this platform" - ) - - @staticmethod - def _process_identity(process) -> tuple[int, float]: - return process.pid, process.create_time() - - def _initialize_linux_subreaper(self) -> None: - if signal.getsignal(signal.SIGCHLD) != signal.SIG_DFL: - raise RuntimeError( - "Linux child-subreaper containment requires the default SIGCHLD handler" - ) - try: - import psutil - except ModuleNotFoundError as exc: - raise RuntimeError( - "Linux child-subreaper containment requires psutil" - ) from exc - try: - runner_process = psutil.Process(os.getpid()) - existing_descendants = runner_process.children(recursive=True) - except (OSError, psutil.Error) as exc: - raise RuntimeError( - "Could not verify an idle process tree before enabling containment" - ) from exc - if existing_descendants: - raise RuntimeError( - "Cannot enable Linux child-subreaper containment with pre-existing " - "runner descendants" - ) - _acquire_linux_child_subreaper() - self._linux_runner_process = runner_process - self._linux_subreaper_lease = True - - @staticmethod - def popen_kwargs() -> dict[str, object]: - return {"start_new_session": True} - - def start_process(self, command: list[str], **kwargs) -> subprocess.Popen: - if not command: - raise ValueError("Contained command must not be empty") - read_gate, write_gate = os.pipe() - process = None - assigned = False - try: - process = subprocess.Popen( - [ - sys.executable, - "-I", - "-S", - "-c", - _POSIX_GATE_PROGRAM, - str(read_gate), - *command, - ], - pass_fds=(read_gate,), - **kwargs, - **self.popen_kwargs(), - ) - os.close(read_gate) - read_gate = -1 - self.assign(process) - assigned = True - _raise_if_cancelled() - os.write(write_gate, b"1") - return process - except BaseException as exc: - cleanup_deadline = ( - exc.cleanup_deadline - if isinstance(exc, _ValidationCancelled) - else time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS - ) - if assigned: - self.abort(cleanup_deadline) - elif process is not None: - _reap_direct_process(process, cleanup_deadline) - raise - finally: - if read_gate >= 0: - os.close(read_gate) - os.close(write_gate) - - def assign(self, process: subprocess.Popen) -> None: - process_group = process.pid - if process_group == os.getpgrp(): - raise RuntimeError( - "Refusing to contain the validation runner process group" - ) - if process_group in self._process_groups: - raise RuntimeError("Process group was assigned more than once") - self._process_groups.append(process_group) - self._remember_root_process(process.pid) - try: - actual_process_group = os.getpgid(process.pid) - except ProcessLookupError: - self.refresh_descendants() - return - if actual_process_group != process_group: - raise RuntimeError("Child process did not start in an isolated session") - self.refresh_descendants() - - def _remember_root_process(self, process_id: int) -> None: - try: - import psutil - except ModuleNotFoundError: - self._tracking_complete = False - return - try: - process = psutil.Process(process_id) - identity = self._process_identity(process) - except (psutil.NoSuchProcess, psutil.ZombieProcess): - return - except (OSError, psutil.Error): - self._tracking_complete = False - return - self._root_processes.setdefault(identity, process) - - def refresh_descendants(self) -> bool: - if ( - not self._root_processes - and not self._retained_descendants - and self._linux_runner_process is None - ): - return self._tracking_complete - try: - import psutil - except ModuleNotFoundError: - self._tracking_complete = False - return False - - complete = True - if self._linux_runner_process is not None: - try: - adopted_processes = self._linux_runner_process.children(recursive=False) - except (OSError, psutil.Error): - adopted_processes = [] - complete = False - root_identities = set(self._root_processes) - for adopted_process in adopted_processes: - try: - identity = self._process_identity(adopted_process) - except (psutil.NoSuchProcess, psutil.ZombieProcess): - continue - except (OSError, psutil.Error): - complete = False - continue - if identity not in root_identities: - self._retained_descendants.setdefault(identity, adopted_process) - discovery_roots = list(self._root_processes.values()) + list( - self._retained_descendants.values() - ) - for root_process in discovery_roots: - try: - descendants = root_process.children(recursive=True) - except (psutil.NoSuchProcess, psutil.ZombieProcess): - continue - except (OSError, psutil.Error): - complete = False - continue - for descendant in descendants: - if descendant.pid == os.getpid(): - complete = False - continue - try: - identity = self._process_identity(descendant) - except (psutil.NoSuchProcess, psutil.ZombieProcess): - continue - except (OSError, psutil.Error): - complete = False - continue - self._retained_descendants.setdefault(identity, descendant) - if self._linux_runner_process is not None: - for identity, descendant in self._retained_descendants.items(): - try: - if descendant.ppid() != os.getpid(): - continue - waited_pid, _status = os.waitpid(descendant.pid, os.WNOHANG) - if waited_pid not in (0, descendant.pid): - complete = False - except ChildProcessError: - try: - if ( - self._process_identity(descendant) == identity - and descendant.is_running() - and descendant.status() - not in {psutil.STATUS_DEAD, psutil.STATUS_ZOMBIE} - ): - complete = False - except (psutil.NoSuchProcess, psutil.ZombieProcess): - pass - except (OSError, psutil.Error): - complete = False - except (psutil.NoSuchProcess, psutil.ZombieProcess): - continue - except (OSError, psutil.Error): - complete = False - if not complete: - self._tracking_complete = False - return complete - - def _release_kernel_tracking(self) -> bool: - if not self._linux_subreaper_lease: - return True - try: - _release_linux_child_subreaper() - except (OSError, RuntimeError): - self._tracking_complete = False - return False - self._linux_subreaper_lease = False - return True - - def _live_retained_descendants(self) -> list[object]: - try: - import psutil - except ModuleNotFoundError: - self._tracking_complete = False - return list(self._retained_descendants.values()) - - dead_statuses = {psutil.STATUS_DEAD, psutil.STATUS_ZOMBIE} - live_processes = [] - for identity, process in self._retained_descendants.items(): - try: - if ( - self._process_identity(process) == identity - and process.is_running() - and process.status() not in dead_statuses - ): - live_processes.append(process) - except (psutil.NoSuchProcess, psutil.ZombieProcess): - continue - except (OSError, psutil.Error): - self._tracking_complete = False - live_processes.append(process) - return live_processes - - def _signal_retained_descendants( - self, - processes: list[object], - selected_signal: int, - ) -> None: - try: - import psutil - except ModuleNotFoundError: - self._tracking_complete = False - return - for process in processes: - try: - process.send_signal(selected_signal) - except (psutil.NoSuchProcess, psutil.ZombieProcess): - continue - except (OSError, psutil.Error): - self._tracking_complete = False - continue - - @staticmethod - def _has_live_members(process_group: int) -> bool: - def probe_process_group() -> bool: - try: - os.killpg(process_group, 0) - except ProcessLookupError: - return False - except OSError: - return True - return True - - try: - import psutil - except ModuleNotFoundError: - return probe_process_group() - - dead_statuses = {psutil.STATUS_DEAD, psutil.STATUS_ZOMBIE} - try: - for process in psutil.process_iter(("pid", "status")): - try: - if ( - process.info["status"] not in dead_statuses - and os.getpgid(process.pid) == process_group - ): - return True - except (OSError, psutil.Error): - continue - except (OSError, psutil.Error): - return probe_process_group() - return False - - def _wait_for_exit( - self, - process_groups: list[int], - timeout: float, - *, - stop_on_cancellation: bool = False, - ) -> bool: - deadline = time.monotonic() + timeout - while True: - if stop_on_cancellation and _cancellation_pending(): - return False - self.refresh_descendants() - if ( - not any(self._has_live_members(group) for group in process_groups) - and not self._live_retained_descendants() - ): - return True - if time.monotonic() >= deadline: - return False - time.sleep(0.05) - - @classmethod - def _signal_groups(cls, process_groups: list[int], selected_signal: int) -> None: - for process_group in process_groups: - if not cls._has_live_members(process_group): - continue - try: - os.killpg(process_group, selected_signal) - except OSError: - continue - - def close(self) -> bool: - if self._cleanup_finished: - return self._close_result - self.refresh_descendants() - live_groups = [ - group for group in self._process_groups if self._has_live_members(group) - ] - live_descendants = self._live_retained_descendants() - had_survivors = bool(live_groups or live_descendants) - self._signal_groups(live_groups, signal.SIGTERM) - self._signal_retained_descendants(live_descendants, signal.SIGTERM) - if not self._wait_for_exit( - live_groups, - 5, - stop_on_cancellation=True, - ): - remaining_groups = [ - group for group in live_groups if self._has_live_members(group) - ] - remaining_descendants = self._live_retained_descendants() - self._signal_groups(remaining_groups, signal.SIGKILL) - self._signal_retained_descendants( - remaining_descendants, - signal.SIGKILL, - ) - self._wait_for_exit( - remaining_groups, - 5, - stop_on_cancellation=True, - ) - self.refresh_descendants() - cleanup_finished = ( - self._tracking_complete - and not any(self._has_live_members(group) for group in self._process_groups) - and not self._live_retained_descendants() - ) - if cleanup_finished and not self._release_kernel_tracking(): - cleanup_finished = False - self._cleanup_finished = cleanup_finished - clean = not had_survivors and cleanup_finished and self._tracking_complete - self._close_result = clean if self._close_result is None else False - return self._close_result - - def abort(self, deadline: float) -> bool: - if self._cleanup_finished: - return self._close_result - self.refresh_descendants() - live_groups = [ - group for group in self._process_groups if self._has_live_members(group) - ] - live_descendants = self._live_retained_descendants() - had_survivors = bool(live_groups or live_descendants) - self._signal_groups(live_groups, signal.SIGTERM) - self._signal_retained_descendants(live_descendants, signal.SIGTERM) - term_timeout = min( - EXCEPTIONAL_TERM_GRACE_SECONDS, - max(0.0, deadline - time.monotonic()), - ) - if not self._wait_for_exit(live_groups, term_timeout): - remaining_groups = [ - group for group in live_groups if self._has_live_members(group) - ] - remaining_descendants = self._live_retained_descendants() - self._signal_groups(remaining_groups, signal.SIGKILL) - self._signal_retained_descendants( - remaining_descendants, - signal.SIGKILL, - ) - kill_timeout = max(0.0, deadline - time.monotonic()) - self._wait_for_exit(remaining_groups, kill_timeout) - self.refresh_descendants() - cleanup_finished = ( - self._tracking_complete - and not any(self._has_live_members(group) for group in self._process_groups) - and not self._live_retained_descendants() - ) - if cleanup_finished and not self._release_kernel_tracking(): - cleanup_finished = False - self._cleanup_finished = cleanup_finished - clean = not had_survivors and cleanup_finished and self._tracking_complete - self._close_result = clean if self._close_result is None else False - return self._close_result - - -def _verified_darwin_ephemeral_runner( - *, - allow_darwin_github_hosted_ephemeral_runner: bool, - backend: str | None, - target: str | None, - environment: dict[str, str] | None = None, -) -> bool: - if sys.platform != "darwin": - return False - selected_environment = os.environ if environment is None else environment - if not allow_darwin_github_hosted_ephemeral_runner: - return False - if backend != "metal" or target != "macos-metal": - raise RuntimeError( - "The Darwin ephemeral-runner exception is restricted to macos-metal" - ) - if ( - selected_environment.get(DARWIN_EPHEMERAL_RUNNER_CONTEXT_KEY) - != DARWIN_EPHEMERAL_RUNNER_CONTEXT - or selected_environment.get("GITHUB_ACTIONS") != "true" - ): - raise RuntimeError( - "The Darwin ephemeral-runner exception requires the trusted " - "github-hosted macos-latest workflow context" - ) - return True - - -def _create_process_containment( - *, - allow_darwin_github_hosted_ephemeral_runner: bool = False, - backend: str | None = None, - target: str | None = None, -): - if os.name == "nt": - return _WindowsKillOnCloseJob.create() - allow_untracked_descendants = _verified_darwin_ephemeral_runner( - allow_darwin_github_hosted_ephemeral_runner=( - allow_darwin_github_hosted_ephemeral_runner - ), - backend=backend, - target=target, - ) - return _PosixProcessGroupContainment( - allow_untracked_descendants=allow_untracked_descendants - ) - - -def _start_contained_process(containment, command: list[str], **kwargs): - start_process = getattr(containment, "start_process", None) - if start_process is not None: - return start_process(command, **kwargs) - process = subprocess.Popen( - command, - **kwargs, - **containment.popen_kwargs(), - ) - containment.assign(process) - return process - - -def _close_process_containment(containment) -> bool: - try: - return containment.close() - except Exception as exc: # pylint: disable=broad-exception-caught - print(f"Process containment cleanup failed: {exc}", file=sys.stderr) - return False - - -def _abort_process_containment(containment, deadline: float) -> bool: - try: - return containment.abort(deadline) - except Exception as exc: # pylint: disable=broad-exception-caught - print(f"Process containment abort failed: {exc}", file=sys.stderr) - return False - - -def _refresh_process_containment(containment) -> bool: - refresh = getattr(containment, "refresh_descendants", None) - return True if refresh is None else bool(refresh()) - - -def _csv_values(value: str) -> list[str]: - return [item.strip() for item in value.split(",") if item.strip()] - - -def build_validation_command( - *, - python_executable: Path, - backend: str, - channel: str, - target: str, - models_csv: str, - lite: bool, - capability_profile: str, - capability_models_csv: str, - port: int, - output_path: Path | None = None, -) -> list[str]: - if not TARGET_PATTERN.fullmatch(target): - raise ValueError( - "target must contain only lowercase letters, digits, and hyphens" - ) - models = _csv_values(models_csv) - capability_models = _csv_values(capability_models_csv) - if models and lite: - raise ValueError("Explicit models and lite mode are mutually exclusive") - if bool(capability_profile) != bool(capability_models): - raise ValueError("Capability profile and capability models must be paired") - if capability_profile and capability_profile != K2_HORIZON_PROFILE: - raise ValueError(f"Unsupported capability profile: {capability_profile}") - if models: - selected_model_ids = {model.removeprefix("builtin.") for model in models} - missing = [ - model - for model in capability_models - if model.removeprefix("builtin.") not in selected_model_ids - ] - if missing: - raise ValueError( - "Capability models must be present in explicit models: " - + ", ".join(missing) - ) - - command = [ - str(python_executable), - str((ROOT / "test" / "validate_llamacpp.py").resolve()), - "--backend", - backend, - "--port", - str(port), - "--output", - str(output_path or f"llamacpp_validation_{target}.json"), - ] - if channel: - command.extend(("--channel", channel)) - for model in models: - command.extend(("--model", model)) - if lite: - command.append("--lite") - if capability_profile: - command.extend(("--capability-profile", capability_profile)) - for model in capability_models: - command.extend(("--capability-model", model)) - return command - - -def build_restart_validation_command( - *, - python_executable: Path, - backend: str, - channel: str, - target: str, - model: str, - port: int, - output_path: Path | None = None, -) -> list[str]: - if not model.strip() or "," in model: - raise ValueError("Restart validation requires exactly one model") - command = build_validation_command( - python_executable=python_executable, - backend=backend, - channel=channel, - target=target, - models_csv=model, - lite=False, - capability_profile="", - capability_models_csv="", - port=port, - output_path=output_path, - ) - if output_path is None: - command[command.index("--output") + 1] = ( - f"llamacpp_restart_validation_{target}.json" - ) - command.append("--skip-install") - return command - - -def build_prepare_validation_command( - *, - python_executable: Path, - backend: str, - channel: str, - target: str, - models_csv: str, - lite: bool, - capability_profile: str, - capability_models_csv: str, - port: int, - output_path: Path | None = None, -) -> list[str]: - command = build_validation_command( - python_executable=python_executable, - backend=backend, - channel=channel, - target=target, - models_csv=models_csv, - lite=lite, - capability_profile=capability_profile, - capability_models_csv=capability_models_csv, - port=port, - output_path=output_path, - ) - command.append("--prepare-only") - return command - - -def _require_unused_port(port: int) -> None: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: - probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - probe.bind(("127.0.0.1", port)) - except OSError as exc: - raise RuntimeError(f"Port {port} is already in use") from exc - - -def _require_idle_runner() -> None: - import psutil - - conflicts = [] - for process in psutil.process_iter(("pid", "name")): - try: - name = (process.info.get("name") or "").lower().removesuffix(".exe") - if name in INFERENCE_PROCESS_NAMES: - conflicts.append(f"{name}:{process.pid}") - except psutil.Error: - continue - if conflicts: - raise RuntimeError("Runner is not idle: " + ", ".join(sorted(conflicts))) - - -def _is_link_or_junction(path: Path) -> bool: - if path.is_symlink(): - return True - is_junction = getattr(os.path, "isjunction", None) - if is_junction is not None and is_junction(path): - return True - try: - path_status = path.lstat() - except OSError: - return False - reparse_attribute = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) - file_attributes = getattr(path_status, "st_file_attributes", 0) - return bool(reparse_attribute and file_attributes & reparse_attribute) - - -def _capture_directory(path: Path, allowed_root: Path | None) -> _DirectoryIdentity: - if _is_link_or_junction(path): - raise RuntimeError(f"Directory must not be a symlink or junction: {path}") - try: - resolved_path = path.resolve(strict=True) - path_status = path.lstat() - except OSError as exc: - raise RuntimeError(f"Directory does not exist: {path}") from exc - if not stat.S_ISDIR(path_status.st_mode): - raise RuntimeError(f"Path is not a directory: {path}") - if allowed_root is not None: - try: - resolved_path.relative_to(allowed_root) - except ValueError as exc: - raise RuntimeError(f"Directory escapes its allowed root: {path}") from exc - return _DirectoryIdentity( - path=path, - resolved_path=resolved_path, - device=path_status.st_dev, - inode=path_status.st_ino, - ) - - -def create_validation_cache(temporary_root: Path, target: str) -> ValidationCache: - if not TARGET_PATTERN.fullmatch(target): - raise ValueError( - "target must contain only lowercase letters, digits, and hyphens" - ) - temporary_identity = _capture_directory(temporary_root, None) - validation_root = Path( - tempfile.mkdtemp( - prefix=f"llamacpp-validation-{target}-", - dir=temporary_identity.resolved_path, - ) - ) - validation_identity = _capture_directory( - validation_root, temporary_identity.resolved_path - ) - cache_directory = validation_identity.resolved_path / "lemonade-cache" - config_directory = validation_identity.resolved_path / "lemonade-config" - cache_directory.mkdir(mode=0o700) - hf_cache_directory = cache_directory / "hf-hub" - hf_cache_directory.mkdir(mode=0o700) - config_directory.mkdir(mode=0o700) - private_temp_directory = validation_identity.resolved_path / "private-temp" - private_temp_directory.mkdir(mode=0o700) - results_directory = validation_identity.resolved_path / "results" - results_directory.mkdir(mode=0o700) - logs_directory = validation_identity.resolved_path / "logs" - logs_directory.mkdir(mode=0o700) - return ValidationCache( - temporary_root=temporary_identity, - validation_root=validation_identity, - cache_directory=_capture_directory( - cache_directory, validation_identity.resolved_path - ), - config_directory=_capture_directory( - config_directory, validation_identity.resolved_path - ), - hf_cache_directory=_capture_directory( - hf_cache_directory, validation_identity.resolved_path - ), - private_temp_directory=_capture_directory( - private_temp_directory, validation_identity.resolved_path - ), - results_directory=_capture_directory( - results_directory, validation_identity.resolved_path - ), - logs_directory=_capture_directory( - logs_directory, validation_identity.resolved_path - ), - ) - - -def revalidate_validation_cache(cache: ValidationCache) -> None: - current_temporary_root = _capture_directory(cache.temporary_root.path, None) - current_validation_root = _capture_directory( - cache.validation_root.path, - current_temporary_root.resolved_path, - ) - expected_and_current = ( - (cache.temporary_root, current_temporary_root), - (cache.validation_root, current_validation_root), - ( - cache.cache_directory, - _capture_directory( - cache.cache_directory.path, - current_validation_root.resolved_path, - ), - ), - ( - cache.config_directory, - _capture_directory( - cache.config_directory.path, - current_validation_root.resolved_path, - ), - ), - ( - cache.hf_cache_directory, - _capture_directory( - cache.hf_cache_directory.path, - current_validation_root.resolved_path, - ), - ), - ( - cache.private_temp_directory, - _capture_directory( - cache.private_temp_directory.path, - current_validation_root.resolved_path, - ), - ), - ( - cache.results_directory, - _capture_directory( - cache.results_directory.path, - current_validation_root.resolved_path, - ), - ), - ( - cache.logs_directory, - _capture_directory( - cache.logs_directory.path, - current_validation_root.resolved_path, - ), - ), - ) - for expected, current in expected_and_current: - if expected != current: - raise RuntimeError( - f"Validation cache directory changed after creation: {expected.path}" - ) - - -def _file_state(file_status: os.stat_result) -> tuple[int, ...]: - return ( - file_status.st_dev, - file_status.st_ino, - file_status.st_mode, - file_status.st_size, - file_status.st_mtime_ns, - file_status.st_ctime_ns, - ) - - -def _open_binary_no_follow(path: Path): - flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) - no_follow = getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(path, flags | no_follow) - return os.fdopen(descriptor, "rb") - - -def _capture_regular_file( - path: Path, allowed_root: Path, *, require_nonempty: bool -) -> _FileIdentity: - try: - allowed_root = allowed_root.resolve(strict=True) - except OSError as exc: - raise RuntimeError( - f"Validation output root does not exist: {allowed_root}" - ) from exc - if _is_link_or_junction(path): - raise RuntimeError(f"Validation output must not be a link: {path}") - try: - resolved_path = path.resolve(strict=True) - resolved_path.relative_to(allowed_root) - with _open_binary_no_follow(path) as output_file: - file_status = os.fstat(output_file.fileno()) - path_after = path.resolve(strict=True) - path_after.relative_to(allowed_root) - path_status = path_after.stat() - except (OSError, ValueError) as exc: - raise RuntimeError( - f"Validation output is absent or escapes its root: {path}" - ) from exc - if _is_link_or_junction(path): - raise RuntimeError(f"Validation output must not be a link: {path}") - if not stat.S_ISREG(file_status.st_mode) or ( - require_nonempty and file_status.st_size == 0 - ): - qualifier = "nonempty " if require_nonempty else "" - raise RuntimeError( - f"Validation output must be a {qualifier}regular file: {path}" - ) - if path_after != resolved_path or _file_state(file_status) != _file_state( - path_status - ): - raise RuntimeError(f"Validation output changed while inspected: {path}") - return _FileIdentity( - path=path, - resolved_path=resolved_path, - device=file_status.st_dev, - inode=file_status.st_ino, - size=file_status.st_size, - modified_ns=file_status.st_mtime_ns, - changed_ns=file_status.st_ctime_ns, - ) - - -def _revalidate_regular_file( - identity: _FileIdentity, allowed_root: Path, *, require_nonempty: bool -) -> None: - if ( - _capture_regular_file( - identity.path, - allowed_root, - require_nonempty=require_nonempty, - ) - != identity - ): - raise RuntimeError(f"Validation output changed after creation: {identity.path}") - - -def _scan_stable_log( - path: Path, allowed_root: Path, handle_line, byte_budget: int -) -> tuple[_FileIdentity, int]: - try: - allowed_root = allowed_root.resolve(strict=True) - except OSError as exc: - raise RuntimeError( - f"Validation log root does not exist: {allowed_root}" - ) from exc - if _is_link_or_junction(path): - raise RuntimeError(f"Validation log must not be a link: {path}") - try: - resolved_path = path.resolve(strict=True) - resolved_path.relative_to(allowed_root) - with _open_binary_no_follow(path) as log_file: - status_before = os.fstat(log_file.fileno()) - if not stat.S_ISREG(status_before.st_mode): - raise RuntimeError(f"Validation log must be a regular file: {path}") - consumed = 0 - while True: - raw_line = log_file.readline(MAX_ATTESTATION_LINE_BYTES + 1) - if not raw_line: - break - consumed += len(raw_line) - if consumed > byte_budget: - raise RuntimeError( - "Accelerator attestation log exceeds byte budget" - ) - if len(raw_line) > MAX_ATTESTATION_LINE_BYTES or ( - len(raw_line) == MAX_ATTESTATION_LINE_BYTES - and not raw_line.endswith(b"\n") - ): - raise RuntimeError("Accelerator attestation log line is too long") - handle_line(raw_line.decode("utf-8", errors="replace")) - status_after = os.fstat(log_file.fileno()) - path_after = path.resolve(strict=True) - path_after.relative_to(allowed_root) - path_status = path_after.stat() - except RuntimeError: - raise - except (OSError, ValueError) as exc: - raise RuntimeError(f"Could not safely scan validation log: {path}") from exc - if _is_link_or_junction(path): - raise RuntimeError(f"Validation log must not be a link: {path}") - if ( - path_after != resolved_path - or _file_state(status_before) != _file_state(status_after) - or _file_state(status_after) != _file_state(path_status) - ): - raise RuntimeError(f"Validation log changed while scanned: {path}") - identity = _FileIdentity( - path=path, - resolved_path=resolved_path, - device=status_after.st_dev, - inode=status_after.st_ino, - size=status_after.st_size, - modified_ns=status_after.st_mtime_ns, - changed_ns=status_after.st_ctime_ns, - ) - return identity, consumed - - -def require_accelerator_attestation( - log_paths: list[Path], - allowed_root: Path, - phase: str, - expected_artifacts: dict[str, VerifiedValidationArtifact], - expected_backend: str, - *, - selected_models: list[str] | None = None, -) -> list[_FileIdentity]: - cpu_backend = expected_backend == "cpu" - backend_family = expected_backend.split("-", maxsplit=1)[0] - expected_device_prefix = ACCELERATOR_DEVICE_PREFIXES.get(backend_family) - if expected_backend not in ATTESTED_BACKENDS: - raise RuntimeError( - f"{phase} accelerator attestation has unsupported backend: " - f"{expected_backend}" - ) - expected_models = {} - for model_id, artifact in expected_artifacts.items(): - canonical_model_id = model_id.removeprefix("builtin.") - if ( - not isinstance(artifact, VerifiedValidationArtifact) - or artifact.model_id.removeprefix("builtin.") != canonical_model_id - ): - raise RuntimeError( - f"{phase} accelerator attestation has an invalid artifact binding" - ) - expected_models[canonical_model_id] = artifact - canonical_selected_models = [ - model.removeprefix("builtin.") - for model in ( - selected_models if selected_models is not None else list(expected_models) - ) - if isinstance(model, str) and model - ] - if ( - not canonical_selected_models - or len(canonical_selected_models) - != len(selected_models if selected_models is not None else expected_models) - or len(canonical_selected_models) != len(set(canonical_selected_models)) - or not set(expected_models).issubset(canonical_selected_models) - ): - raise RuntimeError( - f"{phase} accelerator attestation has an invalid selected-model binding" - ) - selected_model_ids = set(canonical_selected_models) - attested_models = set() - scanned_identities = [] - total_bytes = 0 - for log_path in log_paths: - current_model = None - current_backend_seen = False - current_identity_seen = False - current_device_seen = False - current_cpu_buffer_seen = False - current_offload = None - - def handle_line(line: str) -> None: - nonlocal current_model, current_identity_seen - nonlocal current_backend_seen, current_device_seen - nonlocal current_cpu_buffer_seen, current_offload - backend_marker = BACKEND_MARKER.search(line) - offload = OFFLOAD_MARKER.search(line) - device = DEVICE_MARKER.search(line) - cpu_buffer = CPU_BUFFER_MARKER.search(line) - if cpu_backend and (DEVICE_PREFIX in line or OFFLOAD_PREFIX in line): - raise RuntimeError( - f"{phase} CPU attestation contains accelerator markers" - ) - if OFFLOAD_PREFIX in line: - if line.count(OFFLOAD_PREFIX) != 1 or offload is None: - raise RuntimeError( - f"{phase} accelerator attestation has malformed offload proof" - ) - offloaded_layers, total_layers = map(int, offload.groups()) - if not 0 < offloaded_layers <= total_layers: - raise RuntimeError( - f"{phase} accelerator attestation has invalid layer offload proof" - ) - - loading = LOADING_MODEL_MARKER.search(line) - if loading is not None: - if current_model is not None: - raise RuntimeError( - f"{phase} accelerator attestation has an incomplete model load" - ) - current_model = loading.group("model").removeprefix("builtin.") - if current_model not in selected_model_ids: - raise RuntimeError( - f"{phase} accelerator attestation has an unexpected model " - f"load: {current_model}" - ) - current_backend_seen = False - current_identity_seen = False - current_device_seen = False - current_cpu_buffer_seen = False - current_offload = None - return - if current_model not in expected_models: - if current_model is None and BACKEND_PREFIX in line: - raise RuntimeError( - f"{phase} accelerator attestation has an unbound backend proof" - ) - if current_model is not None and LOADED_MODEL_MARKER.search(line): - current_model = None - return - if BACKEND_PREFIX in line: - if line.count(BACKEND_PREFIX) != 1 or backend_marker is None: - raise RuntimeError( - f"{phase} accelerator attestation has malformed backend proof" - ) - if current_backend_seen: - raise RuntimeError( - f"{phase} accelerator attestation has duplicate backend proof" - ) - if ( - current_identity_seen - or current_device_seen - or current_cpu_buffer_seen - or current_offload is not None - ): - raise RuntimeError( - f"{phase} accelerator attestation has late backend proof" - ) - actual_backend = backend_marker.group("backend") - if actual_backend != expected_backend: - raise RuntimeError( - f"{phase} accelerator attestation has backend " - f"'{actual_backend}', expected '{expected_backend}'" - ) - current_backend_seen = True - return - gguf = GGUF_MARKER.search(line) - if gguf is not None: - if not current_backend_seen: - raise RuntimeError( - f"{phase} accelerator attestation is missing backend proof" - ) - loaded_path = Path(gguf.group("path")) - try: - if not loaded_path.is_absolute(): - raise ValueError("artifact path is not absolute") - resolved_loaded_path = loaded_path.resolve(strict=True) - except (OSError, ValueError) as exc: - raise RuntimeError( - f"{phase} accelerator attestation has the wrong artifact" - ) from exc - expected_path = expected_models[current_model].resolved_path - if os.path.normcase(os.path.normpath(str(resolved_loaded_path))) != ( - os.path.normcase(os.path.normpath(str(expected_path))) - ): - raise RuntimeError( - f"{phase} accelerator attestation has the wrong artifact" - ) - current_identity_seen = True - if ( - cpu_backend - and cpu_buffer is not None - and current_identity_seen - and float(cpu_buffer.group("size")) > 0 - ): - current_cpu_buffer_seen = True - if current_offload is None: - if device is not None and current_identity_seen: - device_name = device.group("device") - if ( - re.fullmatch( - rf"{re.escape(expected_device_prefix)}[0-9]+", device_name - ) - is None - ): - raise RuntimeError( - f"{phase} accelerator attestation has device " - f"'{device_name}' for the wrong backend" - ) - current_device_seen = True - if offload is not None: - if not current_identity_seen or not current_device_seen: - raise RuntimeError( - f"{phase} accelerator attestation has unbound device/offload proof" - ) - current_offload = tuple(map(int, offload.groups())) - if LOADED_MODEL_MARKER.search(line) is None: - return - if not current_identity_seen: - raise RuntimeError( - f"{phase} accelerator attestation is missing artifact identity" - ) - if not current_backend_seen: - raise RuntimeError( - f"{phase} accelerator attestation is missing backend proof" - ) - if cpu_backend and not current_cpu_buffer_seen: - raise RuntimeError( - f"{phase} CPU attestation is missing a CPU model buffer" - ) - if not cpu_backend and current_offload is None: - raise RuntimeError( - f"{phase} accelerator attestation is missing layer offload proof" - ) - attested_models.add(current_model) - current_model = None - current_backend_seen = False - current_identity_seen = False - current_device_seen = False - current_cpu_buffer_seen = False - current_offload = None - - identity, consumed = _scan_stable_log( - log_path, - allowed_root, - handle_line, - MAX_ATTESTATION_LOG_BYTES - total_bytes, - ) - scanned_identities.append(identity) - total_bytes += consumed - if current_model is not None: - raise RuntimeError( - f"{phase} accelerator attestation has an incomplete model load" - ) - missing_models = sorted(set(expected_models) - attested_models) - if missing_models: - raise RuntimeError( - f"{phase} accelerator attestation is missing models: " - + ", ".join(missing_models) - ) - return scanned_identities - - -def resolve_attestation_backend(backend: str, channel: str) -> str: - if backend == "rocm": - if channel not in {"stable", "nightly"}: - raise RuntimeError( - "ROCm validation requires an exact stable or nightly channel" - ) - return f"rocm-{channel}" - if channel: - raise RuntimeError( - f"{backend} validation must not specify a ROCm release channel" - ) - if backend not in ATTESTED_BACKENDS: - raise RuntimeError(f"Unsupported validation backend: {backend}") - return backend - - -def _emit_workflow_outputs(outputs: dict[str, Path]) -> None: - github_output = os.environ.get("GITHUB_OUTPUT") - if not github_output: - return - with Path(github_output).open("a", encoding="utf-8") as output_file: - for name, path in outputs.items(): - output_file.write(f"{name}={path}\n") - - -def _cache_processes(cache_directory: Path, excluded_pid: int) -> _CacheProcessScan: - try: - import psutil - except ModuleNotFoundError: - return _CacheProcessScan([], False) - - cache_prefix = os.path.normcase(str(cache_directory.resolve()) + os.sep) - matches = [] - try: - for process in psutil.process_iter(("pid", "exe")): - if process.pid == excluded_pid: - continue - try: - executable = process.info.get("exe") - if executable and os.path.normcase( - str(Path(executable).resolve()) - ).startswith(cache_prefix): - matches.append(process) - except (OSError, psutil.Error): - continue - except (OSError, psutil.Error): - return _CacheProcessScan(matches, False) - return _CacheProcessScan(matches, True) - - -def _stop_processes(processes: list[object]) -> bool: - import psutil - - alive = list(processes) - for process in processes: - try: - process.terminate() - except psutil.Error: - pass - alive = _wait_for_psutil_processes(alive, timeout=5) - for process in alive: - try: - process.kill() - except psutil.Error: - pass - alive = _wait_for_psutil_processes(alive, timeout=5) - return not alive - - -def _force_kill_processes(processes: list[object], deadline: float) -> bool: - import psutil - - alive = list(processes) - for process in alive: - try: - process.kill() - except psutil.Error: - pass - reap_timeout = min( - PROCESS_WAIT_POLL_SECONDS, - max(0.0, deadline - time.monotonic()), - ) - try: - _, alive = psutil.wait_procs(alive, timeout=reap_timeout) - except (OSError, psutil.Error): - return False - return not alive - - -def _wait_for_psutil_processes( - processes: list[object], - timeout: float, -) -> list[object]: - import psutil - - deadline = time.monotonic() + timeout - alive = list(processes) - while alive: - _raise_if_cancelled() - remaining = deadline - time.monotonic() - if remaining <= 0: - break - _, alive = psutil.wait_procs( - alive, - timeout=min(PROCESS_WAIT_POLL_SECONDS, remaining), - ) - _raise_if_cancelled() - return alive - - -def _open_local_url(request: urllib.request.Request, timeout: int): - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - return opener.open(request, timeout=timeout) - - -def _open_local_url_with_cancellation( - request: urllib.request.Request, - timeout: int, - containment=None, -) -> None: - outcome: list[BaseException | None] = [] - - def open_url() -> None: - try: - with _open_local_url(request, timeout=timeout): - pass - except BaseException as exc: # pylint: disable=broad-exception-caught - outcome.append(exc) - else: - outcome.append(None) - - worker = threading.Thread(target=open_url, daemon=True) - worker.start() - deadline = time.monotonic() + timeout - while worker.is_alive(): - if containment is not None: - _refresh_process_containment(containment) - _raise_if_cancelled() - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("Timed out requesting graceful server shutdown") - worker.join(min(PROCESS_WAIT_POLL_SECONDS, remaining)) - _raise_if_cancelled() - if not outcome: - raise RuntimeError("Graceful server shutdown request returned no result") - if outcome[0] is not None: - raise outcome[0] - - -def _wait_for_process_with_cancellation( - process: subprocess.Popen, - timeout: float, - containment=None, -) -> int: - deadline = time.monotonic() + timeout - while True: - if containment is not None: - _refresh_process_containment(containment) - _raise_if_cancelled() - remaining = deadline - time.monotonic() - if remaining <= 0: - raise subprocess.TimeoutExpired( - getattr(process, "args", "process"), timeout - ) - try: - exit_code = process.wait(timeout=min(PROCESS_WAIT_POLL_SECONDS, remaining)) - except subprocess.TimeoutExpired: - continue - _raise_if_cancelled() - return exit_code - - -def _shutdown_server( - process: subprocess.Popen, - port: int, - cache_directory: Path, - containment=None, -) -> bool: - orphan_processes = [] - try: - running_on_entry = process.poll() is None - clean = running_on_entry - if running_on_entry: - request = urllib.request.Request( - f"http://127.0.0.1:{port}/internal/shutdown", - data=b"", - method="POST", - ) - try: - _open_local_url_with_cancellation( - request, - timeout=10, - containment=containment, - ) - if ( - _wait_for_process_with_cancellation( - process, - timeout=10, - containment=containment, - ) - != 0 - ): - clean = False - except (OSError, subprocess.TimeoutExpired): - clean = False - if process.poll() is None: - process.terminate() - try: - _wait_for_process_with_cancellation( - process, - timeout=5, - containment=containment, - ) - except subprocess.TimeoutExpired: - process.kill() - _wait_for_process_with_cancellation( - process, - timeout=5, - containment=containment, - ) - clean = False - - _raise_if_cancelled() - if containment is not None and not _close_process_containment(containment): - clean = False - _raise_if_cancelled() - orphan_scan = _cache_processes(cache_directory, process.pid) - orphan_processes = list(orphan_scan.processes) - if not orphan_scan.complete: - clean = False - if orphan_processes: - clean = False - _stop_processes(orphan_processes) - _raise_if_cancelled() - return clean - except _ValidationCancelled as exc: - cleanup_deadline = exc.cleanup_deadline - cancellation_orphans = list(orphan_processes) - cancellation_scan = _cache_processes(cache_directory, process.pid) - retained_orphan_pids = { - getattr(orphan, "pid", None) for orphan in cancellation_orphans - } - for orphan in cancellation_scan.processes: - if getattr(orphan, "pid", None) not in retained_orphan_pids: - cancellation_orphans.append(orphan) - if cancellation_orphans: - _force_kill_processes(cancellation_orphans, cleanup_deadline) - if containment is not None: - _abort_process_containment(containment, cleanup_deadline) - _reap_direct_process(process, cleanup_deadline) - raise - - -def _reap_direct_process(process: subprocess.Popen, deadline: float) -> None: - try: - if process.poll() is None: - process.kill() - except OSError: - pass - try: - process.wait(timeout=max(0.0, deadline - time.monotonic())) - except (OSError, subprocess.TimeoutExpired): - pass - - -def _wait_for_contained_process( - process: subprocess.Popen, - timeout_seconds: float, - containment, -) -> int: - if timeout_seconds <= 0: - raise ValueError("Validation phase timeout must be positive") - deadline = time.monotonic() + timeout_seconds - while True: - _refresh_process_containment(containment) - _raise_if_cancelled() - remaining = deadline - time.monotonic() - if remaining <= 0: - raise RuntimeError( - f"Validation command timed out after {timeout_seconds:g} seconds" - ) - try: - exit_code = process.wait(timeout=min(PROCESS_WAIT_POLL_SECONDS, remaining)) - except subprocess.TimeoutExpired: - continue - _raise_if_cancelled() - return exit_code - - -def _run_contained_command( - command: list[str], - *, - env: dict[str, str], - containment, - timeout_seconds: float, -) -> int: - process = None - try: - process = _start_contained_process( - containment, - command, - env=env, - ) - return _wait_for_contained_process(process, timeout_seconds, containment) - except BaseException as exc: - cleanup_deadline = ( - exc.cleanup_deadline - if isinstance(exc, _ValidationCancelled) - else time.monotonic() + EXCEPTIONAL_CLEANUP_TIMEOUT_SECONDS - ) - _abort_process_containment(containment, cleanup_deadline) - if process is not None: - _reap_direct_process(process, cleanup_deadline) - if isinstance(exc, _ValidationCancelled): - raise - _raise_if_cancelled() - raise - - -def build_server_environment( - hf_cache_directory: Path, - inherited: dict[str, str] | None = None, - *, - private_temp_directory: Path, - candidate_defaults_path: Path | None = None, - allow_download_credentials: bool = False, -) -> dict[str, str]: - inherited_environment = os.environ if inherited is None else inherited - environment = {} - for key, value in inherited_environment.items(): - normalized_key = key.upper() - if normalized_key in INHERITED_RUNTIME_ENVIRONMENT_KEYS: - environment[normalized_key] = value - if allow_download_credentials and inherited_environment.get("HF_TOKEN"): - environment["HF_TOKEN"] = inherited_environment["HF_TOKEN"] - environment["HF_HUB_CACHE"] = str(hf_cache_directory.resolve()) - environment["LEMONADE_CI_MODE"] = "True" - environment["LLAMA_ARG_LOG_VERBOSITY"] = "4" - environment["NO_PROXY"] = "127.0.0.1" - environment["no_proxy"] = "127.0.0.1" - resolved_private_temp = str(private_temp_directory.resolve()) - environment["TMPDIR"] = resolved_private_temp - environment["TMP"] = resolved_private_temp - environment["TEMP"] = resolved_private_temp - environment["HOME"] = resolved_private_temp - environment["USERPROFILE"] = resolved_private_temp - environment["APPDATA"] = resolved_private_temp - environment["LOCALAPPDATA"] = resolved_private_temp - if candidate_defaults_path is not None: - environment["LEMONADE_DEFAULTS_PATH"] = str(candidate_defaults_path.resolve()) - return environment - - -def resolve_candidate_defaults(lemond: Path) -> Path: - resource_directory = ( - lemond.parent.parent / "resources" - if lemond.suffix.lower() == ".exe" - else lemond.parent / "resources" - ) - defaults_path = resource_directory / "defaults.json" - if _is_link_or_junction(resource_directory): - raise RuntimeError( - f"Candidate defaults directory is not a regular build resource: " - f"{resource_directory}" - ) - try: - resolved_resource_directory = resource_directory.resolve(strict=True) - resolved_defaults = defaults_path.resolve(strict=True) - resolved_defaults.relative_to(resolved_resource_directory) - except (OSError, ValueError) as exc: - raise RuntimeError( - f"Candidate defaults file was not found in build resources: {defaults_path}" - ) from exc - if _is_link_or_junction(defaults_path) or not resolved_defaults.is_file(): - raise RuntimeError( - f"Candidate defaults file is not a regular build resource: {defaults_path}" - ) - return resolved_defaults - - -def _run_validation_process( - *, - lemond: Path, - cache_directory: Path, - config_directory: Path, - command: list[str], - environment: dict[str, str], - port: int, - pid_path: Path, - stdout_path: Path, - stderr_path: Path, - backend: str | None = None, - target: str | None = None, - allow_darwin_github_hosted_ephemeral_runner: bool = False, -) -> tuple[int, bool]: - containment = None - cleanup_succeeded = False - try: - with stdout_path.open("x", encoding="utf-8") as stdout_log, stderr_path.open( - "x", encoding="utf-8" - ) as stderr_log: - containment = _create_process_containment( - allow_darwin_github_hosted_ephemeral_runner=( - allow_darwin_github_hosted_ephemeral_runner - ), - backend=backend, - target=target, - ) - process = _start_contained_process( - containment, - [ - str(lemond), - str(cache_directory.resolve()), - str(config_directory.resolve()), - "--port", - str(port), - "--host", - "127.0.0.1", - ], - stdout=stdout_log, - stderr=stderr_log, - env=environment, - ) - try: - with pid_path.open("x", encoding="ascii") as pid_file: - pid_file.write(f"{process.pid}\n") - print(f"Started lemond PID {process.pid}", flush=True) - time.sleep(0.5) - if process.poll() is not None: - raise RuntimeError( - f"lemond exited before validation started; see {stderr_path}" - ) - exit_code = _run_contained_command( - command, - env=environment, - containment=containment, - timeout_seconds=VALIDATION_PHASE_TIMEOUT_SECONDS, - ) - finally: - cleanup_succeeded = _shutdown_server( - process, - port, - cache_directory, - containment, - ) - finally: - if containment is not None and not _close_process_containment(containment): - cleanup_succeeded = False - _raise_if_cancelled() - if cleanup_succeeded: - pid_path.unlink(missing_ok=True) - return exit_code, cleanup_succeeded - - -def require_validation_result_evidence( - identity: _FileIdentity, - allowed_root: Path, - expected_models: list[str], - *, - capability_profile: str = "", - capability_models: list[str] | None = None, -) -> None: - try: - load_and_validate_result_file( - identity.resolved_path, - expected_models, - capability_profile=capability_profile, - capability_models=capability_models, - ) - except ValidationEvidenceError as exc: - raise RuntimeError(str(exc)) from exc - _revalidate_regular_file(identity, allowed_root, require_nonempty=True) - - -def run_validation(args: argparse.Namespace) -> int: - lemond = args.lemond.resolve() - python_executable = args.python.resolve() - allow_huggingface_download_credentials = bool( - getattr(args, "allow_huggingface_download_credentials", False) - ) - allow_darwin_github_hosted_ephemeral_runner = bool( - getattr(args, "allow_darwin_github_hosted_ephemeral_runner", False) - ) - if not lemond.is_file(): - raise RuntimeError(f"lemond executable not found: {lemond}") - if not python_executable.is_file(): - raise RuntimeError(f"Python executable not found: {python_executable}") - attestation_backend = resolve_attestation_backend(args.backend, args.channel) - - command = build_validation_command( - python_executable=python_executable, - backend=args.backend, - channel=args.channel, - target=args.target, - models_csv=args.models, - lite=args.lite, - capability_profile=args.capability_profile, - capability_models_csv=args.capability_models, - port=args.port, - ) - command.append("--skip-install") - prepare_command = build_prepare_validation_command( - python_executable=python_executable, - backend=args.backend, - channel=args.channel, - target=args.target, - models_csv=args.models, - lite=args.lite, - capability_profile=args.capability_profile, - capability_models_csv=args.capability_models, - port=args.port, - ) - selected_models = _csv_values(args.models) - capability_models = _csv_values(args.capability_models) - restart_models = capability_models or selected_models - if not selected_models: - raise ValueError("Validation requires explicit selected models") - if not restart_models: - raise ValueError( - "A capability model or explicit model is required for restart validation" - ) - artifact_locks = load_model_catalog(VALIDATION_ARTIFACT_LOCKS) - locked_restart_models = [ - model - for model in restart_models - if model.removeprefix("builtin.") in artifact_locks - ] - restart_model_value = (locked_restart_models or restart_models)[0] - restart_command = build_restart_validation_command( - python_executable=python_executable, - backend=args.backend, - channel=args.channel, - target=args.target, - model=restart_model_value, - port=args.port, - ) - _require_idle_runner() - _require_unused_port(args.port) - candidate_defaults_path = resolve_candidate_defaults(lemond) - - temporary_root_value = os.environ.get("RUNNER_TEMP") - if not temporary_root_value: - raise RuntimeError("RUNNER_TEMP is required for validation cache isolation") - validation_cache = create_validation_cache(Path(temporary_root_value), args.target) - cache_directory = validation_cache.cache_directory.resolved_path - config_directory = validation_cache.config_directory.resolved_path - hf_cache_directory = validation_cache.hf_cache_directory.resolved_path - private_temp_directory = validation_cache.private_temp_directory.resolved_path - results_directory = validation_cache.results_directory.resolved_path - logs_directory = validation_cache.logs_directory.resolved_path - validation_result_path = ( - results_directory / f"llamacpp_validation_{args.target}.json" - ) - restart_result_path = ( - results_directory / f"llamacpp_restart_validation_{args.target}.json" - ) - command[command.index("--output") + 1] = str(validation_result_path) - prepare_command[prepare_command.index("--output") + 1] = str(validation_result_path) - restart_command[restart_command.index("--output") + 1] = str(restart_result_path) - prepare_environment = build_server_environment( - hf_cache_directory, - private_temp_directory=private_temp_directory, - candidate_defaults_path=candidate_defaults_path, - allow_download_credentials=allow_huggingface_download_credentials, - ) - environment = build_server_environment( - hf_cache_directory, - private_temp_directory=private_temp_directory, - candidate_defaults_path=candidate_defaults_path, - ) - pid_path = validation_cache.validation_root.resolved_path / "lemond.pid" - prepare_stdout_path = logs_directory / "lemond.prepare.stdout.log" - prepare_stderr_path = logs_directory / "lemond.prepare.stderr.log" - validation_stdout_path = logs_directory / "lemond.stdout.log" - validation_stderr_path = logs_directory / "lemond.stderr.log" - restart_stdout_path = logs_directory / "lemond.restart.stdout.log" - restart_stderr_path = logs_directory / "lemond.restart.stderr.log" - - revalidate_validation_cache(validation_cache) - validation_exit_code, cleanup_succeeded = _run_validation_process( - lemond=lemond, - cache_directory=cache_directory, - config_directory=config_directory, - command=prepare_command, - environment=prepare_environment, - port=args.port, - pid_path=pid_path, - stdout_path=prepare_stdout_path, - stderr_path=prepare_stderr_path, - backend=args.backend, - target=args.target, - allow_darwin_github_hosted_ephemeral_runner=( - allow_darwin_github_hosted_ephemeral_runner - ), - ) - revalidate_validation_cache(validation_cache) - if validation_exit_code: - return validation_exit_code - if not cleanup_succeeded: - print("Prepare process cleanup did not complete", file=sys.stderr) - return 1 - - def verify_selected_artifacts() -> dict[str, VerifiedValidationArtifact]: - revalidate_validation_cache(validation_cache) - verified = verify_validation_artifacts( - hf_cache_directory, - VALIDATION_MODEL_CATALOG, - VALIDATION_ARTIFACT_LOCKS, - selected_models, - ) - if verified: - print( - "Verified immutable validation artifacts: " + ", ".join(verified), - flush=True, - ) - return verified - - verified_artifacts = verify_selected_artifacts() - require_backend_proof = bool(verified_artifacts) - attested_log_identities = [] - _require_unused_port(args.port) - revalidate_validation_cache(validation_cache) - validation_exit_code, cleanup_succeeded = _run_validation_process( - lemond=lemond, - cache_directory=cache_directory, - config_directory=config_directory, - command=command, - environment=environment, - port=args.port, - pid_path=pid_path, - stdout_path=validation_stdout_path, - stderr_path=validation_stderr_path, - backend=args.backend, - target=args.target, - allow_darwin_github_hosted_ephemeral_runner=( - allow_darwin_github_hosted_ephemeral_runner - ), - ) - revalidate_validation_cache(validation_cache) - if validation_exit_code: - return validation_exit_code - if not cleanup_succeeded: - print("Validation process cleanup did not complete", file=sys.stderr) - return 1 - validation_result = _capture_regular_file( - validation_result_path, - results_directory, - require_nonempty=True, - ) - require_validation_result_evidence( - validation_result, - results_directory, - selected_models, - capability_profile=args.capability_profile, - capability_models=capability_models, - ) - verified_artifacts = verify_selected_artifacts() - if require_backend_proof: - attested_log_identities.extend( - require_accelerator_attestation( - [validation_stdout_path, validation_stderr_path], - logs_directory, - "validation", - verified_artifacts, - attestation_backend, - selected_models=selected_models, - ) - ) - _require_unused_port(args.port) - revalidate_validation_cache(validation_cache) - restart_exit_code, restart_cleanup_succeeded = _run_validation_process( - lemond=lemond, - cache_directory=cache_directory, - config_directory=config_directory, - command=restart_command, - environment=environment, - port=args.port, - pid_path=pid_path, - stdout_path=restart_stdout_path, - stderr_path=restart_stderr_path, - backend=args.backend, - target=args.target, - allow_darwin_github_hosted_ephemeral_runner=( - allow_darwin_github_hosted_ephemeral_runner - ), - ) - revalidate_validation_cache(validation_cache) - if restart_exit_code: - return restart_exit_code - if not restart_cleanup_succeeded: - print("Restart validation process cleanup did not complete", file=sys.stderr) - return 1 - restart_result = _capture_regular_file( - restart_result_path, - results_directory, - require_nonempty=True, - ) - require_validation_result_evidence( - restart_result, - results_directory, - [restart_model_value], - ) - verified_artifacts = verify_selected_artifacts() - restart_model = restart_model_value.removeprefix("builtin.") - if require_backend_proof and restart_model in verified_artifacts: - attested_log_identities.extend( - require_accelerator_attestation( - [restart_stdout_path, restart_stderr_path], - logs_directory, - "restart validation", - {restart_model: verified_artifacts[restart_model]}, - attestation_backend, - selected_models=[restart_model_value], - ) - ) - revalidate_validation_cache(validation_cache) - _revalidate_regular_file( - validation_result, - results_directory, - require_nonempty=True, - ) - _revalidate_regular_file( - restart_result, - results_directory, - require_nonempty=True, - ) - log_paths = ( - prepare_stdout_path, - prepare_stderr_path, - validation_stdout_path, - validation_stderr_path, - restart_stdout_path, - restart_stderr_path, - ) - log_identities_by_path = { - identity.path: identity for identity in attested_log_identities - } - for path in log_paths: - if path not in log_identities_by_path: - log_identities_by_path[path] = _capture_regular_file( - path, - logs_directory, - require_nonempty=False, - ) - log_identities = list(log_identities_by_path.values()) - for identity in log_identities: - _revalidate_regular_file(identity, logs_directory, require_nonempty=False) - output_log_paths = { - identity.path: identity.resolved_path for identity in log_identities - } - _emit_workflow_outputs( - { - "validation_result": validation_result.resolved_path, - "restart_result": restart_result.resolved_path, - "prepare_stdout_log": output_log_paths[prepare_stdout_path], - "prepare_stderr_log": output_log_paths[prepare_stderr_path], - "validation_stdout_log": output_log_paths[validation_stdout_path], - "validation_stderr_log": output_log_paths[validation_stderr_path], - "restart_stdout_log": output_log_paths[restart_stdout_path], - "restart_stderr_log": output_log_paths[restart_stderr_path], - } - ) - return 0 - - -def _bool_argument(value: str) -> bool: - normalized = value.strip().lower() - if normalized == "true": - return True - if normalized in {"", "false"}: - return False - raise argparse.ArgumentTypeError("expected true or false") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--lemond", required=True, type=Path) - parser.add_argument("--python", required=True, type=Path) - parser.add_argument( - "--backend", - default=os.environ.get("LLAMACPP_BACKEND"), - choices=("cpu", "cuda", "metal", "rocm", "vulkan"), - ) - parser.add_argument( - "--channel", - default=os.environ.get("LLAMACPP_CHANNEL", ""), - choices=("", "stable", "nightly"), - ) - parser.add_argument("--target", default=os.environ.get("TARGET")) - parser.add_argument("--models", default=os.environ.get("LLAMACPP_TEST_MODELS", "")) - parser.add_argument( - "--lite", - default=_bool_argument(os.environ.get("LITE_MODE", "false")), - type=_bool_argument, - ) - parser.add_argument( - "--capability-profile", default=os.environ.get("CAPABILITY_PROFILE", "") - ) - parser.add_argument( - "--capability-models", default=os.environ.get("CAPABILITY_MODELS", "") - ) - parser.add_argument( - "--allow-huggingface-download-credentials", - action="store_true", - help="allow the prepare phase to inherit the canonical HF_TOKEN value", - ) - parser.add_argument( - "--allow-darwin-github-hosted-ephemeral-runner", - action="store_true", - help=( - "accept github-hosted macos-latest VM disposal as the Darwin " - "descendant-containment boundary" - ), - ) - parser.add_argument("--port", default=13305, type=int) - args = parser.parse_args() - if not args.backend: - parser.error("--backend or LLAMACPP_BACKEND is required") - if not args.target: - parser.error("--target or TARGET is required") - try: - with _CancellationSignalGuard(): - exit_code = run_validation(args) - _raise_if_cancelled() - except _ValidationCancelled as exc: - signal_name = signal.Signals(exc.signum).name - parser.exit( - 128 + exc.signum, - f"ERROR: validation cancelled by {signal_name}\n", - ) - except (OSError, ValueError, RuntimeError) as exc: - parser.exit(1, f"ERROR: {exc}\n") - raise SystemExit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/validate_backend_bench.py b/.github/scripts/validate_backend_bench.py index ae749840df..19b71543df 100644 --- a/.github/scripts/validate_backend_bench.py +++ b/.github/scripts/validate_backend_bench.py @@ -4,9 +4,9 @@ Uses `lemonade bench` against each tracked fork's prebuilt binary. The fork binary is downloaded, then routed into lemond via -`lemonade config set llamacpp._bin=` (done by the workflow +`lemonade config set llamacpp._bin ` (done by the workflow before lemond starts). lemond then serves --backend from that -binary instead of downloading its own. +binary instead of downloading its own. There is no LEMONADE_*_BIN env var. Usage: python .github/scripts/validate_backend_bench.py --forks src/cpp/resources/benchmark_forks.json --output ci/results @@ -15,6 +15,7 @@ """ import argparse +import hashlib import json import os import platform @@ -160,7 +161,7 @@ def find_lemonade_bin() -> str: def get_models_from_registry(base_url: str) -> list[str]: - """Select hot llama.cpp models from the live registry.""" + """Mirror validate_llamacpp.py:get_hot_llamacpp_models() — reads live registry.""" try: req = urllib.request.Request(f"{base_url}/api/v1/models?show_all=true") with urllib.request.urlopen(req, timeout=30) as r: @@ -209,7 +210,7 @@ def install_fork_binary( fork_id = fork["fork_id"] exe_name = fork["binary_exe_windows"] if IS_WINDOWS else fork["binary_exe_linux"] if not exe_name: - print(" [skip] No binary for this platform") + print(f" [skip] No binary for this platform") return None # One slot per fork — overwrite on version change, no disk accumulation @@ -303,8 +304,9 @@ def run_bench( # llamacpp exposes {system, cuda, vulkan, rocm, cpu} — "rocm-nightly" and # "rocm-gfx11" are NOT backend keys (nightly/stable are rocm *channels*), so # forks declare install_as to map to the real key. The fork binary itself is - # routed via `lemonade config set llamacpp._bin=` by the - # workflow before lemond starts. + # routed via `lemonade config set llamacpp._bin ` (done by the + # workflow before lemond starts); there is no LEMONADE_*_BIN env var — + # lemond never reads one. bench_as = fork.get("install_as", backend) cmd = [ @@ -608,7 +610,7 @@ def main() -> int: print(f"[skip] {fork_id} requires {runner_os}, running on {current_os}") continue - print(f"{'=' * 60}") + print(f"{'='*60}") print(f"Fork: {fork_id} ({fork['label']})") print(f"Repo: {fork['repo']}") @@ -651,7 +653,7 @@ def main() -> int: continue else: print( - " Skipping POST /install — fork provides its own prebuilt binary" + f" Skipping POST /install — fork provides its own prebuilt binary" ) # Fetch model list from registry now that lemond is confirmed running @@ -750,7 +752,7 @@ def main() -> int: ) print(f"Regression : {rr_out}") - print(f"\n{'=' * 60}") + print(f"\n{'='*60}") print("SUMMARY") criticals = [a for a in all_alerts if a["severity"] == "CRITICAL"] warns = [a for a in all_alerts if a["severity"] == "WARN"] diff --git a/.github/scripts/verify_llamacpp_pin_manifest.sh b/.github/scripts/verify_llamacpp_pin_manifest.sh deleted file mode 100755 index f9847e92c6..0000000000 --- a/.github/scripts/verify_llamacpp_pin_manifest.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ $# -ne 4 ]]; then - echo "Usage: $0 TRUSTED_REPOSITORY CANDIDATE_REPOSITORY BASE_SHA CANDIDATE_SHA" >&2 - exit 2 -fi - -trusted_repository=$1 -candidate_repository=$2 -base_sha=$3 -candidate_sha=$4 -oid_pattern='^[0-9a-fA-F]{40}$' - -if ! [[ "$base_sha" =~ $oid_pattern && "$candidate_sha" =~ $oid_pattern ]]; then - echo "Protected validation commits must be full Git OIDs." >&2 - exit 1 -fi -if [[ $(git -C "$trusted_repository" rev-parse HEAD) != "$base_sha" ]]; then - echo "Trusted checkout does not match the protected base commit." >&2 - exit 1 -fi -if [[ $(git -C "$candidate_repository" rev-parse HEAD) != "$candidate_sha" ]]; then - echo "Candidate checkout does not match the protected merge commit." >&2 - exit 1 -fi - -script_directory=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -repository_root=$(cd -- "${script_directory}/../.." && pwd) -manifest_tool="${repository_root}/test/utils/llamacpp_release_manifest.py" -capture_tool="${script_directory}/capture_llamacpp_release_manifest.sh" -snapshot_directory=$(mktemp -d) -base_versions="${snapshot_directory}/base-versions.json" -candidate_versions="${snapshot_directory}/candidate-versions.json" -candidate_manifest="${snapshot_directory}/candidate-manifest.json" -manifest_path=.github/llamacpp_release_manifest.json - -git -C "$trusted_repository" show \ - "${base_sha}:src/cpp/resources/backend_versions.json" >"$base_versions" -git -C "$candidate_repository" show \ - "${candidate_sha}:src/cpp/resources/backend_versions.json" >"$candidate_versions" - -changes=$(python "$manifest_tool" \ - --managed-pin-changes "$base_versions" "$candidate_versions") -if [[ "$changes" == '{}' ]]; then - base_manifest_entry=$(git -C "$trusted_repository" ls-tree \ - "$base_sha" -- "$manifest_path") - candidate_manifest_entry=$(git -C "$candidate_repository" ls-tree \ - "$candidate_sha" -- "$manifest_path") - if [[ "$base_manifest_entry" != "$candidate_manifest_entry" ]]; then - echo "Release manifest changes require changed managed llama.cpp pins." >&2 - exit 1 - fi - echo "No changed managed llama.cpp pins; no release manifest is required." - exit 0 -fi - -if ! git -C "$candidate_repository" show \ - "${candidate_sha}:${manifest_path}" \ - >"$candidate_manifest"; then - echo "Changed managed llama.cpp pins require a committed release manifest." >&2 - exit 1 -fi -python "$manifest_tool" \ - --require-managed-pin-manifest \ - "$base_versions" "$candidate_versions" "$candidate_manifest" >/dev/null -LLAMACPP_RELEASE_MANIFEST_TOOL="$manifest_tool" \ - bash "$capture_tool" "$candidate_manifest" "${snapshot_directory}/live" \ - >/dev/null - -echo "Verified publisher source claims and asset metadata for changed managed llama.cpp pins." diff --git a/.github/workflows/cpp_server_build_test_release.yml b/.github/workflows/cpp_server_build_test_release.yml index 47b6cdfbda..a4f49e7a10 100644 --- a/.github/workflows/cpp_server_build_test_release.yml +++ b/.github/workflows/cpp_server_build_test_release.yml @@ -127,17 +127,17 @@ jobs: if ($failures.Count -gt 0) { exit 1 } Write-Host "Build and packaging successful!" -ForegroundColor Green - - name: Run Windows process unit tests + - name: Run MCP client unit test on Windows shell: PowerShell run: | $ErrorActionPreference = "Stop" # Exercise the Windows-only process, quoting, environment-block, and # pipe/handle paths that Linux and macOS cannot cover. - cmake --build build --config Release --target test_mcp_client_config test_process_manager_windows + cmake --build build --config Release --target test_mcp_client_config if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - ctest --test-dir build -C Release --output-on-failure -R "^(mcp_client_config|ProcessManagerWindowsTest)$" + ctest --test-dir build -C Release --output-on-failure -R "^mcp_client_config$" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Upload Lemonade Server Installers diff --git a/.github/workflows/docs_and_style.yml b/.github/workflows/docs_and_style.yml index ec0ca82f5a..b9690d679b 100644 --- a/.github/workflows/docs_and_style.yml +++ b/.github/workflows/docs_and_style.yml @@ -75,30 +75,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.x' - - run: pip install pre-commit psutil requests openai pyyaml - - name: Verify focused worker imports - run: python -c "import test.utils.server_base" - - name: Run focused Python unit tests - run: >- - python -m unittest - test.test_comment_slop - test.test_llamacpp_capabilities - test.test_llamacpp_check_reporting - test.test_llamacpp_release_manifest - test.test_llamacpp_release_assets - test.test_llamacpp_validation_artifacts - test.test_llamacpp_validation_evidence - test.test_llamacpp_pr_file_classification - test.test_process_inheritance_contract - test.test_publish_llamacpp_update - test.test_run_llamacpp_validation - test.test_validation_model_catalog - test.test_validate_llamacpp_action_pins - test.test_validate_llamacpp_selection - test.test_validate_llamacpp_trusted_execution - test.test_validate_llamacpp_workflow_plan - test.test_verify_llamacpp_pin_manifest - -v + - run: pip install pre-commit + - name: Run the comment-slop unit tests + run: python -m unittest test.test_comment_slop -v + - name: Run pre-commit on the changed files # Scoped to what the change touches; running repo-wide would fail on drift that # predates it and is not the contributor's to fix. diff --git a/.github/workflows/validate_llamacpp.yml b/.github/workflows/validate_llamacpp.yml new file mode 100644 index 0000000000..0862c4fdb6 --- /dev/null +++ b/.github/workflows/validate_llamacpp.yml @@ -0,0 +1,1154 @@ +name: Validate New llama.cpp Release + +on: + workflow_dispatch: + inputs: + lite: + description: 'Run in lite mode (smallest model only)' + type: boolean + default: false + pull_request: + types: [opened, synchronize, reopened, labeled] + merge_group: + schedule: + # Sunday at 12:00 PM ET (17:00 UTC, 16:00 UTC during EDT) + - cron: "0 16 * * 0" + +permissions: + contents: write + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + LEMONADE_CI_MODE: "True" + PYTHONIOENCODING: utf-8 + HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} + LITE_MODE: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' || (github.event_name == 'workflow_dispatch' && inputs.lite == true) }} + +jobs: + # ======================================================================== + # Discover the latest llama.cpp release for each upstream repo + # ======================================================================== + get-latest-releases: + name: Get latest releases + # Only the auto-update runs resolve upstream releases. Everywhere else the + # build validates the pins committed in backend_versions.json, so a bad + # upstream release can't fail a merge or hide a broken pin. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + outputs: + llamacpp_release: ${{ steps.llamacpp.outputs.release }} + llamacpp_rocm_release: ${{ steps.llamacpp_rocm.outputs.release }} + llamacpp_lemonade_release: ${{ steps.llamacpp_lemonade.outputs.release }} + steps: + - name: Get latest llama.cpp release + id: llamacpp + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + # Upstream's semver releases (v0.3.0) are what /releases/latest now + # returns, and they carry no binaries -- the bNNNN nightlies hold the + # assets and are flagged as prereleases. + TAGS=$(gh api repos/ggml-org/llama.cpp/releases \ + --jq '.[] | select(.draft | not) | .tag_name') + RELEASE=$(printf '%s\n' "$TAGS" | grep -m1 -E '^b[0-9]+$' || true) + if [ -z "$RELEASE" ]; then + echo "::error::No bNNNN release found in ggml-org/llama.cpp." + exit 1 + fi + echo "Latest llama.cpp release: $RELEASE" + echo "release=$RELEASE" >> "$GITHUB_OUTPUT" + + - name: Get latest llamacpp-rocm release + id: llamacpp_rocm + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RELEASE=$(gh api repos/lemonade-sdk/llamacpp-rocm/releases/latest --jq '.tag_name') + echo "Latest llamacpp-rocm release: $RELEASE" + echo "release=$RELEASE" >> "$GITHUB_OUTPUT" + + - name: Get latest lemonade-sdk/llama.cpp release + id: llamacpp_lemonade + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RELEASE=$(gh api repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name') + echo "Latest lemonade-sdk/llama.cpp release: $RELEASE" + echo "release=$RELEASE" >> "$GITHUB_OUTPUT" + + # ======================================================================== + # Build binaries with updated backend versions + # ======================================================================== + verify-release-assets: + name: Verify release assets + needs: get-latest-releases + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + outputs: + ggml_update_backends: ${{ steps.assets.outputs.ggml_update_backends }} + lemonade_update_backends: ${{ steps.assets.outputs.lemonade_update_backends }} + rocm_update_backends: ${{ steps.assets.outputs.rocm_update_backends }} + ggml_missing_count: ${{ steps.assets.outputs.ggml_missing_count }} + rocm_nightly_missing_count: ${{ steps.assets.outputs.rocm_nightly_missing_count }} + rocm_stable_missing_count: ${{ steps.assets.outputs.rocm_stable_missing_count }} + cuda_missing_count: ${{ steps.assets.outputs.cuda_missing_count }} + vulkan_available: ${{ steps.assets.outputs.vulkan_available }} + cpu_available: ${{ steps.assets.outputs.cpu_available }} + metal_available: ${{ steps.assets.outputs.metal_available }} + rocm_nightly_available: ${{ steps.assets.outputs.rocm_nightly_available }} + rocm_stable_available: ${{ steps.assets.outputs.rocm_stable_available }} + cuda_available: ${{ steps.assets.outputs.cuda_available }} + steps: + - uses: actions/checkout@v5 + + - name: Fetch release asset lists + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GGML_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} + ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} + LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} + run: | + set -euo pipefail + + for value in GGML_RELEASE ROCM_RELEASE LEMONADE_RELEASE; do + if [ -z "${!value}" ]; then + echo "::error::$value was not resolved." + exit 1 + fi + done + + gh api "repos/ggml-org/llama.cpp/releases/tags/${GGML_RELEASE}" \ + --jq '.assets[].name' | sort > ggml_assets.txt + gh api "repos/lemonade-sdk/llamacpp-rocm/releases/tags/${ROCM_RELEASE}" \ + --jq '.assets[].name' | sort > rocm_assets.txt + gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LEMONADE_RELEASE}" \ + --jq '.assets[].name' | sort > lemonade_assets.txt + + echo "ggml-org/llama.cpp assets for ${GGML_RELEASE}:" + cat ggml_assets.txt + echo "lemonade-sdk/llamacpp-rocm assets for ${ROCM_RELEASE}:" + cat rocm_assets.txt + echo "lemonade-sdk/llama.cpp assets for ${LEMONADE_RELEASE}:" + cat lemonade_assets.txt + + - name: Verify expected asset families + id: assets + env: + GGML_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} + ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} + LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} + run: | + set -euo pipefail + + python3 - <<'PY' + import json + import os + import re + from pathlib import Path + + RELEASE_RE = re.compile(r"^b[0-9]+$") + CUDA_SMS = ( + "sm_75", + "sm_80", + "sm_86", + "sm_89", + "sm_90", + "sm_100", + "sm_120", + ) + ROCM_NIGHTLY_ARCHES = ( + "gfx1151", + "gfx1150", + "gfx120X", + "gfx110X", + "gfx103X", + "gfx90a", + "gfx908", + ) + + def require_release(name): + value = os.environ[name].strip() + if not RELEASE_RE.match(value): + raise SystemExit(f"Invalid {name}: {value!r}") + return value + + def read_assets(path): + return set(Path(path).read_text(encoding="utf-8").splitlines()) + + def update_outputs(values): + github_output = os.environ.get("GITHUB_OUTPUT") + if not github_output: + return + with open(github_output, "a", encoding="utf-8") as out: + for key, value in values.items(): + out.write(f"{key}={value}\n") + + def report_group(name, release, backend_requirements): + update_backends = [] + missing_by_backend = {} + for backend, required in backend_requirements.items(): + missing = [asset for asset in required if asset not in release] + missing_by_backend[backend] = missing + if missing: + print(f"{name}: {backend} assets incomplete; keeping existing pin.") + for asset in missing: + print(f" missing: {asset}") + else: + print(f"{name}: {backend} assets complete; pin may update.") + update_backends.append(backend) + return update_backends, missing_by_backend + + ggml_release = require_release("GGML_RELEASE") + rocm_release = require_release("ROCM_RELEASE") + lemonade_release = require_release("LEMONADE_RELEASE") + + ggml_assets = read_assets("ggml_assets.txt") + rocm_assets = read_assets("rocm_assets.txt") + lemonade_assets = read_assets("lemonade_assets.txt") + + with open("src/cpp/resources/backend_versions.json", encoding="utf-8") as f: + versions = json.load(f) + + therock = str(versions.get("therock", {}).get("version", "")).strip() + if therock.startswith("v"): + therock = therock[1:] + therock_parts = therock.split(".") + if len(therock_parts) >= 2: + therock = ".".join(therock_parts[:2]) + if not therock: + raise SystemExit("backend_versions.json is missing therock.version") + + asset_families = versions.get("rocm_asset_families", {}) + if not isinstance(asset_families, dict): + raise SystemExit("backend_versions.json rocm_asset_families must be an object") + + rocm_nightly_targets = sorted({asset_families.get(arch, arch) for arch in ROCM_NIGHTLY_ARCHES}) + + ggml_requirements = { + "vulkan": [ + f"llama-{ggml_release}-bin-win-vulkan-x64.zip", + f"llama-{ggml_release}-bin-ubuntu-vulkan-x64.tar.gz", + f"llama-{ggml_release}-bin-ubuntu-vulkan-arm64.tar.gz", + ], + "cpu": [ + f"llama-{ggml_release}-bin-win-cpu-x64.zip", + f"llama-{ggml_release}-bin-ubuntu-x64.tar.gz", + f"llama-{ggml_release}-bin-ubuntu-arm64.tar.gz", + ], + "metal": [ + f"llama-{ggml_release}-bin-macos-arm64.tar.gz", + ], + } + + rocm_nightly_requirements = { + "rocm-nightly": [ + asset + for target in rocm_nightly_targets + for asset in ( + f"llama-{rocm_release}-windows-rocm-{target}-x64.zip", + f"llama-{rocm_release}-ubuntu-rocm-{target}-x64.zip", + ) + ], + } + + lemonade_requirements = { + "rocm-stable": [ + f"llama-{lemonade_release}-bin-win-rocm-{therock}-x64.zip", + f"llama-{lemonade_release}-bin-ubuntu-rocm-{therock}-x64.tar.gz", + ], + "cuda": [ + asset + for sm in CUDA_SMS + for asset in ( + f"llama-{lemonade_release}-windows-cuda-{sm}-x64.7z", + f"llama-{lemonade_release}-ubuntu-cuda-{sm}-x64.tar.xz", + f"llama-{lemonade_release}-ubuntu-cuda-{sm}-arm64.tar.xz", + ) + ], + } + + ggml_updates, ggml_missing = report_group("ggml-org/llama.cpp", ggml_assets, ggml_requirements) + rocm_updates, rocm_missing = report_group("lemonade-sdk/llamacpp-rocm", rocm_assets, rocm_nightly_requirements) + lemonade_updates, lemonade_missing = report_group("lemonade-sdk/llama.cpp", lemonade_assets, lemonade_requirements) + + values = { + "ggml_update_backends": ",".join(ggml_updates), + "lemonade_update_backends": ",".join(lemonade_updates), + "rocm_update_backends": ",".join(rocm_updates), + "ggml_missing_count": str(sum(len(items) for items in ggml_missing.values())), + "rocm_nightly_missing_count": str(len(rocm_missing["rocm-nightly"])), + "rocm_stable_missing_count": str(len(lemonade_missing["rocm-stable"])), + "cuda_missing_count": str(len(lemonade_missing["cuda"])), + "vulkan_available": "true" if not ggml_missing["vulkan"] else "false", + "cpu_available": "true" if not ggml_missing["cpu"] else "false", + "metal_available": "true" if not ggml_missing["metal"] else "false", + "rocm_nightly_available": "true" if not rocm_missing["rocm-nightly"] else "false", + "rocm_stable_available": "true" if not lemonade_missing["rocm-stable"] else "false", + "cuda_available": "true" if not lemonade_missing["cuda"] else "false", + } + update_outputs(values) + + print("Eligible ggml-org/llama.cpp backend updates: " + (values["ggml_update_backends"] or "none")) + print("Eligible lemonade-sdk/llamacpp-rocm backend updates: " + (values["rocm_update_backends"] or "none")) + print("Eligible lemonade-sdk/llama.cpp backend updates: " + (values["lemonade_update_backends"] or "none")) + PY + + build: + name: Build + needs: [get-latest-releases, verify-release-assets] + # Label a PR `ci:upgrades` to opt it back in. + if: (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:upgrades')) && always() && !failure() && !cancelled() + runs-on: windows-latest + outputs: + lite: ${{ steps.lite.outputs.lite }} + steps: + - name: Resolve lite mode + # The env context is unavailable in a job's `strategy` block, so the + # validate matrix reads LITE_MODE through this output to size its + # runners from the same source of truth the validation itself uses. + id: lite + shell: PowerShell + run: | + "lite=$env:LITE_MODE" | + Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Prepare Windows runner for long paths and stale workspace caches + if: runner.os == 'Windows' + shell: PowerShell + run: | + $ErrorActionPreference = "Continue" + + # 1. Kill any running/orphaned processes to release file locks + $patterns = @("lemonade", "lemond", "llama-server", "llama", "flm", "ort-server", "moonshine-server", "wscript", "LemonadeServer") + foreach ($p in $patterns) { + Get-Process | Where-Object { $_.ProcessName -like "*$p*" } | Stop-Process -Force -ErrorAction SilentlyContinue + } + + + - uses: actions/checkout@v5 + with: + clean: true + fetch-depth: 0 + + - name: Update backend_versions.json + if: needs.get-latest-releases.result == 'success' + shell: PowerShell + env: + LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} + LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.ggml_update_backends }} + LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.rocm_update_backends }} + LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.lemonade_update_backends }} + run: | + $ErrorActionPreference = "Stop" + + $script = @' + import json + import os + import re + from pathlib import Path + + RELEASE_RE = re.compile(r"^b[0-9]+$") + + def parse_csv(value): + return [item.strip() for item in (value or "").split(",") if item.strip()] + + def update(section, release, backends): + if not backends: + return [] + if not RELEASE_RE.match(release): + raise SystemExit(f"Invalid llama.cpp release tag: {release}") + changed = [] + for key in backends: + if key not in section: + raise SystemExit(f"Refusing to create missing llamacpp.{key}") + if not isinstance(section[key], str): + raise SystemExit(f"llamacpp.{key} must be a string") + old = section[key] + section[key] = release + changed.append((key, old, release)) + return changed + + path = Path("src/cpp/resources/backend_versions.json") + data = json.loads(path.read_text(encoding="utf-8")) + section = data.get("llamacpp") + if not isinstance(section, dict): + raise SystemExit("backend_versions.json is missing a llamacpp object") + + changes = [] + changes += update(section, os.environ["LLAMACPP_RELEASE"], parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) + changes += update(section, os.environ["LLAMACPP_ROCM_RELEASE"], parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) + changes += update(section, os.environ["LLAMACPP_LEMONADE_RELEASE"], parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) + + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + print("Updated llamacpp backend versions:") + if changes: + for key, old, new in changes: + print(f" llamacpp.{key}: {old} -> {new}") + else: + print(" none") + '@ + + $script | python - + + - name: Build lemond + shell: PowerShell + run: | + $ErrorActionPreference = "Stop" + Write-Host "Building Lemonade server binaries..." -ForegroundColor Cyan + if (Test-Path "build") { Remove-Item -Recurse -Force "build" } + # The validation drives lemond over HTTP and never invokes the CLI, + # so lemond is the only target it needs. + cmake --preset vs18 -DBUILD_WEB_APP=OFF + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + # /m:2 goes straight to MSBuild rather than through --parallel, which + # makes CMake set CL_MPCount=1 and cancel the /MP in CMakeLists.txt. + # Capped at 2 because /MP already spawns one cl.exe per core. + cmake --build build --config Release --target lemond -- /m:2 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if (-not (Test-Path "build\Release\lemond.exe")) { + Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red + exit 1 + } + Write-Host "Build successful!" -ForegroundColor Green + + - name: Upload build artifacts + uses: actions/upload-artifact@v7 + with: + name: llamacpp-build + path: | + build/Release/ + build/resources/ + retention-days: 7 + + # ======================================================================== + # Validate backends on self-hosted runner + # ======================================================================== + validate: + name: Validate ${{ matrix.backend }}${{ matrix.channel && format(' ({0})', matrix.channel) || '' }} + needs: [get-latest-releases, build] + if: always() && needs.build.result == 'success' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: ${{ github.event_name == 'merge_group' }} + matrix: + # Full runs walk every hot model, topping out at gpt-oss-120b (59 GiB), + # which only fits the 128gb runner. Lite runs test just the smallest hot + # model (0.6 GiB), so they stay off that single contended machine. + include: + - backend: vulkan + runner: >- + ${{ fromJSON(needs.build.outputs.lite == 'true' + && '["self-hosted","Windows","stx-halo","vulkan","lemon-prod"]' + || '["self-hosted","Windows","128gb","vulkan","lemon-prod"]') }} + - backend: rocm + channel: stable + runner: >- + ${{ fromJSON(needs.build.outputs.lite == 'true' + && '["self-hosted","Windows","stx-halo","rocm","lemon-prod"]' + || '["self-hosted","Windows","128gb","rocm","lemon-prod"]') }} + - backend: rocm + channel: nightly + runner: >- + ${{ fromJSON(needs.build.outputs.lite == 'true' + && '["self-hosted","Windows","stx-halo","rocm","lemon-prod"]' + || '["self-hosted","Windows","128gb","rocm","lemon-prod"]') }} + # CUDA validation is prepared but intentionally disabled until an + # NVIDIA self-hosted runner is available. To enable it, uncomment this + # matrix entry. + # + # - backend: cuda + # runner: [self-hosted, Windows, 128gb, cuda] + steps: + - name: Capture Vulkan runner configuration + if: matrix.backend == 'vulkan' + shell: PowerShell + run: | + $ErrorActionPreference = "Continue" + + $outDir = Join-Path $env:RUNNER_TEMP "vulkan-runner-config" + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + + # General system / Windows / BIOS information + $os = Get-CimInstance Win32_OperatingSystem + $computer = Get-CimInstance Win32_ComputerSystem + $bios = Get-CimInstance Win32_BIOS + + [pscustomobject]@{ + CapturedUtc = (Get-Date).ToUniversalTime() + RunnerName = $env:RUNNER_NAME + Manufacturer = $computer.Manufacturer + Model = $computer.Model + PhysicalMemoryGiB = [math]::Round( + $computer.TotalPhysicalMemory / 1GB, + 2 + ) + WindowsCaption = $os.Caption + WindowsVersion = $os.Version + WindowsBuild = $os.BuildNumber + OSArchitecture = $os.OSArchitecture + LastBootUpTime = $os.LastBootUpTime + UptimeHours = [math]::Round( + ((Get-Date) - $os.LastBootUpTime).TotalHours, + 2 + ) + BIOSManufacturer = $bios.Manufacturer + BIOSVersion = $bios.SMBIOSBIOSVersion + BIOSReleaseDate = $bios.ReleaseDate + } | + Format-List | + Out-File (Join-Path $outDir "system.txt") -Encoding utf8 + + # GPU model, PCI device ID and Windows display-driver information + Get-CimInstance Win32_VideoController | + Select-Object ` + Name, + VideoProcessor, + PNPDeviceID, + DriverVersion, + DriverDate, + Status | + Format-List | + Out-File (Join-Path $outDir "gpu.txt") -Encoding utf8 + + Get-CimInstance Win32_PnPSignedDriver | + Where-Object { $_.DeviceClass -eq "DISPLAY" } | + Select-Object ` + DeviceName, + Manufacturer, + DriverProviderName, + DriverVersion, + DriverDate, + InfName, + DeviceID | + Format-List | + Out-File (Join-Path $outDir "display-drivers.txt") -Encoding utf8 + + # Current Windows power scheme + powercfg /getactivescheme 2>&1 | + Out-File (Join-Path $outDir "power.txt") -Encoding utf8 + + # Reboot-frequency proxy plus exact current boot time above + $bootEvents = @( + Get-WinEvent -FilterHashtable @{ + LogName = "System" + Id = 6005 + StartTime = (Get-Date).AddDays(-30) + } -ErrorAction SilentlyContinue + ) + + "Event Log service starts in last 30 days: $($bootEvents.Count)" | + Out-File (Join-Path $outDir "reboots.txt") -Encoding utf8 + + # Relevant workloads present before validation cleanup + $processes = @( + Get-Process -Name @( + "lemond", + "llama-server", + "lemonade", + "LemonadeServer", + "flm", + "ort-server", + "moonshine-server" + ) -ErrorAction SilentlyContinue + ) + + if ($processes.Count -gt 0) { + $processes | + Select-Object ProcessName, Id, StartTime, CPU, WorkingSet64 | + Format-Table -AutoSize | + Out-File (Join-Path $outDir "processes-before-cleanup.txt") ` + -Encoding utf8 + } else { + "No known inference processes detected." | + Out-File (Join-Path $outDir "processes-before-cleanup.txt") ` + -Encoding utf8 + } + + # Best-effort AMD cache-directory state. + # This does not modify or clear any cache. + $amdRoot = if ($env:LOCALAPPDATA) { + Join-Path $env:LOCALAPPDATA "AMD" + } else { + $null + } + + if ($amdRoot -and (Test-Path $amdRoot)) { + Get-ChildItem $amdRoot -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -match "cache" } | + Select-Object Name, CreationTimeUtc, LastWriteTimeUtc | + Format-Table -AutoSize | + Out-File (Join-Path $outDir "amd-cache-state.txt") ` + -Encoding utf8 + } else { + "AMD local cache directory not found." | + Out-File (Join-Path $outDir "amd-cache-state.txt") ` + -Encoding utf8 + } + + # Vulkan device, driver, memory heaps and capabilities + $vulkaninfo = Get-Command vulkaninfo -ErrorAction SilentlyContinue + + if ($vulkaninfo) { + & $vulkaninfo.Source --summary *>&1 | + Out-File (Join-Path $outDir "vulkaninfo-summary.txt") ` + -Encoding utf8 + $summaryExitCode = $LASTEXITCODE + + & $vulkaninfo.Source *>&1 | + Out-File (Join-Path $outDir "vulkaninfo.txt") ` + -Encoding utf8 + $fullExitCode = $LASTEXITCODE + + Write-Host "=== vulkaninfo --summary ===" + Get-Content (Join-Path $outDir "vulkaninfo-summary.txt") + + Write-Host "vulkaninfo summary exit code: $summaryExitCode" + Write-Host "vulkaninfo full exit code: $fullExitCode" + } else { + "vulkaninfo was not found in PATH." | + Out-File (Join-Path $outDir "vulkaninfo-unavailable.txt") ` + -Encoding utf8 + Write-Warning "vulkaninfo was not found in PATH." + } + + Write-Host "=== Runner configuration ===" + Get-Content (Join-Path $outDir "system.txt") + Get-Content (Join-Path $outDir "gpu.txt") + Get-Content (Join-Path $outDir "display-drivers.txt") + Get-Content (Join-Path $outDir "power.txt") + Get-Content (Join-Path $outDir "reboots.txt") + Get-Content (Join-Path $outDir "processes-before-cleanup.txt") + + # Environment collection should never make validation fail. + exit 0 + + - name: Upload Vulkan runner configuration + if: always() && matrix.backend == 'vulkan' + uses: actions/upload-artifact@v7 + with: + name: vulkan-runner-config + path: ${{ runner.temp }}/vulkan-runner-config/ + retention-days: 30 + if-no-files-found: warn + + - name: Cleanup prior validation processes + if: runner.os == 'Windows' + shell: PowerShell + run: | + $ErrorActionPreference = "Stop" + $workspaceRoot = [IO.Path]::GetFullPath($env:GITHUB_WORKSPACE) + $lemondPath = [IO.Path]::GetFullPath( + (Join-Path $workspaceRoot "build\Release\lemond.exe") + ) + $cachePrefix = [IO.Path]::GetFullPath( + (Join-Path $workspaceRoot "ci-cache-") + ) + $staleProcesses = @( + Get-CimInstance Win32_Process ` + -Filter "Name = 'lemond.exe' OR Name = 'llama-server.exe'" ` + -ErrorAction SilentlyContinue | Where-Object { + $_.ExecutablePath -and ( + ($_.Name -eq "lemond.exe" -and $_.ExecutablePath.Equals( + $lemondPath, + [StringComparison]::OrdinalIgnoreCase + )) -or + ($_.Name -eq "llama-server.exe" -and $_.ExecutablePath.StartsWith( + $cachePrefix, + [StringComparison]::OrdinalIgnoreCase + )) + ) + } + ) + + foreach ($process in $staleProcesses) { + Write-Host "Stopping stale $($process.Name) PID $($process.ProcessId)." ` + -ForegroundColor Yellow + Stop-Process -Id $process.ProcessId -Force -ErrorAction SilentlyContinue + } + if ($staleProcesses.Count -gt 0) { + Start-Sleep -Seconds 1 + foreach ($process in $staleProcesses) { + if (Get-Process -Id $process.ProcessId -ErrorAction SilentlyContinue) { + throw "Failed to stop workspace process PID $($process.ProcessId)." + } + } + } + + - uses: actions/checkout@v5 + with: + clean: true + + - name: Cleanup processes + uses: ./.github/actions/cleanup-processes-windows + + - name: Download build artifacts + uses: actions/download-artifact@v7 + with: + name: llamacpp-build + path: build + + - name: Verify binaries + shell: PowerShell + run: | + $lemondExe = "build\Release\lemond.exe" + if (-not (Test-Path $lemondExe)) { + Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red + Get-ChildItem -Recurse build | Select-Object FullName + exit 1 + } + & $lemondExe --version + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "Binaries verified!" -ForegroundColor Green + + - name: Setup Python and virtual environment + uses: ./.github/actions/setup-venv + with: + venv-name: '.venv' + python-version: '3.10' + requirements-file: 'test/requirements.txt' + + - name: Run validation with lemond + shell: PowerShell + run: | + $ErrorActionPreference = "Stop" + $port = 13305 + $lemondExe = (Resolve-Path "build\Release\lemond.exe").Path + $backend = "${{ matrix.backend }}" + $channel = "${{ matrix.channel }}" + $label = if ($channel) { "$backend-$channel" } else { $backend } + $logsDir = "server-logs-$label" + $cacheDir = Join-Path $PWD "ci-cache-$label" + $cachePrefix = [IO.Path]::GetFullPath($cacheDir) + $separator = [string][IO.Path]::DirectorySeparatorChar + if (-not $cachePrefix.EndsWith($separator)) { $cachePrefix += $separator } + $venvPython = ".\.venv\Scripts\python.exe" + $tempRoot = if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { $PWD } + $pidFile = Join-Path $tempRoot "lemond-validate-llamacpp.pid" + + function Get-ValidationLlamaServers { + Get-CimInstance Win32_Process ` + -Filter "Name = 'llama-server.exe'" ` + -ErrorAction SilentlyContinue | Where-Object { + $_.ExecutablePath -and $_.ExecutablePath.StartsWith( + $cachePrefix, + [StringComparison]::OrdinalIgnoreCase + ) + } + } + + New-Item -ItemType Directory -Force -Path $logsDir | Out-Null + New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null + + $stdoutLog = Join-Path $PWD "$logsDir\lemond.stdout.log" + $stderrLog = Join-Path $PWD "$logsDir\lemond.stderr.log" + + # Do not benchmark while another known inference process is active. + $conflicts = @( + Get-Process -Name @( + "lemond", "llama-server", "lemonade", "LemonadeServer", + "flm", "ort-server", "moonshine-server" + ) -ErrorAction SilentlyContinue + ) + if ($conflicts.Count -gt 0) { + $details = ($conflicts | ForEach-Object { + "$($_.ProcessName):$($_.Id)" + }) -join ", " + throw "Runner is not idle: $details" + } + + if (Get-NetTCPConnection -State Listen -LocalPort $port -ErrorAction SilentlyContinue) { + throw "Port $port is already in use." + } + + $proc = $null + $validationExitCode = 0 + + try { + $proc = Start-Process ` + -FilePath $lemondExe ` + -ArgumentList @($cacheDir, "--port", $port, "--host", "127.0.0.1") ` + -RedirectStandardOutput $stdoutLog ` + -RedirectStandardError $stderrLog ` + -PassThru + + # Same-job fallback for the existing final cleanup action. + Set-Content -Path $pidFile -Value $proc.Id -Encoding ASCII + Write-Host "Started lemond PID $($proc.Id)" -ForegroundColor Cyan + + if ($proc.WaitForExit(500)) { + throw "lemond exited before validation started. See $stderrLog." + } + + $validationArgs = @( + "test/validate_llamacpp.py", + "--backend", $backend, + "--port", $port, + "--output", "llamacpp_validation_$label.json", + "--logs-dir", $logsDir + ) + + if ($channel) { + $validationArgs += "--channel" + $validationArgs += $channel + } + + if ("${{ env.LITE_MODE }}" -eq "true") { + $validationArgs += "--lite" + } + + & $venvPython @validationArgs + $validationExitCode = $LASTEXITCODE + } finally { + if ($null -ne $proc) { + try { + if (-not $proc.HasExited) { + Invoke-WebRequest -Uri "http://127.0.0.1:$port/internal/shutdown" ` + -Method POST -TimeoutSec 10 -UseBasicParsing | Out-Null + $null = $proc.WaitForExit(10000) + } + } catch { + Write-Host "Graceful shutdown failed; forcing targeted cleanup." ` + -ForegroundColor Yellow + } + + if (-not $proc.HasExited) { + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + $null = $proc.WaitForExit(5000) + } + + # Catch a llama-server orphan even if its lemond parent exited first. + $orphans = @(Get-ValidationLlamaServers) + foreach ($orphan in $orphans) { + Write-Host "Stopping validation llama-server PID $($orphan.ProcessId)." ` + -ForegroundColor Yellow + Stop-Process -Id $orphan.ProcessId -Force -ErrorAction SilentlyContinue + } + + # Process teardown can lag slightly on Windows, especially after + # releasing Vulkan resources. Give it a short bounded grace period. + $remainingOrphans = @(Get-ValidationLlamaServers) + $cleanupDeadline = (Get-Date).AddSeconds(5) + + while ( + $remainingOrphans.Count -gt 0 -and + (Get-Date) -lt $cleanupDeadline + ) { + Start-Sleep -Milliseconds 500 + $remainingOrphans = @(Get-ValidationLlamaServers) + } + + if ($proc.HasExited -and $remainingOrphans.Count -eq 0) { + Remove-Item $pidFile -Force -ErrorAction SilentlyContinue + } else { + if (-not $proc.HasExited) { + Write-Warning "lemond PID $($proc.Id) is still running after cleanup." + } + + foreach ($orphan in $remainingOrphans) { + Write-Warning ( + "llama-server still present after cleanup: " + + "PID=$($orphan.ProcessId), Path=$($orphan.ExecutablePath)" + ) + } + + Write-Warning ( + "Validation cleanup did not fully complete. " + + "The validation result will be preserved." + ) + } + } + } + + if ($validationExitCode -ne 0) { + exit $validationExitCode + } + + - name: Upload results + if: always() + uses: actions/upload-artifact@v7 + with: + name: validation-results-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }} + path: llamacpp_validation_${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}.json + retention-days: 30 + + - name: Upload server logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }} + path: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}/ + retention-days: 30 + if-no-files-found: ignore + + - name: Cleanup + if: always() + uses: ./.github/actions/cleanup-processes-windows + + # ======================================================================== + # Create PR if both validations passed (schedule or workflow_dispatch only) + # ======================================================================== + create-pr: + name: Create update PR + needs: [get-latest-releases, verify-release-assets, validate] + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + if: >- + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && + needs.validate.result == 'success' && + needs.verify-release-assets.result == 'success' + steps: + - uses: actions/checkout@v5 + + - name: Download validation results + uses: actions/download-artifact@v7 + with: + pattern: validation-results-* + merge-multiple: true + + - name: Update backend_versions.json with verified releases + shell: bash + env: + LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} + LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.ggml_update_backends }} + LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.rocm_update_backends }} + LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.lemonade_update_backends }} + run: | + set -euo pipefail + + python3 - <<'PY' + import json + import os + import re + from pathlib import Path + + RELEASE_RE = re.compile(r"^b[0-9]+$") + + def parse_csv(value): + return [item.strip() for item in (value or "").split(",") if item.strip()] + + def update(section, release, backends): + if not backends: + return [] + if not RELEASE_RE.match(release): + raise SystemExit(f"Invalid llama.cpp release tag: {release}") + changed = [] + for key in backends: + if key not in section: + raise SystemExit(f"Refusing to create missing llamacpp.{key}") + if not isinstance(section[key], str): + raise SystemExit(f"llamacpp.{key} must be a string") + old = section[key] + section[key] = release + changed.append((key, old, release)) + return changed + + path = Path("src/cpp/resources/backend_versions.json") + data = json.loads(path.read_text(encoding="utf-8")) + section = data.get("llamacpp") + if not isinstance(section, dict): + raise SystemExit("backend_versions.json is missing a llamacpp object") + + changes = [] + changes += update(section, os.environ["LLAMACPP_RELEASE"], parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) + changes += update(section, os.environ["LLAMACPP_ROCM_RELEASE"], parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) + changes += update(section, os.environ["LLAMACPP_LEMONADE_RELEASE"], parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) + + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + print("Updated llamacpp backend versions:") + if changes: + for key, old, new in changes: + print(f" llamacpp.{key}: {old} -> {new}") + else: + print(" none") + PY + + - name: Generate PR body + shell: bash + env: + LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} + LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.ggml_update_backends }} + LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.rocm_update_backends }} + LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.lemonade_update_backends }} + VULKAN_AVAILABLE: ${{ needs.verify-release-assets.outputs.vulkan_available }} + CPU_AVAILABLE: ${{ needs.verify-release-assets.outputs.cpu_available }} + METAL_AVAILABLE: ${{ needs.verify-release-assets.outputs.metal_available }} + ROCM_NIGHTLY_AVAILABLE: ${{ needs.verify-release-assets.outputs.rocm_nightly_available }} + ROCM_STABLE_AVAILABLE: ${{ needs.verify-release-assets.outputs.rocm_stable_available }} + CUDA_AVAILABLE: ${{ needs.verify-release-assets.outputs.cuda_available }} + GGML_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.ggml_missing_count }} + ROCM_NIGHTLY_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.rocm_nightly_missing_count }} + ROCM_STABLE_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.rocm_stable_missing_count }} + CUDA_MISSING_COUNT: ${{ needs.verify-release-assets.outputs.cuda_missing_count }} + run: | + python3 <<'PYEOF' + import json + import os + + def parse_csv(value): + return [item.strip() for item in (value or "").split(",") if item.strip()] + + def yes(name): + return os.environ.get(name, "false") == "true" + + def pin_action(backend, release, enabled): + return release if enabled else "unchanged" + + body_lines = [] + llamacpp = os.environ.get("LLAMACPP_RELEASE", "unknown") + rocm_nightly = os.environ.get("LLAMACPP_ROCM_RELEASE", "unknown") + lemonade_release = os.environ.get("LLAMACPP_LEMONADE_RELEASE", "unknown") + ggml_updates = set(parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) + rocm_updates = set(parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) + lemonade_updates = set(parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) + + pin_plan = { + "vulkan": pin_action("vulkan", llamacpp, "vulkan" in ggml_updates), + "cpu": pin_action("cpu", llamacpp, "cpu" in ggml_updates), + "metal": pin_action("metal", llamacpp, "metal" in ggml_updates), + "rocm-nightly": pin_action("rocm-nightly", rocm_nightly, "rocm-nightly" in rocm_updates), + "rocm-stable": pin_action("rocm-stable", lemonade_release, "rocm-stable" in lemonade_updates), + "cuda": pin_action("cuda", lemonade_release, "cuda" in lemonade_updates), + } + + body_lines.append("## Auto-update llama.cpp backends") + body_lines.append("") + body_lines.append("This PR updates only backend pins whose expected release assets were found.") + body_lines.append("") + body_lines.append(f"- **ggml-org/llama.cpp candidate**: `{llamacpp}`") + body_lines.append(f"- **lemonade-sdk/llamacpp-rocm candidate**: `{rocm_nightly}`") + body_lines.append(f"- **lemonade-sdk/llama.cpp candidate**: `{lemonade_release}`") + body_lines.append("") + body_lines.append("| Backend | Pin action | Asset check |") + body_lines.append("|---|---|---|") + body_lines.append(f"| `vulkan` | `{pin_plan['vulkan']}` | {'complete' if yes('VULKAN_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `cpu` | `{pin_plan['cpu']}` | {'complete' if yes('CPU_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `metal` | `{pin_plan['metal']}` | {'complete' if yes('METAL_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `rocm-nightly` | `{pin_plan['rocm-nightly']}` | {'complete' if yes('ROCM_NIGHTLY_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `rocm-stable` | `{pin_plan['rocm-stable']}` | {'complete' if yes('ROCM_STABLE_AVAILABLE') else 'incomplete'} |") + body_lines.append(f"| `cuda` | `{pin_plan['cuda']}` | {'complete' if yes('CUDA_AVAILABLE') else 'incomplete'} |") + body_lines.append("") + body_lines.append("Missing asset counts:") + body_lines.append(f"- ggml-org/llama.cpp: `{os.environ.get('GGML_MISSING_COUNT', 'unknown')}`") + body_lines.append(f"- lemonade-sdk/llamacpp-rocm rocm-nightly: `{os.environ.get('ROCM_NIGHTLY_MISSING_COUNT', 'unknown')}`") + body_lines.append(f"- lemonade-sdk/llama.cpp rocm-stable: `{os.environ.get('ROCM_STABLE_MISSING_COUNT', 'unknown')}`") + body_lines.append(f"- lemonade-sdk/llama.cpp cuda: `{os.environ.get('CUDA_MISSING_COUNT', 'unknown')}`") + body_lines.append("") + body_lines.append("CUDA remains asset-gated only until a CUDA self-hosted runner is enabled.") + body_lines.append("") + body_lines.append("## Validation Results") + body_lines.append("") + body_lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |") + body_lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|") + + for backend in ["vulkan", "rocm-stable", "rocm-nightly"]: + # Labels match the artifact names produced by the validate job + results_file = f"llamacpp_validation_{backend}.json" + if not os.path.isfile(results_file): + body_lines.append(f"| _{backend}: no results_ | | | | | | |") + continue + with open(results_file, "r") as f: + results = json.load(f) + for r in results: + status = "PASS" if r["pass"] else "FAIL" + # Truncate response for table readability + resp = str(r.get("response", ""))[:80].replace("|", "\\|").replace("\n", " ") + ttft = r.get("time_to_first_token", "N/A") + if isinstance(ttft, float): + ttft = f"{ttft:.3f}s" + tps = r.get("tokens_per_second", "N/A") + if isinstance(tps, float): + tps = f"{tps:.1f}" + body_lines.append( + f"| {r['model']} ({backend}) | {status} | {resp} | " + f"{r.get('input_tokens', 'N/A')} | {r.get('output_tokens', 'N/A')} | " + f"{ttft} | {tps} |" + ) + + body_lines.append("") + body_lines.append("---") + body_lines.append("*Auto-generated by validate_llamacpp workflow*") + + body = "\n".join(body_lines) + with open("pr_body.md", "w") as f: + f.write(body) + PYEOF + + - name: Create Pull Request + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} + LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} + LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} + run: | + BRANCH="auto/llamacpp-update-${LLAMACPP_RELEASE}-${LLAMACPP_LEMONADE_RELEASE}-${LLAMACPP_ROCM_RELEASE}" + + # Check if a PR already exists for this update + EXISTING_PR=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || echo "") + if [ -n "$EXISTING_PR" ]; then + echo "PR #$EXISTING_PR already exists for branch $BRANCH. Skipping." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Skip if backend_versions.json is already up to date + if git diff --quiet src/cpp/resources/backend_versions.json; then + echo "backend_versions.json is unchanged — nothing to update." + exit 0 + fi + + git checkout -b "$BRANCH" + git add src/cpp/resources/backend_versions.json + git commit -m "Update llama.cpp to ${LLAMACPP_RELEASE}, rocm-stable to ${LLAMACPP_LEMONADE_RELEASE}, rocm-nightly to ${LLAMACPP_ROCM_RELEASE}" + git push origin "$BRANCH" + + gh pr create \ + --title "Update llama.cpp to ${LLAMACPP_RELEASE}" \ + --body-file pr_body.md \ + --base main \ + --head "$BRANCH" + + # Gate job that ensures: 1. in the merge queue, all jobs in `needs:` ran + # successfully (otherwise the merge is blocked), and 2. these jobs do not need + # to run on ordinary pull request pushes. + validation-gate: + name: llama.cpp validation + needs: [build, validate] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check gated jobs + env: + NEEDS: ${{ toJSON(needs) }} + run: | + # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...} + # Fail if any job broke. In the merge queue, also fail if any never ran. + echo "$NEEDS" + broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS") + if [ -n "$broke" ]; then + echo "FAILED: $broke" + exit 1 + fi + if [ "${{ github.event_name }}" = "merge_group" ]; then + absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS") + if [ -n "$absent" ]; then + echo "DID NOT RUN IN MERGE QUEUE: $absent" + exit 1 + fi + fi diff --git a/.github/workflows/validate_llamacpp_core.yml b/.github/workflows/validate_llamacpp_core.yml deleted file mode 100644 index 3a39b0a9e1..0000000000 --- a/.github/workflows/validate_llamacpp_core.yml +++ /dev/null @@ -1,1498 +0,0 @@ -name: Validate llama.cpp - -on: - workflow_call: - inputs: - authorization_epoch: - description: 'Trusted authorization issue-event ID; zero outside pull requests' - type: string - default: '0' - lite: - description: 'Run only the K2-Horizon 0.9B smoke model' - type: boolean - default: false - models: - description: 'Additional comma-separated IDs; empty uses hot models + K2 0.9B' - type: string - required: false - default: '' - secrets: - HUGGINGFACE_ACCESS_TOKEN: - required: false - outputs: - check_run_id: - value: ${{ jobs.validate-invocation.outputs.check_run_id }} - promotion_eligible: - value: ${{ jobs.plan.outputs.promotion_eligible }} - validation_matrix: - value: ${{ jobs.plan.outputs.matrix }} - llamacpp_release: - value: ${{ jobs.get-latest-releases.outputs.llamacpp_release }} - llamacpp_rocm_release: - value: ${{ jobs.get-latest-releases.outputs.llamacpp_rocm_release }} - llamacpp_lemonade_release: - value: ${{ jobs.get-latest-releases.outputs.llamacpp_lemonade_release }} - release_asset_manifest: - value: ${{ jobs.verify-release-assets.outputs.release_asset_manifest }} - ggml_update_backends: - value: ${{ jobs.verify-release-assets.outputs.ggml_update_backends }} - lemonade_update_backends: - value: ${{ jobs.verify-release-assets.outputs.lemonade_update_backends }} - rocm_update_backends: - value: ${{ jobs.verify-release-assets.outputs.rocm_update_backends }} - ggml_missing_count: - value: ${{ jobs.verify-release-assets.outputs.ggml_missing_count }} - rocm_nightly_missing_count: - value: ${{ jobs.verify-release-assets.outputs.rocm_nightly_missing_count }} - rocm_stable_missing_count: - value: ${{ jobs.verify-release-assets.outputs.rocm_stable_missing_count }} - cuda_missing_count: - value: ${{ jobs.verify-release-assets.outputs.cuda_missing_count }} - vulkan_available: - value: ${{ jobs.verify-release-assets.outputs.vulkan_available }} - cpu_available: - value: ${{ jobs.verify-release-assets.outputs.cpu_available }} - metal_available: - value: ${{ jobs.verify-release-assets.outputs.metal_available }} - rocm_nightly_available: - value: ${{ jobs.verify-release-assets.outputs.rocm_nightly_available }} - rocm_stable_available: - value: ${{ jobs.verify-release-assets.outputs.rocm_stable_available }} - cuda_available: - value: ${{ jobs.verify-release-assets.outputs.cuda_available }} -permissions: - contents: read - -env: - DEPENDABOT_PR: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.user.login == 'dependabot[bot]' && github.event.pull_request.user.id == 49699333 && github.event.pull_request.user.type == 'Bot' }} - LEMONADE_CI_MODE: "True" - PYTHONIOENCODING: utf-8 - -jobs: - validate-invocation: - name: Validate invocation - runs-on: ubuntu-latest - permissions: - checks: write - contents: read - outputs: - check_run_id: ${{ steps.start-check.outputs.check_run_id }} - steps: - - name: Reject untrusted caller events - env: - EVENT_NAME: ${{ github.event_name }} - run: | - set -euo pipefail - - case "$EVENT_NAME" in - merge_group|pull_request_target|schedule|workflow_dispatch) - ;; - *) - echo "::error::Unsupported invocation event: $EVENT_NAME" - exit 1 - ;; - esac - - - name: Start authorized merge check - id: start-check - if: (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && env.DEPENDABOT_PR != 'true' - env: - AUTHORIZATION_EPOCH: ${{ inputs.authorization_epoch }} - CHECK_NAME: llama.cpp authorized validation v2 - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MERGE_SHA: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.event.pull_request.merge_commit_sha }} - RUN_ATTEMPT: ${{ github.run_attempt }} - RUN_ID: ${{ github.run_id }} - RUN_NUMBER: ${{ github.run_number }} - TARGET_KEY: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}', github.event.pull_request.number) }} - run: | - set -euo pipefail - EXTERNAL_PREFIX="lemonade-llamacpp-required-${TARGET_KEY}-" - EXTERNAL_ID="${EXTERNAL_PREFIX}${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}" - if ! [[ "$AUTHORIZATION_EPOCH" =~ ^[0-9]+$ ]] || \ - ! [[ "$MERGE_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::Protected validation requires an exact candidate commit." - exit 1 - fi - if [ "$EVENT_NAME" = "pull_request_target" ] && \ - ! [[ "$AUTHORIZATION_EPOCH" =~ ^[1-9][0-9]*$ ]]; then - echo "::error::Pull-request validation requires a live authorization epoch." - exit 1 - fi - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" \ - -f filter=all \ - -F per_page=100) - - IFS=$'\t' read -r newest_number newest_attempt < <( - jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$EXTERNAL_PREFIX" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {epoch: (.epoch | tonumber), number: (.number | tonumber), - attempt: (.attempt | tonumber)}) catch empty] - | sort_by(.number, .attempt) - | (last // {epoch: 0, number: 0, attempt: 0}) - | [.number, .attempt] | @tsv' <<<"$check_runs" - ) - if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ - { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ - [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then - echo "::error::A newer GitHub Actions validation run already owns this exact-commit gate." - exit 1 - fi - - check_run_id=$(jq -r --arg check_name "$CHECK_NAME" \ - --arg external_id "$EXTERNAL_ID" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions" and - .external_id == $external_id) - | .id] | last // empty' <<<"$check_runs") - - if [ -n "$check_run_id" ] && ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid existing check-run ID." - exit 1 - fi - if [ -z "$check_run_id" ]; then - response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f head_sha="$MERGE_SHA" \ - -f external_id="$EXTERNAL_ID" \ - -f status=in_progress \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}") - check_run_id=$(jq -r '.id // empty' <<<"$response") - else - gh api --method PATCH \ - "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f status=in_progress \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >/dev/null - fi - if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid check-run ID." - exit 1 - fi - - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" \ - -f filter=all \ - -F per_page=100) - IFS=$'\t' read -r newest_number newest_attempt < <( - jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$EXTERNAL_PREFIX" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {epoch: (.epoch | tonumber), number: (.number | tonumber), - attempt: (.attempt | tonumber)}) catch empty] - | sort_by(.number, .attempt) - | (last // {epoch: 0, number: 0, attempt: 0}) - | [.number, .attempt] | @tsv' <<<"$check_runs" - ) - if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ - { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ - [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then - gh api --method PATCH \ - "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f status=completed \ - -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=Superseded by a newer policy decision." >/dev/null - echo "::error::A newer GitHub Actions validation run superseded this check." - exit 1 - fi - - prior_check_ids=$(jq -r \ - --arg check_name "$CHECK_NAME" \ - --arg external_prefix "$EXTERNAL_PREFIX" \ - --argjson current_number "$RUN_NUMBER" --argjson current_attempt "$RUN_ATTEMPT" \ - '.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | . as $check - | try (($check.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {epoch: (.epoch | tonumber), number: (.number | tonumber), - attempt: (.attempt | tonumber)}) catch empty - | select(.number < $current_number or - (.number == $current_number and .attempt < $current_attempt)) - | $check.id' <<<"$check_runs") - while IFS= read -r prior_check_id; do - [ -n "$prior_check_id" ] || continue - if ! [[ "$prior_check_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid prior check-run ID." - exit 1 - fi - gh api --method PATCH \ - "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" \ - -f status=completed \ - -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=Superseded by a newer authorized validation run." >/dev/null - done <<<"$prior_check_ids" - - echo "check_run_id=$check_run_id" >> "$GITHUB_OUTPUT" - - plan: - name: Plan validation - needs: validate-invocation - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.protected-plan.outputs.matrix || steps.branch-plan.outputs.matrix }} - promotion_eligible: ${{ steps.protected-plan.outputs.promotion_eligible || steps.branch-plan.outputs.promotion_eligible }} - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' - with: - repository: ${{ github.repository }} - ref: ${{ github.sha }} - persist-credentials: false - fetch-depth: 2 - - name: Build trusted protected-change matrix - id: protected-plan - if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - run: | - set -euo pipefail - matrix=$(jq -cn '{ - include: [ - { - target: "windows-vulkan", - backend: "vulkan", - channel: "", - managed_pin: "vulkan", - build_platform: "windows", - runner: ["self-hosted", "Windows", "128gb", "vulkan", "lemon-prod"], - models: [ - "K2-Horizon-0.9B-GGUF", - "K2-Horizon-3.7B-GGUF", - "K2-Horizon-7B-GGUF" - ], - capability_profile: "k2-horizon-v1", - capability_models: ["K2-Horizon-0.9B-GGUF"], - lite: false - }, - { - target: "linux-vulkan", - backend: "vulkan", - channel: "", - managed_pin: "vulkan", - build_platform: "linux", - runner: ["self-hosted", "Linux", "vulkan", "lemon-prod"], - models: ["K2-Horizon-0.9B-GGUF"], - capability_profile: "k2-horizon-v1", - capability_models: ["K2-Horizon-0.9B-GGUF"], - lite: false - }, - { - target: "linux-cpu", - backend: "cpu", - channel: "", - managed_pin: "cpu", - build_platform: "linux", - runner: ["self-hosted", "Linux", "lemon-prod"], - models: ["K2-Horizon-0.9B-GGUF"], - capability_profile: "k2-horizon-v1", - capability_models: ["K2-Horizon-0.9B-GGUF"], - lite: false - }, - { - target: "macos-metal", - backend: "metal", - channel: "", - managed_pin: "metal", - build_platform: "macos", - runner: ["macos-latest"], - models: ["K2-Horizon-0.9B-GGUF"], - capability_profile: "k2-horizon-v1", - capability_models: ["K2-Horizon-0.9B-GGUF"], - lite: false - }, - { - target: "windows-cuda", - backend: "cuda", - channel: "", - managed_pin: "cuda", - build_platform: "windows", - runner: ["self-hosted", "Windows", "cuda", "lemon-prod"], - models: ["K2-Horizon-0.9B-GGUF"], - capability_profile: "k2-horizon-v1", - capability_models: ["K2-Horizon-0.9B-GGUF"], - lite: false - }, - { - target: "windows-rocm-stable", - backend: "rocm", - channel: "stable", - managed_pin: "rocm-stable", - build_platform: "windows", - runner: ["self-hosted", "Windows", "stx-halo", "rocm", "lemon-prod"], - models: ["K2-Horizon-0.9B-GGUF"], - capability_profile: "k2-horizon-v1", - capability_models: ["K2-Horizon-0.9B-GGUF"], - lite: false - }, - { - target: "windows-rocm-nightly", - backend: "rocm", - channel: "nightly", - managed_pin: "rocm-nightly", - build_platform: "windows", - runner: ["self-hosted", "Windows", "stx-halo", "rocm", "lemon-prod"], - models: ["K2-Horizon-0.9B-GGUF"], - capability_profile: "k2-horizon-v1", - capability_models: ["K2-Horizon-0.9B-GGUF"], - lite: false - } - ] - }') - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - echo "promotion_eligible=false" >> "$GITHUB_OUTPUT" - - name: Build branch validation matrix - id: branch-plan - if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' - env: - EVENT_NAME: ${{ github.event_name }} - MANUAL_MODELS: ${{ inputs.models || '' }} - MANUAL_LITE: ${{ inputs.lite || false }} - run: >- - python -m test.utils.llamacpp_validation_plan - --event "$EVENT_NAME" - --models "$MANUAL_MODELS" - --lite "$MANUAL_LITE" - - # ======================================================================== - # Discover the latest llama.cpp release for each upstream repo - # ======================================================================== - get-latest-releases: - name: Get latest releases - # Only the auto-update runs resolve upstream releases. Everywhere else the - # build validates the pins committed in backend_versions.json, so a bad - # upstream release can't fail a merge or hide a broken pin. - needs: validate-invocation - if: needs.validate-invocation.result == 'success' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - outputs: - llamacpp_release: ${{ steps.llamacpp.outputs.release }} - llamacpp_rocm_release: ${{ steps.llamacpp_rocm.outputs.release }} - llamacpp_lemonade_release: ${{ steps.llamacpp_lemonade.outputs.release }} - steps: - - name: Get latest llama.cpp release - id: llamacpp - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - # Upstream's semver releases (v0.3.0) are what /releases/latest now - # returns, and they carry no binaries -- the bNNNN nightlies hold the - # assets and are flagged as prereleases. - TAGS=$(gh api repos/ggml-org/llama.cpp/releases \ - --jq '.[] | select(.draft | not) | .tag_name') - RELEASE=$(printf '%s\n' "$TAGS" | grep -m1 -E '^b[0-9]+$' || true) - if [ -z "$RELEASE" ]; then - echo "::error::No bNNNN release found in ggml-org/llama.cpp." - exit 1 - fi - echo "Latest llama.cpp release: $RELEASE" - echo "release=$RELEASE" >> "$GITHUB_OUTPUT" - - - name: Get latest llamacpp-rocm release - id: llamacpp_rocm - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - RELEASE=$(gh api repos/lemonade-sdk/llamacpp-rocm/releases/latest --jq '.tag_name') - echo "Latest llamacpp-rocm release: $RELEASE" - echo "release=$RELEASE" >> "$GITHUB_OUTPUT" - - - name: Get latest lemonade-sdk/llama.cpp release - id: llamacpp_lemonade - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - RELEASE=$(gh api repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name') - echo "Latest lemonade-sdk/llama.cpp release: $RELEASE" - echo "release=$RELEASE" >> "$GITHUB_OUTPUT" - - # ======================================================================== - # Build binaries with updated backend versions - # ======================================================================== - verify-release-assets: - name: Verify release assets - needs: get-latest-releases - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - outputs: - ggml_update_backends: ${{ steps.assets.outputs.ggml_update_backends }} - lemonade_update_backends: ${{ steps.assets.outputs.lemonade_update_backends }} - rocm_update_backends: ${{ steps.assets.outputs.rocm_update_backends }} - ggml_missing_count: ${{ steps.assets.outputs.ggml_missing_count }} - rocm_nightly_missing_count: ${{ steps.assets.outputs.rocm_nightly_missing_count }} - rocm_stable_missing_count: ${{ steps.assets.outputs.rocm_stable_missing_count }} - cuda_missing_count: ${{ steps.assets.outputs.cuda_missing_count }} - vulkan_available: ${{ steps.assets.outputs.vulkan_available }} - cpu_available: ${{ steps.assets.outputs.cpu_available }} - metal_available: ${{ steps.assets.outputs.metal_available }} - rocm_nightly_available: ${{ steps.assets.outputs.rocm_nightly_available }} - rocm_stable_available: ${{ steps.assets.outputs.rocm_stable_available }} - cuda_available: ${{ steps.assets.outputs.cuda_available }} - release_asset_manifest: ${{ steps.manifest.outputs.release_asset_manifest }} - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.sha }} - persist-credentials: false - - - name: Fetch release asset lists - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GGML_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} - ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} - LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} - run: | - set -euo pipefail - - for value in GGML_RELEASE ROCM_RELEASE LEMONADE_RELEASE; do - if [ -z "${!value}" ]; then - echo "::error::$value was not resolved." - exit 1 - fi - done - - gh api "repos/ggml-org/llama.cpp/releases/tags/${GGML_RELEASE}" \ - > ggml_release.json - gh api "repos/lemonade-sdk/llamacpp-rocm/releases/tags/${ROCM_RELEASE}" \ - > rocm_release.json - gh api "repos/lemonade-sdk/llama.cpp/releases/tags/${LEMONADE_RELEASE}" \ - > lemonade_release.json - - source_repository=ggml-org/llama.cpp - source_manifest_max_bytes=65536 - source_manifest_directory=$(mktemp -d) - source_default_branch=$(gh api "repos/${source_repository}" --jq '.default_branch') - if [ -z "$source_default_branch" ]; then - echo "::error::Could not resolve the ${source_repository} default branch." - exit 1 - fi - upstream_reference="refs/heads/${source_default_branch}" - upstream_reference_head=$(gh api \ - "repos/${source_repository}/commits/heads/${source_default_branch}" \ - --jq '.sha') - if ! [[ "$upstream_reference_head" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::Could not resolve ${upstream_reference} to a full commit." - exit 1 - fi - - attach_release_claims() { - local repository="$1" - local tag="$2" - local path="$3" - local release_tag_commit publisher_claim_type - local publisher_claimed_source_commit comparison_path - local source_manifest_asset_id source_manifest_path - local build_target_attestation_path - release_tag_commit=$(gh api \ - "repos/${repository}/commits/tags/${tag}" --jq '.sha') - if ! [[ "$release_tag_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::Could not resolve ${repository} ${tag} to a release tag commit." - exit 1 - fi - if [ "$repository" = "$source_repository" ]; then - publisher_claim_type="source-release-tag" - publisher_claimed_source_commit="$release_tag_commit" - build_target_attestation_path="${source_manifest_directory}/source-release-targets.json" - printf '[]\n' > "$build_target_attestation_path" - else - publisher_claim_type="immutable-source-manifest" - source_manifest_asset_id=$(python -m test.utils.llamacpp_release_manifest \ - --source-manifest-asset-id "$path" "$repository") - source_manifest_path="${source_manifest_directory}/source-${source_manifest_asset_id}.json" - build_target_attestation_path="${source_manifest_path}.build-targets" - gh api "repos/${repository}/releases/assets/${source_manifest_asset_id}" \ - -H "Accept: application/octet-stream" | - head -c "$((source_manifest_max_bytes + 1))" > "$source_manifest_path" - publisher_claimed_source_commit=$(python -m test.utils.llamacpp_release_manifest \ - --validate-source-manifest "$source_manifest_path" "$path" \ - "$repository" "$tag" "$release_tag_commit" \ - --build-target-attestation-output "$build_target_attestation_path") - fi - if ! [[ "$publisher_claimed_source_commit" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::${repository} ${tag} did not bind one full upstream source commit." - exit 1 - fi - comparison_path="${path}.upstream-comparison.json" - gh api \ - "repos/${source_repository}/compare/${publisher_claimed_source_commit}...${upstream_reference_head}" \ - > "$comparison_path" - python -m test.utils.llamacpp_release_manifest \ - --require-upstream-ancestry "$comparison_path" "$source_repository" \ - "$publisher_claimed_source_commit" "$upstream_reference_head" - jq \ - --slurpfile build_target_attestations "$build_target_attestation_path" \ - --arg publisher_claim_type "$publisher_claim_type" \ - --arg publisher_claimed_source_commit "$publisher_claimed_source_commit" \ - --arg release_tag_commit "$release_tag_commit" \ - --arg source_repository "$source_repository" \ - --arg upstream_reference "$upstream_reference" \ - --arg upstream_reference_head "$upstream_reference_head" \ - '.build_target_attestations = $build_target_attestations[0] - | .publisher_claim_type = $publisher_claim_type - | .publisher_claimed_source_commit = $publisher_claimed_source_commit - | .release_tag_commit = $release_tag_commit - | .source_repository = $source_repository - | .upstream_reference = $upstream_reference - | .upstream_reference_head = $upstream_reference_head - | del(.source_commit)' \ - "$path" > "${path}.with-source" - mv "${path}.with-source" "$path" - } - attach_release_claims ggml-org/llama.cpp "$GGML_RELEASE" ggml_release.json - attach_release_claims lemonade-sdk/llamacpp-rocm \ - "$ROCM_RELEASE" rocm_release.json - attach_release_claims lemonade-sdk/llama.cpp \ - "$LEMONADE_RELEASE" lemonade_release.json - - jq -r '.assets[].name' ggml_release.json | sort > ggml_assets.txt - jq -r '.assets[].name' rocm_release.json | sort > rocm_assets.txt - jq -r '.assets[].name' lemonade_release.json | sort > lemonade_assets.txt - - echo "ggml-org/llama.cpp assets for ${GGML_RELEASE}:" - cat ggml_assets.txt - echo "lemonade-sdk/llamacpp-rocm assets for ${ROCM_RELEASE}:" - cat rocm_assets.txt - echo "lemonade-sdk/llama.cpp assets for ${LEMONADE_RELEASE}:" - cat lemonade_assets.txt - - - name: Capture release asset manifest - id: manifest - env: - GGML_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} - ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} - LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} - run: >- - python -m test.utils.llamacpp_release_manifest - --release ggml-org/llama.cpp "$GGML_RELEASE" ggml_release.json - --release lemonade-sdk/llamacpp-rocm "$ROCM_RELEASE" rocm_release.json - --release lemonade-sdk/llama.cpp "$LEMONADE_RELEASE" lemonade_release.json - --github-output "$GITHUB_OUTPUT" - - - name: Verify expected asset families - id: assets - env: - GGML_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} - ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} - LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} - run: | - set -euo pipefail - - python3 - <<'PY' - import json - import os - from pathlib import Path - - from test.utils.llamacpp_release_assets import ( - build_asset_requirements, - build_rocm_asset_target_requirements, - evaluate_asset_group, - ) - - def read_asset_metadata(path): - release = json.loads(Path(path).read_text(encoding="utf-8")) - asset_sizes = {} - for asset in release["assets"]: - asset_sizes[asset["name"]] = asset["size"] - attested_targets = { - attestation["name"]: attestation["build_targets"] - for attestation in release.get("build_target_attestations", []) - } - return asset_sizes, attested_targets - - def update_outputs(values): - github_output = os.environ.get("GITHUB_OUTPUT") - if not github_output: - return - with open(github_output, "a", encoding="utf-8") as out: - for key, value in values.items(): - out.write(f"{key}={value}\n") - - def report_group( - name, - available_assets, - backend_requirements, - **evaluation_options, - ): - update_backends, missing_by_backend = evaluate_asset_group( - available_assets, backend_requirements, **evaluation_options - ) - for backend, required in backend_requirements.items(): - missing = missing_by_backend[backend] - if missing: - print(f"{name}: {backend} assets incomplete; keeping existing pin.") - for asset in missing: - print(f" missing: {asset}") - else: - print(f"{name}: {backend} assets complete; pin may update.") - return update_backends, missing_by_backend - - ggml_release = os.environ["GGML_RELEASE"] - rocm_release = os.environ["ROCM_RELEASE"] - lemonade_release = os.environ["LEMONADE_RELEASE"] - - ggml_assets, _ = read_asset_metadata("ggml_release.json") - rocm_assets, rocm_attested_targets = read_asset_metadata("rocm_release.json") - lemonade_assets, _ = read_asset_metadata("lemonade_release.json") - - with open("src/cpp/resources/backend_versions.json", encoding="utf-8") as f: - versions = json.load(f) - - requirements = build_asset_requirements( - ggml_release=ggml_release, - rocm_release=rocm_release, - lemonade_release=lemonade_release, - backend_versions=versions, - ) - rocm_target_requirements = build_rocm_asset_target_requirements( - rocm_release=rocm_release, - backend_versions=versions, - ) - - ggml_updates, ggml_missing = report_group( - "ggml-org/llama.cpp", ggml_assets, requirements["ggml"] - ) - rocm_updates, rocm_missing = report_group( - "lemonade-sdk/llamacpp-rocm", - rocm_assets, - requirements["rocm"], - required_asset_targets=rocm_target_requirements, - attested_asset_targets=rocm_attested_targets, - ) - lemonade_updates, lemonade_missing = report_group( - "lemonade-sdk/llama.cpp", lemonade_assets, requirements["lemonade"] - ) - - values = { - "ggml_update_backends": ",".join(ggml_updates), - "lemonade_update_backends": ",".join(lemonade_updates), - "rocm_update_backends": ",".join(rocm_updates), - "ggml_missing_count": str(sum(len(items) for items in ggml_missing.values())), - "rocm_nightly_missing_count": str(len(rocm_missing["rocm-nightly"])), - "rocm_stable_missing_count": str(len(lemonade_missing["rocm-stable"])), - "cuda_missing_count": str(len(lemonade_missing["cuda"])), - "vulkan_available": "true" if not ggml_missing["vulkan"] else "false", - "cpu_available": "true" if not ggml_missing["cpu"] else "false", - "metal_available": "true" if not ggml_missing["metal"] else "false", - "rocm_nightly_available": "true" if not rocm_missing["rocm-nightly"] else "false", - "rocm_stable_available": "true" if not lemonade_missing["rocm-stable"] else "false", - "cuda_available": "true" if not lemonade_missing["cuda"] else "false", - } - update_outputs(values) - - print("Eligible ggml-org/llama.cpp backend updates: " + (values["ggml_update_backends"] or "none")) - print("Eligible lemonade-sdk/llamacpp-rocm backend updates: " + (values["rocm_update_backends"] or "none")) - print("Eligible lemonade-sdk/llama.cpp backend updates: " + (values["lemonade_update_backends"] or "none")) - PY - - verify-protected-release-manifest: - name: Verify protected release manifest - needs: validate-invocation - if: >- - needs.validate-invocation.result == 'success' && - (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') - runs-on: ubuntu-latest - steps: - - name: Check out the protected candidate commit - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} - allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} - path: candidate - persist-credentials: false - fetch-depth: 1 - - name: Check out trusted validation tools - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - path: trusted - persist-credentials: false - fetch-depth: 1 - - name: Verify managed llama.cpp pins - env: - BASE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - CANDIDATE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: >- - bash trusted/.github/scripts/verify_llamacpp_pin_manifest.sh - "$GITHUB_WORKSPACE/trusted" "$GITHUB_WORKSPACE/candidate" - "$BASE_SHA" "$CANDIDATE_SHA" - - build: - name: Build ${{ matrix.platform }} - needs: [validate-invocation, get-latest-releases, verify-release-assets, verify-protected-release-manifest, plan] - if: always() && !failure() && !cancelled() - runs-on: ${{ matrix.runner }} - permissions: - contents: read - env: - VALIDATION_CANDIDATE_ROOT: ${{ (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && format('{0}/candidate', github.workspace) || github.workspace }} - strategy: - fail-fast: false - matrix: - include: - - platform: windows - runner: windows-latest - - platform: linux - runner: ubuntu-latest - - platform: macos - runner: macos-latest - steps: - - name: Prepare Windows runner for long paths and stale workspace caches - if: runner.os == 'Windows' - shell: PowerShell - run: | - $ErrorActionPreference = "Continue" - - # 1. Kill any running/orphaned processes to release file locks - $patterns = @("lemonade", "lemond", "llama-server", "llama", "flm", "ort-server", "moonshine-server", "wscript", "LemonadeServer") - foreach ($p in $patterns) { - Get-Process | Where-Object { $_.ProcessName -like "*$p*" } | Stop-Process -Force -ErrorAction SilentlyContinue - } - - - - name: Check out validation control plane - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.sha }} - persist-credentials: false - clean: true - fetch-depth: 2 - - - name: Check out the validation candidate - if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} - allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} - path: candidate - persist-credentials: false - clean: true - fetch-depth: 2 - - - name: Verify authorized merge commit - if: github.event_name == 'pull_request_target' - shell: bash - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -euo pipefail - test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^1)" = "$BASE_SHA" || { - echo "::error::The merge commit does not contain the authorized base commit." - exit 1 - } - test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^2)" = "$HEAD_SHA" || { - echo "::error::The merge commit does not contain the authorized pull request head." - exit 1 - } - - - name: Verify merge-group commit - if: github.event_name == 'merge_group' - shell: bash - env: - MERGE_GROUP_SHA: ${{ github.event.merge_group.head_sha }} - run: | - set -euo pipefail - test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD)" = "$MERGE_GROUP_SHA" || { - echo "::error::The checkout does not match the authorized merge-group commit." - exit 1 - } - - - name: Update backend_versions.json - if: needs.get-latest-releases.result == 'success' - shell: bash - working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} - env: - LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_lemonade_release }} - LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.ggml_update_backends }} - LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.rocm_update_backends }} - LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.verify-release-assets.outputs.lemonade_update_backends }} - run: | - set -euo pipefail - python - <<'PY' - import json - import os - import re - from pathlib import Path - - RELEASE_RE = re.compile(r"^b[0-9]+$") - - def parse_csv(value): - return [item.strip() for item in (value or "").split(",") if item.strip()] - - def update(section, release, backends): - if not backends: - return [] - if not RELEASE_RE.match(release): - raise SystemExit(f"Invalid llama.cpp release tag: {release}") - changed = [] - for key in backends: - if key not in section: - raise SystemExit(f"Refusing to create missing llamacpp.{key}") - if not isinstance(section[key], str): - raise SystemExit(f"llamacpp.{key} must be a string") - old = section[key] - section[key] = release - changed.append((key, old, release)) - return changed - - path = Path("src/cpp/resources/backend_versions.json") - data = json.loads(path.read_text(encoding="utf-8")) - section = data.get("llamacpp") - if not isinstance(section, dict): - raise SystemExit("backend_versions.json is missing a llamacpp object") - - changes = [] - changes += update(section, os.environ["LLAMACPP_RELEASE"], parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) - changes += update(section, os.environ["LLAMACPP_ROCM_RELEASE"], parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) - changes += update(section, os.environ["LLAMACPP_LEMONADE_RELEASE"], parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) - - path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") - - print("Updated llamacpp backend versions:") - if changes: - for key, old, new in changes: - print(f" llamacpp.{key}: {old} -> {new}") - else: - print(" none") - PY - - - name: Stage protected validation-only model catalog - if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - shell: bash - working-directory: ${{ github.workspace }} - env: - PYTHONPATH: ${{ github.workspace }} - run: | - set -euo pipefail - candidate_catalog="$GITHUB_WORKSPACE/candidate/src/cpp/resources/server_models.json" - python - "$GITHUB_WORKSPACE/candidate" "$candidate_catalog" <<'PY' - import os - import stat - import sys - from pathlib import Path - - candidate_root = Path(sys.argv[1]).resolve(strict=True) - catalog_path = Path(sys.argv[2]) - resolved_catalog = catalog_path.resolve(strict=True) - try: - resolved_catalog.relative_to(candidate_root) - except ValueError as exc: - raise SystemExit("candidate catalog escapes its checkout") from exc - if not stat.S_ISREG(os.lstat(catalog_path).st_mode): - raise SystemExit("candidate catalog must be a regular file") - PY - python "$GITHUB_WORKSPACE/test/utils/validation_model_catalog.py" \ - --base "$candidate_catalog" \ - --overlay "$GITHUB_WORKSPACE/test/fixtures/llamacpp_validation_models.json" \ - --output "$candidate_catalog" - - - name: Stage branch validation-only model catalog - if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' - working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} - run: >- - python -m test.utils.validation_model_catalog - --base src/cpp/resources/server_models.json - --overlay test/fixtures/llamacpp_validation_models.json - --output src/cpp/resources/server_models.json - - - name: Install Linux build dependencies - if: runner.os == 'Linux' - shell: bash - working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} - run: ./setup.sh - - - name: Install macOS build dependencies - if: runner.os == 'macOS' - shell: bash - run: | - set -euo pipefail - command -v ninja >/dev/null 2>&1 || brew install ninja - - - name: Build lemond on Windows - if: runner.os == 'Windows' - shell: PowerShell - working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} - run: | - $ErrorActionPreference = "Stop" - Write-Host "Building Lemonade server binaries..." -ForegroundColor Cyan - if (Test-Path "build") { Remove-Item -Recurse -Force "build" } - # The validation drives lemond over HTTP and never invokes the CLI, - # so lemond is the only target it needs. - cmake --preset vs18 -DBUILD_WEB_APP=OFF - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - # /m:2 goes straight to MSBuild rather than through --parallel, which - # makes CMake set CL_MPCount=1 and cancel the /MP in CMakeLists.txt. - # Capped at 2 because /MP already spawns one cl.exe per core. - cmake --build build --config Release --target lemond -- /m:2 - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if (-not (Test-Path "build\Release\lemond.exe")) { - Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red - exit 1 - } - Write-Host "Build successful!" -ForegroundColor Green - - - name: Build lemond on Linux or macOS - if: runner.os != 'Windows' - shell: bash - working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }} - run: | - set -euo pipefail - cmake --preset default -DBUILD_WEB_APP=OFF -DBUILD_TESTING=OFF - cmake --build --preset default --target lemond --parallel 2 - test -x build/lemond - build/lemond --version - - - name: Upload Windows build artifacts - if: runner.os == 'Windows' - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: llamacpp-build-${{ matrix.platform }} - path: | - ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/Release/ - ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/resources/ - retention-days: 7 - - - name: Upload Linux or macOS build artifacts - if: runner.os != 'Windows' - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: llamacpp-build-${{ matrix.platform }} - path: | - ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/lemond - ${{ env.VALIDATION_CANDIDATE_ROOT }}/build/resources/ - retention-days: 7 - - # ======================================================================== - # Validate backends on self-hosted runner - # ======================================================================== - validate: - name: Validate ${{ matrix.target }} - needs: [get-latest-releases, build, plan] - if: always() && needs.build.result == 'success' - runs-on: ${{ matrix.runner }} - permissions: - contents: read - env: - VALIDATION_CANDIDATE_ROOT: ${{ (github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && format('{0}/candidate', github.workspace) || github.workspace }} - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - steps: - - name: Capture Vulkan runner configuration - if: matrix.backend == 'vulkan' && runner.os == 'Windows' - shell: PowerShell - run: | - $ErrorActionPreference = "Continue" - - $outDir = Join-Path $env:RUNNER_TEMP "vulkan-runner-config" - New-Item -ItemType Directory -Force -Path $outDir | Out-Null - - # General system / Windows / BIOS information - $os = Get-CimInstance Win32_OperatingSystem - $computer = Get-CimInstance Win32_ComputerSystem - $bios = Get-CimInstance Win32_BIOS - - [pscustomobject]@{ - CapturedUtc = (Get-Date).ToUniversalTime() - RunnerName = $env:RUNNER_NAME - Manufacturer = $computer.Manufacturer - Model = $computer.Model - PhysicalMemoryGiB = [math]::Round( - $computer.TotalPhysicalMemory / 1GB, - 2 - ) - WindowsCaption = $os.Caption - WindowsVersion = $os.Version - WindowsBuild = $os.BuildNumber - OSArchitecture = $os.OSArchitecture - LastBootUpTime = $os.LastBootUpTime - UptimeHours = [math]::Round( - ((Get-Date) - $os.LastBootUpTime).TotalHours, - 2 - ) - BIOSManufacturer = $bios.Manufacturer - BIOSVersion = $bios.SMBIOSBIOSVersion - BIOSReleaseDate = $bios.ReleaseDate - } | - Format-List | - Out-File (Join-Path $outDir "system.txt") -Encoding utf8 - - # GPU model, PCI device ID and Windows display-driver information - Get-CimInstance Win32_VideoController | - Select-Object ` - Name, - VideoProcessor, - PNPDeviceID, - DriverVersion, - DriverDate, - Status | - Format-List | - Out-File (Join-Path $outDir "gpu.txt") -Encoding utf8 - - Get-CimInstance Win32_PnPSignedDriver | - Where-Object { $_.DeviceClass -eq "DISPLAY" } | - Select-Object ` - DeviceName, - Manufacturer, - DriverProviderName, - DriverVersion, - DriverDate, - InfName, - DeviceID | - Format-List | - Out-File (Join-Path $outDir "display-drivers.txt") -Encoding utf8 - - # Current Windows power scheme - powercfg /getactivescheme 2>&1 | - Out-File (Join-Path $outDir "power.txt") -Encoding utf8 - - # Reboot-frequency proxy plus exact current boot time above - $bootEvents = @( - Get-WinEvent -FilterHashtable @{ - LogName = "System" - Id = 6005 - StartTime = (Get-Date).AddDays(-30) - } -ErrorAction SilentlyContinue - ) - - "Event Log service starts in last 30 days: $($bootEvents.Count)" | - Out-File (Join-Path $outDir "reboots.txt") -Encoding utf8 - - # Relevant workloads present before validation cleanup - $processes = @( - Get-Process -Name @( - "lemond", - "llama-server", - "lemonade", - "LemonadeServer", - "flm", - "ort-server", - "moonshine-server" - ) -ErrorAction SilentlyContinue - ) - - if ($processes.Count -gt 0) { - $processes | - Select-Object ProcessName, Id, StartTime, CPU, WorkingSet64 | - Format-Table -AutoSize | - Out-File (Join-Path $outDir "processes-before-cleanup.txt") ` - -Encoding utf8 - } else { - "No known inference processes detected." | - Out-File (Join-Path $outDir "processes-before-cleanup.txt") ` - -Encoding utf8 - } - - # Best-effort AMD cache-directory state. - # This does not modify or clear any cache. - $amdRoot = if ($env:LOCALAPPDATA) { - Join-Path $env:LOCALAPPDATA "AMD" - } else { - $null - } - - if ($amdRoot -and (Test-Path $amdRoot)) { - Get-ChildItem $amdRoot -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.Name -match "cache" } | - Select-Object Name, CreationTimeUtc, LastWriteTimeUtc | - Format-Table -AutoSize | - Out-File (Join-Path $outDir "amd-cache-state.txt") ` - -Encoding utf8 - } else { - "AMD local cache directory not found." | - Out-File (Join-Path $outDir "amd-cache-state.txt") ` - -Encoding utf8 - } - - # Vulkan device, driver, memory heaps and capabilities - $vulkaninfo = Get-Command vulkaninfo -ErrorAction SilentlyContinue - - if ($vulkaninfo) { - & $vulkaninfo.Source --summary *>&1 | - Out-File (Join-Path $outDir "vulkaninfo-summary.txt") ` - -Encoding utf8 - $summaryExitCode = $LASTEXITCODE - - & $vulkaninfo.Source *>&1 | - Out-File (Join-Path $outDir "vulkaninfo.txt") ` - -Encoding utf8 - $fullExitCode = $LASTEXITCODE - - Write-Host "=== vulkaninfo --summary ===" - Get-Content (Join-Path $outDir "vulkaninfo-summary.txt") - - Write-Host "vulkaninfo summary exit code: $summaryExitCode" - Write-Host "vulkaninfo full exit code: $fullExitCode" - } else { - "vulkaninfo was not found in PATH." | - Out-File (Join-Path $outDir "vulkaninfo-unavailable.txt") ` - -Encoding utf8 - Write-Warning "vulkaninfo was not found in PATH." - } - - Write-Host "=== Runner configuration ===" - Get-Content (Join-Path $outDir "system.txt") - Get-Content (Join-Path $outDir "gpu.txt") - Get-Content (Join-Path $outDir "display-drivers.txt") - Get-Content (Join-Path $outDir "power.txt") - Get-Content (Join-Path $outDir "reboots.txt") - Get-Content (Join-Path $outDir "processes-before-cleanup.txt") - - # Environment collection should never make validation fail. - exit 0 - - - name: Capture Vulkan runner configuration on Linux - if: matrix.backend == 'vulkan' && runner.os == 'Linux' - shell: bash - run: | - set +e - output_directory="$RUNNER_TEMP/vulkan-runner-config" - mkdir -p "$output_directory" - uname -a > "$output_directory/system.txt" - if command -v vulkaninfo >/dev/null 2>&1; then - vulkaninfo --summary > "$output_directory/vulkaninfo-summary.txt" 2>&1 - vulkaninfo > "$output_directory/vulkaninfo.txt" 2>&1 - else - echo "vulkaninfo was not found in PATH." \ - > "$output_directory/vulkaninfo-unavailable.txt" - fi - exit 0 - - - name: Upload Vulkan runner configuration - if: always() && matrix.backend == 'vulkan' - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: vulkan-runner-config-${{ matrix.target }} - path: ${{ runner.temp }}/vulkan-runner-config/ - retention-days: 30 - if-no-files-found: warn - - - name: Cleanup prior validation processes - if: runner.os == 'Windows' - shell: PowerShell - run: | - $ErrorActionPreference = "Stop" - $workspaceRoot = [IO.Path]::GetFullPath( - $env:VALIDATION_CANDIDATE_ROOT - ) - $lemondPath = [IO.Path]::GetFullPath( - (Join-Path $workspaceRoot "build\Release\lemond.exe") - ) - $cachePrefix = [IO.Path]::GetFullPath( - (Join-Path $workspaceRoot "ci-cache-") - ) - $staleProcesses = @( - Get-CimInstance Win32_Process ` - -Filter "Name = 'lemond.exe' OR Name = 'llama-server.exe'" ` - -ErrorAction SilentlyContinue | Where-Object { - $_.ExecutablePath -and ( - ($_.Name -eq "lemond.exe" -and $_.ExecutablePath.Equals( - $lemondPath, - [StringComparison]::OrdinalIgnoreCase - )) -or - ($_.Name -eq "llama-server.exe" -and $_.ExecutablePath.StartsWith( - $cachePrefix, - [StringComparison]::OrdinalIgnoreCase - )) - ) - } - ) - - foreach ($process in $staleProcesses) { - Write-Host "Stopping stale $($process.Name) PID $($process.ProcessId)." ` - -ForegroundColor Yellow - Stop-Process -Id $process.ProcessId -Force -ErrorAction SilentlyContinue - } - if ($staleProcesses.Count -gt 0) { - Start-Sleep -Seconds 1 - foreach ($process in $staleProcesses) { - if (Get-Process -Id $process.ProcessId -ErrorAction SilentlyContinue) { - throw "Failed to stop workspace process PID $($process.ProcessId)." - } - } - } - - - name: Check out validation control plane - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.sha }} - persist-credentials: false - clean: true - fetch-depth: 2 - - - name: Check out the validation candidate - if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} - allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} - path: candidate - persist-credentials: false - clean: true - fetch-depth: 2 - - - name: Verify authorized merge commit - if: github.event_name == 'pull_request_target' - shell: bash - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -euo pipefail - test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^1)" = "$BASE_SHA" || { - echo "::error::The merge commit does not contain the authorized base commit." - exit 1 - } - test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD^2)" = "$HEAD_SHA" || { - echo "::error::The merge commit does not contain the authorized pull request head." - exit 1 - } - - - name: Verify merge-group commit - if: github.event_name == 'merge_group' - shell: bash - env: - MERGE_GROUP_SHA: ${{ github.event.merge_group.head_sha }} - run: | - set -euo pipefail - test "$(git -C "$VALIDATION_CANDIDATE_ROOT" rev-parse HEAD)" = "$MERGE_GROUP_SHA" || { - echo "::error::The checkout does not match the authorized merge-group commit." - exit 1 - } - - - name: Cleanup processes - if: runner.os == 'Windows' - uses: ./.github/actions/cleanup-processes-windows - - - name: Cleanup processes on Linux - if: runner.os == 'Linux' - uses: ./.github/actions/cleanup-processes-linux - - - name: Download build artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: llamacpp-build-${{ matrix.build_platform }} - path: ${{ env.VALIDATION_CANDIDATE_ROOT }}/build - - - name: Verify Windows binary artifact - if: runner.os == 'Windows' - shell: PowerShell - run: | - $lemondExe = Join-Path ` - $env:VALIDATION_CANDIDATE_ROOT "build\Release\lemond.exe" - if (-not (Test-Path $lemondExe)) { - Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red - Get-ChildItem -Recurse ` - (Join-Path $env:VALIDATION_CANDIDATE_ROOT "build") | - Select-Object FullName - exit 1 - } - if ((Get-Item $lemondExe).Attributes -band ` - [IO.FileAttributes]::ReparsePoint) { - Write-Host "ERROR: lemond.exe must not be a link!" ` - -ForegroundColor Red - exit 1 - } - Write-Host "Binary artifact verified!" -ForegroundColor Green - - - name: Verify Linux or macOS binary artifact - if: runner.os != 'Windows' - shell: bash - run: | - set -euo pipefail - lemond="$VALIDATION_CANDIDATE_ROOT/build/lemond" - test -f "$lemond" - test ! -L "$lemond" - chmod +x "$lemond" - - - name: Set up trusted or branch validation environment - id: validation-venv - uses: ./.github/actions/setup-venv - with: - venv-name: '.venv' - python-version: '3.10' - requirements-file: '${{ github.workspace }}/test/requirements.txt' - - - name: Run validation with lemond - id: run-validation - env: - CAPABILITY_MODELS: ${{ join(matrix.capability_models || fromJSON('[]'), ',') }} - CAPABILITY_PROFILE: ${{ matrix.capability_profile || '' }} - CANDIDATE_LEMOND: ${{ runner.os == 'Windows' && format('{0}/build/Release/lemond.exe', env.VALIDATION_CANDIDATE_ROOT) || format('{0}/build/lemond', env.VALIDATION_CANDIDATE_ROOT) }} - HF_TOKEN: ${{ github.event_name == 'schedule' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }} - HUGGINGFACE_TOKEN: '' - HUGGING_FACE_HUB_TOKEN: '' - LLAMACPP_BACKEND: ${{ matrix.backend }} - LLAMACPP_CHANNEL: ${{ matrix.channel }} - LLAMACPP_TEST_MODELS: ${{ join(matrix.models, ',') }} - LITE_MODE: ${{ matrix.lite }} - LEMONADE_PROTECTED_RUNNER_CONTEXT: ${{ matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && join(matrix.runner, ',') == 'macos-latest' && 'github-hosted:macos-latest:macos-metal' || '' }} - PYTHONPATH: ${{ github.workspace }} - TARGET: "${{ matrix.target }}" - VALIDATION_ROOT: ${{ github.workspace }} - shell: bash - working-directory: ${{ github.workspace }} - run: >- - python "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py" - --lemond "$CANDIDATE_LEMOND" - --python "${{ steps.validation-venv.outputs.python-path }}" - ${{ github.event_name == 'schedule' && '--allow-huggingface-download-credentials' || '' }} - ${{ matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && join(matrix.runner, ',') == 'macos-latest' && '--allow-darwin-github-hosted-ephemeral-runner' || '' }} - - - name: Upload results - if: >- - steps.run-validation.outcome == 'success' && - !((github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && - matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && - join(matrix.runner, ',') == 'macos-latest') - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: validation-results-${{ matrix.target }} - path: ${{ steps.run-validation.outputs.validation_result }} - retention-days: 30 - if-no-files-found: error - - - name: Upload restart results - if: >- - steps.run-validation.outcome == 'success' && - !((github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && - matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && - join(matrix.runner, ',') == 'macos-latest') - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: restart-validation-results-${{ matrix.target }} - path: ${{ steps.run-validation.outputs.restart_result }} - retention-days: 30 - if-no-files-found: error - - - name: Upload server logs - if: >- - steps.run-validation.outcome == 'success' && - !((github.event_name == 'pull_request_target' || github.event_name == 'merge_group') && - matrix.target == 'macos-metal' && runner.environment == 'github-hosted' && - join(matrix.runner, ',') == 'macos-latest') - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: server-logs-${{ matrix.target }} - path: | - ${{ steps.run-validation.outputs.prepare_stdout_log }} - ${{ steps.run-validation.outputs.prepare_stderr_log }} - ${{ steps.run-validation.outputs.validation_stdout_log }} - ${{ steps.run-validation.outputs.validation_stderr_log }} - ${{ steps.run-validation.outputs.restart_stdout_log }} - ${{ steps.run-validation.outputs.restart_stderr_log }} - retention-days: 30 - if-no-files-found: error - - - name: Cleanup - if: >- - always() && runner.os == 'Windows' && - github.event_name != 'pull_request_target' && github.event_name != 'merge_group' - uses: ./.github/actions/cleanup-processes-windows - - - name: Cleanup on Linux - if: >- - always() && runner.os == 'Linux' && - github.event_name != 'pull_request_target' && github.event_name != 'merge_group' - uses: ./.github/actions/cleanup-processes-linux - - reverify-protected-release-manifest: - name: Reverify protected release manifest - needs: [validate, verify-protected-release-manifest] - if: >- - always() && needs.validate.result == 'success' && - (needs.verify-protected-release-manifest.result == 'success' || - needs.verify-protected-release-manifest.result == 'skipped') - runs-on: ubuntu-latest - steps: - - name: No protected manifest recheck is needed - if: github.event_name != 'pull_request_target' && github.event_name != 'merge_group' - run: echo "This invocation does not review a protected pin change." - - name: Check out the protected candidate commit - if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} - allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} - path: candidate - persist-credentials: false - fetch-depth: 1 - - name: Check out trusted validation tools - if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - path: trusted - persist-credentials: false - fetch-depth: 1 - - name: Reverify managed llama.cpp pins - if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' - env: - BASE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - CANDIDATE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || github.event.merge_group.head_sha }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: >- - bash trusted/.github/scripts/verify_llamacpp_pin_manifest.sh - "$GITHUB_WORKSPACE/trusted" "$GITHUB_WORKSPACE/candidate" - "$BASE_SHA" "$CANDIDATE_SHA" - - # Keep a stable terminal job for callers that require one validation result. - validation-gate: - name: llama.cpp validation - needs: [validate-invocation, build, validate, reverify-protected-release-manifest, plan] - if: always() - runs-on: ubuntu-latest - steps: - - name: Check gated jobs - env: - NEEDS: ${{ toJSON(needs) }} - run: | - set -euo pipefail - echo "$NEEDS" - - incomplete=$(jq -r \ - 'to_entries[] | select(.value.result != "success") | .key' \ - <<<"$NEEDS" | paste -sd ',' -) - if [ -n "$incomplete" ]; then - echo "FAILED OR INCOMPLETE: $incomplete" - exit 1 - fi diff --git a/.github/workflows/validate_llamacpp_manual.yml b/.github/workflows/validate_llamacpp_manual.yml deleted file mode 100644 index d7be8812e7..0000000000 --- a/.github/workflows/validate_llamacpp_manual.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Validate llama.cpp Manually - -on: - workflow_dispatch: - inputs: - lite: - description: 'Run only the K2-Horizon 0.9B smoke model' - type: boolean - default: false - models: - description: 'Additional comma-separated IDs; empty uses hot models + K2 0.9B' - type: string - required: false - default: '' - -permissions: - contents: read - -jobs: - validate: - name: Run manual validation - permissions: - checks: write - contents: read - uses: ./.github/workflows/validate_llamacpp_core.yml - with: - lite: ${{ inputs.lite }} - models: ${{ inputs.models }} diff --git a/.github/workflows/validate_llamacpp_pr.yml b/.github/workflows/validate_llamacpp_pr.yml deleted file mode 100644 index c872c3f0a8..0000000000 --- a/.github/workflows/validate_llamacpp_pr.yml +++ /dev/null @@ -1,1303 +0,0 @@ -name: Validate llama.cpp protected change - -on: - pull_request_target: - types: [opened, reopened, synchronize, edited, labeled, unlabeled, closed] - merge_group: - types: [checks_requested] - -permissions: - contents: read - -env: - DEPENDABOT_PR: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.user.login == 'dependabot[bot]' && github.event.pull_request.user.id == 49699333 && github.event.pull_request.user.type == 'Bot' }} - -concurrency: - group: ${{ github.event_name == 'merge_group' && format('{0}-merge-group-{1}-{2}-{3}', github.workflow, github.event.merge_group.head_sha, github.run_number, github.run_attempt) || ((github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || github.event.action == 'edited' || github.event.action == 'labeled' || github.event.action == 'unlabeled' || github.event.action == 'closed') && format('{0}-pr-{1}-event-{2}-{3}', github.workflow, github.event.pull_request.number, github.run_number, github.run_attempt) || format('{0}-unsupported-{1}-{2}', github.workflow, github.run_number, github.run_attempt)) }} - cancel-in-progress: false - -jobs: - policy: - name: Resolve protected validation policy - runs-on: ubuntu-latest - permissions: - actions: read - checks: read - contents: read - issues: read - pull-requests: read - outputs: - authorization_actor: ${{ steps.decision.outputs.authorization_actor }} - authorization_actor_id: ${{ steps.decision.outputs.authorization_actor_id }} - authorization_boundary: ${{ steps.decision.outputs.authorization_boundary }} - authorization_created_at: ${{ steps.decision.outputs.authorization_created_at }} - authorization_event_id: ${{ steps.decision.outputs.authorization_event_id }} - base_ref: ${{ steps.identity.outputs.base_ref }} - base_sha: ${{ steps.identity.outputs.base_sha }} - conclusion: ${{ steps.decision.outputs.conclusion }} - head_sha: ${{ steps.identity.outputs.head_sha }} - merge_sha: ${{ steps.identity.outputs.merge_sha }} - observed_merge_sha: ${{ steps.identity.outputs.observed_merge_sha }} - secondary_observed_merge_sha: ${{ steps.identity.outputs.secondary_observed_merge_sha }} - mode: ${{ steps.decision.outputs.mode }} - run_validation: ${{ steps.decision.outputs.run_validation }} - summary: ${{ steps.decision.outputs.summary }} - steps: - - name: Resolve the live candidate identity - id: identity - env: - EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }} - EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} - EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} - EVENT_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha || '' }} - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha || '' }} - MERGE_GROUP_HEAD_SHA: ${{ github.event.merge_group.head_sha || '' }} - PR_NUMBER: ${{ github.event.pull_request.number || '' }} - run: | - set -euo pipefail - oid_pattern='^[0-9a-fA-F]{40}$' - - if [ "$EVENT_NAME" = "merge_group" ]; then - if ! [[ "$MERGE_GROUP_BASE_SHA" =~ $oid_pattern && "$MERGE_GROUP_HEAD_SHA" =~ $oid_pattern ]]; then - echo "::error::Merge-group commits must be full Git OIDs." - exit 1 - fi - { - echo "base_sha=$MERGE_GROUP_BASE_SHA" - echo "base_ref=" - echo "head_sha=$MERGE_GROUP_HEAD_SHA" - echo "merge_sha=$MERGE_GROUP_HEAD_SHA" - echo "observed_merge_sha=" - echo "secondary_observed_merge_sha=" - echo "merge_candidate_available=true" - echo "state=open" - echo "merged=false" - echo "changed_files=0" - echo "authorization_label_present=false" - echo "identity_matches=true" - } >> "$GITHUB_OUTPUT" - exit 0 - fi - - if [ "$EVENT_NAME" != "pull_request_target" ] || ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then - echo "::error::Unsupported protected-validation invocation." - exit 1 - fi - - pull_request="" - for attempt in 1 2 3; do - pull_request=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") - snapshot_state=$(jq -r '.state // empty' <<<"$pull_request") - snapshot_merged=$(jq -r '.merged | tostring' <<<"$pull_request") - snapshot_mergeable=$(jq -r \ - '.mergeable | if . == null then "null" else tostring end' \ - <<<"$pull_request") - if [ "$snapshot_state" != "open" ] || [ "$snapshot_merged" = "true" ] || \ - [ "$snapshot_mergeable" = "true" ] || [ "$snapshot_mergeable" = "false" ] || \ - [ "$attempt" -eq 3 ]; then - break - fi - sleep 2 - done - current_number=$(jq -er '.number' <<<"$pull_request") - current_state=$(jq -er '.state' <<<"$pull_request") - current_merged=$(jq -er \ - '.merged | if type == "boolean" then tostring else error("invalid merged state") end' \ - <<<"$pull_request") - current_base=$(jq -er '.base.sha' <<<"$pull_request") - current_base_ref=$(jq -er \ - '.base.ref | select(type == "string" and test("^[0-9A-Za-z._/-]+$"))' \ - <<<"$pull_request") - current_head=$(jq -er '.head.sha' <<<"$pull_request") - current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") - current_mergeable=$(jq -r \ - '.mergeable | if . == null then "null" else tostring end' \ - <<<"$pull_request") - current_changed_files=$(jq -er '.changed_files' <<<"$pull_request") - current_authorized=$(jq -r \ - 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") - - if [ "$current_number" != "$PR_NUMBER" ] || \ - ! [[ "$current_state" =~ ^(open|closed)$ ]] || \ - ! [[ "$current_merged" =~ ^(true|false)$ ]] || \ - ! [[ "$current_mergeable" =~ ^(true|false|null)$ ]] || \ - ! [[ "$current_changed_files" =~ ^[0-9]+$ ]] || \ - ! [[ "$current_base" =~ $oid_pattern && "$current_head" =~ $oid_pattern ]] || \ - ! git check-ref-format "refs/heads/${current_base_ref}" >/dev/null || \ - { [ -n "$current_merge" ] && ! [[ "$current_merge" =~ $oid_pattern ]]; }; then - echo "::error::The live pull request returned an invalid identity." - exit 1 - fi - if [ "$current_merged" = "true" ] && [ "$current_state" != "closed" ]; then - echo "::error::The live pull request returned an inconsistent lifecycle state." - exit 1 - fi - - candidate_sha="$current_head" - merge_candidate_available=false - observed_merge_sha="" - secondary_observed_merge_sha="" - if [ "$current_mergeable" = "true" ] && [[ "$current_merge" =~ $oid_pattern ]]; then - candidate_sha="$current_merge" - merge_candidate_available=true - elif [[ "$current_merge" =~ $oid_pattern ]]; then - observed_merge_sha="$current_merge" - fi - if [ "$merge_candidate_available" = "false" ] && \ - [[ "$EVENT_MERGE_SHA" =~ $oid_pattern ]] && \ - [ "$EVENT_MERGE_SHA" != "$observed_merge_sha" ]; then - if [ -z "$observed_merge_sha" ]; then - observed_merge_sha="$EVENT_MERGE_SHA" - else - secondary_observed_merge_sha="$EVENT_MERGE_SHA" - fi - fi - - merge_identity_matches=true - if [ "$merge_candidate_available" = "true" ] && \ - [ "$current_merge" != "$EVENT_MERGE_SHA" ]; then - merge_identity_matches=false - fi - identity_matches=false - if [ "$current_base_ref" = "$EVENT_BASE_REF" ] && \ - [ "$current_base" = "$EVENT_BASE_SHA" ] && \ - [ "$current_head" = "$EVENT_HEAD_SHA" ] && \ - [ "$merge_identity_matches" = "true" ]; then - identity_matches=true - fi - { - echo "base_ref=$current_base_ref" - echo "base_sha=$current_base" - echo "head_sha=$current_head" - echo "merge_sha=$candidate_sha" - echo "observed_merge_sha=$observed_merge_sha" - echo "secondary_observed_merge_sha=$secondary_observed_merge_sha" - echo "merge_candidate_available=$merge_candidate_available" - echo "state=$current_state" - echo "merged=$current_merged" - echo "changed_files=$current_changed_files" - echo "authorization_label_present=$current_authorized" - echo "identity_matches=$identity_matches" - } >> "$GITHUB_OUTPUT" - - - name: Check out trusted policy code - if: github.event_name == 'pull_request_target' - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ steps.identity.outputs.base_sha }} - path: trusted - persist-credentials: false - fetch-depth: 1 - - - name: Classify the complete pull request file set - id: classify - if: github.event_name == 'pull_request_target' && steps.identity.outputs.state == 'open' - env: - BASE_SHA: ${{ steps.identity.outputs.base_sha }} - EXPECTED_COUNT: ${{ steps.identity.outputs.changed_files }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MERGE_SHA: ${{ steps.identity.outputs.merge_sha }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - set -euo pipefail - test "$(git -C trusted rev-parse HEAD)" = "$BASE_SHA" - if ! [[ "$EXPECTED_COUNT" =~ ^[0-9]+$ ]]; then - echo "::error::The pull-request file count is invalid." - exit 1 - fi - if [ "$EXPECTED_COUNT" -gt 300 ]; then - echo "relevant=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - comparison_path="$RUNNER_TEMP/llamacpp-pr-comparison-${PR_NUMBER}.json" - gh api --method GET \ - "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${MERGE_SHA}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - > "$comparison_path" - python trusted/.github/scripts/llamacpp_required_check_policy.py \ - classify-compare \ - --response "$comparison_path" \ - --expected-base-sha "$BASE_SHA" \ - --expected-target-sha "$MERGE_SHA" \ - --expected-count "$EXPECTED_COUNT" \ - --github-output "$GITHUB_OUTPUT" - - - name: Decide whether protected validation may run - id: decision - env: - AUTHORIZATION_LABEL_PRESENT: ${{ steps.identity.outputs.authorization_label_present }} - BASE_REF_CHANGED: ${{ github.event.action == 'edited' && github.event.changes.base.ref.from != '' }} - CHECK_NAME: llama.cpp authorized validation v2 - CURRENT_RUN_ATTEMPT: ${{ github.run_attempt }} - CURRENT_RUN_ID: ${{ github.run_id }} - CURRENT_RUN_NUMBER: ${{ github.run_number }} - DEPENDABOT_PR: ${{ env.DEPENDABOT_PR }} - EVENT_ACTION: ${{ github.event.action || '' }} - EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }} - EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} - EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} - EVENT_LABEL: ${{ github.event.label.name || '' }} - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IDENTITY_MATCHES: ${{ steps.identity.outputs.identity_matches }} - MERGED: ${{ steps.identity.outputs.merged }} - MERGE_CANDIDATE_AVAILABLE: ${{ steps.identity.outputs.merge_candidate_available }} - MERGE_SHA: ${{ steps.identity.outputs.merge_sha }} - PR_NUMBER: ${{ github.event.pull_request.number || '' }} - RELEVANT: ${{ steps.classify.outputs.relevant || 'true' }} - STATE: ${{ steps.identity.outputs.state }} - TRIGGERING_ACTOR: ${{ github.triggering_actor || '' }} - run: | - set -euo pipefail - if [ "$EVENT_NAME" = "merge_group" ]; then - { - echo "authorization_actor=" - echo "authorization_actor_id=0" - echo "authorization_created_at=" - echo "authorization_event_id=0" - echo "mode=validate" - echo "run_validation=true" - echo "conclusion=" - echo "summary=Merge-queue validation is mandatory." - } >> "$GITHUB_OUTPUT" - exit 0 - fi - - authorization_actor="" - authorization_actor_id=0 - authorization_boundary="" - authorization_created_at="" - authorization_epoch_valid=false - authorization_event_id=0 - authorization_rerun_matches=false - actor_permission="" - has_existing_check=false - has_newer_check=false - if [ "$MERGED" != "true" ] && [ "$STATE" = "open" ]; then - if [[ "$CURRENT_RUN_ATTEMPT" =~ ^([2-9]|[1-9][0-9]+)$ ]] && \ - [[ "$CURRENT_RUN_ID" =~ ^[1-9][0-9]*$ ]] && \ - [[ "$CURRENT_RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] && \ - { [ "$EVENT_ACTION" = "opened" ] || \ - [ "$EVENT_ACTION" = "reopened" ] || \ - [ "$EVENT_ACTION" = "synchronize" ]; }; then - first_attempt=$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${CURRENT_RUN_ID}/attempts/1" \ - 2>/dev/null || true) - if jq -e --arg base_ref "$EVENT_BASE_REF" \ - --arg head_sha "$EVENT_HEAD_SHA" \ - --arg path ".github/workflows/validate_llamacpp_pr.yml" \ - --arg repository "$GITHUB_REPOSITORY" \ - --argjson run_id "$CURRENT_RUN_ID" \ - --argjson run_number "$CURRENT_RUN_NUMBER" \ - '.id == $run_id and .run_number == $run_number and - .run_attempt == 1 and .event == "pull_request_target" and - (.path == $path or .path == ($path + "@" + $base_ref) or - .path == ($path + "@refs/heads/" + $base_ref)) and - .head_sha == $head_sha and - .repository.full_name == $repository and - ((.pull_requests | type) == "array") and - .status == "completed" and - .conclusion == "failure" and - (.created_at | type == "string" and length > 0) and - (.run_started_at | type == "string" and length > 0)' \ - <<<"$first_attempt" >/dev/null 2>&1; then - authorization_boundary=$(jq -r '.run_started_at' <<<"$first_attempt") - fi - fi - events_path="$RUNNER_TEMP/llamacpp-pr-events-${PR_NUMBER}.json" - gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path" - provenance_arguments=(provenance --pages "$events_path") - if [ -n "$authorization_boundary" ]; then - provenance_arguments+=(--authorization-after "$authorization_boundary") - fi - provenance=$(python \ - trusted/.github/scripts/llamacpp_required_check_policy.py \ - "${provenance_arguments[@]}") - authorization_actor=$(jq -er '.authorization_actor' <<<"$provenance") - authorization_actor_id=$(jq -er '.authorization_actor_id' <<<"$provenance") - authorization_created_at=$(jq -er '.authorization_created_at' <<<"$provenance") - authorization_epoch_valid=$(jq -er \ - '.epoch_valid | if type == "boolean" then tostring else error("invalid epoch") end' \ - <<<"$provenance") - authorization_event_id=$(jq -er '.authorization_event_id' <<<"$provenance") - if [ -n "$authorization_actor" ]; then - permission_response=$(gh api \ - "repos/${GITHUB_REPOSITORY}/collaborators/${authorization_actor}/permission" \ - 2>/dev/null || true) - if jq -e --argjson actor_id "$authorization_actor_id" \ - '.user.id == $actor_id and (.permission | type == "string")' \ - <<<"$permission_response" >/dev/null 2>&1; then - actor_permission=$(jq -r '.permission' <<<"$permission_response") - fi - fi - if [ -n "$authorization_boundary" ] && \ - [ "$TRIGGERING_ACTOR" = "$authorization_actor" ]; then - authorization_rerun_matches=true - fi - - if [ "$DEPENDABOT_PR" != "true" ]; then - external_prefix="lemonade-llamacpp-required-pr-${PR_NUMBER}-" - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) - IFS=$'\t' read -r newest_check_epoch newest_check_number \ - newest_check_attempt newest_check_status newest_check_conclusion < <( - jq -r --arg check_name "$CHECK_NAME" \ - --arg external_prefix "$external_prefix" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | . as $check - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {epoch: (.epoch | tonumber), number: (.number | tonumber), - attempt: (.attempt | tonumber), - status: ($check.status // ""), - conclusion: ($check.conclusion // "")}) catch empty] - | sort_by(.number, .attempt) - | (last // {epoch: 0, number: 0, attempt: 0, - status: "", conclusion: ""}) - | [.epoch, .number, .attempt, .status, .conclusion] - | @tsv' <<<"$check_runs" - ) - if [ "$newest_check_epoch" = "$authorization_event_id" ] && \ - [ "$newest_check_status" = "completed" ] && \ - [ "$newest_check_conclusion" = "success" ]; then - has_existing_check=true - fi - if [ "$newest_check_number" -gt "$CURRENT_RUN_NUMBER" ] || \ - { [ "$newest_check_number" -eq "$CURRENT_RUN_NUMBER" ] && \ - [ "$newest_check_attempt" -gt "$CURRENT_RUN_ATTEMPT" ]; }; then - has_newer_check=true - fi - fi - fi - - python trusted/.github/scripts/llamacpp_required_check_policy.py decide \ - --action "$EVENT_ACTION" --event-label "$EVENT_LABEL" \ - --base-ref-changed "$BASE_REF_CHANGED" --state "$STATE" \ - --merged "$MERGED" \ - --merge-candidate-available "$MERGE_CANDIDATE_AVAILABLE" \ - --relevant "$RELEVANT" \ - --identity-matches "$IDENTITY_MATCHES" \ - --authorization-label-present "$AUTHORIZATION_LABEL_PRESENT" \ - --authorization-epoch-valid "$authorization_epoch_valid" \ - --authorization-rerun-matches "$authorization_rerun_matches" \ - --actor-permission "$actor_permission" \ - --has-existing-check "$has_existing_check" \ - --has-newer-check "$has_newer_check" \ - --required-workflow-only "$DEPENDABOT_PR" \ - --github-output "$GITHUB_OUTPUT" - { - echo "authorization_actor=$authorization_actor" - echo "authorization_actor_id=$authorization_actor_id" - echo "authorization_boundary=$authorization_boundary" - echo "authorization_created_at=$authorization_created_at" - echo "authorization_event_id=$authorization_event_id" - } >> "$GITHUB_OUTPUT" - - validate: - name: Run authorized validation - needs: policy - if: needs.policy.result == 'success' && needs.policy.outputs.run_validation == 'true' - permissions: - checks: write - contents: read - uses: ./.github/workflows/validate_llamacpp_core.yml - with: - authorization_epoch: ${{ needs.policy.outputs.authorization_event_id }} - - reconcile: - name: Reconcile required validation check - needs: [policy, validate] - if: always() - runs-on: ubuntu-latest - permissions: - checks: write - contents: read - issues: read - pull-requests: read - steps: - - name: Check out trusted reconciliation policy code - if: >- - needs.policy.result == 'success' && - (needs.policy.outputs.mode == 'validate' || - needs.policy.outputs.mode == 'preserve') - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - repository: ${{ github.repository }} - ref: ${{ needs.policy.outputs.base_sha }} - path: trusted-reconcile - persist-credentials: false - fetch-depth: 1 - - - name: Assess policy and validation - id: assess-policy - if: always() - env: - EVENT_ACTION: ${{ github.event.action || '' }} - EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }} - EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} - EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} - EVENT_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha || '' }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MODE: ${{ needs.policy.outputs.mode }} - POLICY_CONCLUSION: ${{ needs.policy.outputs.conclusion }} - POLICY_RESULT: ${{ needs.policy.result }} - POLICY_SUMMARY: ${{ needs.policy.outputs.summary }} - PR_NUMBER: ${{ github.event.pull_request.number || '' }} - VALIDATION_RESULT: ${{ needs.validate.result }} - run: | - set -euo pipefail - mode="${MODE:-policy_error}" - conclusion="${POLICY_CONCLUSION:-failure}" - summary="${POLICY_SUMMARY:-Protected validation policy resolution failed.}" - passed=false - fallback_base_ref="" - fallback_base_sha="" - fallback_head_sha="" - fallback_merge_sha="" - fallback_observed_merge_sha="" - fallback_secondary_observed_merge_sha="" - - if [ "$POLICY_RESULT" != "success" ]; then - live_closed=false - live_identity_valid=false - if [[ "${PR_NUMBER:-}" =~ ^[0-9]+$ ]]; then - pull_request=$(gh api \ - "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" 2>/dev/null || true) - if jq -e --argjson number "$PR_NUMBER" \ - '.number == $number and (.state == "open" or .state == "closed") and - (.merged | type == "boolean") and (.base.ref | type == "string") and - (.base.sha | type == "string") and (.head.sha | type == "string") and - ((.mergeable == null) or (.mergeable | type == "boolean")) and - ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string"))' \ - <<<"$pull_request" >/dev/null 2>&1; then - current_state=$(jq -r '.state' <<<"$pull_request") - current_merged=$(jq -r '.merged' <<<"$pull_request") - current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") - current_base=$(jq -r '.base.sha' <<<"$pull_request") - current_head=$(jq -r '.head.sha' <<<"$pull_request") - current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") - current_mergeable=$(jq -r \ - '.mergeable | if . == null then "null" else tostring end' \ - <<<"$pull_request") - if git check-ref-format "refs/heads/${current_base_ref}" >/dev/null && \ - [[ "$current_base" =~ ^[0-9a-fA-F]{40}$ ]] && \ - [[ "$current_head" =~ ^[0-9a-fA-F]{40}$ ]] && \ - { [ -z "$current_merge" ] || \ - [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; } && \ - { [ "$current_merged" = "false" ] || \ - [ "$current_state" = "closed" ]; }; then - live_identity_valid=true - if [ "$current_state" = "closed" ]; then - live_closed=true - else - fallback_base_ref="$current_base_ref" - fallback_base_sha="$current_base" - fallback_head_sha="$current_head" - fallback_merge_sha="$current_head" - if [ "$current_mergeable" = "true" ] && \ - [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; then - fallback_merge_sha="$current_merge" - elif [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; then - fallback_observed_merge_sha="$current_merge" - fi - if [ "$current_mergeable" != "true" ] && \ - [[ "$EVENT_MERGE_SHA" =~ ^[0-9a-fA-F]{40}$ ]] && \ - [ "$EVENT_MERGE_SHA" != "$fallback_observed_merge_sha" ]; then - if [ -z "$fallback_observed_merge_sha" ]; then - fallback_observed_merge_sha="$EVENT_MERGE_SHA" - else - fallback_secondary_observed_merge_sha="$EVENT_MERGE_SHA" - fi - fi - fi - fi - fi - fi - if [ "$live_identity_valid" = "false" ] && \ - [[ "${EVENT_BASE_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] && \ - [[ "${EVENT_HEAD_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] && \ - git check-ref-format "refs/heads/${EVENT_BASE_REF:-}" >/dev/null; then - fallback_base_ref="$EVENT_BASE_REF" - fallback_base_sha="$EVENT_BASE_SHA" - fallback_head_sha="$EVENT_HEAD_SHA" - fallback_merge_sha="$EVENT_HEAD_SHA" - if [[ "${EVENT_MERGE_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]]; then - fallback_observed_merge_sha="$EVENT_MERGE_SHA" - fi - fi - if [ "$live_closed" = "true" ]; then - mode=skip_closed - conclusion="" - summary="The pull request is still closed; no commit check was changed." - passed=true - else - mode=policy_error - conclusion=failure - summary="Protected validation policy resolution failed closed." - fi - else - case "$mode" in - validate) - if [ "$VALIDATION_RESULT" = "success" ]; then - conclusion=success - summary="All authorized llama.cpp validation jobs passed." - passed=true - else - conclusion=failure - summary="Authorized validation result: $VALIDATION_RESULT." - fi - ;; - neutral) - [ "$conclusion" = "success" ] && passed=true - ;; - awaiting_rerun) - conclusion=failure - ;; - superseded) - conclusion=failure - ;; - stale_required) - conclusion=failure - ;; - preserve|stale|merged|closed) - passed=true - ;; - *) - conclusion=failure - ;; - esac - fi - { - echo "mode=$mode" - echo "conclusion=$conclusion" - echo "summary=$summary" - echo "passed=$passed" - echo "fallback_base_ref=$fallback_base_ref" - echo "fallback_base_sha=$fallback_base_sha" - echo "fallback_head_sha=$fallback_head_sha" - echo "fallback_merge_sha=$fallback_merge_sha" - echo "fallback_observed_merge_sha=$fallback_observed_merge_sha" - echo "fallback_secondary_observed_merge_sha=$fallback_secondary_observed_merge_sha" - } >> "$GITHUB_OUTPUT" - - - name: Preserve the exact-commit gate for unrelated metadata - id: preserve-gate - if: needs.policy.outputs.mode == 'preserve' && env.DEPENDABOT_PR != 'true' - env: - AUTHORIZATION_ACTOR: ${{ needs.policy.outputs.authorization_actor }} - AUTHORIZATION_ACTOR_ID: ${{ needs.policy.outputs.authorization_actor_id }} - AUTHORIZATION_CREATED_AT: ${{ needs.policy.outputs.authorization_created_at }} - AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id }} - BASE_REF: ${{ needs.policy.outputs.base_ref }} - BASE_SHA: ${{ needs.policy.outputs.base_sha }} - CHECK_NAME: llama.cpp authorized validation v2 - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_SHA: ${{ needs.policy.outputs.head_sha }} - MERGE_SHA: ${{ needs.policy.outputs.merge_sha }} - POLICY_HELPER: trusted-reconcile/.github/scripts/llamacpp_required_check_policy.py - PR_NUMBER: ${{ github.event.pull_request.number }} - RUN_ATTEMPT: ${{ github.run_attempt }} - RUN_NUMBER: ${{ github.run_number }} - run: | - set -euo pipefail - preserve_valid=false - external_prefix="lemonade-llamacpp-required-pr-${PR_NUMBER}-" - pull_request=$(gh api \ - "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" 2>/dev/null || true) - if jq -e --argjson number "$PR_NUMBER" \ - '.number == $number and .state == "open" and .merged == false and - (.base.ref | type == "string") and (.base.sha | type == "string") and - (.head.sha | type == "string") and - ((.mergeable == null) or (.mergeable | type == "boolean")) and - ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string")) and - any(.labels[]?; .name == "ci:upgrades")' \ - <<<"$pull_request" >/dev/null 2>&1; then - current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") - current_base=$(jq -r '.base.sha' <<<"$pull_request") - current_head=$(jq -r '.head.sha' <<<"$pull_request") - current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") - current_mergeable=$(jq -r \ - '.mergeable | if . == null then "null" else tostring end' \ - <<<"$pull_request") - current_candidate="$current_head" - if [ "$current_mergeable" = "true" ] && \ - [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; then - current_candidate="$current_merge" - fi - if git check-ref-format "refs/heads/${current_base_ref}" >/dev/null && \ - [[ "$current_base" =~ ^[0-9a-fA-F]{40}$ ]] && \ - [[ "$current_head" =~ ^[0-9a-fA-F]{40}$ ]] && \ - { [ -z "$current_merge" ] || \ - [[ "$current_merge" =~ ^[0-9a-fA-F]{40}$ ]]; } && \ - [ "$current_base_ref" = "$BASE_REF" ] && \ - [ "$current_base" = "$BASE_SHA" ] && \ - [ "$current_head" = "$HEAD_SHA" ] && \ - [ "$current_candidate" = "$MERGE_SHA" ] && \ - [[ "$AUTHORIZATION_EPOCH" =~ ^[1-9][0-9]*$ ]] && \ - [[ "$AUTHORIZATION_ACTOR_ID" =~ ^[1-9][0-9]*$ ]] && \ - [[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] && \ - [[ "$RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] && \ - [ -f "$POLICY_HELPER" ]; then - events_path=$(mktemp "${RUNNER_TEMP}/llamacpp-preserve-events.XXXXXX") - if gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path"; then - provenance=$(python "$POLICY_HELPER" provenance \ - --pages "$events_path" 2>/dev/null || true) - if jq -e --arg actor "$AUTHORIZATION_ACTOR" \ - --arg created_at "$AUTHORIZATION_CREATED_AT" \ - --argjson actor_id "$AUTHORIZATION_ACTOR_ID" \ - --argjson epoch "$AUTHORIZATION_EPOCH" \ - '.authorization_actor == $actor and - .authorization_actor_id == $actor_id and - .authorization_created_at == $created_at and - .authorization_event_id == $epoch and - .authorization_event == "labeled" and .epoch_valid == true' \ - <<<"$provenance" >/dev/null 2>&1; then - permission_response=$(gh api \ - "repos/${GITHUB_REPOSITORY}/collaborators/${AUTHORIZATION_ACTOR}/permission" \ - 2>/dev/null || true) - if jq -e --argjson actor_id "$AUTHORIZATION_ACTOR_ID" \ - '.user.id == $actor_id and - (.permission == "admin" or .permission == "maintain" or - .permission == "push" or .permission == "write")' \ - <<<"$permission_response" >/dev/null 2>&1; then - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" -f filter=all -F per_page=100 \ - 2>/dev/null || true) - IFS=$'\t' read -r newest_check_epoch newest_check_number \ - newest_check_attempt newest_check_status \ - newest_check_conclusion < <( - jq -r --arg check_name "$CHECK_NAME" \ - --arg external_prefix "$external_prefix" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and - .app.slug == "github-actions") - | select((.external_id // "") | - startswith($external_prefix)) - | . as $check - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {epoch: (.epoch | tonumber), - number: (.number | tonumber), - attempt: (.attempt | tonumber), - status: ($check.status // ""), - conclusion: ($check.conclusion // "")}) catch empty] - | sort_by(.number, .attempt) - | (last // {epoch: 0, number: 0, attempt: 0, - status: "", conclusion: ""}) - | [.epoch, .number, .attempt, .status, .conclusion] - | @tsv' <<<"$check_runs" - ) - if [ "$newest_check_epoch" = "$AUTHORIZATION_EPOCH" ] && \ - [ "$newest_check_status" = "completed" ] && \ - [ "$newest_check_conclusion" = "success" ] && \ - { [ "$newest_check_number" -lt "$RUN_NUMBER" ] || \ - { [ "$newest_check_number" -eq "$RUN_NUMBER" ] && \ - [ "$newest_check_attempt" -le "$RUN_ATTEMPT" ]; }; }; then - preserve_valid=true - fi - fi - fi - fi - rm -f "$events_path" - fi - fi - echo "valid=$preserve_valid" >> "$GITHUB_OUTPUT" - if [ "$preserve_valid" != "true" ]; then - echo "::error::The live authorization epoch cannot preserve this exact-commit gate." - fi - - - name: Revalidate an authorized Dependabot result - if: >- - always() && - env.DEPENDABOT_PR == 'true' && - steps.assess-policy.outputs.mode == 'validate' && - steps.assess-policy.outputs.conclusion == 'success' - env: - AUTHORIZATION_ACTOR: ${{ needs.policy.outputs.authorization_actor }} - AUTHORIZATION_ACTOR_ID: ${{ needs.policy.outputs.authorization_actor_id }} - AUTHORIZATION_BOUNDARY: ${{ needs.policy.outputs.authorization_boundary }} - AUTHORIZATION_CREATED_AT: ${{ needs.policy.outputs.authorization_created_at }} - AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id }} - BASE_REF: ${{ needs.policy.outputs.base_ref }} - BASE_SHA: ${{ needs.policy.outputs.base_sha }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_SHA: ${{ needs.policy.outputs.head_sha }} - MERGE_SHA: ${{ needs.policy.outputs.merge_sha }} - POLICY_HELPER: trusted-reconcile/.github/scripts/llamacpp_required_check_policy.py - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - set -euo pipefail - oid_pattern='^[0-9a-fA-F]{40}$' - pull_request=$(gh api \ - "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") - if ! jq -e --argjson number "$PR_NUMBER" \ - '.number == $number and .state == "open" and .merged == false and - (.base.ref | type == "string") and (.base.sha | type == "string") and - (.head.sha | type == "string") and - ((.mergeable == null) or (.mergeable | type == "boolean")) and - ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string"))' \ - <<<"$pull_request" >/dev/null; then - echo "::error::The live Dependabot pull request identity is invalid." - exit 1 - fi - current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") - current_base=$(jq -r '.base.sha' <<<"$pull_request") - current_head=$(jq -r '.head.sha' <<<"$pull_request") - current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") - current_mergeable=$(jq -r \ - '.mergeable | if . == null then "null" else tostring end' \ - <<<"$pull_request") - if ! git check-ref-format "refs/heads/${current_base_ref}" >/dev/null || \ - ! [[ "$current_base" =~ $oid_pattern && "$current_head" =~ $oid_pattern ]] || \ - { [ -n "$current_merge" ] && ! [[ "$current_merge" =~ $oid_pattern ]]; }; then - echo "::error::The live Dependabot pull request identity is invalid." - exit 1 - fi - current_candidate="$current_head" - if [ "$current_mergeable" = "true" ] && \ - [[ "$current_merge" =~ $oid_pattern ]]; then - current_candidate="$current_merge" - fi - if [ "$current_base_ref" != "$BASE_REF" ] || \ - [ "$current_base" != "$BASE_SHA" ] || \ - [ "$current_head" != "$HEAD_SHA" ] || \ - [ "$current_candidate" != "$MERGE_SHA" ]; then - echo "::error::The Dependabot candidate changed during validation." - exit 1 - fi - current_authorized=$(jq -r \ - 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") - if [ "$current_authorized" != "true" ] || \ - ! [[ "$AUTHORIZATION_EPOCH" =~ ^[1-9][0-9]*$ ]] || \ - ! [[ "$AUTHORIZATION_ACTOR_ID" =~ ^[1-9][0-9]*$ ]] || \ - [ -z "$AUTHORIZATION_BOUNDARY" ] || [ ! -f "$POLICY_HELPER" ]; then - echo "::error::The Dependabot authorization is no longer live." - exit 1 - fi - - events_path=$(mktemp "${RUNNER_TEMP:-/tmp}/llamacpp-dependabot-events.XXXXXX") - trap 'rm -f "$events_path"' EXIT - gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path" - provenance=$(python "$POLICY_HELPER" provenance --pages "$events_path" \ - --authorization-after "$AUTHORIZATION_BOUNDARY") - authorization_actor=$(jq -er '.authorization_actor' <<<"$provenance") - authorization_actor_id=$(jq -er '.authorization_actor_id' <<<"$provenance") - authorization_created_at=$(jq -er '.authorization_created_at' <<<"$provenance") - authorization_event_id=$(jq -er '.authorization_event_id' <<<"$provenance") - authorization_event=$(jq -er '.authorization_event' <<<"$provenance") - authorization_epoch_valid=$(jq -er \ - '.epoch_valid | if type == "boolean" then tostring else error("invalid epoch") end' \ - <<<"$provenance") - if [ "$authorization_actor" != "$AUTHORIZATION_ACTOR" ] || \ - [ "$authorization_actor_id" != "$AUTHORIZATION_ACTOR_ID" ] || \ - [ "$authorization_created_at" != "$AUTHORIZATION_CREATED_AT" ] || \ - [ "$authorization_event_id" != "$AUTHORIZATION_EPOCH" ] || \ - [ "$authorization_event" != "labeled" ] || \ - [ "$authorization_epoch_valid" != "true" ]; then - echo "::error::The Dependabot authorization epoch changed during validation." - exit 1 - fi - permission_response=$(gh api \ - "repos/${GITHUB_REPOSITORY}/collaborators/${authorization_actor}/permission") - if ! jq -e --argjson actor_id "$authorization_actor_id" \ - '.user.id == $actor_id and - (.permission == "admin" or .permission == "maintain" or - .permission == "push" or .permission == "write")' \ - <<<"$permission_response" >/dev/null; then - echo "::error::The Dependabot authorization actor lost write permission." - exit 1 - fi - - - name: Invalidate an unconfirmed observed merge commit - if: >- - always() && - env.DEPENDABOT_PR != 'true' && - (needs.policy.outputs.observed_merge_sha != '' || - needs.policy.outputs.secondary_observed_merge_sha != '' || - steps.assess-policy.outputs.fallback_observed_merge_sha != '' || - steps.assess-policy.outputs.fallback_secondary_observed_merge_sha != '') && - steps.assess-policy.outputs.mode != 'preserve' && - steps.assess-policy.outputs.mode != 'awaiting_rerun' && - steps.assess-policy.outputs.mode != 'superseded' && - steps.assess-policy.outputs.mode != 'stale_required' && - steps.assess-policy.outputs.mode != 'stale' && - steps.assess-policy.outputs.mode != 'merged' && - steps.assess-policy.outputs.mode != 'skip_closed' - env: - AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id || 0 }} - CHECK_NAME: llama.cpp authorization diagnostics v2 - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OBSERVED_MERGE_SHA: ${{ steps.assess-policy.outputs.fallback_observed_merge_sha || needs.policy.outputs.observed_merge_sha }} - PR_NUMBER: ${{ github.event.pull_request.number }} - RUN_ATTEMPT: ${{ github.run_attempt }} - RUN_ID: ${{ github.run_id }} - RUN_NUMBER: ${{ github.run_number }} - SECONDARY_OBSERVED_MERGE_SHA: ${{ steps.assess-policy.outputs.fallback_secondary_observed_merge_sha || needs.policy.outputs.secondary_observed_merge_sha }} - run: | - set -euo pipefail - external_prefix="lemonade-llamacpp-metadata-pr-${PR_NUMBER}-" - if ! [[ "$AUTHORIZATION_EPOCH" =~ ^[0-9]+$ ]]; then - AUTHORIZATION_EPOCH=0 - fi - - invalidate_observed_merge() { - local observed_merge_sha="$1" - local check_runs check_run_id newest_number newest_attempt prior_check_ids - local external_id summary - if ! [[ "$observed_merge_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::The observed merge commit must be a full Git OID." - return 1 - fi - - external_id="${external_prefix}${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}" - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${observed_merge_sha}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) - IFS=$'\t' read -r newest_number newest_attempt < <( - jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] - | sort_by(.number, .attempt) - | (last // {number: 0, attempt: 0}) - | [.number, .attempt] | @tsv' <<<"$check_runs" - ) - if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ - { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ - [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then - echo "::notice::A newer policy run already invalidated ${observed_merge_sha}." - return 0 - fi - - check_run_id=$(jq -r --arg check_name "$CHECK_NAME" \ - --arg external_id "$external_id" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions" and - .external_id == $external_id) - | .id] | last // empty' <<<"$check_runs") - summary="GitHub did not confirm this observed synthetic merge commit as executable." - if [[ "$check_run_id" =~ ^[0-9]+$ ]]; then - gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f status=completed -f conclusion=failure \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - -f "output[title]=$CHECK_NAME" -f "output[summary]=$summary" >/dev/null - else - response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f head_sha="$observed_merge_sha" \ - -f external_id="$external_id" -f status=completed -f conclusion=failure \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - -f "output[title]=$CHECK_NAME" -f "output[summary]=$summary") - check_run_id=$(jq -r '.id // empty' <<<"$response") - fi - if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid observed-merge check-run ID." - return 1 - fi - - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${observed_merge_sha}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) - IFS=$'\t' read -r newest_number newest_attempt < <( - jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] - | sort_by(.number, .attempt) - | (last // {number: 0, attempt: 0}) - | [.number, .attempt] | @tsv' <<<"$check_runs" - ) - if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ - { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ - [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then - gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=Superseded by a newer policy decision." >/dev/null - return 0 - fi - - prior_check_ids=$(jq -r --arg check_name "$CHECK_NAME" \ - --arg external_prefix "$external_prefix" \ - --argjson current_number "$RUN_NUMBER" --argjson current_attempt "$RUN_ATTEMPT" \ - '.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | . as $check - | try (($check.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty - | select(.number < $current_number or - (.number == $current_number and .attempt < $current_attempt)) - | $check.id' <<<"$check_runs") - while IFS= read -r prior_check_id; do - [ -n "$prior_check_id" ] || continue - [[ "$prior_check_id" =~ ^[0-9]+$ ]] || return 1 - gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=Superseded by a newer policy decision." >/dev/null - done <<<"$prior_check_ids" - } - - invalidate_observed_merge "$OBSERVED_MERGE_SHA" - if [ -n "$SECONDARY_OBSERVED_MERGE_SHA" ] && \ - [ "$SECONDARY_OBSERVED_MERGE_SHA" != "$OBSERVED_MERGE_SHA" ]; then - invalidate_observed_merge "$SECONDARY_OBSERVED_MERGE_SHA" - fi - - - name: Report result on the exact candidate commit - if: >- - always() && - env.DEPENDABOT_PR != 'true' && - (steps.assess-policy.outputs.mode != 'preserve' || - steps.preserve-gate.outputs.valid != 'true') && - steps.assess-policy.outputs.mode != 'awaiting_rerun' && - steps.assess-policy.outputs.mode != 'superseded' && - steps.assess-policy.outputs.mode != 'stale_required' && - steps.assess-policy.outputs.mode != 'stale' && - steps.assess-policy.outputs.mode != 'merged' && - steps.assess-policy.outputs.mode != 'skip_closed' - env: - AUTHORIZATION_ACTOR: ${{ needs.policy.outputs.authorization_actor || '' }} - AUTHORIZATION_ACTOR_ID: ${{ needs.policy.outputs.authorization_actor_id || 0 }} - AUTHORIZATION_BOUNDARY: ${{ needs.policy.outputs.authorization_boundary || '' }} - AUTHORIZATION_CREATED_AT: ${{ needs.policy.outputs.authorization_created_at || '' }} - AUTHORIZATION_EPOCH: ${{ needs.policy.outputs.authorization_event_id || 0 }} - BASE_REF: ${{ steps.assess-policy.outputs.fallback_base_ref || needs.policy.outputs.base_ref || github.event.pull_request.base.ref || '' }} - BASE_SHA: ${{ steps.assess-policy.outputs.fallback_base_sha || needs.policy.outputs.base_sha || github.event.pull_request.base.sha || '' }} - CHECK_NAME: llama.cpp authorized validation v2 - CHECK_RUN_ID: ${{ needs.validate.outputs.check_run_id }} - CONCLUSION: ${{ steps.assess-policy.outputs.conclusion || 'failure' }} - EVENT_ACTION: ${{ github.event.action || '' }} - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_SHA: ${{ steps.assess-policy.outputs.fallback_head_sha || needs.policy.outputs.head_sha || github.event.pull_request.head.sha || '' }} - MERGE_SHA: ${{ steps.assess-policy.outputs.fallback_merge_sha || needs.policy.outputs.merge_sha || github.event.merge_group.head_sha || github.event.pull_request.head.sha || '' }} - METADATA_CHECK_NAME: llama.cpp authorization diagnostics v2 - MODE: ${{ steps.assess-policy.outputs.mode }} - POLICY_HELPER: trusted-reconcile/.github/scripts/llamacpp_required_check_policy.py - PRESERVE_VALID: ${{ steps.preserve-gate.outputs.valid || 'false' }} - PR_NUMBER: ${{ github.event.pull_request.number || '' }} - RUN_ATTEMPT: ${{ github.run_attempt }} - RUN_ID: ${{ github.run_id }} - RUN_NUMBER: ${{ github.run_number }} - SUMMARY: ${{ steps.assess-policy.outputs.summary || 'Validation assessment failed.' }} - TARGET_KEY: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || format('pr-{0}', github.event.pull_request.number) }} - run: | - set -euo pipefail - oid_pattern='^[0-9a-fA-F]{40}$' - if ! [[ "$MERGE_SHA" =~ $oid_pattern ]]; then - echo "::error::Cannot report a required check without an exact candidate commit." - exit 1 - fi - - if ! [[ "$AUTHORIZATION_EPOCH" =~ ^[0-9]+$ ]]; then - AUTHORIZATION_EPOCH=0 - fi - if [ "$MODE" = "preserve" ] && [ "$PRESERVE_VALID" != "true" ]; then - CONCLUSION=failure - SUMMARY="The live authorization epoch could not preserve this exact-commit gate." - fi - - verify_live_candidate() { - local pull_request current_number current_state current_merged - local current_base_ref current_base current_head current_merge - local current_mergeable current_candidate current_authorized - local events_path provenance permission_response - local authorization_actor authorization_actor_id - local authorization_created_at authorization_event_id - local authorization_event authorization_epoch_valid - pull_request=$(gh api \ - "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}") || return 1 - if ! jq -e --argjson number "$PR_NUMBER" \ - '.number == $number and .state == "open" and .merged == false and - (.base.ref | type == "string") and (.base.sha | type == "string") and - (.head.sha | type == "string") and - ((.mergeable == null) or (.mergeable | type == "boolean")) and - ((.merge_commit_sha == null) or (.merge_commit_sha | type == "string"))' \ - <<<"$pull_request" >/dev/null; then - return 1 - fi - current_number=$(jq -r '.number' <<<"$pull_request") - current_state=$(jq -r '.state' <<<"$pull_request") - current_merged=$(jq -r '.merged' <<<"$pull_request") - current_base_ref=$(jq -r '.base.ref' <<<"$pull_request") - current_base=$(jq -r '.base.sha' <<<"$pull_request") - current_head=$(jq -r '.head.sha' <<<"$pull_request") - current_merge=$(jq -r '.merge_commit_sha // empty' <<<"$pull_request") - current_mergeable=$(jq -r \ - '.mergeable | if . == null then "null" else tostring end' \ - <<<"$pull_request") - if [ "$current_number" != "$PR_NUMBER" ] || \ - [ "$current_state" != "open" ] || [ "$current_merged" != "false" ] || \ - ! git check-ref-format "refs/heads/${current_base_ref}" >/dev/null || \ - ! [[ "$current_base" =~ $oid_pattern && "$current_head" =~ $oid_pattern ]] || \ - { [ -n "$current_merge" ] && ! [[ "$current_merge" =~ $oid_pattern ]]; }; then - return 1 - fi - current_candidate="$current_head" - if [ "$current_mergeable" = "true" ] && [[ "$current_merge" =~ $oid_pattern ]]; then - current_candidate="$current_merge" - fi - if [ "$current_base_ref" != "$BASE_REF" ] || \ - [ "$current_base" != "$BASE_SHA" ] || \ - [ "$current_head" != "$HEAD_SHA" ] || \ - [ "$current_candidate" != "$MERGE_SHA" ]; then - return 1 - fi - - if [ "$MODE" != "validate" ]; then - return 0 - fi - current_authorized=$(jq -r \ - 'any(.labels[]?; .name == "ci:upgrades")' <<<"$pull_request") - [ "$current_authorized" = "true" ] || return 1 - [ -n "$AUTHORIZATION_BOUNDARY" ] || return 1 - [ "$AUTHORIZATION_EPOCH" -gt 0 ] || return 1 - [ -f "$POLICY_HELPER" ] || return 1 - events_path=$(mktemp "${RUNNER_TEMP:-/tmp}/llamacpp-reconcile-events.XXXXXX") - if ! gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/events?per_page=100" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" > "$events_path"; then - rm -f "$events_path" - return 1 - fi - provenance=$(python "$POLICY_HELPER" provenance --pages "$events_path" \ - --authorization-after "$AUTHORIZATION_BOUNDARY") || { - rm -f "$events_path" - return 1 - } - rm -f "$events_path" - authorization_actor=$(jq -er '.authorization_actor' <<<"$provenance") || return 1 - authorization_actor_id=$(jq -er '.authorization_actor_id' <<<"$provenance") || return 1 - authorization_created_at=$(jq -er '.authorization_created_at' <<<"$provenance") || return 1 - authorization_event_id=$(jq -er '.authorization_event_id' <<<"$provenance") || return 1 - authorization_event=$(jq -er '.authorization_event' <<<"$provenance") || return 1 - authorization_epoch_valid=$(jq -er \ - '.epoch_valid | if type == "boolean" then tostring else error("invalid epoch") end' \ - <<<"$provenance") || return 1 - if [ "$authorization_actor" != "$AUTHORIZATION_ACTOR" ] || \ - [ "$authorization_actor_id" != "$AUTHORIZATION_ACTOR_ID" ] || \ - [ "$authorization_created_at" != "$AUTHORIZATION_CREATED_AT" ] || \ - [ "$authorization_event_id" != "$AUTHORIZATION_EPOCH" ] || \ - [ "$authorization_event" != "labeled" ] || \ - [ "$authorization_epoch_valid" != "true" ]; then - return 1 - fi - permission_response=$(gh api \ - "repos/${GITHUB_REPOSITORY}/collaborators/${authorization_actor}/permission") || \ - return 1 - if ! jq -e --argjson actor_id "$authorization_actor_id" \ - '.user.id == $actor_id and - (.permission == "admin" or .permission == "maintain" or - .permission == "push" or .permission == "write")' \ - <<<"$permission_response" >/dev/null; then - return 1 - fi - } - - stale=false - if [ "$CONCLUSION" = "success" ] && \ - [ "$EVENT_NAME" = "pull_request_target" ] && \ - ! verify_live_candidate; then - stale=true - CONCLUSION=failure - SUMMARY="The pull request identity or authorization changed before reporting." - fi - - decision_scope=metadata - if [ "$EVENT_NAME" = "merge_group" ] || \ - [ "$EVENT_ACTION" = "opened" ] || \ - [ "$EVENT_ACTION" = "reopened" ] || \ - [ "$EVENT_ACTION" = "synchronize" ]; then - decision_scope=required - fi - if [ "$decision_scope" = "metadata" ]; then - CHECK_NAME="$METADATA_CHECK_NAME" - fi - external_prefix="lemonade-llamacpp-${decision_scope}-${TARGET_KEY}-" - external_id="${external_prefix}${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}" - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) - - IFS=$'\t' read -r newest_number newest_attempt < <( - jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] - | sort_by(.number, .attempt) - | (last // {number: 0, attempt: 0}) - | [.number, .attempt] | @tsv' <<<"$check_runs" - ) - if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ - { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ - [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then - echo "::error::A newer policy run superseded this required-workflow run." - exit 1 - fi - - if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then - CHECK_RUN_ID=$(jq -r --argjson check_run_id "$CHECK_RUN_ID" \ - --arg check_name "$CHECK_NAME" --arg external_id "$external_id" \ - '[.[] | .check_runs[]? - | select(.id == $check_run_id and .name == $check_name and - .app.slug == "github-actions" and .external_id == $external_id) - | .id] | last // empty' <<<"$check_runs") - fi - if ! [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then - CHECK_RUN_ID=$(jq -r --arg check_name "$CHECK_NAME" \ - --arg external_id "$external_id" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions" and - .external_id == $external_id) - | .id] | last // empty' <<<"$check_runs") - fi - - if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then - gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f status=completed -f conclusion="$CONCLUSION" \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - -f "output[title]=$CHECK_NAME" -f "output[summary]=$SUMMARY" >/dev/null - else - response=$(gh api --method POST "repos/${GITHUB_REPOSITORY}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f head_sha="$MERGE_SHA" -f external_id="$external_id" \ - -f status=completed -f conclusion="$CONCLUSION" \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - -f "output[title]=$CHECK_NAME" -f "output[summary]=$SUMMARY") - CHECK_RUN_ID=$(jq -r '.id // empty' <<<"$response") - fi - if ! [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then - echo "::error::Checks API returned an invalid check-run ID." - exit 1 - fi - - check_runs=$(gh api --method GET --paginate --slurp \ - "repos/${GITHUB_REPOSITORY}/commits/${MERGE_SHA}/check-runs" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f check_name="$CHECK_NAME" -f filter=all -F per_page=100) - IFS=$'\t' read -r newest_number newest_attempt < <( - jq -r --arg check_name "$CHECK_NAME" --arg external_prefix "$external_prefix" \ - '[.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | try ((.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty] - | sort_by(.number, .attempt) - | (last // {number: 0, attempt: 0}) - | [.number, .attempt] | @tsv' <<<"$check_runs" - ) - if [ "$newest_number" -gt "$RUN_NUMBER" ] || \ - { [ "$newest_number" -eq "$RUN_NUMBER" ] && \ - [ "$newest_attempt" -gt "$RUN_ATTEMPT" ]; }; then - gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=Superseded by a newer policy decision." >/dev/null - echo "::error::A newer policy run superseded this required-workflow run." - exit 1 - fi - - if [ "$CONCLUSION" = "success" ] && \ - [ "$EVENT_NAME" = "pull_request_target" ] && \ - ! verify_live_candidate; then - stale=true - CONCLUSION=failure - SUMMARY="The pull request identity or authorization changed after reporting." - gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${CHECK_RUN_ID}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f status=completed -f conclusion=failure \ - -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - -f "output[title]=$CHECK_NAME" -f "output[summary]=$SUMMARY" >/dev/null - fi - - prior_check_ids=$(jq -r \ - --arg check_name "$CHECK_NAME" \ - --arg external_prefix "$external_prefix" \ - --argjson current_number "$RUN_NUMBER" --argjson current_attempt "$RUN_ATTEMPT" \ - '.[] | .check_runs[]? - | select(.name == $check_name and .app.slug == "github-actions") - | select((.external_id // "") | startswith($external_prefix)) - | . as $check - | try (($check.external_id | ltrimstr($external_prefix) - | capture("^(?[0-9]+)-(?[0-9]+)-(?[0-9]+)-(?[0-9]+)$")) - | {number: (.number | tonumber), attempt: (.attempt | tonumber)}) catch empty - | select(.number < $current_number or - (.number == $current_number and .attempt < $current_attempt)) - | $check.id' <<<"$check_runs") - while IFS= read -r prior_check_id; do - [ -n "$prior_check_id" ] || continue - [[ "$prior_check_id" =~ ^[0-9]+$ ]] || exit 1 - gh api --method PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${prior_check_id}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -f name="$CHECK_NAME" -f status=completed -f conclusion=cancelled \ - -f "output[title]=$CHECK_NAME" \ - -f "output[summary]=Superseded by a newer policy decision." >/dev/null - done <<<"$prior_check_ids" - [ "$stale" = "false" ] - - - name: Enforce policy result - if: >- - always() && - (steps.assess-policy.outputs.passed != 'true' || - (steps.assess-policy.outputs.mode == 'preserve' && - env.DEPENDABOT_PR != 'true' && - steps.preserve-gate.outputs.valid != 'true')) - run: | - echo "${{ steps.assess-policy.outputs.summary || 'Validation assessment failed.' }}" - exit 1 diff --git a/.github/workflows/validate_llamacpp_schedule.yml b/.github/workflows/validate_llamacpp_schedule.yml deleted file mode 100644 index c939a5b8da..0000000000 --- a/.github/workflows/validate_llamacpp_schedule.yml +++ /dev/null @@ -1,280 +0,0 @@ -name: Update llama.cpp Releases - -on: - schedule: - - cron: "0 16 * * 0" - -permissions: - contents: read - -concurrency: - group: llamacpp-auto-update - cancel-in-progress: false - -jobs: - validate: - name: Validate release candidates - permissions: - checks: write - contents: read - uses: ./.github/workflows/validate_llamacpp_core.yml - secrets: - HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} - - publish: - name: Publish validated update - needs: validate - if: >- - needs.validate.result == 'success' && - needs.validate.outputs.promotion_eligible == 'true' - runs-on: ubuntu-latest - concurrency: - group: llamacpp-auto-update-publication - cancel-in-progress: false - permissions: - contents: read - steps: - - name: Require dedicated publication identity - shell: bash - env: - GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} - EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }} - run: | - set -euo pipefail - - : "${GH_TOKEN:?LLAMACPP_UPDATE_TOKEN repository secret is required}" - : "${EXPECTED_PUBLICATION_ACTOR:?LLAMACPP_UPDATE_ACTOR repository variable is required}" - if ! authenticated_actor=$(gh api graphql \ - -f 'query=query { viewer { login } }' \ - --jq '.data.viewer.login'); then - echo "Could not authenticate the dedicated publication token." >&2 - exit 1 - fi - if [[ -z "$authenticated_actor" || - "$authenticated_actor" != "$EXPECTED_PUBLICATION_ACTOR" ]]; then - echo "The authenticated publication actor does not match LLAMACPP_UPDATE_ACTOR." >&2 - exit 1 - fi - - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - fetch-depth: 0 - persist-credentials: false - token: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} - - - name: Download primary validation results - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - pattern: validation-results-* - merge-multiple: true - - - name: Download restart validation results - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - pattern: restart-validation-results-* - merge-multiple: true - - - name: Validate complete validation evidence - env: - EXPECTED_VALIDATION_MATRIX: ${{ needs.validate.outputs.validation_matrix }} - run: >- - python -m test.utils.llamacpp_validation_evidence - --directory . - --expected-matrix-json "$EXPECTED_VALIDATION_MATRIX" - - - name: Update backend_versions.json from release manifest - shell: bash - env: - LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} - LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.validate.outputs.ggml_update_backends }} - LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.validate.outputs.rocm_update_backends }} - LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.validate.outputs.lemonade_update_backends }} - run: | - set -euo pipefail - - python3 - <<'PY' - import json - import os - import re - from pathlib import Path - - RELEASE_RE = re.compile(r"^b[0-9]+$") - - def parse_csv(value): - return [item.strip() for item in (value or "").split(",") if item.strip()] - - def update(section, release, backends): - if not backends: - return [] - if not RELEASE_RE.match(release): - raise SystemExit(f"Invalid llama.cpp release tag: {release}") - changed = [] - for key in backends: - if key not in section: - raise SystemExit(f"Refusing to create missing llamacpp.{key}") - if not isinstance(section[key], str): - raise SystemExit(f"llamacpp.{key} must be a string") - old = section[key] - section[key] = release - changed.append((key, old, release)) - return changed - - path = Path("src/cpp/resources/backend_versions.json") - data = json.loads(path.read_text(encoding="utf-8")) - section = data.get("llamacpp") - if not isinstance(section, dict): - raise SystemExit("backend_versions.json is missing a llamacpp object") - - changes = [] - changes += update(section, os.environ["LLAMACPP_RELEASE"], parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) - changes += update(section, os.environ["LLAMACPP_ROCM_RELEASE"], parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) - changes += update(section, os.environ["LLAMACPP_LEMONADE_RELEASE"], parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) - - path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") - - print("Updated llamacpp backend versions:") - if changes: - for key, old, new in changes: - print(f" llamacpp.{key}: {old} -> {new}") - else: - print(" none") - PY - - - name: Generate PR body - shell: bash - env: - LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} - LLAMACPP_GGML_UPDATE_BACKENDS: ${{ needs.validate.outputs.ggml_update_backends }} - LLAMACPP_ROCM_UPDATE_BACKENDS: ${{ needs.validate.outputs.rocm_update_backends }} - LLAMACPP_LEMONADE_UPDATE_BACKENDS: ${{ needs.validate.outputs.lemonade_update_backends }} - VULKAN_AVAILABLE: ${{ needs.validate.outputs.vulkan_available }} - CPU_AVAILABLE: ${{ needs.validate.outputs.cpu_available }} - METAL_AVAILABLE: ${{ needs.validate.outputs.metal_available }} - ROCM_NIGHTLY_AVAILABLE: ${{ needs.validate.outputs.rocm_nightly_available }} - ROCM_STABLE_AVAILABLE: ${{ needs.validate.outputs.rocm_stable_available }} - CUDA_AVAILABLE: ${{ needs.validate.outputs.cuda_available }} - GGML_MISSING_COUNT: ${{ needs.validate.outputs.ggml_missing_count }} - ROCM_NIGHTLY_MISSING_COUNT: ${{ needs.validate.outputs.rocm_nightly_missing_count }} - ROCM_STABLE_MISSING_COUNT: ${{ needs.validate.outputs.rocm_stable_missing_count }} - CUDA_MISSING_COUNT: ${{ needs.validate.outputs.cuda_missing_count }} - VALIDATION_MATRIX: ${{ needs.validate.outputs.validation_matrix }} - run: | - python3 <<'PYEOF' - import json - import os - - def parse_csv(value): - return [item.strip() for item in (value or "").split(",") if item.strip()] - - def yes(name): - return os.environ.get(name, "false") == "true" - - def pin_action(backend, release, enabled): - return release if enabled else "unchanged" - - body_lines = [] - llamacpp = os.environ.get("LLAMACPP_RELEASE", "unknown") - rocm_nightly = os.environ.get("LLAMACPP_ROCM_RELEASE", "unknown") - lemonade_release = os.environ.get("LLAMACPP_LEMONADE_RELEASE", "unknown") - ggml_updates = set(parse_csv(os.environ.get("LLAMACPP_GGML_UPDATE_BACKENDS", ""))) - rocm_updates = set(parse_csv(os.environ.get("LLAMACPP_ROCM_UPDATE_BACKENDS", ""))) - lemonade_updates = set(parse_csv(os.environ.get("LLAMACPP_LEMONADE_UPDATE_BACKENDS", ""))) - - pin_plan = { - "vulkan": pin_action("vulkan", llamacpp, "vulkan" in ggml_updates), - "cpu": pin_action("cpu", llamacpp, "cpu" in ggml_updates), - "metal": pin_action("metal", llamacpp, "metal" in ggml_updates), - "rocm-nightly": pin_action("rocm-nightly", rocm_nightly, "rocm-nightly" in rocm_updates), - "rocm-stable": pin_action("rocm-stable", lemonade_release, "rocm-stable" in lemonade_updates), - "cuda": pin_action("cuda", lemonade_release, "cuda" in lemonade_updates), - } - - body_lines.append("## Auto-update llama.cpp backends") - body_lines.append("") - body_lines.append("This PR updates only backend pins whose expected release assets were found.") - body_lines.append("") - body_lines.append(f"- **ggml-org/llama.cpp candidate**: `{llamacpp}`") - body_lines.append(f"- **lemonade-sdk/llamacpp-rocm candidate**: `{rocm_nightly}`") - body_lines.append(f"- **lemonade-sdk/llama.cpp candidate**: `{lemonade_release}`") - body_lines.append("") - body_lines.append("| Backend | Pin action | Asset check |") - body_lines.append("|---|---|---|") - body_lines.append(f"| `vulkan` | `{pin_plan['vulkan']}` | {'complete' if yes('VULKAN_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `cpu` | `{pin_plan['cpu']}` | {'complete' if yes('CPU_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `metal` | `{pin_plan['metal']}` | {'complete' if yes('METAL_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `rocm-nightly` | `{pin_plan['rocm-nightly']}` | {'complete' if yes('ROCM_NIGHTLY_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `rocm-stable` | `{pin_plan['rocm-stable']}` | {'complete' if yes('ROCM_STABLE_AVAILABLE') else 'incomplete'} |") - body_lines.append(f"| `cuda` | `{pin_plan['cuda']}` | {'complete' if yes('CUDA_AVAILABLE') else 'incomplete'} |") - body_lines.append("") - body_lines.append("Missing asset counts:") - body_lines.append(f"- ggml-org/llama.cpp: `{os.environ.get('GGML_MISSING_COUNT', 'unknown')}`") - body_lines.append(f"- lemonade-sdk/llamacpp-rocm rocm-nightly: `{os.environ.get('ROCM_NIGHTLY_MISSING_COUNT', 'unknown')}`") - body_lines.append(f"- lemonade-sdk/llama.cpp rocm-stable: `{os.environ.get('ROCM_STABLE_MISSING_COUNT', 'unknown')}`") - body_lines.append(f"- lemonade-sdk/llama.cpp cuda: `{os.environ.get('CUDA_MISSING_COUNT', 'unknown')}`") - body_lines.append("") - body_lines.append("## Validation Results") - body_lines.append("") - body_lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |") - body_lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|") - - validation_matrix = json.loads(os.environ["VALIDATION_MATRIX"]) - for lane in validation_matrix["include"]: - target = lane["target"] - results_file = f"llamacpp_validation_{target}.json" - with open(results_file, "r") as f: - results = json.load(f) - for r in results: - status = "PASS" if r["pass"] else "FAIL" - # Truncate response for table readability - resp = str(r.get("response", ""))[:80].replace("|", "\\|").replace("\n", " ") - ttft = r.get("time_to_first_token", "N/A") - if isinstance(ttft, float): - ttft = f"{ttft:.3f}s" - tps = r.get("tokens_per_second", "N/A") - if isinstance(tps, float): - tps = f"{tps:.1f}" - body_lines.append( - f"| {r['model']} ({target}) | {status} | {resp} | " - f"{r.get('input_tokens', 'N/A')} | {r.get('output_tokens', 'N/A')} | " - f"{ttft} | {tps} |" - ) - - body_lines.append("") - body_lines.append("---") - body_lines.append("*Auto-generated by validate_llamacpp workflow*") - - body = "\n".join(body_lines) - with open("pr_body.md", "w") as f: - f.write(body) - PYEOF - - - name: Verify release asset manifest - env: - GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} - EXPECTED_RELEASE_ASSET_MANIFEST: ${{ needs.validate.outputs.release_asset_manifest }} - run: | - set -euo pipefail - - snapshot_directory=$(mktemp -d) - expected_path="${snapshot_directory}/expected-input.json" - python -m test.utils.llamacpp_release_manifest \ - --materialize-manifest "$expected_path" \ - --manifest-json "$EXPECTED_RELEASE_ASSET_MANIFEST" >/dev/null - bash .github/scripts/capture_llamacpp_release_manifest.sh \ - "$expected_path" "${snapshot_directory}/live" >/dev/null - - - name: Publish update pull request - env: - BASE_BRANCH: ${{ github.event.repository.default_branch }} - GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }} - EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }} - EXPECTED_RELEASE_ASSET_MANIFEST: ${{ needs.validate.outputs.release_asset_manifest }} - LLAMACPP_RELEASE: ${{ needs.validate.outputs.llamacpp_release }} - LLAMACPP_ROCM_RELEASE: ${{ needs.validate.outputs.llamacpp_rocm_release }} - LLAMACPP_LEMONADE_RELEASE: ${{ needs.validate.outputs.llamacpp_lemonade_release }} - VALIDATED_BASE_SHA: ${{ github.sha }} - run: bash .github/scripts/publish_llamacpp_update.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 3858d35522..92a6893b11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1192,16 +1192,6 @@ if(BUILD_TESTING AND EXISTS "${MCP_CLIENT_CONFIG_TEST_SOURCE}") ) target_link_libraries(test_mcp_client_config PRIVATE lemonade-server-core) - if(NOT WIN32) - set_property(SOURCE src/cpp/server/mcp_client.cpp APPEND PROPERTY - COMPILE_DEFINITIONS - LEMONADE_MCP_PROCESS_TEST_HOOK - ) - target_compile_definitions(test_mcp_client_config PRIVATE - LEMONADE_MCP_PROCESS_TEST_HOOK - ) - endif() - if(Python3_Interpreter_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/fixtures/mock_mcp_stdio_server.py") file(TO_CMAKE_PATH "${Python3_EXECUTABLE}" LEMONADE_TEST_PYTHON_PATH) file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/fixtures/mock_mcp_stdio_server.py" LEMONADE_TEST_MCP_STDIO_SERVER_PATH) @@ -1213,20 +1203,6 @@ if(BUILD_TESTING AND EXISTS "${MCP_CLIENT_CONFIG_TEST_SOURCE}") add_cpp_ci_test(mcp_client_config CI ON COMMAND test_mcp_client_config) endif() -set(MCP_CLIENT_PROCESS_CONTRACT_TEST_SOURCE - "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_mcp_client_process_contract.cpp" -) -if(BUILD_TESTING AND EXISTS "${MCP_CLIENT_PROCESS_CONTRACT_TEST_SOURCE}") - add_executable(test_mcp_client_process_contract - ${MCP_CLIENT_PROCESS_CONTRACT_TEST_SOURCE} - ) - target_compile_definitions(test_mcp_client_process_contract PRIVATE - "LEMONADE_MCP_CLIENT_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/server/mcp_client.cpp\"" - ) - add_cpp_ci_test(mcp_client_process_contract CI ON - COMMAND test_mcp_client_process_contract) -endif() - # libwebsockets for the realtime and log-stream WebSocket server if(USE_SYSTEM_LWS) target_link_libraries(lemonade-server-core PUBLIC ${LWS_LIBRARIES}) @@ -2254,57 +2230,27 @@ if(BUILD_TESTING AND EXISTS "${_ARCHIVE_PLATFORM_TEST_SRC}") add_cpp_ci_test(ArchivePlatformTest CI ON COMMAND test_archive_platform) endif() -# ProcessManager lifecycle and output-capture contracts. -if(BUILD_TESTING AND (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR - CMAKE_SYSTEM_NAME STREQUAL "Darwin")) +# ProcessManager zombie detection and non-mutating reap contract. +if(BUILD_TESTING AND CMAKE_SYSTEM_NAME STREQUAL "Linux") set(_PROCESS_MANAGER_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_process_manager.cpp" ) if(EXISTS "${_PROCESS_MANAGER_TEST_SRC}") - if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - set(_PROCESS_MANAGER_PLATFORM_SRC - src/cpp/server/utils/platform/process_linux.cpp - ) - else() - set(_PROCESS_MANAGER_PLATFORM_SRC - src/cpp/server/utils/platform/process_macos.cpp - ) - endif() add_executable(test_process_manager test/cpp/test_process_manager.cpp src/cpp/server/utils/process_manager.cpp - ${_PROCESS_MANAGER_PLATFORM_SRC} + src/cpp/server/utils/platform/process_linux.cpp ) target_include_directories(test_process_manager PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/include ${CMAKE_CURRENT_BINARY_DIR}/include ) - target_compile_definitions(test_process_manager PRIVATE - LEMONADE_PROCESS_TEST_HOOK - ) find_package(Threads REQUIRED) target_link_libraries(test_process_manager PRIVATE Threads::Threads) add_cpp_ci_test(ProcessManagerTest CI ON COMMAND test_process_manager) endif() endif() -set(_PROCESS_MANAGER_WINDOWS_TEST_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_process_manager_windows.cpp" -) -if(BUILD_TESTING AND WIN32 AND EXISTS "${_PROCESS_MANAGER_WINDOWS_TEST_SRC}") - add_executable(test_process_manager_windows - test/cpp/test_process_manager_windows.cpp - src/cpp/server/utils/process_manager.cpp - src/cpp/server/utils/platform/process_windows.cpp - ) - target_include_directories(test_process_manager_windows PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/include - ${CMAKE_CURRENT_BINARY_DIR}/include - ) - add_cpp_ci_test(ProcessManagerWindowsTest CI ON - COMMAND test_process_manager_windows) -endif() - # FastFlowLM argument validation. Keep this test independent from the server set(_FLM_ARG_RESOLVER_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_flm_arg_resolver.cpp" @@ -3276,13 +3222,6 @@ if(BUILD_TESTING AND EXISTS "${_STREAMING_PROXY_CANCEL_TEST_SRC}") add_cpp_ci_test(StreamingProxyCancelTest CI OFF COMMAND test_streaming_proxy_cancel) endif() -set(_LLAMACPP_STREAM_MODEL_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_llamacpp_stream_model.cpp") -if(BUILD_TESTING AND EXISTS "${_LLAMACPP_STREAM_MODEL_TEST_SRC}") - add_executable(test_llamacpp_stream_model test/cpp/test_llamacpp_stream_model.cpp) - target_link_libraries(test_llamacpp_stream_model PRIVATE lemonade-server-core) - add_cpp_ci_test(LlamaCppStreamModelTest CI ON COMMAND test_llamacpp_stream_model) -endif() - set(_STREAMING_HEARTBEAT_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_streaming_heartbeat.cpp") if(BUILD_TESTING AND EXISTS "${_STREAMING_HEARTBEAT_TEST_SRC}") add_executable(test_streaming_heartbeat test/cpp/test_streaming_heartbeat.cpp) diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 315c4ab14a..7334ddea88 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -145,55 +145,16 @@ Every PR runs the C++ `cpp-ci` tests, the endpoint/CLI suites on Windows and Lin ### What defers to the merge queue -Packaging, distro, PPA, backend-validation, self-hosted inference and most macOS jobs do **not** run on PR pushes. They run in the merge queue, so a break there blocks the merge rather than every push. Each group reports an aggregate gate signal. The llama.cpp custom checks are diagnostic only; the trusted wrapper workflow is their enforcement boundary. +Packaging, distro, PPA, backend-validation, self-hosted inference and most macOS jobs do **not** run on PR pushes. They run in the merge queue, so a break there blocks the merge rather than every push. Each group reports through one aggregate check whose name is stable enough to be required: -| Group | Gate signal | Opt in on a PR with | +| Group | Gate check | Opt in on a PR with | |---|---|---| | Fedora RPM, Debian 13, Arch, openSUSE, Launchpad PPA, `Build Lemonade Desktop Installer` | `Packaging builds`, `Linux distro builds`, `Launchpad PPA builds` | `ci:distros` | | macOS `.dmg`, `Test CLI/Endpoints (macos-latest)`, `Test Embeddable (macOS)`, `Test .dmg - macOS inference` | `macOS builds` | `ci:macos` | -| llama.cpp, vLLM, stable-diffusion.cpp validation | Required workflow `Validate llama.cpp protected change`; `vLLM validation`; `stable-diffusion.cpp validation` | `ci:upgrades` | +| llama.cpp, vLLM, stable-diffusion.cpp validation | `llama.cpp validation`, `vLLM validation`, `stable-diffusion.cpp validation` | `ci:upgrades` | | `Test .exe - *` and `Test .deb - *` inference suites on the self-hosted rigs | `Inference backend tests` | `ci:backends` | -The base-branch llama.cpp wrapper reports `llama.cpp authorized validation v2` for required default-event and merge-queue decisions, and `llama.cpp authorization diagnostics v2` for label and lifecycle metadata. Each check targets GitHub's synthetic test-merge commit when one is available and falls back to the immutable current head while GitHub is still computing mergeability or the PR conflicts; protected validation never executes without a test-merge commit. Each provides exact-candidate diagnostics and stale-run reconciliation, while the wrapper's own conclusion fails whenever policy resolution, authorization, validation, or reconciliation fails. Changes confined to `docs/`, `src/app/`, or `src/web-app/` receive a neutral success; `src/app/src/renderer/utils/toolDefinitions.json` is an exception because it is copied into the server resources. Every other path is treated as protected, including unknown future paths, because the validation builds or executes candidate-controlled inputs. Protected changes use the authorization and rerun procedure below. Shared CMake or runtime changes may need both `ci:upgrades` and the other matching surface label. - -Dependabot-authored `pull_request_target` runs receive a read-only `GITHUB_TOKEN`, including when a maintainer re-runs them. These runs rely solely on the organization required-workflow conclusion and intentionally omit custom diagnostic Checks API runs. Authorization-required and awaiting-rerun states still fail the required workflow, while an authorized same-run rerun can pass. - -Do not require `llama.cpp validation`; that is the reusable workflow's internal aggregate job and does not prove that a PR was authorized. Do **not** require `llama.cpp authorized validation v2` during bootstrap, and do not require `llama.cpp authorization diagnostics v2` either. Never configure `llama.cpp authorized validation v2` as a required status check, including with **GitHub Actions** selected as its expected source: the shared `github-actions` App can also report a same-name check from candidate-controlled workflow code, so its App identity does not identify the workflow or event that produced the result. Both custom checks are diagnostic only. - -The validation-infrastructure change that introduces the trusted wrapper and policy helper must land on the default branch as a separate bootstrap merge before an organization required-workflow ruleset is created or activated; a rule cannot enforce a workflow that is not yet on the default branch. Enforcement therefore starts with later protected pull requests, not with the bootstrap pull request itself. Before relying on that enforcement, create the `ci:upgrades` label and provision ephemeral protected self-hosted runners that are discarded after every job. - -The current `.github/CODEOWNERS` covers only agent configuration files. It does not cover the trusted llama.cpp workflows, policy and publication scripts, validation runner, validator, or trusted test helpers and fixtures. A maintainer must add explicit entries for every trusted validation and publication surface, using real eligible owners, and a repository administrator must enforce code-owner review on the protected branch. Do not invent an owner in repository code. Missing exact CODEOWNERS coverage or missing enforced code-owner review is an external activation **NO-GO**. - -Scheduled publication has two additional external activation blockers. Add the repository secret `LLAMACPP_UPDATE_TOKEN` containing a separately provisioned, repository-scoped fine-grained PAT with **Contents: read and write** and **Pull requests: read and write**. Add the repository variable `LLAMACPP_UPDATE_ACTOR` containing that PAT's exact `viewer.login`. The current workflow does not support storing a GitHub App installation token in this secret: installation tokens expire after one hour, so App support would have to mint a fresh token inside each publication job. The workflow and publisher both fail before mutation when either value is absent or the authenticated actor differs. They use only this credential for checkout, release and pull-request APIs, push, pull-request creation, and draft promotion. Never substitute the repository's `GITHUB_TOKEN` or enable **Allow GitHub Actions to create and approve pull requests** as a workaround: that token does not trigger the required workflow for the pull request it creates. The publisher deliberately creates a draft first so its `opened` event records the expected initial failed required-workflow run, then marks it ready for a maintainer to authorize with the procedure below. Provisioning and rotation of the dedicated credential cannot be completed by repository code, so missing secret or variable configuration is an activation **NO-GO**. - -The validation runner uses Linux child-subreaper adoption plus process groups. On Windows it starts a trusted gate wrapper, assigns the wrapper to a kill-on-close Job Object before releasing candidate execution, and relies on Job inheritance for candidate descendants. macOS has no equivalent kernel API that can retain a fast double-fork after it creates a new session, so Darwin validation fails closed by default. The trusted core workflow may pass `--allow-darwin-github-hosted-ephemeral-runner` only for its static `macos-latest` Metal lane when GitHub reports `runner.environment == 'github-hosted'`; for that lane, disposal of the GitHub-hosted VM is the final containment boundary. Ephemeral workers remain part of the Windows trust boundary because process containment is not an adversarial sandbox. - -For protected validation, the base-branch checkout supplies the setup and cleanup actions, Python runner, validator, fixtures, and every imported `test.utils` helper. The candidate supplies only its build inputs, built `lemond` and adjacent build resources (including `defaults.json`), plus its base model catalog before the trusted overlay is applied. This prevents a pull request from substituting validation code, but it does **not** make candidate native code an adversarial sandbox: candidate build scripts and `lemond` execute as the runner's OS user and can access anything that user can. Jobs that execute protected candidate code must therefore use disposable GitHub-hosted workers or single-job ephemeral protected self-hosted workers, expose no repository secrets, and grant only a read-only repository token. The protected GitHub-hosted `macos-latest` lane receives no Hugging Face credential and runs no artifact upload, cache upload, or other token-bearing step after candidate runtime begins. - -Before K2 promotion, all three pinned release repositories must provide immutable K2-capable backend releases. The Lemonade-controlled publishers must enable release immutability. Because the current `ggml-org/llama.cpp` b-release history is not immutable, upstream must enable immutable releases or Lemonade must use a controlled immutable mirror and revise the trust contract accordingly. The `lemonade-sdk/llama.cpp` publisher must verify its complete CUDA matrix before making a release immutable, including all three `sm_121` artifacts: Windows x64 plus Ubuntu x64 and arm64. - -The `lemonade-sdk/llamacpp-rocm` publisher must emit the full advertised nightly matrix and immutable source sidecars that bind each exact asset to its concrete ROCm build targets; the `gfx103X` assets must cover `gfx1033`, `gfx1035`, and `gfx1036`. In particular, it currently needs Windows `gfx1152` plus Ubuntu `gfx1152` and `gfx942` artifacts. The scheduled updater intentionally keeps ROCm nightly ineligible while an asset or required build-target attestation is absent; do not weaken the matrix to make an incomplete release pass. Land and validate the matching managed pins plus validation-only model catalog (or temporarily phase K2 out of the gate). - -Only after those prerequisites pass may an LLM360 organization owner create an organization-level branch ruleset for `LLM360/lemonade`. Target the protected `main` branch and add **Require workflows to pass before merging** with source repository `LLM360/lemonade`, source branch `main`, and workflow `.github/workflows/validate_llamacpp_pr.yml`. That workflow declares the supported `pull_request_target` and `merge_group` events, and its failing reconciliation job makes a policy failure fail the required workflow itself. Do not substitute a required status-check name for this workflow-bound rule. See GitHub's documentation for [organization ruleset setup](https://docs.github.com/en/enterprise-cloud@latest/organizations/managing-organization-settings/creating-rulesets-for-repositories-in-your-organization) and [workflow-rule event semantics](https://docs.github.com/en/enterprise-cloud@latest/repositories/configuring-branches-and-merges-in-your-repository/managing-rulesets/available-rules-for-rulesets#require-workflows-to-pass-before-merging). - -GitHub ignores configured activity filters for a ruleset workflow. It starts required `pull_request_target` runs only for `opened`, `synchronize`, and `reopened`. Thus, an authorization-label run cannot satisfy the workflow rule. Use this procedure for a protected PR: - -1. Find the newest failed **Validate llama.cpp protected change** run that GitHub created for `opened`, `synchronize`, or `reopened`. Do not use a run that a label or metadata change created. -2. If `ci:upgrades` is present, remove it. Apply `ci:upgrades` strictly after the original attempt's recorded run start time, not its queued creation time. A label applied while the run was queued is stale. The maintainer who applies the label must have current write permission. -3. In the selected run, select **Re-run all jobs**. The same maintainer must start the rerun. Do not select **Re-run failed jobs**, because that option does not rerun the successful policy job. -4. If the head, base branch, base commit, merge commit, or authorization actor changes, or a newer `opened`, `synchronize`, or `reopened` run starts, stop. Use the newest failed default-event run and repeat the procedure. - -Metadata diagnostic runs do not supersede the selected required default-event run. `llama.cpp authorization diagnostics v2` uses a distinct visible name, so a red metadata result does not conflict with a recovered required result after the label is reapplied and the selected run is rerun. The `unlabeled` and `labeled` runs produced by step 2 are expected. If authorization is removed later, apply a fresh label and rerun the same selected default-event run, provided no newer default-event run exists. A removal during validation makes that attempt fail its live authorization recheck. - -Removal after a completed green rerun is diagnostic only: because the workflow rule ignores `unlabeled`, that metadata run may not revoke the already-green organization workflow result. The authorization epoch applies only to reruns of one exact default-event run and candidate while the live identity remains current; do not describe it as durable merge state. Activation is **NO-GO** unless Evaluate mode proves that post-success removal revokes the required result, or an exclusive GitHub App or equivalent separately trusted status primitive is provisioned to enforce revocation. - -A base-branch retarget starts an `edited` run, which the workflow rule cannot require. After a retarget, push a new commit or close and reopen the PR to create a supported default-event run. Then, apply a new label and rerun all jobs on that run. The policy rejects attempt 1, a rerun of a label event, a label at or before the original run start time, a rerun by a different actor, and an older run after an identity or lifecycle change. - -The synthetic merge SHA proves one exact base-plus-head candidate. Before activation, require a merge queue, strict up-to-date enforcement, or a live-proven equivalent so the exact validated base-plus-head candidate is the candidate that will eventually merge. In Evaluate mode, test what happens when the default branch advances after a green rerun: the PR must become unmergeable until the new candidate passes a fresh required workflow, or a mandatory merge queue must validate the eventual merge-group candidate. Activation is **NO-GO** if a prior green result can carry over to a different base-plus-head candidate on the ordinary merge path. - -After the bootstrap merge is on the default branch, an organization owner can create the rule in **Evaluate** mode. Exercise it with separate, unmerged live fork PRs before making it **Active** with no bypass for ordinary merges. The live checks must cover a neutral docs change, an unauthorized protected change, the full authorization and rerun procedure, label removal both during and after a successful run, a subsequent head update, a default-branch advance after a green rerun, a base retarget followed by a supported retrigger, and a merge-queue entry. Do not combine validation-infrastructure bootstrap and the protected feature change that depends on it into one activation step. Ruleset activation is an external deployment blocker; repository code and a repository administrator cannot complete it. If the organization-level workflow rule cannot enforce this lifecycle, the alternative is a separately provisioned GitHub App whose identity is exclusive to this gate, not the repository's ordinary `GITHUB_TOKEN` or the shared GitHub Actions expected source. - -Every gated job is reachable from a PR by label — nothing is merge-queue-only. Apply the label when your change plausibly affects that surface (a backend version pin wants `ci:upgrades`, packaging or install-path changes want `ci:distros`, `#ifdef __APPLE__` or CMake changes want `ci:macos`, a wrapped-server or inference-path change wants `ci:backends`). For llama.cpp, complete the required-workflow rerun procedure after you apply `ci:upgrades`. Other surface labels take effect immediately without a push. Fork contributors cannot apply labels themselves — ask a maintainer (or the [Discord](https://discord.gg/5xXzkMu8Zk)) to add one. Note that `Test .dmg - macOS inference` exercises several of the same wrapped servers (llama.cpp, whisper.cpp, moonshine, kokoro) on Metal but lives in the macOS group — a change to one of those that could break on Metal wants `ci:macos` too. `Build Embeddable Lemonade (macOS)` still runs on every PR as the AppleClang compile check. +Every gated job is reachable from a PR by label — nothing is merge-queue-only. Apply the label when your change plausibly affects that surface (a backend version pin wants `ci:upgrades`, packaging or install-path changes want `ci:distros`, `#ifdef __APPLE__` or CMake changes want `ci:macos`, a wrapped-server or inference-path change wants `ci:backends`); the label takes effect immediately, without a push. Fork contributors can't apply labels themselves — ask a maintainer (or the [Discord](https://discord.gg/5xXzkMu8Zk)) to add one. Note that `Test .dmg - macOS inference` exercises several of the same wrapped servers (llama.cpp, whisper.cpp, moonshine, kokoro) on Metal but lives in the macOS group — a change to one of those that could break on Metal wants `ci:macos` too. `Build Embeddable Lemonade (macOS)` still runs on every PR as the AppleClang compile check. Two consequences worth knowing: diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index 13aa714fa3..882d52c61f 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -159,7 +159,7 @@ For example, to use your own Vulkan `llama-server` in place of Lemonade's: lemond.exe ./ REM Set the llama-server vulkan binary path - lemonade.exe config set llamacpp.vulkan_bin=C:\path\to\llama-server.exe + lemonade.exe config set "llamacpp.vulkan_bin=C:\path\to\llama-server.exe" ``` === "Linux (bash)" @@ -173,3 +173,63 @@ For example, to use your own Vulkan `llama-server` in place of Lemonade's: ``` See the `*_bin` settings in the [Configuration Guide](../guide/configuration/README.md) for the full set of customization options. + +### IFM K2-Horizon with a custom llama.cpp build + +K2-Horizon can use the existing `llamacpp` recipe with a build of IFM's +[`model/K2Horizon` branch](https://github.com/MBZUAI-IFM/llama.cpp/tree/model/K2Horizon). +No Lemonade source changes or managed-backend version updates are required. + +Build `llama-server` using the branch's [build instructions](https://github.com/MBZUAI-IFM/llama.cpp/blob/model/K2Horizon/docs/build.md) +for your accelerator. For example, on macOS with Metal: + +```bash +git clone --branch model/K2Horizon --single-branch https://github.com/MBZUAI-IFM/llama.cpp.git llama.cpp-ifm +cmake -S llama.cpp-ifm -B llama.cpp-ifm/build -DGGML_METAL=ON -DCMAKE_BUILD_TYPE=Release +cmake --build llama.cpp-ifm/build --config Release --target llama-server -j +``` + +With `lemond` running, select the matching backend and the **full executable path** +(not its directory). Keep the build's shared libraries alongside the executable. + +```bash +lemonade config set llamacpp.backend=metal "llamacpp.metal_bin=$(pwd)/llama.cpp-ifm/build/bin/llama-server" +``` + +For a Vulkan build on Windows or Linux, use `llamacpp.backend=vulkan` and +`llamacpp.vulkan_bin` instead; Windows builds normally place `llama-server.exe` +under `build/bin/Release`. The override applies to all models using that backend. +Unload any already-loaded model before switching binaries. + +Register and download a model through the existing [custom-model CLI](../guide/configuration/custom-models.md): + +```bash +lemonade pull user.K2-Horizon-0.9B --recipe llamacpp --checkpoint main IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf +``` + +For the larger models, substitute the corresponding name and checkpoint: + +| Name | Checkpoint | BF16 weight size | +|---|---|---:| +| `user.K2-Horizon-0.9B` | `IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf` | 2.16 GB | +| `user.K2-Horizon-3.7B` | `IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf` | 10.13 GB | +| `user.K2-Horizon-7B` | `IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf` | 18.01 GB | + +Context and compute buffers require additional memory. Start with a bounded context: + +```bash +curl http://localhost:8000/v1/load -H "Content-Type: application/json" \ + -d '{"model_name":"user.K2-Horizon-0.9B","ctx_size":4096}' +curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" \ + -d '{"model":"user.K2-Horizon-0.9B","messages":[{"role":"user","content":"What is 2 + 2?"}],"reasoning_effort":"high","max_tokens":1024}' +``` + +These are custom models, not a claim of support in Lemonade's managed binaries. +IFM commit `35999d101cf2233fc54f09c3c8d599da7303ce02` was used for development +validation; its medium/low reasoning modes can leak control markers. Prefer high +reasoning and the default XML tool format, and verify any other modes against your +build. K2-Horizon Uno is not covered by this example. + +To return to Lemonade's managed Metal binary, unload the model and run +`lemonade config set llamacpp.metal_bin=builtin` (use the corresponding `*_bin` +key for another backend). diff --git a/docs/guide/configuration/README.md b/docs/guide/configuration/README.md index adb8a00eef..4d42ad5e38 100644 --- a/docs/guide/configuration/README.md +++ b/docs/guide/configuration/README.md @@ -345,7 +345,7 @@ Every `*_bin` key (e.g. `llamacpp.vulkan_bin`, `whispercpp.cpu_bin`, `sdcpp.rocm | `""` | Same as `"builtin"`. | | `"latest"` | Resolve to the most-recent upstream GitHub release on first install or first status query for that backend, then install on demand. The resolved tag is recorded in `/bin///version.txt`. | | `"b8664"` / `"v1.8.2"` / etc. | A specific upstream release tag. Lemonade downloads that exact version from GitHub. | -| `"/full/path/to/backend-executable"` | The full path to an executable you built or installed. Lemonade uses this file and never downloads a replacement. The path must exist when set. | +| `"/path/to/bin"` | A directory you populated yourself (e.g. a local build). Lemonade uses the executable inside this directory and never downloads. The path must exist when set. | > Note: the `latest` setting is experimental. @@ -361,7 +361,7 @@ lemonade config set llamacpp.vulkan_bin=latest lemonade config set llamacpp.vulkan_bin=b8664 # Use your own llama.cpp build -lemonade config set llamacpp.vulkan_bin=/home/me/llama.cpp/build/bin/llama-server +lemonade config set llamacpp.vulkan_bin=/home/me/llama.cpp/build/bin # Revert to the version lemonade ships lemonade config set llamacpp.vulkan_bin=builtin diff --git a/src/cpp/include/lemon/streaming_proxy.h b/src/cpp/include/lemon/streaming_proxy.h index 23605a9bd8..851ea47533 100644 --- a/src/cpp/include/lemon/streaming_proxy.h +++ b/src/cpp/include/lemon/streaming_proxy.h @@ -1,8 +1,7 @@ #pragma once -#include -#include #include +#include #include #include #include "utils/http_client.h" @@ -15,11 +14,6 @@ using json = nlohmann::json; class StreamingProxy { public: - static constexpr std::size_t kMaxSseFrameBytes = 1024 * 1024; - - using SseFrameTransform = - std::function; - struct TelemetryData { int input_tokens = 0; int output_tokens = 0; @@ -60,21 +54,8 @@ class StreamingProxy { httplib::DataSink& sink, std::function on_complete = nullptr, long timeout_seconds = 300, - std::function on_backend_progress = nullptr, - long heartbeat_interval_ms = 1000, - std::function on_frame = nullptr - ); - - static void forward_transformed_sse_stream( - const std::string& backend_url, - const std::string& request_body, - httplib::DataSink& sink, - SseFrameTransform frame_transform, - std::function on_complete = nullptr, - long timeout_seconds = 300, - std::function on_backend_progress = nullptr, - long heartbeat_interval_ms = 1000, - std::function on_frame = nullptr + std::function on_chunk = nullptr, + long heartbeat_interval_ms = 1000 ); static void forward_byte_stream( @@ -97,17 +78,6 @@ class StreamingProxy { static void accumulate_responses_delta(const nlohmann::json& parsed, std::string& accumulated_text); private: - static void forward_sse_stream_impl( - const std::string& backend_url, - const std::string& request_body, - httplib::DataSink& sink, - SseFrameTransform frame_transform, - std::function on_complete, - long timeout_seconds, - std::function on_backend_progress, - long heartbeat_interval_ms, - std::function on_frame - ); }; } // namespace lemon diff --git a/src/cpp/include/lemon/utils/process_manager.h b/src/cpp/include/lemon/utils/process_manager.h index 02eb392b4a..4cd9553124 100644 --- a/src/cpp/include/lemon/utils/process_manager.h +++ b/src/cpp/include/lemon/utils/process_manager.h @@ -1,36 +1,15 @@ #pragma once -#include -#include -#include -#include -#include #include #include +#include namespace lemon { namespace utils { -class ProcessOutputCapture { -public: - explicit ProcessOutputCapture(std::size_t max_bytes); - - void append(const char* data, std::size_t size); - void finish_reader(); - std::string read_tail(std::size_t max_bytes, int wait_timeout_ms); - -private: - const std::size_t max_bytes_; - std::mutex mutex_; - std::condition_variable completion_cv_; - std::string tail_; - int active_readers_ = 2; -}; - struct ProcessHandle { void* handle; int pid; - std::shared_ptr output_capture; }; // Returns true to continue, false to kill the process @@ -44,8 +23,7 @@ class ProcessManager { const std::string& working_dir = "", bool inherit_output = false, bool filter_health_logs = false, - const std::vector>& env_vars = {}, - std::size_t capture_output_bytes = 0); + const std::vector>& env_vars = {}); // Blocks until process exits or callback returns false (which kills the process) // Returns exit code, or -1 if killed by callback @@ -74,9 +52,7 @@ class ProcessManager { // read-only while lifecycle cleanup can reliably remove zombies. static int reap_process(ProcessHandle handle); - static std::string read_output(ProcessHandle handle, - int max_bytes = 4096, - int wait_timeout_ms = 1000); + static std::string read_output(ProcessHandle handle, int max_bytes = 4096); // Kill process forcefully and wait/close owned handles. static void kill_process(ProcessHandle handle); diff --git a/src/cpp/include/lemon/utils/process_platform.h b/src/cpp/include/lemon/utils/process_platform.h index d137804c62..406a75d865 100644 --- a/src/cpp/include/lemon/utils/process_platform.h +++ b/src/cpp/include/lemon/utils/process_platform.h @@ -1,10 +1,9 @@ #pragma once #include - -#include #include #include +#include namespace lemon::utils { @@ -20,8 +19,7 @@ class ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars, - std::shared_ptr output_capture) = 0; + const std::vector>& env_vars) = 0; virtual void terminate(ProcessHandle handle) = 0; virtual bool is_running(ProcessHandle handle) = 0; diff --git a/src/cpp/include/lemon/wrapped_server.h b/src/cpp/include/lemon/wrapped_server.h index 2d11b7a161..56b07da4ba 100644 --- a/src/cpp/include/lemon/wrapped_server.h +++ b/src/cpp/include/lemon/wrapped_server.h @@ -531,8 +531,8 @@ class WrappedServer : public ICompletionServer { } // Mark observable backend progress. Streaming proxies call this for every - // nonempty successful backend body chunk; non-streaming requests call it - // on start/finish and when the watchdog observes a healthy out-of-band probe. + // delivered chunk; non-streaming requests call it on start/finish and when + // the watchdog observes a healthy out-of-band probe. void note_backend_activity(); void set_telemetry(int input_tokens, int output_tokens, @@ -612,15 +612,6 @@ class WrappedServer : public ICompletionServer { // Common method to forward requests to the wrapped server (non-streaming) json forward_request(const std::string& endpoint, const json& request, long timeout_seconds = 0); - void forward_streaming_request_impl( - const std::string& endpoint, - const std::string& request_body, - httplib::DataSink& sink, - bool sse, - long timeout_seconds, - TelemetryCallback telemetry_callback, - StreamingProxy::SseFrameTransform frame_transform); - json forward_get_request(const std::string& endpoint, long timeout_seconds = 0); // Forward multipart form data to the wrapped server diff --git a/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp b/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp index 8457745082..14cf086515 100644 --- a/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp +++ b/src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp @@ -160,7 +160,17 @@ std::vector flm_installed_checkpoints() { int rc = lemon::utils::ProcessManager::run_command(command, output); #else std::string command = "\"" + flm_path + "\" list --filter installed --quiet --json 2>/dev/null"; - (void)lemon::utils::ProcessManager::run_command(command, output); + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) { + return installed_models; + } + + char buffer[256]; + while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { + output += buffer; + } + + pclose(pipe); #endif // Parse output: { "models": [ { "name": "modelname:tag", ... }, ... ] } @@ -230,7 +240,17 @@ std::vector flm_discover_models() { } #else std::string command = "\"" + flm_path + "\" list --json 2>/dev/null"; - (void)lemon::utils::ProcessManager::run_command(command, output); + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) { + return flm_models; + } + + char buffer[256]; + while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { + output += buffer; + } + + pclose(pipe); #endif // Parse output: { "models": [ { "name": "modelname:tag", "footprint": 1.23, ... }, ... ] } @@ -545,7 +565,17 @@ std::string flm_version() { int rc = lemon::utils::ProcessManager::run_command(command, output); #else std::string command = "\"" + flm_path + "\" version --json 2>/dev/null"; - (void)lemon::utils::ProcessManager::run_command(command, output); + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) { + return "unknown"; + } + + char buffer[256]; + while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { + output += buffer; + } + + pclose(pipe); #endif // Parse JSON output: { "version": "0.9.34" } @@ -650,7 +680,18 @@ bool run_flm_validate(const std::string& flm_path, std::string& error_message) { #ifdef _WIN32 exit_code = lemon::utils::ProcessManager::run_command(command, output); #else - exit_code = lemon::utils::ProcessManager::run_command(command, output); + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) { + error_message = "Failed to execute " + flm_exe; + return false; + } + + char buffer[1024]; + while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { + output += buffer; + } + + exit_code = pclose(pipe); if (exit_code != -1) { exit_code = WEXITSTATUS(exit_code); } diff --git a/src/cpp/server/backends/llamacpp/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp/llamacpp_server.cpp index 4bb03e6ced..0f8e021f3a 100644 --- a/src/cpp/server/backends/llamacpp/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp/llamacpp_server.cpp @@ -11,8 +11,8 @@ #include "lemon/model_manager.h" #include #include -#include #include +#include #include #include #include "lemon/auto_tune.h" @@ -81,108 +81,6 @@ static void push_arg(std::vector& args, push_reserved(reserved, key, aliases); } -static std::string normalize_sse_response_model_frame( - const std::string& frame, - const std::string& model_name) { - struct Line { - std::string text; - std::size_t content_end = 0; - std::size_t payload_start = 0; - bool is_data = false; - }; - - std::vector lines; - std::size_t line_start = 0; - while (line_start < frame.size()) { - const std::size_t terminator = frame.find_first_of("\r\n", line_start); - std::size_t line_end = frame.size(); - if (terminator != std::string::npos) { - line_end = terminator + 1; - if (frame[terminator] == '\r' && line_end < frame.size() && - frame[line_end] == '\n') { - ++line_end; - } - } - - Line line; - line.text = frame.substr(line_start, line_end - line_start); - line.content_end = line.text.size(); - if (line.content_end > 0 && line.text[line.content_end - 1] == '\n') { - --line.content_end; - } - if (line.content_end > 0 && line.text[line.content_end - 1] == '\r') { - --line.content_end; - } - if (line.text.rfind("data:", 0) == 0) { - line.is_data = true; - line.payload_start = std::strlen("data:"); - if (line.payload_start < line.content_end && - line.text[line.payload_start] == ' ') { - ++line.payload_start; - } - } - lines.push_back(std::move(line)); - line_start = line_end; - } - - std::string event_data; - std::size_t first_data_line = std::string::npos; - for (std::size_t index = 0; index < lines.size(); ++index) { - const Line& line = lines[index]; - if (!line.is_data) { - continue; - } - if (first_data_line == std::string::npos) { - first_data_line = index; - } else { - event_data.push_back('\n'); - } - event_data.append( - line.text, line.payload_start, - line.content_end - line.payload_start); - } - - if (first_data_line == std::string::npos || event_data.empty() || - event_data == "[DONE]") { - return frame; - } - - json payload = json::parse(event_data, nullptr, false); - if (!payload.is_object()) { - throw std::runtime_error( - "backend connection failed during SSE stream before DONE: " - "invalid OpenAI SSE data event"); - } - - bool normalized = false; - if (payload.contains("model")) { - payload["model"] = model_name; - normalized = true; - } - if (payload.contains("response") && payload["response"].is_object() && - payload["response"].contains("model")) { - payload["response"]["model"] = model_name; - normalized = true; - } - if (!normalized) { - return frame; - } - - std::string output; - for (std::size_t index = 0; index < lines.size(); ++index) { - const Line& line = lines[index]; - if (!line.is_data) { - output += line.text; - } else if (index == first_data_line) { - output.append(line.text, 0, line.payload_start); - output += payload.dump(); - output.append(line.text, line.content_end, - line.text.size() - line.content_end); - } - } - return output; -} - static std::string resolve_llamacpp_backend(const std::string& backend) { if (backend == "rocm") { // Map "rocm" to the appropriate channel based on config @@ -195,93 +93,6 @@ static std::string resolve_llamacpp_backend(const std::string& backend) { return backend; } -static std::string resolve_system_llamacpp_executable_path( - const std::string& executable) { - std::error_code ec; - fs::path executable_path = utils::path_from_utf8(executable); - if (executable_path.has_parent_path()) { - fs::path absolute_path = fs::absolute(executable_path, ec); - return ec ? executable : path_to_utf8(absolute_path); - } - -#ifdef _WIN32 - const std::string resolved = utils::find_executable_in_path(executable); - return resolved.empty() ? executable : resolved; -#else - const char* path_env = std::getenv("PATH"); - if (!path_env || path_env[0] == '\0') { - return executable; - } - - std::string path_value(path_env); - size_t start = 0; - while (start <= path_value.size()) { - const size_t end = path_value.find(':', start); - const std::string directory = path_value.substr( - start, end == std::string::npos ? std::string::npos : end - start); - if (!directory.empty()) { - fs::path candidate = fs::path(directory) / executable; - ec.clear(); - if (fs::is_regular_file(candidate, ec) && - access(candidate.c_str(), X_OK) == 0) { - fs::path absolute_path = fs::absolute(candidate, ec); - return ec ? path_to_utf8(candidate) : path_to_utf8(absolute_path); - } - } - if (end == std::string::npos) { - break; - } - start = end + 1; - } - return executable; -#endif -} - -static std::string detect_system_llamacpp_version( - const std::string& executable_path) { - constexpr std::size_t max_version_output_bytes = 4096; - ProcessHandle handle{}; - try { - handle = ProcessManager::start_process( - executable_path, {"--version"}, "", false, false, {}, - max_version_output_bytes); - const int exit_code = ProcessManager::wait_for_exit(handle, 10); - if (exit_code < 0) { - ProcessManager::kill_process(handle); - return "unknown"; - } - -#ifndef _WIN32 - handle.pid = 0; -#endif - const std::string output = ProcessManager::read_output( - handle, static_cast(max_version_output_bytes)); -#ifdef _WIN32 - ProcessManager::reap_process(handle); -#endif - handle = {}; - return llamacpp::detail::parse_system_llamacpp_version(output, - exit_code); - } catch (const std::exception&) { - if (handle.pid > 0 || handle.handle != nullptr) { - if (ProcessManager::is_running(handle)) { - ProcessManager::kill_process(handle); - } else { - ProcessManager::reap_process(handle); - } - } - return "unknown"; - } -} - -static bool custom_args_control_llamacpp_logging( - const std::vector& args) { - return std::any_of(args.begin(), args.end(), [](const std::string& arg) { - return arg == "--log-file" || arg.rfind("--log-file=", 0) == 0 || - arg == "--log-disable"; - }); -} - static bool is_llamacpp_rocm_backend(const std::string& backend) { return backend == "rocm-stable" || backend == "rocm-nightly"; } @@ -582,9 +393,6 @@ void LlamaCppServer::load(const std::string& model_name, } push_reserved(reserved_flags, "--reranking", std::vector{"--rerank"}); - bool user_controls_logging = - !utils::get_environment_variable_utf8("LLAMA_ARG_LOG_FILE").empty(); - // Validate and append custom arguments if (!llamacpp_args.empty()) { std::string validation_error = validate_custom_args(llamacpp_args, reserved_flags); @@ -596,8 +404,6 @@ void LlamaCppServer::load(const std::string& model_name, LOG(DEBUG, "LlamaCpp") << "Adding custom arguments: " << llamacpp_args << std::endl; std::vector custom_args_vec = parse_custom_args(llamacpp_args); - user_controls_logging = user_controls_logging || - custom_args_control_llamacpp_logging(custom_args_vec); args.insert(args.end(), custom_args_vec.begin(), custom_args_vec.end()); } @@ -767,51 +573,16 @@ void LlamaCppServer::load(const std::string& model_name, #endif bool inherit_llama_output = (log_level_ == "info") || is_debug(); - constexpr std::size_t max_startup_output_bytes = 64 * 1024; - const std::size_t startup_output_bytes = - llamacpp_backend == "system" && !user_controls_logging && - llamacpp::detail::identifies_k2_horizon_model( - model_name, model_info.checkpoint(), - model_info.gguf.architecture) - ? max_startup_output_bytes - : 0; - const ProcessHandle started_handle = ProcessManager::start_process( - process_executable, args, working_dir, inherit_llama_output, true, - env_vars, startup_output_bytes); - set_process_handle(started_handle, process_executable, args); + set_process_handle(ProcessManager::start_process( + process_executable, args, working_dir, inherit_llama_output, true, env_vars), + process_executable, args); if (!wait_for_ready("/health")) { const ProcessHandle handle = consume_process_handle_for_cleanup(); - const bool process_still_running = - has_process_handle(handle) && ProcessManager::is_running(handle); if (has_process_handle(handle)) { ProcessManager::stop_process(handle); } - const std::string startup_error = "llama-server failed to start"; - const bool load_cancelled = load_cancel_ && load_cancel_->load(); - const bool k2_system_failure = - startup_output_bytes > 0 && !load_cancelled && !process_still_running; - const std::string startup_output = - k2_system_failure - ? ProcessManager::read_output(started_handle, 64 * 1024) - : ""; - if (llamacpp::detail::should_report_k2_system_startup_diagnostic( - llamacpp_backend, load_cancelled, model_name, - model_info.checkpoint(), model_info.gguf.architecture, - startup_output)) { - const std::string system_executable = - resolve_system_llamacpp_executable_path(executable); - const std::string original_error = - llamacpp::detail::k2_horizon_unsupported_architecture_error( - startup_output); - throw std::runtime_error( - llamacpp::detail::k2_horizon_system_startup_error( - model_name, - system_executable, - detect_system_llamacpp_version(system_executable), - original_error)); - } - throw std::runtime_error(startup_error); + throw std::runtime_error("llama-server failed to start"); } LOG(DEBUG, "LlamaCpp") << "Model loaded on port " << get_backend_port() << std::endl; @@ -904,37 +675,16 @@ void LlamaCppServer::forward_streaming_request(const std::string& endpoint, long timeout_seconds, TelemetryCallback telemetry_callback) { std::string body = request_body; - std::string response_model; - const bool is_openai_text_endpoint = - endpoint == "/v1/chat/completions" || - endpoint == "/v1/completions" || endpoint == "/v1/responses"; - if (is_openai_text_endpoint) { + if (endpoint == "/v1/chat/completions" || endpoint == "/v1/responses") { json request = json::parse(request_body, nullptr, false); if (!request.is_discarded()) { - if (request.contains("model") && request["model"].is_string()) { - response_model = request["model"].get(); - } if (endpoint == "/v1/chat/completions") { JsonUtils::add_legacy_max_tokens_alias(request); - request = llamacpp::sanitize_tool_schema_limits(std::move(request)); - body = request.dump(); - } else if (endpoint == "/v1/responses") { - request = llamacpp::sanitize_tool_schema_limits(std::move(request)); - body = request.dump(); } + body = llamacpp::sanitize_tool_schema_limits(std::move(request)).dump(); } } - if (sse && !response_model.empty()) { - WrappedServer::forward_streaming_request_impl( - endpoint, body, sink, sse, timeout_seconds, telemetry_callback, - [response_model](const std::string& frame) { - return normalize_sse_response_model_frame( - frame, response_model); - }); - return; - } - WrappedServer::forward_streaming_request( endpoint, body, sink, sse, timeout_seconds, telemetry_callback); } @@ -951,6 +701,44 @@ std::unique_ptr create(const BackendContext& ctx) { } namespace { +std::string system_llamacpp_version() { + std::string output; + #ifdef _WIN32 + std::string command = "llama-server --version 2>NUL"; + int rc = lemon::utils::ProcessManager::run_command(command, output); + #else + FILE* pipe = popen("llama-server --version 2>/dev/null", "r"); + if (!pipe) { + return "unknown"; + } + + char buffer[256]; + if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { + output = buffer; + } + + pclose(pipe); + #endif + + // Parse version from output like "version: 3432 (e2b2a632)" or "llama.cpp version b3432" + if (!output.empty()) { + // Try to find a version number + std::regex version_regex(R"(version:\s*(\d+)|version\s+b?(\d+))"); + std::smatch match; + if (std::regex_search(output, match, version_regex)) { + for (size_t i = 1; i < match.size(); ++i) { + if (match[i].matched) { + return "b" + match[i].str(); + } + } + } + return "detected"; + } + + return "unknown"; +} + + bool is_ggml_hip_plugin_available() { #ifdef __linux__ // Allow distros/packagers that install outside the FHS paths below @@ -1090,8 +878,7 @@ class LlamaCppOps : public BackendOps { const std::string& file_version) const override { // The PATH-installed "system" llama-server has no version.txt; query it. if (backend == "system") { - return detect_system_llamacpp_version( - resolve_system_llamacpp_executable_path(descriptor.binary)); + return system_llamacpp_version(); } return file_version; } diff --git a/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h b/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h index 268edbb484..53a673e42d 100644 --- a/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h +++ b/src/cpp/server/backends/llamacpp/llamacpp_system_utils.h @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -17,115 +16,6 @@ namespace backends { namespace llamacpp { namespace detail { -inline std::string lowercase_ascii(std::string value) { - std::transform(value.begin(), value.end(), value.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return value; -} - -inline bool identifies_k2_horizon_model(const std::string& model_name, - const std::string& checkpoint, - const std::string& architecture) { - const std::string normalized_architecture = lowercase_ascii(architecture); - if (normalized_architecture == "k2-horizon" || - normalized_architecture == "k2_horizon") { - return true; - } - if (!normalized_architecture.empty()) { - return false; - } - - const auto contains_k2_horizon = [](const std::string& value) { - const std::string normalized = lowercase_ascii(value); - return normalized.find("k2-horizon") != std::string::npos || - normalized.find("k2_horizon") != std::string::npos; - }; - return contains_k2_horizon(model_name) || contains_k2_horizon(checkpoint); -} - -inline std::string k2_horizon_unsupported_architecture_error( - const std::string& startup_output) { - size_t start = 0; - while (start <= startup_output.size()) { - const size_t end = startup_output.find_first_of("\r\n", start); - std::string line = startup_output.substr( - start, end == std::string::npos ? std::string::npos : end - start); - const std::string normalized = lowercase_ascii(line); - const bool names_k2 = - normalized.find("k2-horizon") != std::string::npos || - normalized.find("k2_horizon") != std::string::npos; - if (names_k2 && - normalized.find("unknown model architecture") != std::string::npos) { - const size_t first = line.find_first_not_of(" \t"); - const size_t last = line.find_last_not_of(" \t"); - return first == std::string::npos - ? std::string() - : line.substr(first, last - first + 1); - } - if (end == std::string::npos) { - break; - } - start = end + 1; - if (startup_output[end] == '\r' && start < startup_output.size() && - startup_output[start] == '\n') { - ++start; - } - } - return ""; -} - -inline bool should_report_k2_system_startup_diagnostic( - const std::string& backend, - bool load_cancelled, - const std::string& model_name, - const std::string& checkpoint, - const std::string& architecture, - const std::string& startup_output = "") { - return backend == "system" && !load_cancelled && - identifies_k2_horizon_model(model_name, checkpoint, architecture) && - !k2_horizon_unsupported_architecture_error(startup_output).empty(); -} - -inline std::string parse_system_llamacpp_version(const std::string& output, - int exit_code = 0) { - if (exit_code != 0) { - return "unknown"; - } - - std::smatch match; - if (std::regex_search(output, match, std::regex(R"(build\s+(\d+))"))) { - return "b" + match[1].str(); - } - - if (std::regex_search( - output, - match, - std::regex(R"(version:\s*(\d+)|version\s+b?(\d+))"))) { - for (size_t i = 1; i < match.size(); ++i) { - if (match[i].matched) { - return "b" + match[i].str(); - } - } - } - return output.empty() ? "unknown" : "detected"; -} - -inline std::string k2_horizon_system_startup_error( - const std::string& model_name, - const std::string& executable_path, - const std::string& version, - const std::string& original_error) { - const std::string displayed_path = executable_path.empty() ? "unknown" : executable_path; - const std::string displayed_version = - version.empty() || version == "detected" ? "unknown" : version; - return "System llama-server failed to start while loading K2-Horizon model '" + - model_name + "' using executable '" + displayed_path + "' (version " + - displayed_version + "). Original error: " + original_error + - ". Install a K2-Horizon-capable llama.cpp build from " - "https://github.com/MBZUAI-IFM/llama.cpp/tree/model/K2Horizon and " - "ensure that executable is first on PATH, then retry."; -} - // Private, deterministic pieces of the system llama.cpp HIP lookup. The full // production availability check stays in llamacpp_server.cpp so this header is // only a narrow test seam, not a second implementation of the policy. diff --git a/src/cpp/server/mcp_client.cpp b/src/cpp/server/mcp_client.cpp index a34b0436e0..5f2ca7a46a 100644 --- a/src/cpp/server/mcp_client.cpp +++ b/src/cpp/server/mcp_client.cpp @@ -36,9 +36,6 @@ #include #include #include - #ifdef __APPLE__ - #include - #endif #include #include #include @@ -49,26 +46,6 @@ extern char** environ; namespace fs = std::filesystem; -#if defined(LEMONADE_MCP_PROCESS_TEST_HOOK) && !defined(_WIN32) -using McpProcessTestHook = void (*)(pid_t); - -static std::atomic mcp_exit_observed_hook{nullptr}; -static std::atomic mcp_group_cleanup_hook{nullptr}; -static std::atomic mcp_final_reap_hook{nullptr}; - -extern "C" void lemonade_test_set_mcp_process_hooks( - McpProcessTestHook exit_observed, McpProcessTestHook group_cleanup, - McpProcessTestHook final_reap); - -extern "C" void lemonade_test_set_mcp_process_hooks( - McpProcessTestHook exit_observed, McpProcessTestHook group_cleanup, - McpProcessTestHook final_reap) { - mcp_exit_observed_hook.store(exit_observed, std::memory_order_release); - mcp_group_cleanup_hook.store(group_cleanup, std::memory_order_release); - mcp_final_reap_hook.store(final_reap, std::memory_order_release); -} -#endif - namespace lemon { namespace { @@ -633,76 +610,42 @@ class StdioProcess { #else if (pid_ > 0) { int status = 0; - bool child_identity_retained = true; - auto wait_for_exit_without_reaping = [&](auto deadline) { - while (std::chrono::steady_clock::now() < deadline) { - siginfo_t child_info{}; - int result; - do { - result = ::waitid( - P_PID, static_cast(pid_), &child_info, - WEXITED | WNOHANG | WNOWAIT); - } while (result < 0 && errno == EINTR); - if (result < 0) { - child_identity_retained = false; - return false; - } - if (child_info.si_pid == pid_) return true; - std::this_thread::sleep_for( - std::chrono::milliseconds(25)); - } - return false; - }; - - const bool exit_observed = wait_for_exit_without_reaping( - std::chrono::steady_clock::now() + - std::chrono::seconds(1)); -#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK - if (exit_observed) { - if (const auto hook = mcp_exit_observed_hook.load( - std::memory_order_acquire)) { - hook(pid_); + bool reaped = false; + auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(1); + while (std::chrono::steady_clock::now() < deadline) { + const pid_t result = ::waitpid(pid_, &status, WNOHANG); + if (result == pid_ || (result < 0 && errno == ECHILD)) { + reaped = true; + break; } + std::this_thread::sleep_for(std::chrono::milliseconds(25)); } -#endif - if (child_identity_retained) { + if (!reaped) { ::kill(-pid_, SIGTERM); - if (exit_observed) { - std::this_thread::sleep_for( - std::chrono::milliseconds(50)); - } else { - const bool observed_after_terminate = - wait_for_exit_without_reaping( - std::chrono::steady_clock::now() + - std::chrono::seconds(1)); -#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK - if (observed_after_terminate) { - if (const auto hook = mcp_exit_observed_hook.load( - std::memory_order_acquire)) { - hook(pid_); - } + deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(1); + while (std::chrono::steady_clock::now() < deadline) { + const pid_t result = ::waitpid(pid_, &status, WNOHANG); + if (result == pid_ || + (result < 0 && errno == ECHILD)) { + reaped = true; + break; } -#else - (void)observed_after_terminate; -#endif + std::this_thread::sleep_for( + std::chrono::milliseconds(25)); } } - if (child_identity_retained) { + if (!reaped) { ::kill(-pid_, SIGKILL); -#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK - if (const auto hook = mcp_group_cleanup_hook.load( - std::memory_order_acquire)) { - hook(pid_); - } -#endif while (::waitpid(pid_, &status, 0) < 0 && errno == EINTR) { } -#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK - if (const auto hook = mcp_final_reap_hook.load( - std::memory_order_acquire)) { - hook(pid_); - } -#endif + } else { + // Clean up descendants in the process group even when the direct + // child exited after stdin was closed. + ::kill(-pid_, SIGTERM); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + ::kill(-pid_, SIGKILL); } } #endif @@ -1001,40 +944,19 @@ class StdioProcess { static int create_cloexec_pipe(int fds[2]) { #ifdef __linux__ - if (::pipe2(fds, O_CLOEXEC) != 0) return -1; + return ::pipe2(fds, O_CLOEXEC); #else if (::pipe(fds) != 0) return -1; - if (!set_close_on_exec(fds[0]) || !set_close_on_exec(fds[1])) { - const int saved_errno = errno; - ::close(fds[0]); - ::close(fds[1]); - fds[0] = fds[1] = -1; - errno = saved_errno; - return -1; + if (set_close_on_exec(fds[0]) && set_close_on_exec(fds[1])) { + return 0; } + const int saved_errno = errno; + ::close(fds[0]); + ::close(fds[1]); + fds[0] = fds[1] = -1; + errno = saved_errno; + return -1; #endif - - for (int index = 0; index < 2; ++index) { - int& fd = fds[index]; - if (fd > STDERR_FILENO) continue; - - int replacement; - do { - replacement = - ::fcntl(fd, F_DUPFD_CLOEXEC, STDERR_FILENO + 1); - } while (replacement < 0 && errno == EINTR); - if (replacement < 0) { - const int saved_errno = errno; - ::close(fds[0]); - ::close(fds[1]); - fds[0] = fds[1] = -1; - errno = saved_errno; - return -1; - } - ::close(fd); - fd = replacement; - } - return 0; } void start_posix(const McpServerConfig& config) { @@ -1065,108 +987,27 @@ class StdioProcess { int stdin_pipe[2] = {-1, -1}; int stdout_pipe[2] = {-1, -1}; int stderr_pipe[2] = {-1, -1}; -#ifndef __APPLE__ int exec_error_pipe[2] = {-1, -1}; -#endif auto close_pair = [](int pair[2]) { if (pair[0] >= 0) ::close(pair[0]); if (pair[1] >= 0) ::close(pair[1]); pair[0] = pair[1] = -1; }; - bool pipe_creation_failed = create_cloexec_pipe(stdin_pipe) != 0 || - create_cloexec_pipe(stdout_pipe) != 0 || - create_cloexec_pipe(stderr_pipe) != 0; -#ifndef __APPLE__ - pipe_creation_failed = pipe_creation_failed || - create_cloexec_pipe(exec_error_pipe) != 0; -#endif - if (pipe_creation_failed) { + if (create_cloexec_pipe(stdin_pipe) != 0 || + create_cloexec_pipe(stdout_pipe) != 0 || + create_cloexec_pipe(stderr_pipe) != 0 || + create_cloexec_pipe(exec_error_pipe) != 0) { const int error = errno; close_pair(stdin_pipe); close_pair(stdout_pipe); close_pair(stderr_pipe); -#ifndef __APPLE__ close_pair(exec_error_pipe); -#endif throw std::runtime_error( std::string("pipe creation failed: ") + std::strerror(error)); } -#ifdef __APPLE__ - posix_spawn_file_actions_t file_actions; - bool file_actions_initialized = false; - int launch_error = posix_spawn_file_actions_init(&file_actions); - if (launch_error == 0) { - file_actions_initialized = true; - launch_error = posix_spawn_file_actions_adddup2( - &file_actions, stdin_pipe[0], STDIN_FILENO); - } - if (launch_error == 0) { - launch_error = posix_spawn_file_actions_adddup2( - &file_actions, stdout_pipe[1], STDOUT_FILENO); - } - if (launch_error == 0) { - launch_error = posix_spawn_file_actions_adddup2( - &file_actions, stderr_pipe[1], STDERR_FILENO); - } - for (int fd : {stdin_pipe[0], stdin_pipe[1], stdout_pipe[0], - stdout_pipe[1], stderr_pipe[0], stderr_pipe[1]}) { - if (launch_error == 0) { - launch_error = - posix_spawn_file_actions_addclose(&file_actions, fd); - } - } - if (launch_error == 0 && !config.working_dir.empty()) { -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdeprecated-declarations" - launch_error = posix_spawn_file_actions_addchdir_np( - &file_actions, config.working_dir.c_str()); -#pragma clang diagnostic pop - } - - posix_spawnattr_t attributes; - bool attributes_initialized = false; - if (launch_error == 0) { - launch_error = posix_spawnattr_init(&attributes); - attributes_initialized = launch_error == 0; - } - if (launch_error == 0) { - launch_error = posix_spawnattr_setpgroup(&attributes, 0); - } - if (launch_error == 0) { - launch_error = posix_spawnattr_setflags( - &attributes, POSIX_SPAWN_CLOEXEC_DEFAULT | - POSIX_SPAWN_SETPGROUP); - } - - pid_t child = 0; - if (launch_error == 0) { - launch_error = posix_spawn( - &child, executable.c_str(), &file_actions, &attributes, - argv.data(), envp.data()); - } - if (attributes_initialized) { - posix_spawnattr_destroy(&attributes); - } - if (file_actions_initialized) { - posix_spawn_file_actions_destroy(&file_actions); - } - - if (launch_error != 0) { - close_pair(stdin_pipe); - close_pair(stdout_pipe); - close_pair(stderr_pipe); - throw std::runtime_error( - "Failed to launch MCP server '" + config.command + "': " + - std::strerror(launch_error)); - } - - ::close(stdin_pipe[0]); - ::close(stdout_pipe[1]); - ::close(stderr_pipe[1]); -#else const pid_t child = ::fork(); if (child < 0) { const int error = errno; @@ -1248,7 +1089,6 @@ class StdioProcess { std::string("Failed to verify MCP exec: ") + std::strerror(error)); } -#endif pid_ = child; stdin_write_ = stdin_pipe[1]; diff --git a/src/cpp/server/streaming_proxy.cpp b/src/cpp/server/streaming_proxy.cpp index 1b492fd334..17c0e432bd 100644 --- a/src/cpp/server/streaming_proxy.cpp +++ b/src/cpp/server/streaming_proxy.cpp @@ -1,13 +1,9 @@ #include "lemon/streaming_proxy.h" -#include +#include +#include #include #include -#include -#include -#include -#include #include -#include #include #include @@ -93,209 +89,6 @@ bool is_gpu_hang_or_compute_error(const std::string& message) { lowered.find("gpu hang") != std::string::npos; } -std::optional sse_event_data(const std::string& frame) { - std::string event_data; - bool has_data_field = false; - std::size_t line_start = 0; - while (line_start < frame.size()) { - const std::size_t terminator = - frame.find_first_of("\r\n", line_start); - const std::size_t content_end = - terminator == std::string::npos ? frame.size() : terminator; - const std::size_t content_length = content_end - line_start; - const bool empty_data_field = - content_length == 4 && - frame.compare(line_start, content_length, "data") == 0; - const bool data_field = - content_length >= 5 && frame.compare(line_start, 5, "data:") == 0; - if (empty_data_field || data_field) { - if (has_data_field) { - event_data.push_back('\n'); - } - if (data_field) { - std::size_t payload_start = line_start + 5; - if (payload_start < content_end && - frame[payload_start] == ' ') { - ++payload_start; - } - event_data.append( - frame, payload_start, content_end - payload_start); - } - has_data_field = true; - } - - if (terminator == std::string::npos) { - break; - } - line_start = terminator + 1; - if (frame[terminator] == '\r' && line_start < frame.size() && - frame[line_start] == '\n') { - ++line_start; - } - } - - if (!has_data_field) { - return std::nullopt; - } - return event_data; -} - -bool is_sse_done_frame(const std::string& frame) { - const std::optional event_data = sse_event_data(frame); - return event_data.has_value() && *event_data == "[DONE]"; -} - -class SseFrameBuffer { -public: - explicit SseFrameBuffer(std::size_t max_frame_bytes) - : max_frame_bytes_(max_frame_bytes) {} - - template - bool append(const char* data, - std::size_t length, - FrameCallback&& frame_callback, - LineCallback&& line_callback) { - std::size_t offset = 0; - while (offset < length) { - const std::size_t buffer_limit = max_frame_bytes_ + 1; - const std::size_t available = buffer_limit - buffer_.size(); - const std::size_t append_size = - std::min(length - offset, available); - buffer_.append(data + offset, append_size); - offset += append_size; - if (!drain(false, frame_callback, line_callback)) { - return false; - } - } - return true; - } - - template - bool finish(FrameCallback&& frame_callback, - LineCallback&& line_callback) { - return drain(true, frame_callback, line_callback); - } - - bool empty() const { - return buffer_.empty(); - } - - void clear() { - buffer_.clear(); - scan_position_ = 0; - line_start_ = 0; - } - -private: - bool prepare_stream_start(bool end_of_stream) { - if (!stream_start_) { - return true; - } - - static constexpr char utf8_bom[] = "\xEF\xBB\xBF"; - constexpr std::size_t utf8_bom_size = sizeof(utf8_bom) - 1; - const std::size_t comparison_size = - std::min(buffer_.size(), utf8_bom_size); - if (buffer_.compare( - 0, comparison_size, utf8_bom, comparison_size) != 0) { - stream_start_ = false; - return true; - } - if (buffer_.size() < utf8_bom_size && !end_of_stream) { - return false; - } - if (buffer_.size() >= utf8_bom_size) { - if (utf8_bom_removed_) { - throw std::runtime_error( - "backend connection failed during SSE stream before DONE: " - "multiple leading UTF-8 BOMs"); - } - buffer_.erase(0, utf8_bom_size); - utf8_bom_removed_ = true; - return prepare_stream_start(end_of_stream); - } - stream_start_ = false; - return true; - } - - template - bool drain(bool end_of_stream, - FrameCallback&& frame_callback, - LineCallback&& line_callback) { - if (!prepare_stream_start(end_of_stream)) { - return true; - } - - std::size_t consumed = 0; - while (scan_position_ < buffer_.size()) { - const std::size_t terminator = - buffer_.find_first_of("\r\n", scan_position_); - if (terminator == std::string::npos) { - scan_position_ = buffer_.size(); - break; - } - - std::size_t terminator_length = 1; - if (buffer_[terminator] == '\r') { - if (terminator + 1 == buffer_.size() && !end_of_stream) { - scan_position_ = terminator; - break; - } - if (terminator + 1 < buffer_.size() && - buffer_[terminator + 1] == '\n') { - terminator_length = 2; - } - } - - const std::size_t terminated_end = - terminator + terminator_length; - if (terminated_end - consumed > max_frame_bytes_) { - throw_frame_too_large(); - } - - line_callback( - buffer_.substr(line_start_, terminator - line_start_)); - const bool frame_complete = terminator == line_start_; - scan_position_ = terminated_end; - line_start_ = terminated_end; - if (!frame_complete) { - continue; - } - - if (!frame_callback( - buffer_.substr(consumed, terminated_end - consumed))) { - clear(); - return false; - } - consumed = terminated_end; - } - - if (buffer_.size() - consumed > max_frame_bytes_) { - throw_frame_too_large(); - } - if (consumed > 0) { - buffer_.erase(0, consumed); - scan_position_ -= consumed; - line_start_ -= consumed; - } - return true; - } - - [[noreturn]] void throw_frame_too_large() const { - throw std::runtime_error( - "backend connection failed during SSE stream before DONE: " - "SSE frame exceeds the maximum size of " + - std::to_string(max_frame_bytes_) + " bytes"); - } - - const std::size_t max_frame_bytes_; - std::string buffer_; - std::size_t scan_position_ = 0; - std::size_t line_start_ = 0; - bool stream_start_ = true; - bool utf8_bom_removed_ = false; -}; - } // namespace @@ -305,42 +98,8 @@ void StreamingProxy::forward_sse_stream( httplib::DataSink& sink, std::function on_complete, long timeout_seconds, - std::function on_backend_progress, - long heartbeat_interval_ms, - std::function on_frame) { - forward_sse_stream_impl( - backend_url, request_body, sink, nullptr, std::move(on_complete), - timeout_seconds, std::move(on_backend_progress), heartbeat_interval_ms, - std::move(on_frame)); -} - -void StreamingProxy::forward_transformed_sse_stream( - const std::string& backend_url, - const std::string& request_body, - httplib::DataSink& sink, - SseFrameTransform frame_transform, - std::function on_complete, - long timeout_seconds, - std::function on_backend_progress, - long heartbeat_interval_ms, - std::function on_frame) { - forward_sse_stream_impl( - backend_url, request_body, sink, std::move(frame_transform), - std::move(on_complete), timeout_seconds, - std::move(on_backend_progress), heartbeat_interval_ms, - std::move(on_frame)); -} - -void StreamingProxy::forward_sse_stream_impl( - const std::string& backend_url, - const std::string& request_body, - httplib::DataSink& sink, - SseFrameTransform frame_transform, - std::function on_complete, - long timeout_seconds, - std::function on_backend_progress, - long heartbeat_interval_ms, - std::function on_frame) { + std::function on_chunk, + long heartbeat_interval_ms) { TelemetryData telemetry; try { @@ -349,14 +108,13 @@ void StreamingProxy::forward_sse_stream_impl( telemetry.model_name = req_json["model"].get(); } } catch (...) {} - SseFrameBuffer frame_buffer(kMaxSseFrameBytes); - std::exception_ptr frame_processing_error; + std::string line_buffer; bool stream_error = false; bool has_done_marker = false; bool has_first_token = false; double time_to_first_token = 0.0; const auto start_time = std::chrono::steady_clock::now(); - auto last_downstream_write_time = start_time; + auto last_activity_time = start_time; int backend_status = 200; std::string error_body; @@ -377,54 +135,13 @@ void StreamingProxy::forward_sse_stream_impl( } }; - auto process_frame = [&](const std::string& source_frame) { - const bool is_done_frame = is_sse_done_frame(source_frame); - has_done_marker = has_done_marker || is_done_frame; - std::string frame = frame_transform - ? frame_transform(source_frame) - : source_frame; - if (frame.empty()) { - return true; - } - if (!sink.write(frame.data(), frame.size())) { - return false; - } - last_downstream_write_time = std::chrono::steady_clock::now(); - - if (on_frame) { - on_frame(); - } - if (!has_first_token && - frame.find("data: ") != std::string::npos) { - has_first_token = true; - time_to_first_token = std::chrono::duration( - std::chrono::steady_clock::now() - start_time) - .count(); - } - return true; - }; - - auto process_frame_bytes = [&](const char* data, - std::size_t length, - bool end_of_stream) { - try { - if (end_of_stream) { - return frame_buffer.finish(process_frame, process_line); - } - return frame_buffer.append( - data, length, process_frame, process_line); - } catch (...) { - frame_processing_error = std::current_exception(); - frame_buffer.clear(); - return false; - } - }; - utils::HttpResponse result = utils::HttpClient::post_stream( backend_url, request_body, - [&backend_status, &error_body, &on_backend_progress, - &process_frame_bytes](const char* data, size_t length) { + [&sink, &line_buffer, &has_done_marker, &has_first_token, &time_to_first_token, + &start_time, &last_activity_time, &on_chunk, &process_line, &backend_status, &error_body](const char* data, size_t length) { + last_activity_time = std::chrono::steady_clock::now(); + if (backend_status != 200) { if (error_body.size() < max_error_body) { error_body.append(data, std::min(length, max_error_body - error_body.size())); @@ -432,18 +149,35 @@ void StreamingProxy::forward_sse_stream_impl( return true; } - if (length > 0 && on_backend_progress) { - on_backend_progress(); + if (on_chunk) { + on_chunk(); } - return process_frame_bytes(data, length, false); + line_buffer.append(data, length); + process_sse_lines(line_buffer, process_line); + + std::string chunk(data, length); + if (!has_first_token && chunk.find("data: ") != std::string::npos) { + has_first_token = true; + time_to_first_token = std::chrono::duration( + std::chrono::steady_clock::now() - start_time).count(); + } + + if (chunk.find("data: [DONE]") != std::string::npos) { + has_done_marker = true; + } + + if (!sink.write(data, length)) { + return false; + } + + return true; }, {}, timeout_seconds, [&backend_status](int status) { backend_status = status; }, utils::HttpSecurityPolicy::TrustedLoopback, - [&sink, &last_downstream_write_time, &backend_status, - heartbeat_interval_ms]() { + [&sink, &last_activity_time, &backend_status, heartbeat_interval_ms]() { if (sink.is_writable && !sink.is_writable()) { return true; } @@ -451,13 +185,13 @@ void StreamingProxy::forward_sse_stream_impl( if (heartbeat_interval_ms > 0 && backend_status == 200) { const auto now = std::chrono::steady_clock::now(); const auto elapsed = std::chrono::duration_cast( - now - last_downstream_write_time).count(); + now - last_activity_time).count(); if (elapsed >= heartbeat_interval_ms) { static constexpr const char* heartbeat = ": ping\n\n"; if (!sink.write(heartbeat, std::strlen(heartbeat))) { return true; } - last_downstream_write_time = now; + last_activity_time = now; } } @@ -465,35 +199,13 @@ void StreamingProxy::forward_sse_stream_impl( } ); - const bool may_have_final_frame = - result.curl_code == CURLE_OK || - result.curl_code == CURLE_PARTIAL_FILE || - result.curl_code == CURLE_RECV_ERROR; - const bool eof_sink_rejected = - backend_status == 200 && may_have_final_frame && - !process_frame_bytes(nullptr, 0, true); - const bool has_incomplete_frame = !frame_buffer.empty(); - frame_buffer.clear(); - if (frame_processing_error) { - std::rethrow_exception(frame_processing_error); - } - if (result.curl_code == CURLE_OK && backend_status == 200 && - has_incomplete_frame) { - throw std::runtime_error( - "backend connection failed during SSE stream before DONE: " - "incomplete final event"); - } - const bool client_disconnected = result.curl_code == CURLE_WRITE_ERROR || - result.curl_code == CURLE_ABORTED_BY_CALLBACK || eof_sink_rejected; + result.curl_code == CURLE_ABORTED_BY_CALLBACK; const bool transport_interrupted = result.curl_code == CURLE_PARTIAL_FILE || result.curl_code == CURLE_RECV_ERROR; - if (eof_sink_rejected && result.curl_code == CURLE_OK) { - stream_error = true; - telemetry.error_message = "Client disconnected during stream"; - } else if (result.curl_code != CURLE_OK) { + if (result.curl_code != CURLE_OK) { if (client_disconnected) { stream_error = true; LOG(WARNING, "StreamingProxy") << "Client disconnected during SSE stream (CURL error: " << result.curl_error << ")" << std::endl; @@ -567,6 +279,13 @@ void StreamingProxy::forward_sse_stream_impl( LOG(INFO, "Server") << "Streaming completed - 200 OK" << std::endl; + if (!line_buffer.empty()) { + if (line_buffer.back() == '\r') { + line_buffer.pop_back(); + } + process_line(line_buffer); + } + if (telemetry.time_to_first_token <= 0.0) { telemetry.time_to_first_token = time_to_first_token; } diff --git a/src/cpp/server/system_info.cpp b/src/cpp/server/system_info.cpp index 9c4a12042b..b14b65f04b 100644 --- a/src/cpp/server/system_info.cpp +++ b/src/cpp/server/system_info.cpp @@ -2398,9 +2398,12 @@ static std::vector query_nvidia_smi() { " --format=csv,noheader,nounits 2>/dev/null"; for (const char* smi : {"nvidia-smi", "/usr/bin/nvidia-smi"}) { std::string cmd = std::string(smi) + smi_query; + FILE* pipe = popen(cmd.c_str(), "r"); + if (!pipe) continue; std::string candidate; - const int rc = lemon::utils::ProcessManager::run_command( - cmd, candidate, 10); + char buffer[512]; + while (fgets(buffer, sizeof(buffer), pipe) != nullptr) candidate += buffer; + int rc = pclose(pipe); if (rc == 0 && !candidate.empty()) { output = candidate; break; @@ -3052,7 +3055,7 @@ CPUInfo LinuxSystemInfo::get_cpu_device() { cpu.available = false; // Execute lscpu command - FILE* pipe = popen("lscpu 2>/dev/null", "re"); + FILE* pipe = popen("lscpu 2>/dev/null", "r"); if (!pipe) { cpu.error = "Failed to execute lscpu command"; return cpu; @@ -3265,7 +3268,7 @@ std::vector LinuxSystemInfo::get_nvidia_gpu_devices() { } // Fallback: lspci (for systems where nvidia-smi is unavailable) - FILE* pipe = popen("lspci 2>/dev/null | grep -iE 'vga|3d|display'", "re"); + FILE* pipe = popen("lspci 2>/dev/null | grep -iE 'vga|3d|display'", "r"); if (!pipe) { GPUInfo gpu; gpu.available = false; @@ -3516,7 +3519,7 @@ std::vector LinuxSystemInfo::detect_amd_gpus(const std::string& gpu_typ std::string LinuxSystemInfo::get_nvidia_driver_version() { // Try nvidia-smi first - FILE* pipe = popen("nvidia-smi --query-gpu=driver_version --format=csv,noheader,nounits 2>/dev/null", "re"); + FILE* pipe = popen("nvidia-smi --query-gpu=driver_version --format=csv,noheader,nounits 2>/dev/null", "r"); if (pipe) { char buffer[128]; if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { @@ -3565,7 +3568,7 @@ std::string LinuxSystemInfo::get_nvidia_driver_version() { } double LinuxSystemInfo::get_nvidia_vram() { - FILE* pipe = popen("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null", "re"); + FILE* pipe = popen("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null", "r"); if (!pipe) { return 0.0; } @@ -3701,7 +3704,7 @@ std::string LinuxSystemInfo::get_os_version() { } std::string LinuxSystemInfo::get_processor_name() { - FILE* pipe = popen("lscpu 2>/dev/null", "re"); + FILE* pipe = popen("lscpu 2>/dev/null", "r"); if (!pipe) { return "ERROR - Failed to execute lscpu"; } @@ -3852,17 +3855,20 @@ std::string MacOSSystemInfo::get_os_version() { std::string result = "macOS"; // Get macOS product version (e.g., "14.3.1") - std::string version; - if (lemon::utils::ProcessManager::run_command( - "sw_vers -productVersion 2>/dev/null", version, 5) == 0) { - // Trim trailing newline - while (!version.empty() && - (version.back() == '\n' || version.back() == '\r')) { - version.pop_back(); - } - if (!version.empty()) { - result += " " + version; + FILE* pipe = popen("sw_vers -productVersion 2>/dev/null", "r"); + if (pipe) { + char buffer[128]; + if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { + std::string version = buffer; + // Trim trailing newline + while (!version.empty() && (version.back() == '\n' || version.back() == '\r')) { + version.pop_back(); + } + if (!version.empty()) { + result += " " + version; + } } + pclose(pipe); } // Append Darwin kernel version diff --git a/src/cpp/server/utils/platform/process_linux.cpp b/src/cpp/server/utils/platform/process_linux.cpp index b1eb392492..8827cfe62c 100644 --- a/src/cpp/server/utils/platform/process_linux.cpp +++ b/src/cpp/server/utils/platform/process_linux.cpp @@ -26,10 +26,6 @@ #include #endif -#ifdef LEMONADE_PROCESS_TEST_HOOK -extern "C" void lemonade_test_run_command_pipe_created(int fd); -#endif - namespace lemon::utils { // Helper functions @@ -59,95 +55,6 @@ static void log_process_line(const std::string& line) { } } -static void close_pipe(int pipe_fds[2]) { - for (int i = 0; i < 2; ++i) { - if (pipe_fds[i] >= 0) { - close(pipe_fds[i]); - pipe_fds[i] = -1; - } - } -} - -static bool create_pipe_above_standard_streams(int pipe_fds[2]) { - int pipe_result; - do { - pipe_result = pipe2(pipe_fds, O_CLOEXEC); - } while (pipe_result < 0 && errno == EINTR); - if (pipe_result < 0) { - return false; - } - - for (int i = 0; i < 2; ++i) { - if (pipe_fds[i] > STDERR_FILENO) { - continue; - } - - int replacement; - do { - replacement = - fcntl(pipe_fds[i], F_DUPFD_CLOEXEC, STDERR_FILENO + 1); - } while (replacement < 0 && errno == EINTR); - if (replacement < 0) { - close_pipe(pipe_fds); - return false; - } - close(pipe_fds[i]); - pipe_fds[i] = replacement; - } - return true; -} - -static void read_process_output( - int fd, - bool log_output, - const std::shared_ptr& output_capture) { - char buffer[4096]; - std::string line_buffer; - ssize_t bytes_read; - - while ((bytes_read = read(fd, buffer, sizeof(buffer))) > 0) { - if (output_capture) { - output_capture->append(buffer, static_cast(bytes_read)); - } - if (!log_output) { - continue; - } - - line_buffer.append(buffer, static_cast(bytes_read)); - size_t pos; - while ((pos = line_buffer.find('\n')) != std::string::npos) { - std::string line = line_buffer.substr(0, pos); - line_buffer.erase(0, pos + 1); - log_process_line(line); - } - } - - if (log_output && !line_buffer.empty()) { - log_process_line(line_buffer); - } - close(fd); - if (output_capture) { - output_capture->finish_reader(); - } -} - -static void start_process_output_reader( - int fd, - bool log_output, - const std::shared_ptr& output_capture) { - try { - std::thread(read_process_output, fd, log_output, output_capture).detach(); - } catch (const std::exception& error) { - close(fd); - if (output_capture) { - output_capture->finish_reader(); - } - LOG(ERROR, "ProcessManager") - << "Failed to start process output reader: " << error.what() - << std::endl; - } -} - #ifdef HAVE_LIBCAP static void preserve_capabilities_for_exec() { cap_t caps = cap_get_proc(); @@ -194,8 +101,7 @@ class LinuxProcessPlatform : public ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars, - std::shared_ptr output_capture) override; + const std::vector>& env_vars) override; void terminate(ProcessHandle handle) override; bool is_running(ProcessHandle handle) override; @@ -223,7 +129,7 @@ class LinuxProcessPlatform : public ProcessPlatform { const std::vector& args, const std::string& working_dir, bool inherit_output, - bool redirect_output, + bool filter_health_logs, const std::vector>& env_vars, int stdout_pipe[2], int stderr_pipe[2]); @@ -235,7 +141,7 @@ pid_t LinuxProcessPlatform::spawn_process( const std::vector& args, const std::string& working_dir, bool inherit_output, - bool redirect_output, + bool filter_health_logs, const std::vector>& env_vars, int stdout_pipe[2], int stderr_pipe[2]) { @@ -259,7 +165,8 @@ pid_t LinuxProcessPlatform::spawn_process( setenv(env_pair.first.c_str(), env_pair.second.c_str(), 1); } - if (redirect_output) { + // Redirect stdout/stderr to pipes if filtering + if (inherit_output && filter_health_logs) { close(stdout_pipe[0]); close(stderr_pipe[0]); dup2(stdout_pipe[1], STDOUT_FILENO); @@ -303,25 +210,18 @@ ProcessHandle LinuxProcessPlatform::spawn( const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars, - std::shared_ptr output_capture) { + const std::vector>& env_vars) { ProcessHandle handle; handle.handle = nullptr; handle.pid = 0; - handle.output_capture = output_capture; int stdout_pipe[2] = {-1, -1}; int stderr_pipe[2] = {-1, -1}; - const bool redirect_output = - (inherit_output && filter_health_logs) || output_capture != nullptr; - if (redirect_output) { - if (!create_pipe_above_standard_streams(stdout_pipe)) { - throw std::runtime_error("Failed to create pipes for output filtering"); - } - if (!create_pipe_above_standard_streams(stderr_pipe)) { - close_pipe(stdout_pipe); + // Create pipes for filtering if requested + if (inherit_output && filter_health_logs) { + if (pipe(stdout_pipe) < 0 || pipe(stderr_pipe) < 0) { throw std::runtime_error("Failed to create pipes for output filtering"); } } @@ -338,17 +238,8 @@ ProcessHandle LinuxProcessPlatform::spawn( } } - pid_t pid; - try { - pid = spawn_process(executable, args, working_dir, inherit_output, - redirect_output, env_vars, stdout_pipe, stderr_pipe); - } catch (...) { - if (redirect_output) { - close_pipe(stdout_pipe); - close_pipe(stderr_pipe); - } - throw; - } + pid_t pid = spawn_process(executable, args, working_dir, inherit_output, + filter_health_logs, env_vars, stdout_pipe, stderr_pipe); handle.pid = pid; @@ -356,14 +247,58 @@ ProcessHandle LinuxProcessPlatform::spawn( LOG(INFO, "ProcessManager") << "Process started successfully, PID: " << pid << std::endl; } - if (redirect_output) { + // Start filter threads if needed + if (inherit_output && filter_health_logs) { close(stdout_pipe[1]); close(stderr_pipe[1]); - start_process_output_reader(stdout_pipe[0], inherit_output, - output_capture); - start_process_output_reader(stderr_pipe[0], inherit_output, - output_capture); + std::thread([fd = stdout_pipe[0]]() { + char buffer[4096]; + std::string line_buffer; + ssize_t bytes_read; + + while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { + buffer[bytes_read] = '\0'; + line_buffer += buffer; + + size_t pos; + while ((pos = line_buffer.find('\n')) != std::string::npos) { + std::string line = line_buffer.substr(0, pos); + line_buffer = line_buffer.substr(pos + 1); + log_process_line(line); + } + } + + if (!line_buffer.empty()) { + log_process_line(line_buffer); + } + + close(fd); + }).detach(); + + std::thread([fd = stderr_pipe[0]]() { + char buffer[4096]; + std::string line_buffer; + ssize_t bytes_read; + + while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { + buffer[bytes_read] = '\0'; + line_buffer += buffer; + + size_t pos; + while ((pos = line_buffer.find('\n')) != std::string::npos) { + std::string line = line_buffer.substr(0, pos); + line_buffer = line_buffer.substr(pos + 1); + log_process_line(line); + } + } + + if (!line_buffer.empty()) { + log_process_line(line_buffer); + } + + close(fd); + }).detach(); } return handle; @@ -565,16 +500,17 @@ int LinuxProcessPlatform::run_with_output( int timeout_seconds, bool capture_stderr) { - int stdout_pipe[2] = {-1, -1}; + int stdout_pipe[2]; - if (!create_pipe_above_standard_streams(stdout_pipe)) { + if (pipe(stdout_pipe) < 0) { throw std::runtime_error("Failed to create pipe"); } pid_t pid = fork(); if (pid < 0) { - close_pipe(stdout_pipe); + close(stdout_pipe[0]); + close(stdout_pipe[1]); throw std::runtime_error("Failed to fork process"); } @@ -740,13 +676,10 @@ int LinuxProcessPlatform::find_free_port(int start_port) { int LinuxProcessPlatform::run_command(const std::string& command, std::string& output, int timeout_seconds) { output.clear(); - FILE* pipe = popen(command.c_str(), "re"); + FILE* pipe = popen(command.c_str(), "r"); if (!pipe) { return -1; } -#ifdef LEMONADE_PROCESS_TEST_HOOK - lemonade_test_run_command_pipe_created(fileno(pipe)); -#endif char buf[4096]; while (fgets(buf, sizeof(buf), pipe)) { diff --git a/src/cpp/server/utils/platform/process_macos.cpp b/src/cpp/server/utils/platform/process_macos.cpp index 186831baaf..a61e060bf2 100644 --- a/src/cpp/server/utils/platform/process_macos.cpp +++ b/src/cpp/server/utils/platform/process_macos.cpp @@ -20,12 +20,6 @@ #include extern char** environ; -#ifdef LEMONADE_PROCESS_TEST_HOOK -extern "C" void lemonade_test_run_with_output_pipe_created(); -extern "C" void lemonade_test_run_command_exit_observed(pid_t pid); -extern "C" void lemonade_test_run_command_group_cleanup_complete(pid_t pid); -extern "C" void lemonade_test_run_command_final_reap_complete(pid_t pid); -#endif namespace lemon::utils { @@ -56,308 +50,6 @@ static void log_process_line(const std::string& line) { } } -static void close_pipe(int pipe_fds[2]) { - for (int i = 0; i < 2; ++i) { - if (pipe_fds[i] >= 0) { - close(pipe_fds[i]); - pipe_fds[i] = -1; - } - } -} - -static bool set_close_on_exec(int fd) { - int flags; - do { - flags = fcntl(fd, F_GETFD); - } while (flags < 0 && errno == EINTR); - if (flags < 0) { - return false; - } - - int result; - do { - result = fcntl(fd, F_SETFD, flags | FD_CLOEXEC); - } while (result < 0 && errno == EINTR); - return result == 0; -} - -static bool create_pipe_above_standard_streams(int pipe_fds[2]) { - if (pipe(pipe_fds) < 0) { - return false; - } -#ifdef LEMONADE_PROCESS_TEST_HOOK - lemonade_test_run_with_output_pipe_created(); -#endif - - for (int i = 0; i < 2; ++i) { - if (pipe_fds[i] > STDERR_FILENO) { - if (!set_close_on_exec(pipe_fds[i])) { - close_pipe(pipe_fds); - return false; - } - continue; - } - - int replacement; - do { - replacement = - fcntl(pipe_fds[i], F_DUPFD_CLOEXEC, STDERR_FILENO + 1); - } while (replacement < 0 && errno == EINTR); - if (replacement < 0) { - close_pipe(pipe_fds); - return false; - } - close(pipe_fds[i]); - pipe_fds[i] = replacement; - } - return true; -} - -static void kill_process_group(pid_t pid, bool fallback_to_process = true) { - int result; - do { - result = ::kill(-pid, SIGKILL); - } while (result < 0 && errno == EINTR); - if (fallback_to_process && result < 0 && errno == ESRCH) { - do { - result = ::kill(pid, SIGKILL); - } while (result < 0 && errno == EINTR); - } -} - -class SpawnFileActions { -public: - SpawnFileActions() { - error_ = posix_spawn_file_actions_init(&actions_); - initialized_ = error_ == 0; - } - - ~SpawnFileActions() { - if (initialized_) { - posix_spawn_file_actions_destroy(&actions_); - } - } - - SpawnFileActions(const SpawnFileActions&) = delete; - SpawnFileActions& operator=(const SpawnFileActions&) = delete; - - void add_close(int fd) { - if (error_ == 0) { - error_ = posix_spawn_file_actions_addclose(&actions_, fd); - } - } - - void add_dup2(int source, int destination) { - if (error_ == 0) { - error_ = - posix_spawn_file_actions_adddup2(&actions_, source, destination); - } - } - - void add_open(int fd, const char* path, int flags, mode_t mode) { - if (error_ == 0) { - error_ = posix_spawn_file_actions_addopen(&actions_, fd, path, - flags, mode); - } - } - - void add_inherit_if_open(int fd) { - if (error_ != 0) { - return; - } - - int flags; - do { - flags = fcntl(fd, F_GETFD); - } while (flags < 0 && errno == EINTR); - if (flags < 0) { - if (errno != EBADF) { - error_ = errno; - } - return; - } - if ((flags & FD_CLOEXEC) == 0) { - error_ = posix_spawn_file_actions_addinherit_np(&actions_, fd); - } - } - - void add_chdir(const std::string& working_dir) { - if (error_ != 0 || working_dir.empty()) { - return; - } -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdeprecated-declarations" - error_ = posix_spawn_file_actions_addchdir_np(&actions_, - working_dir.c_str()); -#pragma clang diagnostic pop - } - - int error() const { return error_; } - posix_spawn_file_actions_t* get() { return &actions_; } - -private: - posix_spawn_file_actions_t actions_; - bool initialized_ = false; - int error_ = 0; -}; - -class SpawnAttributes { -public: - SpawnAttributes() { - error_ = posix_spawnattr_init(&attributes_); - initialized_ = error_ == 0; - } - - ~SpawnAttributes() { - if (initialized_) { - posix_spawnattr_destroy(&attributes_); - } - } - - SpawnAttributes(const SpawnAttributes&) = delete; - SpawnAttributes& operator=(const SpawnAttributes&) = delete; - - void set_default_signals() { - if (error_ != 0) { - return; - } - - sigset_t default_signals; - if (sigfillset(&default_signals) != 0) { - error_ = errno; - return; - } - error_ = posix_spawnattr_setsigdefault(&attributes_, &default_signals); - } - - void set_process_group(pid_t process_group) { - if (error_ == 0) { - error_ = posix_spawnattr_setpgroup(&attributes_, process_group); - } - } - - void set_flags(short flags) { - if (error_ == 0) { - error_ = posix_spawnattr_setflags(&attributes_, flags); - } - } - - int error() const { return error_; } - posix_spawnattr_t* get() { return &attributes_; } - -private: - posix_spawnattr_t attributes_; - bool initialized_ = false; - int error_ = 0; -}; - -static pid_t spawn_capturing_output(const std::string& executable, - const std::vector& args, - const std::string& working_dir, - bool capture_stderr, - bool create_process_group, - int output_pipe[2]) { - SpawnFileActions file_actions; - file_actions.add_inherit_if_open(STDIN_FILENO); - file_actions.add_close(output_pipe[0]); - file_actions.add_dup2(output_pipe[1], STDOUT_FILENO); - if (capture_stderr) { - file_actions.add_dup2(output_pipe[1], STDERR_FILENO); - } else { - file_actions.add_inherit_if_open(STDERR_FILENO); - } - file_actions.add_close(output_pipe[1]); - file_actions.add_chdir(working_dir); - - SpawnAttributes attributes; - short spawn_flags = POSIX_SPAWN_CLOEXEC_DEFAULT; - if (create_process_group) { - attributes.set_process_group(0); - spawn_flags |= POSIX_SPAWN_SETPGROUP; - } - attributes.set_flags(spawn_flags); - - std::vector argv_ptrs; - argv_ptrs.reserve(args.size() + 2); - argv_ptrs.push_back(const_cast(executable.c_str())); - for (const auto& arg : args) { - argv_ptrs.push_back(const_cast(arg.c_str())); - } - argv_ptrs.push_back(nullptr); - - int spawn_result = file_actions.error(); - if (spawn_result == 0) { - spawn_result = attributes.error(); - } - - pid_t pid = 0; - if (spawn_result == 0) { - spawn_result = posix_spawnp(&pid, executable.c_str(), - file_actions.get(), attributes.get(), - argv_ptrs.data(), environ); - } - if (spawn_result != 0) { - close_pipe(output_pipe); - throw std::runtime_error(std::string("posix_spawn failed: ") + - strerror(spawn_result)); - } - - close(output_pipe[1]); - output_pipe[1] = -1; - return pid; -} - -static void read_process_output( - int fd, - bool log_output, - const std::shared_ptr& output_capture) { - char buffer[4096]; - std::string line_buffer; - ssize_t bytes_read; - - while ((bytes_read = read(fd, buffer, sizeof(buffer))) > 0) { - if (output_capture) { - output_capture->append(buffer, static_cast(bytes_read)); - } - if (!log_output) { - continue; - } - - line_buffer.append(buffer, static_cast(bytes_read)); - size_t pos; - while ((pos = line_buffer.find('\n')) != std::string::npos) { - std::string line = line_buffer.substr(0, pos); - line_buffer.erase(0, pos + 1); - log_process_line(line); - } - } - - if (log_output && !line_buffer.empty()) { - log_process_line(line_buffer); - } - close(fd); - if (output_capture) { - output_capture->finish_reader(); - } -} - -static void start_process_output_reader( - int fd, - bool log_output, - const std::shared_ptr& output_capture) { - try { - std::thread(read_process_output, fd, log_output, output_capture).detach(); - } catch (const std::exception& error) { - close(fd); - if (output_capture) { - output_capture->finish_reader(); - } - LOG(ERROR, "ProcessManager") - << "Failed to start process output reader: " << error.what() - << std::endl; - } -} - // Forward declare UnixProcessPlatform base class methods class MacOSProcessPlatform : public ProcessPlatform { public: @@ -367,8 +59,7 @@ class MacOSProcessPlatform : public ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars, - std::shared_ptr output_capture) override; + const std::vector>& env_vars) override; void terminate(ProcessHandle handle) override; bool is_running(ProcessHandle handle) override; @@ -396,25 +87,17 @@ ProcessHandle MacOSProcessPlatform::spawn( const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars, - std::shared_ptr output_capture) { + const std::vector>& env_vars) { ProcessHandle handle; handle.handle = nullptr; handle.pid = 0; - handle.output_capture = output_capture; int stdout_pipe[2] = {-1, -1}; int stderr_pipe[2] = {-1, -1}; - const bool redirect_output = - (inherit_output && filter_health_logs) || output_capture != nullptr; - if (redirect_output) { - if (!create_pipe_above_standard_streams(stdout_pipe)) { - throw std::runtime_error("Failed to create pipes for output filtering"); - } - if (!create_pipe_above_standard_streams(stderr_pipe)) { - close_pipe(stdout_pipe); + if (inherit_output && filter_health_logs) { + if (pipe(stdout_pipe) < 0 || pipe(stderr_pipe) < 0) { throw std::runtime_error("Failed to create pipes for output filtering"); } } @@ -445,30 +128,34 @@ ProcessHandle MacOSProcessPlatform::spawn( // pipe/working-dir semantics. Adds POSIX_SPAWN_CLOEXEC_DEFAULT to avoid // leaking lemond FDs into the child, and POSIX_SPAWN_SETSIGDEF to reset // inherited SIG_IGN dispositions. - SpawnFileActions file_actions; - file_actions.add_inherit_if_open(STDIN_FILENO); - - if (redirect_output) { - file_actions.add_close(stdout_pipe[0]); - file_actions.add_close(stderr_pipe[0]); - file_actions.add_dup2(stdout_pipe[1], STDOUT_FILENO); - file_actions.add_dup2(stderr_pipe[1], STDERR_FILENO); - file_actions.add_close(stdout_pipe[1]); - file_actions.add_close(stderr_pipe[1]); + posix_spawn_file_actions_t file_actions; + posix_spawn_file_actions_init(&file_actions); + + if (inherit_output && filter_health_logs) { + posix_spawn_file_actions_addclose(&file_actions, stdout_pipe[0]); + posix_spawn_file_actions_addclose(&file_actions, stderr_pipe[0]); + posix_spawn_file_actions_adddup2(&file_actions, stdout_pipe[1], STDOUT_FILENO); + posix_spawn_file_actions_adddup2(&file_actions, stderr_pipe[1], STDERR_FILENO); + posix_spawn_file_actions_addclose(&file_actions, stdout_pipe[1]); + posix_spawn_file_actions_addclose(&file_actions, stderr_pipe[1]); } else if (!inherit_output) { - file_actions.add_open(STDOUT_FILENO, "/dev/null", O_WRONLY, 0); - file_actions.add_dup2(STDOUT_FILENO, STDERR_FILENO); - } else { - file_actions.add_inherit_if_open(STDOUT_FILENO); - file_actions.add_inherit_if_open(STDERR_FILENO); + posix_spawn_file_actions_addopen(&file_actions, STDOUT_FILENO, "/dev/null", O_WRONLY, 0); + posix_spawn_file_actions_adddup2(&file_actions, STDOUT_FILENO, STDERR_FILENO); } - file_actions.add_chdir(working_dir); + if (!working_dir.empty()) { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + posix_spawn_file_actions_addchdir_np(&file_actions, working_dir.c_str()); +#pragma clang diagnostic pop + } - SpawnAttributes attributes; - attributes.set_default_signals(); - attributes.set_flags(POSIX_SPAWN_CLOEXEC_DEFAULT | - POSIX_SPAWN_SETSIGDEF); + posix_spawnattr_t attr; + posix_spawnattr_init(&attr); + sigset_t default_signals; + sigfillset(&default_signals); + posix_spawnattr_setsigdefault(&attr, &default_signals); + posix_spawnattr_setflags(&attr, POSIX_SPAWN_CLOEXEC_DEFAULT | POSIX_SPAWN_SETSIGDEF); // Build envp std::vector env_strings; @@ -503,22 +190,19 @@ ProcessHandle MacOSProcessPlatform::spawn( } argv_ptrs.push_back(nullptr); - int spawn_rc = file_actions.error(); - if (spawn_rc == 0) { - spawn_rc = attributes.error(); - } - pid_t pid = 0; - if (spawn_rc == 0) { - spawn_rc = posix_spawnp(&pid, executable.c_str(), file_actions.get(), - attributes.get(), argv_ptrs.data(), - envp.data()); - } + int spawn_rc = posix_spawnp(&pid, executable.c_str(), &file_actions, &attr, + argv_ptrs.data(), envp.data()); + + posix_spawn_file_actions_destroy(&file_actions); + posix_spawnattr_destroy(&attr); if (spawn_rc != 0) { - if (redirect_output) { - close_pipe(stdout_pipe); - close_pipe(stderr_pipe); + if (inherit_output && filter_health_logs) { + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stderr_pipe[0]); + close(stderr_pipe[1]); } throw std::runtime_error(std::string("posix_spawn failed: ") + strerror(spawn_rc)); } @@ -529,14 +213,58 @@ ProcessHandle MacOSProcessPlatform::spawn( LOG(INFO, "ProcessManager") << "Process started successfully, PID: " << pid << std::endl; } - if (redirect_output) { + // Start filter threads if needed + if (inherit_output && filter_health_logs) { close(stdout_pipe[1]); close(stderr_pipe[1]); - start_process_output_reader(stdout_pipe[0], inherit_output, - output_capture); - start_process_output_reader(stderr_pipe[0], inherit_output, - output_capture); + std::thread([fd = stdout_pipe[0]]() { + char buffer[4096]; + std::string line_buffer; + ssize_t bytes_read; + + while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { + buffer[bytes_read] = '\0'; + line_buffer += buffer; + + size_t pos; + while ((pos = line_buffer.find('\n')) != std::string::npos) { + std::string line = line_buffer.substr(0, pos); + line_buffer = line_buffer.substr(pos + 1); + log_process_line(line); + } + } + + if (!line_buffer.empty()) { + log_process_line(line_buffer); + } + + close(fd); + }).detach(); + + std::thread([fd = stderr_pipe[0]]() { + char buffer[4096]; + std::string line_buffer; + ssize_t bytes_read; + + while ((bytes_read = read(fd, buffer, sizeof(buffer) - 1)) > 0) { + buffer[bytes_read] = '\0'; + line_buffer += buffer; + + size_t pos; + while ((pos = line_buffer.find('\n')) != std::string::npos) { + std::string line = line_buffer.substr(0, pos); + line_buffer = line_buffer.substr(pos + 1); + log_process_line(line); + } + } + + if (!line_buffer.empty()) { + log_process_line(line_buffer); + } + + close(fd); + }).detach(); } return handle; @@ -731,13 +459,46 @@ int MacOSProcessPlatform::run_with_output( int timeout_seconds, bool capture_stderr) { - int stdout_pipe[2] = {-1, -1}; + // For simplicity, reuse fork/exec for run_with_output on macOS + // (This is a less critical path than spawn) + int stdout_pipe[2]; - if (!create_pipe_above_standard_streams(stdout_pipe)) { + if (pipe(stdout_pipe) < 0) { throw std::runtime_error("Failed to create pipe"); } - const pid_t pid = spawn_capturing_output( - executable, args, working_dir, capture_stderr, false, stdout_pipe); + + pid_t pid = fork(); + + if (pid < 0) { + close(stdout_pipe[0]); + close(stdout_pipe[1]); + throw std::runtime_error("Failed to fork process"); + } + + if (pid == 0) { + close(stdout_pipe[0]); + dup2(stdout_pipe[1], STDOUT_FILENO); + if (capture_stderr) { + dup2(stdout_pipe[1], STDERR_FILENO); + } + close(stdout_pipe[1]); + + if (!working_dir.empty()) { + chdir(working_dir.c_str()); + } + + std::vector argv_ptrs; + argv_ptrs.push_back(const_cast(executable.c_str())); + for (const auto& arg : args) { + argv_ptrs.push_back(const_cast(arg.c_str())); + } + argv_ptrs.push_back(nullptr); + + execvp(executable.c_str(), argv_ptrs.data()); + _exit(127); + } + + close(stdout_pipe[1]); std::string line_buffer; char buffer[4096]; @@ -867,132 +628,18 @@ int MacOSProcessPlatform::find_free_port(int start_port) { int MacOSProcessPlatform::run_command(const std::string& command, std::string& output, int timeout_seconds) { output.clear(); - int output_pipe[2] = {-1, -1}; - if (!create_pipe_above_standard_streams(output_pipe)) { - return -1; - } - - pid_t pid; - try { - pid = spawn_capturing_output("/bin/sh", {"-c", command}, "", false, - true, output_pipe); - } catch (...) { - return -1; - } - const int flags = fcntl(output_pipe[0], F_GETFL, 0); - if (flags < 0 || - fcntl(output_pipe[0], F_SETFL, flags | O_NONBLOCK) != 0) { - kill_process_group(pid); - close(output_pipe[0]); - int status = 0; - while (waitpid(pid, &status, 0) < 0 && errno == EINTR) { - } + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) { return -1; } - const auto start_time = std::chrono::steady_clock::now(); - const auto deadline = - start_time + std::chrono::seconds(timeout_seconds > 0 - ? timeout_seconds - : 0); - bool timed_out = false; - bool read_failed = false; - char buffer[4096]; - while (true) { - if (timeout_seconds > 0) { - if (std::chrono::steady_clock::now() >= deadline) { - kill_process_group(pid); - timed_out = true; - break; - } - } - - const ssize_t bytes_read = read(output_pipe[0], buffer, sizeof(buffer)); - if (bytes_read > 0) { - output.append(buffer, static_cast(bytes_read)); - continue; - } - if (bytes_read == 0) { - break; - } - if (errno == EINTR) { - continue; - } - if (errno != EAGAIN && errno != EWOULDBLOCK) { - kill_process_group(pid); - read_failed = true; - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - - close(output_pipe[0]); - int status = 0; - pid_t wait_result = -1; - bool child_identity_lost = false; - if (!timed_out && !read_failed) { - while (true) { - siginfo_t child_info{}; - int wait_options = WEXITED | WNOWAIT; - if (timeout_seconds > 0) { - wait_options |= WNOHANG; - } - - int observe_result; - do { - observe_result = - waitid(P_PID, static_cast(pid), &child_info, - wait_options); - } while (observe_result < 0 && errno == EINTR); - - if (observe_result == 0 && child_info.si_pid == pid) { - break; - } - if (observe_result < 0) { - child_identity_lost = errno == ECHILD; - if (!child_identity_lost) { - kill_process_group(pid); - } - read_failed = true; - break; - } - if (timeout_seconds > 0 && - std::chrono::steady_clock::now() >= deadline) { - kill_process_group(pid); - timed_out = true; - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - } - if (timed_out || read_failed) { - if (child_identity_lost) { - return -1; - } - do { - wait_result = waitpid(pid, &status, 0); - } while (wait_result < 0 && errno == EINTR); - return -1; + char buf[4096]; + while (fgets(buf, sizeof(buf), pipe)) { + output += buf; } -#ifdef LEMONADE_PROCESS_TEST_HOOK - lemonade_test_run_command_exit_observed(pid); -#endif - kill_process_group(pid, false); -#ifdef LEMONADE_PROCESS_TEST_HOOK - lemonade_test_run_command_group_cleanup_complete(pid); -#endif - do { - wait_result = waitpid(pid, &status, 0); - } while (wait_result < 0 && errno == EINTR); - if (wait_result != pid) { - return -1; - } -#ifdef LEMONADE_PROCESS_TEST_HOOK - lemonade_test_run_command_final_reap_complete(pid); -#endif - return status; + return pclose(pipe); } std::unique_ptr create_process_platform() { diff --git a/src/cpp/server/utils/platform/process_windows.cpp b/src/cpp/server/utils/platform/process_windows.cpp index 8e95d7c8e1..83b9a3ad33 100644 --- a/src/cpp/server/utils/platform/process_windows.cpp +++ b/src/cpp/server/utils/platform/process_windows.cpp @@ -22,9 +22,8 @@ #include #include #include -#include -#include #include +#include namespace lemon { namespace utils { @@ -52,164 +51,6 @@ static std::string escape_windows_arg(const std::string& arg) { return result; } -static BOOL create_process_with_restricted_handles( - LPSTR command_line, - DWORD creation_flags, - LPVOID environment, - LPCSTR working_directory, - const STARTUPINFOA& startup_info, - PROCESS_INFORMATION* process_information) { - STARTUPINFOEXA extended_startup_info{}; - extended_startup_info.StartupInfo = startup_info; - - HANDLE source_handles[] = { - startup_info.hStdInput, - startup_info.hStdOutput, - startup_info.hStdError, - }; - HANDLE fallback_handles[3] = {}; - HANDLE unique_source_handles[3] = {}; - HANDLE inherited_handles[3] = {}; - DWORD inherited_handle_count = 0; - SIZE_T attribute_list_size = 0; - void* attribute_storage = nullptr; - bool attribute_list_initialized = false; - auto cleanup = [&]() { - if (attribute_list_initialized) { - DeleteProcThreadAttributeList( - extended_startup_info.lpAttributeList); - } - if (attribute_storage != nullptr) { - HeapFree(GetProcessHeap(), 0, attribute_storage); - } - for (DWORD i = 0; i < inherited_handle_count; ++i) { - CloseHandle(inherited_handles[i]); - } - for (DWORD i = 0; i < 3; ++i) { - if (fallback_handles[i] != nullptr && - fallback_handles[i] != INVALID_HANDLE_VALUE) { - CloseHandle(fallback_handles[i]); - } - } - }; - - if ((startup_info.dwFlags & STARTF_USESTDHANDLES) != 0) { - for (DWORD i = 0; i < 3; ++i) { - bool usable = source_handles[i] != nullptr && - source_handles[i] != INVALID_HANDLE_VALUE; - if (usable) { - SetLastError(ERROR_SUCCESS); - const DWORD type = GetFileType(source_handles[i]); - usable = type != FILE_TYPE_UNKNOWN || - GetLastError() == ERROR_SUCCESS; - } - if (usable) { - continue; - } - - const DWORD access = i == 0 ? GENERIC_READ : GENERIC_WRITE; - const HANDLE fallback = CreateFileA( - "NUL", access, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); - if (fallback == INVALID_HANDLE_VALUE) { - const DWORD error = GetLastError(); - cleanup(); - SetLastError(error); - return FALSE; - } - fallback_handles[i] = fallback; - source_handles[i] = fallback; - } - - for (DWORD i = 0; i < 3; ++i) { - const HANDLE source = source_handles[i]; - DWORD existing = 0; - while (existing < inherited_handle_count && - unique_source_handles[existing] != source) { - ++existing; - } - - HANDLE inherited = nullptr; - if (existing < inherited_handle_count) { - inherited = inherited_handles[existing]; - } else { - if (!DuplicateHandle( - GetCurrentProcess(), source, GetCurrentProcess(), - &inherited, 0, TRUE, DUPLICATE_SAME_ACCESS)) { - const DWORD error = GetLastError(); - cleanup(); - SetLastError(error); - return FALSE; - } - unique_source_handles[inherited_handle_count] = source; - inherited_handles[inherited_handle_count] = inherited; - ++inherited_handle_count; - } - - if (i == 0) { - extended_startup_info.StartupInfo.hStdInput = inherited; - } else if (i == 1) { - extended_startup_info.StartupInfo.hStdOutput = inherited; - } else { - extended_startup_info.StartupInfo.hStdError = inherited; - } - } - } - - BOOL inherit_handles = FALSE; - DWORD effective_creation_flags = creation_flags; - if (inherited_handle_count > 0) { - InitializeProcThreadAttributeList( - nullptr, 1, 0, &attribute_list_size); - if (attribute_list_size == 0) { - const DWORD error = GetLastError(); - cleanup(); - SetLastError(error); - return FALSE; - } - - attribute_storage = - HeapAlloc(GetProcessHeap(), 0, attribute_list_size); - if (attribute_storage == nullptr) { - cleanup(); - SetLastError(ERROR_NOT_ENOUGH_MEMORY); - return FALSE; - } - extended_startup_info.lpAttributeList = - static_cast(attribute_storage); - if (!InitializeProcThreadAttributeList( - extended_startup_info.lpAttributeList, 1, 0, - &attribute_list_size)) { - const DWORD error = GetLastError(); - cleanup(); - SetLastError(error); - return FALSE; - } - attribute_list_initialized = true; - if (!UpdateProcThreadAttribute( - extended_startup_info.lpAttributeList, 0, - PROC_THREAD_ATTRIBUTE_HANDLE_LIST, inherited_handles, - inherited_handle_count * sizeof(HANDLE), nullptr, nullptr)) { - const DWORD error = GetLastError(); - cleanup(); - SetLastError(error); - return FALSE; - } - extended_startup_info.StartupInfo.cb = sizeof(STARTUPINFOEXA); - effective_creation_flags |= EXTENDED_STARTUPINFO_PRESENT; - inherit_handles = TRUE; - } - - const BOOL success = CreateProcessA( - nullptr, command_line, nullptr, nullptr, inherit_handles, - effective_creation_flags, environment, working_directory, - &extended_startup_info.StartupInfo, process_information); - const DWORD error = success ? ERROR_SUCCESS : GetLastError(); - cleanup(); - SetLastError(error); - return success; -} - // Helper function to check if a line should be filtered static bool should_filter_line(const std::string& line) { // Filter out health check requests (both /health and /v1/health) @@ -241,77 +82,34 @@ static void log_process_line(const std::string& line) { } } -struct OutputReaderContext { - HANDLE pipe; - bool log_output; - std::shared_ptr output_capture; -}; - +// Thread function to read from pipe and filter output static DWORD WINAPI output_filter_thread(LPVOID param) { - std::unique_ptr context( - static_cast(param)); + HANDLE pipe = static_cast(param); char buffer[4096]; DWORD bytes_read; std::string line_buffer; - while (ReadFile(context->pipe, buffer, sizeof(buffer), &bytes_read, nullptr) && - bytes_read > 0) { - if (context->output_capture) { - context->output_capture->append( - buffer, static_cast(bytes_read)); - } - if (!context->log_output) { - continue; - } - - line_buffer.append(buffer, static_cast(bytes_read)); + while (ReadFile(pipe, buffer, sizeof(buffer) - 1, &bytes_read, nullptr) && bytes_read > 0) { + buffer[bytes_read] = '\0'; + line_buffer += buffer; + // Process complete lines size_t pos; while ((pos = line_buffer.find('\n')) != std::string::npos) { std::string line = line_buffer.substr(0, pos); - line_buffer.erase(0, pos + 1); + line_buffer = line_buffer.substr(pos + 1); + log_process_line(line); } } - if (context->log_output && !line_buffer.empty()) { + // Print any remaining partial line + if (!line_buffer.empty()) { log_process_line(line_buffer); } - CloseHandle(context->pipe); - if (context->output_capture) { - context->output_capture->finish_reader(); - } - return 0; -} - -static void start_output_reader( - HANDLE pipe, - bool log_output, - const std::shared_ptr& output_capture) { - auto* context = - new (std::nothrow) OutputReaderContext{pipe, log_output, output_capture}; - if (context == nullptr) { - CloseHandle(pipe); - if (output_capture) { - output_capture->finish_reader(); - } - LOG(ERROR, "ProcessManager") - << "Failed to allocate process output reader context" << std::endl; - return; - } - - HANDLE thread = CreateThread( - nullptr, 0, output_filter_thread, context, 0, nullptr); - if (thread != nullptr) { - CloseHandle(thread); - return; - } CloseHandle(pipe); - if (output_capture) { - output_capture->finish_reader(); - } - delete context; + return 0; } // Helper function: lowercase ASCII string for case-insensitive comparison @@ -382,13 +180,11 @@ class WindowsProcessPlatform : public ProcessPlatform { const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars, - std::shared_ptr output_capture) override { + const std::vector>& env_vars) override { ProcessHandle handle; handle.handle = nullptr; handle.pid = 0; - handle.output_capture = output_capture; std::string cmdline = escape_windows_arg(executable); for (const auto& arg : args) { @@ -407,8 +203,7 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE stderr_write = nullptr; HANDLE nul_input = nullptr; - bool use_filtered_output = - (inherit_output && filter_health_logs) || output_capture != nullptr; + bool use_filtered_output = (inherit_output && filter_health_logs); if (inherit_output && !filter_health_logs) { const HANDLE std_in = GetStdHandle(STD_INPUT_HANDLE); @@ -422,32 +217,39 @@ class WindowsProcessPlatform : public ProcessPlatform { if (invalid_stdio) { use_filtered_output = true; - if (!output_capture) { - output_capture = - std::make_shared(0); - handle.output_capture = output_capture; - } LOG(WARNING, "ProcessManager") << "Parent std handles are unavailable; enabling filtered output capture" << std::endl; } } - if (use_filtered_output) { - if (!CreatePipe(&stdout_read, &stdout_write, nullptr, 0)) { + // If inherit_output is true, either use pipes with filtering or direct inheritance + if (inherit_output && use_filtered_output) { + // Create pipes for stdout and stderr to filter output + SECURITY_ATTRIBUTES sa; + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + sa.bInheritHandle = TRUE; + sa.lpSecurityDescriptor = nullptr; + + if (!CreatePipe(&stdout_read, &stdout_write, &sa, 0)) { throw std::runtime_error("Failed to create stdout pipe"); } - if (!CreatePipe(&stderr_read, &stderr_write, nullptr, 0)) { + if (!CreatePipe(&stderr_read, &stderr_write, &sa, 0)) { CloseHandle(stdout_read); CloseHandle(stdout_write); throw std::runtime_error("Failed to create stderr pipe"); } + // Make sure the read handles are not inherited + SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0); + SetHandleInformation(stderr_read, HANDLE_FLAG_INHERIT, 0); + si.dwFlags = STARTF_USESTDHANDLES; si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); if (si.hStdInput == nullptr || si.hStdInput == INVALID_HANDLE_VALUE) { nul_input = CreateFileA("NUL", GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (nul_input != INVALID_HANDLE_VALUE) { + SetHandleInformation(nul_input, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT); si.hStdInput = nul_input; } else { si.hStdInput = nullptr; @@ -456,9 +258,7 @@ class WindowsProcessPlatform : public ProcessPlatform { si.hStdOutput = stdout_write; si.hStdError = stderr_write; - if (inherit_output) { - LOG(DEBUG, "ProcessManager") << "Starting process with filtered output: " << cmdline << std::endl; - } + LOG(DEBUG, "ProcessManager") << "Starting process with filtered output: " << cmdline << std::endl; } else if (inherit_output) { // Direct inheritance without filtering si.dwFlags |= STARTF_USESTDHANDLES; @@ -473,6 +273,8 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE hNul = CreateFileA("NUL", GENERIC_WRITE, FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (hNul != INVALID_HANDLE_VALUE) { + // Ensure the NUL handle is inheritable + SetHandleInformation(hNul, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT); si.hStdOutput = hNul; si.hStdError = hNul; } @@ -483,18 +285,21 @@ class WindowsProcessPlatform : public ProcessPlatform { environment_block = build_windows_environment_block(env_vars); } - BOOL success = create_process_with_restricted_handles( + BOOL success = CreateProcessA( + nullptr, const_cast(cmdline.c_str()), + nullptr, + nullptr, + TRUE, // Inherit handles (inherit_output && !use_filtered_output) ? 0 : CREATE_NO_WINDOW, environment_block.empty() ? nullptr : environment_block.data(), working_dir.empty() ? nullptr : working_dir.c_str(), - si, + &si, &pi ); // If we opened a NUL handle, we can close it now (the child process has its own inherited handle) - if (!inherit_output && !use_filtered_output && - si.hStdOutput != nullptr && si.hStdOutput != INVALID_HANDLE_VALUE) { + if (!inherit_output && si.hStdOutput != nullptr && si.hStdOutput != INVALID_HANDLE_VALUE) { CloseHandle(si.hStdOutput); } @@ -531,9 +336,10 @@ class WindowsProcessPlatform : public ProcessPlatform { if (stdout_write) CloseHandle(stdout_write); if (stderr_write) CloseHandle(stderr_write); - if (use_filtered_output) { - start_output_reader(stdout_read, inherit_output, output_capture); - start_output_reader(stderr_read, inherit_output, output_capture); + // Start filter threads if needed + if (inherit_output && use_filtered_output) { + CreateThread(nullptr, 0, output_filter_thread, stdout_read, 0, nullptr); + CreateThread(nullptr, 0, output_filter_thread, stderr_read, 0, nullptr); } if (inherit_output) { @@ -660,10 +466,18 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE stdout_read = nullptr; HANDLE stdout_write = nullptr; - if (!CreatePipe(&stdout_read, &stdout_write, nullptr, 0)) { + SECURITY_ATTRIBUTES sa; + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + sa.bInheritHandle = TRUE; + sa.lpSecurityDescriptor = nullptr; + + if (!CreatePipe(&stdout_read, &stdout_write, &sa, 0)) { throw std::runtime_error("Failed to create stdout pipe"); } + // Make sure the read handle is not inherited + SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0); + STARTUPINFOA si; PROCESS_INFORMATION pi; ZeroMemory(&si, sizeof(si)); @@ -674,12 +488,16 @@ class WindowsProcessPlatform : public ProcessPlatform { si.hStdError = capture_stderr ? stdout_write : GetStdHandle(STD_ERROR_HANDLE); ZeroMemory(&pi, sizeof(pi)); - BOOL success = create_process_with_restricted_handles( + BOOL success = CreateProcessA( + nullptr, const_cast(cmdline.c_str()), + nullptr, + nullptr, + TRUE, // Inherit handles CREATE_NO_WINDOW, nullptr, working_dir.empty() ? nullptr : working_dir.c_str(), - si, + &si, &pi ); @@ -847,9 +665,15 @@ class WindowsProcessPlatform : public ProcessPlatform { HANDLE stdout_read = nullptr; HANDLE stdout_write = nullptr; - if (!CreatePipe(&stdout_read, &stdout_write, nullptr, 0)) { + SECURITY_ATTRIBUTES sa; + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + sa.bInheritHandle = TRUE; + sa.lpSecurityDescriptor = nullptr; + + if (!CreatePipe(&stdout_read, &stdout_write, &sa, 0)) { return -1; } + SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0); STARTUPINFOA si = {}; si.cb = sizeof(si); @@ -861,9 +685,10 @@ class WindowsProcessPlatform : public ProcessPlatform { PROCESS_INFORMATION pi = {}; // Wrap in cmd /c so shell features (redirection, pipes) work std::string cmdline = "cmd /c " + command; - BOOL success = create_process_with_restricted_handles( - const_cast(cmdline.c_str()), CREATE_NO_WINDOW, - nullptr, nullptr, si, &pi); + BOOL success = CreateProcessA( + nullptr, const_cast(cmdline.c_str()), + nullptr, nullptr, TRUE, CREATE_NO_WINDOW, + nullptr, nullptr, &si, &pi); CloseHandle(stdout_write); diff --git a/src/cpp/server/utils/process_manager.cpp b/src/cpp/server/utils/process_manager.cpp index 53778ced22..9a7681e77b 100644 --- a/src/cpp/server/utils/process_manager.cpp +++ b/src/cpp/server/utils/process_manager.cpp @@ -1,97 +1,24 @@ #include #include -#include -#include - namespace lemon { namespace utils { -namespace { - -constexpr int output_reader_drain_timeout_ms = 1000; - -void wait_for_output_readers(const ProcessHandle& handle) { - if (handle.output_capture) { - handle.output_capture->read_tail(0, output_reader_drain_timeout_ms); - } -} - -} // namespace - -ProcessOutputCapture::ProcessOutputCapture(std::size_t max_bytes) - : max_bytes_(max_bytes) { -} - -void ProcessOutputCapture::append(const char* data, std::size_t size) { - if (data == nullptr || size == 0 || max_bytes_ == 0) { - return; - } - - std::lock_guard lock(mutex_); - if (size >= max_bytes_) { - tail_.assign(data + size - max_bytes_, max_bytes_); - return; - } - - const std::size_t overflow = - tail_.size() + size > max_bytes_ ? tail_.size() + size - max_bytes_ : 0; - if (overflow > 0) { - tail_.erase(0, overflow); - } - tail_.append(data, size); -} - -void ProcessOutputCapture::finish_reader() { - std::lock_guard lock(mutex_); - if (active_readers_ > 0) { - --active_readers_; - } - if (active_readers_ == 0) { - completion_cv_.notify_all(); - } -} - -std::string ProcessOutputCapture::read_tail(std::size_t max_bytes, - int wait_timeout_ms) { - std::unique_lock lock(mutex_); - if (wait_timeout_ms > 0 && active_readers_ > 0) { - completion_cv_.wait_for( - lock, - std::chrono::milliseconds(wait_timeout_ms), - [this]() { return active_readers_ == 0; }); - } - - const std::size_t size = std::min(max_bytes, tail_.size()); - return tail_.substr(tail_.size() - size, size); -} - ProcessHandle ProcessManager::start_process( const std::string& executable, const std::vector& args, const std::string& working_dir, bool inherit_output, bool filter_health_logs, - const std::vector>& env_vars, - std::size_t capture_output_bytes) { - - std::shared_ptr output_capture; - if (capture_output_bytes > 0 || - (inherit_output && filter_health_logs)) { - output_capture = - std::make_shared(capture_output_bytes); - } + const std::vector>& env_vars) { auto platform = create_process_platform(); - return platform->spawn(executable, args, working_dir, inherit_output, - filter_health_logs, env_vars, - std::move(output_capture)); + return platform->spawn(executable, args, working_dir, inherit_output, filter_health_logs, env_vars); } void ProcessManager::stop_process(ProcessHandle handle) { auto platform = create_process_platform(); platform->terminate(handle); - wait_for_output_readers(handle); } bool ProcessManager::is_running(ProcessHandle handle) { @@ -106,30 +33,18 @@ int ProcessManager::get_exit_code(ProcessHandle handle) { int ProcessManager::wait_for_exit(ProcessHandle handle, int timeout_seconds) { auto platform = create_process_platform(); - const int exit_code = platform->wait_for_exit(handle, timeout_seconds); - if (exit_code >= 0) { - wait_for_output_readers(handle); - } - return exit_code; + return platform->wait_for_exit(handle, timeout_seconds); } int ProcessManager::reap_process(ProcessHandle handle) { auto platform = create_process_platform(); - const int exit_code = platform->reap(handle); - if (exit_code >= 0) { - wait_for_output_readers(handle); - } - return exit_code; + return platform->reap(handle); } -std::string ProcessManager::read_output(ProcessHandle handle, - int max_bytes, - int wait_timeout_ms) { - if (!handle.output_capture || max_bytes <= 0) { - return ""; - } - return handle.output_capture->read_tail( - static_cast(max_bytes), wait_timeout_ms); +std::string ProcessManager::read_output(ProcessHandle handle, int max_bytes) { + // Note: This is a simplified version. Full implementation would need pipes + // for stdout/stderr capture during process creation. + return ""; } int ProcessManager::run_process_with_output( @@ -147,7 +62,6 @@ int ProcessManager::run_process_with_output( void ProcessManager::kill_process(ProcessHandle handle) { auto platform = create_process_platform(); platform->kill(handle); - wait_for_output_readers(handle); } void ProcessManager::terminate_process(ProcessHandle handle) { diff --git a/src/cpp/server/wrapped_server.cpp b/src/cpp/server/wrapped_server.cpp index 29aa180d5c..38c20527ed 100644 --- a/src/cpp/server/wrapped_server.cpp +++ b/src/cpp/server/wrapped_server.cpp @@ -6,11 +6,10 @@ #include #include #include +#include #include -#include #include -#include -#include +#include #include namespace lemon { @@ -793,19 +792,6 @@ void WrappedServer::forward_streaming_request(const std::string& endpoint, bool sse, long timeout_seconds, TelemetryCallback telemetry_callback) { - forward_streaming_request_impl( - endpoint, request_body, sink, sse, timeout_seconds, - std::move(telemetry_callback), nullptr); -} - -void WrappedServer::forward_streaming_request_impl( - const std::string& endpoint, - const std::string& request_body, - httplib::DataSink& sink, - bool sse, - long timeout_seconds, - TelemetryCallback telemetry_callback, - StreamingProxy::SseFrameTransform frame_transform) { if (!is_backend_alive()) { if (was_watchdog_triggered() || has_backend_process_exited()) { if (!was_watchdog_triggered()) { @@ -825,46 +811,31 @@ void WrappedServer::forward_streaming_request_impl( std::string url = get_base_url() + endpoint; bool streamed_any_bytes = false; - auto mark_client_delivery = [this, &streamed_any_bytes]() { + auto mark_stream_progress = [this, &streamed_any_bytes]() { if (!streamed_any_bytes) { set_streaming(true); } streamed_any_bytes = true; - }; - auto mark_backend_progress = [this]() { note_backend_activity(); }; - auto mark_byte_stream_progress = [&mark_client_delivery, - &mark_backend_progress]() { - mark_client_delivery(); - mark_backend_progress(); - }; try { if (sse) { // Use StreamingProxy to forward the SSE stream with telemetry callback // Use INFERENCE_TIMEOUT_SECONDS (0 = infinite) as chat completions can take a long time - auto report_telemetry = [telemetry_callback]( - const StreamingProxy::TelemetryData& telemetry) { - if (telemetry_callback) { - telemetry_callback(telemetry); - } - }; - if (frame_transform) { - StreamingProxy::forward_transformed_sse_stream( - url, request_body, sink, frame_transform, report_telemetry, - timeout_seconds, mark_backend_progress, 1000, - mark_client_delivery); - } else { - StreamingProxy::forward_sse_stream( - url, request_body, sink, report_telemetry, - timeout_seconds, mark_backend_progress, 1000, - mark_client_delivery); - } + StreamingProxy::forward_sse_stream(url, request_body, sink, + [telemetry_callback](const StreamingProxy::TelemetryData& telemetry) { + if (telemetry_callback) { + telemetry_callback(telemetry); + } + }, + timeout_seconds, + mark_stream_progress + ); } else { StreamingProxy::forward_byte_stream(url, request_body, sink, timeout_seconds, - mark_byte_stream_progress + mark_stream_progress ); } } catch (const std::exception& e) { diff --git a/test/cpp/test_llamacpp_stream_model.cpp b/test/cpp/test_llamacpp_stream_model.cpp deleted file mode 100644 index 6a76f7c07b..0000000000 --- a/test/cpp/test_llamacpp_stream_model.cpp +++ /dev/null @@ -1,893 +0,0 @@ -#include "lemon/backends/llamacpp/llamacpp_server.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -using namespace std::chrono_literals; - -namespace { - -constexpr const char* kRequestedModel = "builtin.K2-Horizon-0.9B-GGUF"; -constexpr const char* kBackendModel = "/private/cache/models/K2-Horizon-1B-BF16.gguf"; -constexpr std::size_t kMaxSseFrameBytes = - lemon::StreamingProxy::kMaxSseFrameBytes; - -int failures = 0; - -void check(bool condition, const char* name) { - if (condition) { - std::printf("[PASS] %s\n", name); - return; - } - std::printf("[FAIL] %s\n", name); - ++failures; -} - -class TestLlamaCppServer : public lemon::backends::LlamaCppServer { -public: - explicit TestLlamaCppServer(int port) - : LlamaCppServer("debug", nullptr, nullptr) { - port_ = port; - } - - bool is_backend_alive() const override { - return true; - } - - void forward_raw_sse(const std::string& endpoint, - const std::string& request_body, - httplib::DataSink& sink) { - lemon::WrappedServer::forward_streaming_request( - endpoint, request_body, sink, true, 5, nullptr); - } -}; - -std::vector data_events(const std::string& stream, - int* malformed_events = nullptr) { - std::vector events; - std::string buffer = stream; - std::string event_data; - bool has_data_line = false; - auto finish_event = [&]() { - if (event_data.empty() || event_data == "[DONE]") { - event_data.clear(); - has_data_line = false; - return; - } - const nlohmann::json parsed = - nlohmann::json::parse(event_data, nullptr, false); - event_data.clear(); - has_data_line = false; - if (parsed.is_discarded()) { - if (malformed_events) { - ++*malformed_events; - } - return; - } - events.push_back(parsed); - }; - lemon::StreamingProxy::process_sse_lines( - buffer, [&event_data, &finish_event, - &has_data_line](const std::string& line) { - if (line.empty()) { - finish_event(); - return; - } - if (line.rfind("data:", 0) != 0) { - return; - } - std::size_t payload_start = std::strlen("data:"); - while (payload_start < line.size() && - (line[payload_start] == ' ' || - line[payload_start] == '\t')) { - ++payload_start; - } - if (has_data_line) { - event_data.push_back('\n'); - } - event_data.append(line, payload_start, std::string::npos); - has_data_line = true; - }); - return events; -} - -std::size_t count_occurrences(const std::string& value, - const std::string& needle) { - std::size_t count = 0; - std::size_t offset = 0; - while ((offset = value.find(needle, offset)) != std::string::npos) { - ++count; - offset += needle.size(); - } - return count; -} - -std::string sized_model_frame(std::size_t frame_size, - const std::string& separator) { - const std::string prefix = - "data: {\"id\":\"sized\",\"model\":\"" + - std::string(kBackendModel) + "\",\"padding\":\""; - const std::string suffix = "\"}" + separator; - if (prefix.size() + suffix.size() > frame_size) { - throw std::runtime_error("test frame size is too small"); - } - return prefix + - std::string(frame_size - prefix.size() - suffix.size(), 'x') + - suffix; -} - -std::string sized_incomplete_model_frame(std::size_t frame_size) { - const std::string prefix = - "data: {\"id\":\"sized\",\"model\":\"" + - std::string(kBackendModel) + "\",\"padding\":\""; - if (prefix.size() > frame_size) { - throw std::runtime_error("test frame size is too small"); - } - return prefix + std::string(frame_size - prefix.size(), 'x'); -} - -struct StreamResult { - std::string output; - bool retryable_reset = false; - int done_calls = 0; -}; - -struct RawStreamResult { - std::string output; - bool interrupted = false; - std::string error_message; - int backend_progress_calls = 0; -}; - -StreamResult forward_stream(TestLlamaCppServer& server, - const std::string& endpoint, - const std::string& fault = "") { - StreamResult result; - httplib::DataSink sink; - sink.write = [&result](const char* data, size_t length) { - result.output.append(data, length); - return true; - }; - sink.done = [&result]() { ++result.done_calls; }; - sink.done_with_trailer = [&result](const httplib::Headers&) { - ++result.done_calls; - }; - sink.is_writable = []() { return true; }; - - nlohmann::json request = { - {"model", kRequestedModel}, - {"stream", true}, - }; - if (!fault.empty()) { - request["fault"] = fault; - } - try { - server.forward_streaming_request( - endpoint, request.dump(), sink, true, 5, nullptr); - } catch (const lemon::BackendStreamRetryableReset&) { - result.retryable_reset = true; - } - return result; -} - -StreamResult forward_wrapped_raw_stream(TestLlamaCppServer& server, - const std::string& endpoint, - const std::string& fault) { - StreamResult result; - httplib::DataSink sink; - sink.write = [&result](const char* data, size_t length) { - result.output.append(data, length); - return true; - }; - sink.done = [&result]() { ++result.done_calls; }; - sink.done_with_trailer = [&result](const httplib::Headers&) { - ++result.done_calls; - }; - sink.is_writable = []() { return true; }; - - const nlohmann::json request = { - {"model", kRequestedModel}, - {"stream", true}, - {"fault", fault}, - }; - try { - server.forward_raw_sse(endpoint, request.dump(), sink); - } catch (const lemon::BackendStreamRetryableReset&) { - result.retryable_reset = true; - } - return result; -} - -RawStreamResult forward_raw_stream(int port, const std::string& fault) { - RawStreamResult result; - httplib::DataSink sink; - sink.write = [&result](const char* data, size_t length) { - result.output.append(data, length); - return true; - }; - sink.done = []() {}; - sink.done_with_trailer = [](const httplib::Headers&) {}; - sink.is_writable = []() { return true; }; - - const nlohmann::json request = { - {"model", kRequestedModel}, - {"stream", true}, - {"fault", fault}, - }; - try { - lemon::StreamingProxy::forward_sse_stream( - "http://127.0.0.1:" + std::to_string(port) + - "/v1/chat/completions", - request.dump(), sink, nullptr, 5, - [&result]() { ++result.backend_progress_calls; }, 0); - } catch (const std::runtime_error& error) { - result.interrupted = true; - result.error_message = error.what(); - } - return result; -} - -int forward_stream_to_disconnected_client(TestLlamaCppServer& server) { - int write_calls = 0; - httplib::DataSink sink; - sink.write = [&write_calls](const char*, size_t) { - ++write_calls; - return false; - }; - sink.done = []() {}; - sink.done_with_trailer = [](const httplib::Headers&) {}; - sink.is_writable = []() { return true; }; - - const nlohmann::json request = { - {"model", kRequestedModel}, - {"stream", true}, - }; - server.forward_streaming_request( - "/v1/chat/completions", request.dump(), sink, true, 5, nullptr); - return write_calls; -} - -} // namespace - -int main() { - httplib::Server backend; - std::atomic partial_before_output_requests{0}; - std::atomic partial_after_output_requests{0}; - const std::vector endpoints = { - "/v1/chat/completions", - "/v1/completions", - "/v1/responses", - }; - - for (const std::string& endpoint : endpoints) { - backend.Post(endpoint, [endpoint, &partial_before_output_requests, - &partial_after_output_requests]( - const httplib::Request& request, - httplib::Response& response) { - const nlohmann::json body = - nlohmann::json::parse(request.body, nullptr, false); - const std::string fault = - body.is_object() ? body.value("fault", "") : ""; - - nlohmann::json event; - if (endpoint == "/v1/responses") { - event = { - {"type", "response.completed"}, - {"response", { - {"id", "resp_mock"}, - {"model", kBackendModel}, - {"status", "completed"}, - }}, - }; - } else { - event = { - {"id", "cmpl_mock"}, - {"model", kBackendModel}, - {"choices", nlohmann::json::array()}, - }; - } - - const std::string separator = - endpoint == "/v1/completions" ? "\r\n\r\n" : "\n\n"; - std::string frame = - ": upstream-comment\n\n" + - std::string("data: ") + event.dump() + separator; - const std::string partial = - "data: {\"id\":\"partial\",\"model\":\"" + - std::string(kBackendModel) + "\""; - - if (fault == "done_literal_partial") { - event["choices"] = nlohmann::json::array({ - {{"delta", {{"content", "data: [DONE]"}}}}, - }); - frame = ": upstream-comment\n\n" + - std::string("data: ") + event.dump() + separator; - } - - if (fault == "multiline_model") { - const std::string multiline_stream = - "id: multiline\n" - "data: {\"id\":\"multi\",\n" - "data: \"model\":\"" + - std::string(kBackendModel) + - "\",\"choices\":[]}\n" - ": retained-comment\n\n" - "data: [DONE]\n\n"; - response.set_content(multiline_stream, "text/event-stream"); - return; - } - if (fault == "bom_model") { - const std::string bom_stream = - std::string("\xEF\xBB\xBF", 3) + "data: " + - event.dump() + separator + "data: [DONE]\n\n"; - response.set_chunked_content_provider( - "text/event-stream", - [bom_stream](size_t, httplib::DataSink& sink) { - for (const char byte : bom_stream) { - if (!sink.write(&byte, 1)) { - return false; - } - } - sink.done(); - return false; - }); - return; - } - if (fault == "bom_done_only") { - response.set_content( - std::string("\xEF\xBB\xBF", 3) + - "data: [DONE]\n\n", - "text/event-stream"); - return; - } - if (fault == "double_bom_model") { - const std::string double_bom_stream = - std::string("\xEF\xBB\xBF\xEF\xBB\xBF", 6) + - "data: " + event.dump() + separator + - "data: [DONE]\n\n"; - response.set_chunked_content_provider( - "text/event-stream", - [double_bom_stream](size_t, httplib::DataSink& sink) { - for (const char byte : double_bom_stream) { - if (!sink.write(&byte, 1)) { - return false; - } - } - sink.done(); - return false; - }); - return; - } - if (fault == "double_bom_done_only") { - response.set_content( - std::string("\xEF\xBB\xBF\xEF\xBB\xBF", 6) + - "data: [DONE]\n\n", - "text/event-stream"); - return; - } - if (fault == "terminal_cr") { - response.set_content( - "data: " + event.dump() + "\r\n\r", - "text/event-stream"); - return; - } - if (fault == "malformed_model") { - response.set_content( - "data: {\"id\":\"broken\",\"model\":\"" + - std::string(kBackendModel) + "\"\n\n", - "text/event-stream"); - return; - } - if (fault == "scalar_model") { - response.set_content( - "data: \"" + std::string(kBackendModel) + "\"\n\n", - "text/event-stream"); - return; - } - if (fault == "frame_at_limit_lf") { - response.set_content( - sized_model_frame(kMaxSseFrameBytes, "\n\n"), - "text/event-stream"); - return; - } - if (fault == "frame_at_limit_crlf") { - response.set_content( - sized_model_frame(kMaxSseFrameBytes, "\r\n\r\n"), - "text/event-stream"); - return; - } - if (fault == "frame_over_limit") { - response.set_content( - sized_model_frame(kMaxSseFrameBytes + 1, "\n\n"), - "text/event-stream"); - return; - } - if (fault == "frame_over_limit_after_frame") { - const std::string safe_frame = - "data: {\"id\":\"safe\",\"model\":\"" + - std::string(kRequestedModel) + "\",\"choices\":[]}\n\n"; - response.set_content( - safe_frame + - sized_model_frame(kMaxSseFrameBytes + 1, "\n\n"), - "text/event-stream"); - return; - } - if (fault == "incomplete_frame_over_limit") { - response.set_content( - sized_incomplete_model_frame(kMaxSseFrameBytes + 1), - "text/event-stream"); - return; - } - if (fault == "incomplete_frame_over_limit_after_frame") { - const std::string safe_frame = - "data: {\"id\":\"safe\",\"model\":\"" + - std::string(kRequestedModel) + "\",\"choices\":[]}\n\n"; - response.set_content( - safe_frame + - sized_incomplete_model_frame(kMaxSseFrameBytes + 1), - "text/event-stream"); - return; - } - if (fault == "one_byte_fragmentation") { - event["padding"] = std::string(4096, 'x'); - const std::string fragmented = - "data: " + event.dump() + "\r\n\r\n" - "data: [DONE]\r\r"; - response.set_chunked_content_provider( - "text/event-stream", - [fragmented](size_t, httplib::DataSink& sink) { - for (const char byte : fragmented) { - if (!sink.write(&byte, 1)) { - return false; - } - } - sink.done(); - return false; - }); - return; - } - if (fault == "incomplete_progress") { - const std::string fragmented = - "data: {\"id\":\"active\",\"model\":\"" + - std::string(kBackendModel) + "\""; - response.set_chunked_content_provider( - "text/event-stream", - [fragmented](size_t, httplib::DataSink& sink) { - for (const char byte : fragmented) { - if (!sink.write(&byte, 1)) { - return false; - } - std::this_thread::sleep_for(1ms); - } - sink.done(); - return false; - }); - return; - } - if (fault.rfind("split_done_", 0) == 0) { - std::string done_separator = "\n\n"; - if (fault == "split_done_crlf") { - done_separator = "\r\n\r\n"; - } else if (fault == "split_done_cr") { - done_separator = "\r\r"; - } - const std::string done = "data: [DONE]" + done_separator; - response.set_chunked_content_provider( - "text/event-stream", - [frame, done](size_t, httplib::DataSink& sink) { - sink.write(frame.data(), frame.size()); - const std::size_t split = done.size() / 2; - sink.write(done.data(), split); - std::this_thread::sleep_for(5ms); - sink.write(done.data() + split, done.size() - split); - sink.done(); - return false; - }); - return; - } - if (fault == "clean_partial_before_output") { - response.set_content(partial, "text/event-stream"); - return; - } - - std::string truncated; - if (fault == "partial_before_output") { - const int attempt = partial_before_output_requests.fetch_add(1) + 1; - if (attempt == 1) { - truncated = partial; - } - } else if (fault == "partial_after_output") { - partial_after_output_requests.fetch_add(1); - truncated = frame + partial; - } else if (fault == "done_literal_partial") { - truncated = frame + partial; - } - - if (!truncated.empty()) { - const std::size_t advertised_length = truncated.size() + 128; - response.set_content_provider( - advertised_length, - "text/event-stream", - [truncated = std::move(truncated)]( - size_t offset, size_t, httplib::DataSink& sink) { - if (offset == 0) { - sink.write(truncated.data(), truncated.size()); - } - return false; - }); - return; - } - - response.set_chunked_content_provider( - "text/event-stream", - [frame](size_t, httplib::DataSink& sink) { - const size_t split = frame.size() / 2; - sink.write(frame.data(), split); - std::this_thread::sleep_for(5ms); - sink.write(frame.data() + split, frame.size() - split); - const std::string done = "data: [DONE]\n\n"; - const size_t done_split = done.size() / 2; - sink.write(done.data(), done_split); - sink.write(done.data() + done_split, - done.size() - done_split); - sink.done(); - return false; - }); - }); - } - - const int port = backend.bind_to_any_port("127.0.0.1"); - if (port <= 0) { - std::printf("[FAIL] failed to bind mock backend\n"); - return 1; - } - - std::thread backend_thread([&backend]() { backend.listen_after_bind(); }); - backend.wait_until_ready(); - - TestLlamaCppServer server(port); - - const std::string chat = - forward_stream(server, "/v1/chat/completions").output; - const auto chat_events = data_events(chat); - check(chat_events.size() == 1, "chat stream keeps one JSON event"); - check(!chat_events.empty() && chat_events[0].value("model", "") == kRequestedModel, - "chat stream exposes the requested model identity"); - check(chat.find(kBackendModel) == std::string::npos, - "chat stream does not expose the backend GGUF path"); - check(chat.find(": upstream-comment\n\n") != std::string::npos, - "chat stream preserves complete non-data frames"); - check(count_occurrences(chat, "data: [DONE]\n\n") == 1, - "chat stream preserves one split DONE frame"); - - const std::string completion = - forward_stream(server, "/v1/completions").output; - const auto completion_events = data_events(completion); - check(completion_events.size() == 1, "completion stream keeps one JSON event"); - check(!completion_events.empty() && - completion_events[0].value("model", "") == kRequestedModel, - "completion stream exposes the requested model identity"); - check(completion.find(kBackendModel) == std::string::npos, - "completion stream does not expose the backend GGUF path"); - check(completion.find("\r\n\r\n") != std::string::npos, - "completion stream preserves CRLF frame boundaries"); - - const std::string responses = - forward_stream(server, "/v1/responses").output; - const auto response_events = data_events(responses); - check(response_events.size() == 1, "Responses stream keeps one JSON event"); - check(!response_events.empty() && - response_events[0].at("response").value("model", "") == kRequestedModel, - "Responses stream exposes the requested model identity"); - check(responses.find(kBackendModel) == std::string::npos, - "Responses stream does not expose the backend GGUF path"); - - TestLlamaCppServer multiline_server(port); - const StreamResult multiline = forward_stream( - multiline_server, "/v1/chat/completions", "multiline_model"); - const auto multiline_events = data_events(multiline.output); - check(multiline_events.size() == 1 && - multiline_events[0].value("model", "") == kRequestedModel, - "multi-line SSE data is normalized as one JSON event"); - check(multiline.output.find(kBackendModel) == std::string::npos, - "multi-line SSE data does not expose the backend GGUF path"); - check(multiline.output.find("id: multiline\n") != std::string::npos && - multiline.output.find(": retained-comment\n\n") != - std::string::npos, - "multi-line normalization preserves non-data fields"); - - TestLlamaCppServer bom_server(port); - const StreamResult bom = forward_stream( - bom_server, "/v1/chat/completions", "bom_model"); - const auto bom_events = data_events(bom.output); - check(!bom.retryable_reset && bom_events.size() == 1 && - bom_events[0].value("model", "") == kRequestedModel, - "a stream-leading UTF-8 BOM preserves the public model identity"); - check(bom.output.find(kBackendModel) == std::string::npos, - "a stream-leading UTF-8 BOM does not expose the backend GGUF path"); - - TestLlamaCppServer bom_done_server(port); - const StreamResult bom_done = forward_stream( - bom_done_server, "/v1/chat/completions", "bom_done_only"); - check(!bom_done.retryable_reset && - count_occurrences(bom_done.output, "data: [DONE]\n\n") == 1, - "a BOM-prefixed terminal event produces exactly one DONE frame"); - - const RawStreamResult raw_bom_done = - forward_raw_stream(port, "bom_done_only"); - check(!raw_bom_done.interrupted && - count_occurrences( - raw_bom_done.output, "data: [DONE]\n\n") == 1, - "a raw BOM-prefixed terminal event produces exactly one DONE frame"); - - TestLlamaCppServer double_bom_server(port); - const StreamResult double_bom = forward_stream( - double_bom_server, "/v1/chat/completions", "double_bom_model"); - check(double_bom.retryable_reset && double_bom.output.empty(), - "two fragmented leading UTF-8 BOMs fail before client output"); - check(double_bom.output.find(kBackendModel) == std::string::npos, - "two leading UTF-8 BOMs cannot expose the backend GGUF path"); - - TestLlamaCppServer double_bom_done_server(port); - const StreamResult double_bom_done = forward_stream( - double_bom_done_server, "/v1/chat/completions", - "double_bom_done_only"); - check(double_bom_done.retryable_reset && double_bom_done.output.empty(), - "two leading UTF-8 BOMs cannot duplicate a transformed DONE frame"); - - const RawStreamResult raw_double_bom_done = - forward_raw_stream(port, "double_bom_done_only"); - check(raw_double_bom_done.interrupted && - raw_double_bom_done.output.empty(), - "two leading UTF-8 BOMs cannot duplicate a raw DONE frame"); - - TestLlamaCppServer terminal_cr_server(port); - const StreamResult terminal_cr = forward_stream( - terminal_cr_server, "/v1/chat/completions", "terminal_cr"); - const auto terminal_cr_events = data_events(terminal_cr.output); - check(!terminal_cr.retryable_reset && terminal_cr_events.size() == 1 && - terminal_cr_events[0].value("model", "") == kRequestedModel, - "a terminal bare CR completes a transformed SSE frame at EOF"); - check(terminal_cr.output.find(kBackendModel) == std::string::npos, - "a terminal bare CR frame does not expose the backend GGUF path"); - - TestLlamaCppServer malformed_server(port); - const StreamResult malformed = forward_stream( - malformed_server, "/v1/chat/completions", "malformed_model"); - check(malformed.retryable_reset && malformed.output.empty(), - "a malformed OpenAI SSE object fails before client output"); - check(malformed.output.find(kBackendModel) == std::string::npos, - "a malformed OpenAI SSE object does not expose the backend GGUF path"); - - TestLlamaCppServer scalar_server(port); - const StreamResult scalar = forward_stream( - scalar_server, "/v1/chat/completions", "scalar_model"); - check(scalar.retryable_reset && scalar.output.empty(), - "a non-object OpenAI SSE value fails before client output"); - check(scalar.output.find(kBackendModel) == std::string::npos, - "a non-object OpenAI SSE value does not expose the backend GGUF path"); - - const std::vector exact_limit_faults = { - "frame_at_limit_lf", - "frame_at_limit_crlf", - }; - for (const std::string& fault : exact_limit_faults) { - TestLlamaCppServer exact_limit_server(port); - const StreamResult transformed_exact_limit = forward_stream( - exact_limit_server, "/v1/chat/completions", fault); - check(!transformed_exact_limit.retryable_reset && - transformed_exact_limit.output.find(kRequestedModel) != - std::string::npos && - transformed_exact_limit.output.find(kBackendModel) == - std::string::npos, - ("transformed stream accepts an exact-limit frame for " + - fault).c_str()); - - const RawStreamResult raw_exact_limit = - forward_raw_stream(port, fault); - check(!raw_exact_limit.interrupted, - ("raw stream accepts an exact-limit frame for " + fault).c_str()); - } - - TestLlamaCppServer oversized_server(port); - const StreamResult transformed_oversized = forward_stream( - oversized_server, "/v1/chat/completions", "frame_over_limit"); - check(transformed_oversized.retryable_reset && - transformed_oversized.output.empty(), - "a transformed oversized frame fails before client output"); - check(transformed_oversized.output.find(kBackendModel) == - std::string::npos, - "a transformed oversized frame does not expose the backend path"); - - const RawStreamResult raw_oversized = - forward_raw_stream(port, "frame_over_limit"); - check(raw_oversized.interrupted && raw_oversized.output.empty(), - "a raw oversized frame fails before client output"); - - TestLlamaCppServer oversized_incomplete_server(port); - const StreamResult transformed_oversized_incomplete = forward_stream( - oversized_incomplete_server, "/v1/chat/completions", - "incomplete_frame_over_limit"); - check(transformed_oversized_incomplete.retryable_reset && - transformed_oversized_incomplete.output.empty() && - transformed_oversized_incomplete.output.find(kBackendModel) == - std::string::npos, - "an incomplete transformed oversized frame fails closed"); - - const RawStreamResult raw_oversized_incomplete = - forward_raw_stream(port, "incomplete_frame_over_limit"); - check(raw_oversized_incomplete.interrupted && - raw_oversized_incomplete.output.empty(), - "an incomplete raw oversized frame fails before client output"); - - for (const std::string& fault : { - "frame_over_limit_after_frame", - "incomplete_frame_over_limit_after_frame", - }) { - TestLlamaCppServer raw_wrapper_server(port); - const StreamResult raw_wrapper = forward_wrapped_raw_stream( - raw_wrapper_server, "/v1/chat/completions", fault); - int malformed_events = 0; - const auto events = data_events(raw_wrapper.output, &malformed_events); - check(!raw_wrapper.retryable_reset && raw_wrapper.done_calls == 1 && - malformed_events == 0 && events.size() == 2 && - events[0].value("model", "") == kRequestedModel && - events[1].contains("error"), - ("a wrapped raw oversized frame ends with a standalone error for " + - fault).c_str()); - check(raw_wrapper.output.find(kBackendModel) == std::string::npos, - ("a wrapped raw oversized frame does not expose buffered data for " + - fault).c_str()); - } - - TestLlamaCppServer fragmented_server(port); - const StreamResult transformed_fragmented = forward_stream( - fragmented_server, "/v1/chat/completions", - "one_byte_fragmentation"); - check(!transformed_fragmented.retryable_reset && - transformed_fragmented.output.find(kRequestedModel) != - std::string::npos && - transformed_fragmented.output.find(kBackendModel) == - std::string::npos && - count_occurrences( - transformed_fragmented.output, "data: [DONE]") == 1, - "a transformed stream accepts one-byte SSE fragmentation"); - - const RawStreamResult raw_fragmented = - forward_raw_stream(port, "one_byte_fragmentation"); - check(!raw_fragmented.interrupted && - count_occurrences(raw_fragmented.output, "data: [DONE]") == 1, - "a raw stream accepts one-byte SSE fragmentation"); - - const RawStreamResult active_incomplete = - forward_raw_stream(port, "incomplete_progress"); - check(active_incomplete.interrupted && active_incomplete.output.empty(), - "an active incomplete frame remains retryable before client output"); - check(active_incomplete.backend_progress_calls > 1, - "raw backend chunks refresh progress before an SSE frame completes"); - - const std::vector split_done_faults = { - "split_done_lf", - "split_done_crlf", - "split_done_cr", - }; - for (const std::string& fault : split_done_faults) { - TestLlamaCppServer transformed_done_server(port); - const StreamResult transformed_done = forward_stream( - transformed_done_server, "/v1/chat/completions", fault); - check(!transformed_done.retryable_reset && - count_occurrences( - transformed_done.output, "data: [DONE]") == 1 && - transformed_done.output.find(kRequestedModel) != - std::string::npos && - transformed_done.output.find(kBackendModel) == - std::string::npos, - ("transformed stream preserves one fragmented DONE for " + - fault).c_str()); - - const RawStreamResult raw_done = forward_raw_stream(port, fault); - check(!raw_done.interrupted && - count_occurrences(raw_done.output, "data: [DONE]") == 1, - ("raw stream preserves one fragmented DONE for " + fault).c_str()); - } - - TestLlamaCppServer literal_done_server(port); - const StreamResult literal_done = forward_stream( - literal_done_server, "/v1/chat/completions", - "done_literal_partial"); - int literal_done_malformed_events = 0; - const auto literal_done_events = - data_events(literal_done.output, &literal_done_malformed_events); - check(!literal_done.retryable_reset && - literal_done_malformed_events == 0 && - literal_done_events.size() == 2 && - literal_done_events[0].value("model", "") == kRequestedModel && - literal_done_events[1].contains("error"), - "DONE text inside JSON does not mask a transformed interruption"); - check(literal_done.output.find(kBackendModel) == std::string::npos, - "a false-DONE interruption does not expose the backend GGUF path"); - - const RawStreamResult raw_literal_done = - forward_raw_stream(port, "done_literal_partial"); - check(raw_literal_done.interrupted, - "DONE text inside JSON does not mask a raw-stream interruption"); - - check(forward_stream_to_disconnected_client(server) == 1, - "a disconnected client receives no buffered writes during cleanup"); - - TestLlamaCppServer retry_server(port); - const StreamResult interrupted_before_output = forward_stream( - retry_server, "/v1/chat/completions", "partial_before_output"); - check(interrupted_before_output.retryable_reset, - "a partial first event preserves the pre-output retry signal"); - check(interrupted_before_output.output.empty(), - "a partial first event emits no client bytes"); - check(interrupted_before_output.output.find(kBackendModel) == - std::string::npos, - "a partial first event does not expose the backend GGUF path"); - - TestLlamaCppServer clean_eof_server(port); - const StreamResult clean_eof_partial = forward_stream( - clean_eof_server, "/v1/chat/completions", - "clean_partial_before_output"); - check(clean_eof_partial.retryable_reset, - "a clean HTTP EOF with an incomplete event remains retryable"); - check(clean_eof_partial.output.empty() && - clean_eof_partial.output.find(kBackendModel) == - std::string::npos, - "a clean HTTP EOF does not flush an incomplete event"); - - StreamResult retried; - if (interrupted_before_output.retryable_reset) { - TestLlamaCppServer reloaded_server(port); - retried = forward_stream( - reloaded_server, "/v1/chat/completions", "partial_before_output"); - } - int malformed_retried_events = 0; - const auto retried_events = - data_events(retried.output, &malformed_retried_events); - check(partial_before_output_requests.load() == 2, - "the caller can replay once after a pre-output interruption"); - check(retried_events.size() == 1 && - retried_events[0].value("model", "") == kRequestedModel, - "the replay produces one normalized event"); - check(malformed_retried_events == 0 && - retried.output.find(kBackendModel) == std::string::npos && - retried.output.find("\"error\"") == std::string::npos && - count_occurrences(retried.output, "data: [DONE]\n\n") == 1, - "the replay produces one complete error-free stream"); - - TestLlamaCppServer no_replay_server(port); - const StreamResult interrupted_after_output = forward_stream( - no_replay_server, "/v1/chat/completions", "partial_after_output"); - int malformed_events = 0; - const auto events_after_output = - data_events(interrupted_after_output.output, &malformed_events); - check(!interrupted_after_output.retryable_reset, - "an interruption after client output is not replayed"); - check(partial_after_output_requests.load() == 1, - "an interruption after client output makes one backend request"); - check(malformed_events == 0, - "the post-output interruption emits only valid JSON events"); - check(events_after_output.size() == 2 && - events_after_output[0].value("model", "") == kRequestedModel && - events_after_output[1].contains("error"), - "the partial event is replaced by a separate streaming error"); - check(interrupted_after_output.output.find(kBackendModel) == - std::string::npos, - "the post-output interruption does not expose the backend GGUF path"); - - backend.stop(); - backend_thread.join(); - - return failures == 0 ? 0 : 1; -} diff --git a/test/cpp/test_llamacpp_system_backend.cpp b/test/cpp/test_llamacpp_system_backend.cpp index 6a3401efc4..7d0ced7bec 100644 --- a/test/cpp/test_llamacpp_system_backend.cpp +++ b/test/cpp/test_llamacpp_system_backend.cpp @@ -208,95 +208,6 @@ int main() { ScopedEnvVar path_env("PATH"); ScopedEnvVar hip_env("LEMONADE_GGML_HIP_PATH"); - check(lemon::backends::llamacpp::detail::identifies_k2_horizon_model( - "builtin.K2-Horizon-0.9B-GGUF", "", ""), - "identifies K2-Horizon from the selected model name"); - check(lemon::backends::llamacpp::detail::identifies_k2_horizon_model( - "user.custom", "IFM/K2_Horizon-7B-GGUF:model.gguf", ""), - "identifies K2-Horizon from the checkpoint"); - check(lemon::backends::llamacpp::detail::identifies_k2_horizon_model( - "user.custom", "example/model:model.gguf", "k2-horizon"), - "identifies K2-Horizon from GGUF architecture metadata"); - check(!lemon::backends::llamacpp::detail::identifies_k2_horizon_model( - "Tiny-Test-Model-GGUF", "unsloth/gemma-3-GGUF:model.gguf", "gemma3"), - "does not classify an ordinary GGUF model as K2-Horizon"); - check(!lemon::backends::llamacpp::detail::identifies_k2_horizon_model( - "user.K2-Horizon-Alias", - "IFM/K2-Horizon-7B-GGUF:model.gguf", - "gemma3"), - "uses nonempty GGUF architecture instead of model-name fallbacks"); - check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( - "system", false, "builtin.K2-Horizon-0.9B-GGUF", "", ""), - "requires unsupported-architecture evidence for the K2 diagnostic"); - const std::string unsupported_k2_architecture = - "error loading model: unknown model architecture: 'k2-horizon'"; - check(lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( - "system", - false, - "builtin.K2-Horizon-0.9B-GGUF", - "", - "", - unsupported_k2_architecture), - "reports a K2 diagnostic for concrete unsupported-architecture evidence"); - check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( - "system", - false, - "builtin.K2-Horizon-0.9B-GGUF", - "", - "", - "error loading model: failed to allocate compute buffers"), - "keeps the ordinary diagnostic for a K2 resource failure"); - check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( - "vulkan", false, "builtin.K2-Horizon-0.9B-GGUF", "", ""), - "does not report the system diagnostic for a managed backend"); - check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( - "system", false, "Tiny-Test-Model-GGUF", "", "gemma3"), - "does not report the K2 diagnostic for an ordinary model"); - check(!lemon::backends::llamacpp::detail::should_report_k2_system_startup_diagnostic( - "system", true, "builtin.K2-Horizon-0.9B-GGUF", "", ""), - "does not classify a cancelled K2 load as binary incompatibility"); - check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version( - "version: 4242 (incompatible mock)") == "b4242", - "parses the incompatible system binary version"); - check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version( - "version: 0.3.0-dev (build 10671, commit 35999d101)\n" - "built with AppleClang 21.0.0.21000101 for Darwin arm64") == - "b10671", - "parses the build number from the required IFM binary version"); - check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version("") == - "unknown", - "reports an unknown system binary version when probing returns no output"); - check(lemon::backends::llamacpp::detail::parse_system_llamacpp_version( - "Failed to execute: missing-llama-server", 1) == "unknown", - "ignores captured stderr when the version probe fails"); - - const std::string diagnostic = - lemon::backends::llamacpp::detail::k2_horizon_system_startup_error( - "builtin.K2-Horizon-0.9B-GGUF", - "/opt/old-llama/bin/llama-server", - "b4242", - unsupported_k2_architecture); - check(diagnostic.find("builtin.K2-Horizon-0.9B-GGUF") != std::string::npos, - "K2 startup diagnostic names the selected model"); - check(diagnostic.find("/opt/old-llama/bin/llama-server") != std::string::npos, - "K2 startup diagnostic names the selected executable"); - check(diagnostic.find("b4242") != std::string::npos, - "K2 startup diagnostic includes the detected version"); - check(diagnostic.find("Original error: " + unsupported_k2_architecture) != - std::string::npos, - "K2 startup diagnostic preserves the original startup error"); - check(diagnostic.find("model/K2Horizon") != std::string::npos && - diagnostic.find("PATH") != std::string::npos, - "K2 startup diagnostic provides the IFM branch remediation"); - const std::string unknown_version_diagnostic = - lemon::backends::llamacpp::detail::k2_horizon_system_startup_error( - "builtin.K2-Horizon-0.9B-GGUF", - "/opt/old-llama/bin/llama-server", - "detected", - "llama-server failed to start"); - check(unknown_version_diagnostic.find("version unknown") != std::string::npos, - "K2 startup diagnostic reports unknown for an unparseable version"); - // PATH discovery is a direct C++ decision: no lemond is needed to prove it. path_env.set(system_bin.string()); check(!lemon::utils::find_executable_in_path("llama-server").empty(), diff --git a/test/cpp/test_mcp_client_config.cpp b/test/cpp/test_mcp_client_config.cpp index 0e75d86af2..eb022301f3 100644 --- a/test/cpp/test_mcp_client_config.cpp +++ b/test/cpp/test_mcp_client_config.cpp @@ -1,6 +1,5 @@ #include "lemon/mcp_client.h" -#include #include #include #include @@ -15,98 +14,11 @@ #include #include -#if defined(LEMONADE_MCP_PROCESS_TEST_HOOK) && !defined(_WIN32) - #include - #include - #include - -using McpProcessTestHook = void (*)(pid_t); -extern "C" void lemonade_test_set_mcp_process_hooks( - McpProcessTestHook exit_observed, McpProcessTestHook group_cleanup, - McpProcessTestHook final_reap); -#endif - namespace fs = std::filesystem; using namespace std::chrono_literals; namespace { -#if defined(LEMONADE_MCP_PROCESS_TEST_HOOK) && \ - defined(LEMONADE_TEST_PYTHON) && \ - defined(LEMONADE_TEST_MCP_STDIO_SERVER) -std::atomic mcp_process_hook_sequence{0}; -std::atomic mcp_exit_observed_order{-1}; -std::atomic mcp_group_cleanup_order{-1}; -std::atomic mcp_final_reap_order{-1}; -std::atomic mcp_leader_reserved{false}; -std::atomic mcp_leader_reserved_after_cleanup{false}; -std::atomic mcp_leader_absent_after_reap{false}; - -bool exited_child_is_reserved(pid_t pid) { - siginfo_t child_info{}; - int result; - do { - result = ::waitid(P_PID, static_cast(pid), &child_info, - WEXITED | WNOHANG | WNOWAIT); - } while (result < 0 && errno == EINTR); - return result == 0 && child_info.si_pid == pid; -} - -void record_mcp_exit_observed(pid_t pid) { - mcp_leader_reserved.store(exited_child_is_reserved(pid)); - mcp_exit_observed_order.store(mcp_process_hook_sequence.fetch_add(1)); -} - -void record_mcp_group_cleanup(pid_t pid) { - mcp_leader_reserved_after_cleanup.store(exited_child_is_reserved(pid)); - mcp_group_cleanup_order.store(mcp_process_hook_sequence.fetch_add(1)); -} - -void record_mcp_final_reap(pid_t pid) { - siginfo_t child_info{}; - errno = 0; - int result; - do { - result = ::waitid(P_PID, static_cast(pid), &child_info, - WEXITED | WNOHANG | WNOWAIT); - } while (result < 0 && errno == EINTR); - mcp_leader_absent_after_reap.store(result < 0 && errno == ECHILD); - mcp_final_reap_order.store(mcp_process_hook_sequence.fetch_add(1)); -} - -class McpProcessHookGuard { -public: - McpProcessHookGuard() { - mcp_process_hook_sequence.store(0); - mcp_exit_observed_order.store(-1); - mcp_group_cleanup_order.store(-1); - mcp_final_reap_order.store(-1); - mcp_leader_reserved.store(false); - mcp_leader_reserved_after_cleanup.store(false); - mcp_leader_absent_after_reap.store(false); - lemonade_test_set_mcp_process_hooks( - record_mcp_exit_observed, record_mcp_group_cleanup, - record_mcp_final_reap); - } - - ~McpProcessHookGuard() { - lemonade_test_set_mcp_process_hooks(nullptr, nullptr, nullptr); - } -}; - -void verify_mcp_process_hook_result() { - const bool order_is_valid = mcp_exit_observed_order.load() == 0 && - mcp_group_cleanup_order.load() == 1 && - mcp_final_reap_order.load() == 2; - if (!mcp_leader_reserved.load() || - !mcp_leader_reserved_after_cleanup.load() || - !mcp_leader_absent_after_reap.load() || !order_is_valid) { - throw std::runtime_error( - "MCP POSIX stop did not retain the leader through group cleanup"); - } -} -#endif - void set_test_env(const std::string& name, const std::string& value) { #ifdef _WIN32 if (_putenv_s(name.c_str(), value.c_str()) != 0) { @@ -322,30 +234,18 @@ int run_tests() { // Reconnect and verify that a child process exiting without a response // wakes the request immediately rather than waiting for its full timeout. manager.connect_server_json(id); - { -#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK - McpProcessHookGuard process_hook_guard; -#endif - const auto exit_start = std::chrono::steady_clock::now(); - const bool exit_failed = throws([&] { - manager.call_tool_json( - id, json{{"name", "exit"}, - {"arguments", json::object()}, - {"timeout_ms", 10000}}); - }); - const auto exit_elapsed = - std::chrono::steady_clock::now() - exit_start; - if (!exit_failed || exit_elapsed >= 5s) { - throw std::runtime_error( - "MCP process exit did not wake the pending request"); - } - - manager.remove_server_json(id); -#ifdef LEMONADE_MCP_PROCESS_TEST_HOOK - verify_mcp_process_hook_result(); - std::cout << "MCP POSIX process lifecycle hook checks passed\n"; -#endif - } + const auto exit_start = std::chrono::steady_clock::now(); + assert(throws([&] { + manager.call_tool_json( + id, json{{"name", "exit"}, + {"arguments", json::object()}, + {"timeout_ms", 10000}}); + })); + const auto exit_elapsed = + std::chrono::steady_clock::now() - exit_start; + assert(exit_elapsed < 5s); + + manager.remove_server_json(id); // Disconnect must also cancel initialization in progress. This covers a // race where disconnect could previously return before connect installed diff --git a/test/cpp/test_mcp_client_process_contract.cpp b/test/cpp/test_mcp_client_process_contract.cpp deleted file mode 100644 index 7a498f263f..0000000000 --- a/test/cpp/test_mcp_client_process_contract.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include -#include -#include -#include -#include - -#ifndef LEMONADE_MCP_CLIENT_SOURCE - #error "LEMONADE_MCP_CLIENT_SOURCE must name mcp_client.cpp" -#endif - -namespace { - -int failures = 0; - -void check(const std::string& name, bool condition) { - if (condition) { - std::cout << "[PASS] " << name << '\n'; - return; - } - std::cerr << "[FAIL] " << name << '\n'; - ++failures; -} - -std::string read_source() { - std::ifstream input(LEMONADE_MCP_CLIENT_SOURCE); - if (!input) { - throw std::runtime_error("failed to open mcp_client.cpp"); - } - return {std::istreambuf_iterator(input), - std::istreambuf_iterator()}; -} - -std::string stop_body(const std::string& source) { - const std::size_t start = source.find("void stop_locked()"); - const std::size_t end = source.find( - "if (stdout_thread_.joinable())", start); - if (start == std::string::npos || end == std::string::npos) { - throw std::runtime_error("failed to locate MCP stop body"); - } - return source.substr(start, end - start); -} - -} // namespace - -int main() { - try { - const std::string body = stop_body(read_source()); - const std::size_t windows_process = - body.find("if (process_info_.hProcess)"); - const std::size_t posix_start = - body.find("#else", windows_process); - if (windows_process == std::string::npos || - posix_start == std::string::npos) { - throw std::runtime_error("failed to locate MCP POSIX stop path"); - } - const std::string posix = body.substr(posix_start); - check("MCP stop uses one POSIX lifecycle path", - posix.find("#ifdef __APPLE__") == std::string::npos); - if (posix.find("#ifdef __APPLE__") != std::string::npos) { - return 1; - } - - const std::size_t observe = posix.find("::waitid("); - const std::size_t terminate = - posix.find("::kill(-pid_, SIGTERM)"); - const std::size_t force_kill = - posix.rfind("::kill(-pid_, SIGKILL)"); - const std::size_t final_reap = posix.find("::waitpid(pid_"); - - check("MCP stop observes the POSIX child without reaping", - observe != std::string::npos && - posix.find("WNOWAIT", observe) != std::string::npos); - check("MCP stop cleans the POSIX process group before final reap", - observe < terminate && terminate < force_kill && - force_kill < final_reap); - check("MCP stop does not use a reaping POSIX status probe", - posix.find("::waitpid(pid_, &status, WNOHANG)") == - std::string::npos && - posix.find("::waitpid(pid_, &status, 0)") == final_reap); - } catch (const std::exception& error) { - std::cerr << "mcp client process contract failed: " << error.what() - << '\n'; - return 1; - } - - if (failures != 0) { - std::cerr << failures << " MCP client process contract check(s) failed\n"; - return 1; - } - std::cout << "MCP client process contract checks passed\n"; - return 0; -} diff --git a/test/cpp/test_process_manager.cpp b/test/cpp/test_process_manager.cpp index dc68a1224f..f609bb4511 100644 --- a/test/cpp/test_process_manager.cpp +++ b/test/cpp/test_process_manager.cpp @@ -1,19 +1,14 @@ #include -#include #include #include #include #include -#include #include -#include #include #include #include -#include -#include #include #include @@ -23,55 +18,6 @@ using lemon::utils::ProcessManager; namespace { int failures = 0; -std::atomic coordinate_forks{false}; -std::atomic first_fork_ready{false}; -std::atomic release_first_fork{false}; -std::atomic second_fork_completed{false}; -std::atomic fork_prepare_count{0}; -std::atomic fork_parent_count{0}; -#ifdef __linux__ -std::atomic run_command_pipe_ready{false}; -std::atomic release_run_command_pipe{false}; -std::atomic run_command_pipe_fd{-1}; -std::atomic run_command_pipe_flags{-1}; -#endif -#ifdef __APPLE__ -std::atomic first_output_pipe_ready{false}; -std::atomic release_first_output_command{false}; -std::atomic output_pipe_hook_count{0}; -std::atomic inspect_run_command_reap_order{false}; -std::atomic run_command_reap_sequence{0}; -std::atomic run_command_exit_observed_order{-1}; -std::atomic run_command_group_cleanup_order{-1}; -std::atomic run_command_final_reap_order{-1}; -std::atomic run_command_leader_reserved{false}; -std::atomic run_command_leader_reserved_after_cleanup{false}; -std::atomic run_command_leader_absent_after_reap{false}; -#endif - -void before_coordinated_fork() { - if (!coordinate_forks.load()) { - return; - } - -#ifdef __linux__ - if (fork_prepare_count.fetch_add(1) == 0) { - first_fork_ready.store(true); - while (coordinate_forks.load() && !release_first_fork.load()) { - std::this_thread::yield(); - } - } -#else - fork_prepare_count.fetch_add(1); -#endif -} - -void after_coordinated_fork_in_parent() { - fork_parent_count.fetch_add(1); - if (coordinate_forks.load() && fork_prepare_count.load() >= 2) { - second_fork_completed.store(true); - } -} void check(const char* name, bool condition) { std::printf("[%s] %s\n", condition ? "PASS" : "FAIL", name); @@ -80,7 +26,6 @@ void check(const char* name, bool condition) { } } -#ifdef __linux__ ProcessHandle make_handle(pid_t pid) { return {nullptr, static_cast(pid)}; } @@ -139,750 +84,9 @@ void kill_and_reap(pid_t pid) { } } -std::string read_fd_target(pid_t pid, int fd) { - const std::string path = "/proc/" + std::to_string(pid) + "/fd/" + - std::to_string(fd); - char target[256]; - const ssize_t length = readlink(path.c_str(), target, sizeof(target) - 1); - if (length < 0) { - return {}; - } - target[length] = '\0'; - return target; -} - -bool wait_for_process_name(pid_t pid, const std::string& expected, - std::chrono::milliseconds timeout) { - const auto deadline = std::chrono::steady_clock::now() + timeout; - do { - std::ifstream comm_file("/proc/" + std::to_string(pid) + "/comm"); - std::string process_name; - if (std::getline(comm_file, process_name) && process_name == expected) { - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (std::chrono::steady_clock::now() < deadline); - return false; -} -#endif - -int capture_with_closed_standard_output() { - const pid_t probe = fork(); - if (probe == 0) { - close(STDOUT_FILENO); - close(STDERR_FILENO); - try { - const ProcessHandle handle = ProcessManager::start_process( - "/bin/sh", - {"-c", "printf 'closed-stdio-error' >&2"}, - "", - false, - false, - {}, - 64); - if (handle.pid <= 0 || - ProcessManager::wait_for_exit(handle, 5) != 0) { - _exit(2); - } - if (ProcessManager::read_output(handle, 64) != - "closed-stdio-error") { - _exit(3); - } - - std::string command_output; - const int result = ProcessManager::run_process_with_output( - "/bin/sh", - {"-c", "printf 'closed-stdio-version' >&2"}, - [&command_output](const std::string& line) { - command_output += line; - return true; - }, - "", - 5); - _exit(result == 0 && command_output == "closed-stdio-version" ? 0 - : 5); - } catch (...) { - _exit(4); - } - } - if (probe <= 0) { - return -1; - } - - int status = 0; - if (waitpid(probe, &status, 0) != probe || !WIFEXITED(status)) { - return -1; - } - return WEXITSTATUS(status); -} - -#ifdef __linux__ -struct ConcurrentCaptureResult { - bool setup_succeeded = false; - int short_exit_code = -1; - long short_read_ms = -1; - std::string short_output; -}; - -struct ConcurrentPopenResult { - bool setup_succeeded = false; - bool close_on_exec = false; - bool descriptor_inherited = false; - int command_status = -1; - std::string command_output; -}; - -ConcurrentPopenResult run_overlapping_popen_and_exec() { - ConcurrentPopenResult result; - char sentinel_path[] = "/tmp/lemonade-popen-overlap-XXXXXX"; - const int sentinel_fd = mkstemp(sentinel_path); - if (sentinel_fd < 0) { - return result; - } - close(sentinel_fd); - unlink(sentinel_path); - - run_command_pipe_ready.store(false); - release_run_command_pipe.store(false); - run_command_pipe_fd.store(-1); - run_command_pipe_flags.store(-1); - - auto command = std::async(std::launch::async, [&]() { - return ProcessManager::run_command("printf popen-output", - result.command_output, 5); - }); - - const auto hook_deadline = - std::chrono::steady_clock::now() + std::chrono::seconds(1); - while (!run_command_pipe_ready.load() && - std::chrono::steady_clock::now() < hook_deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - ProcessHandle long_handle{nullptr, 0, nullptr}; - bool child_exec_observed = false; - std::string parent_pipe_target; - std::string child_pipe_target; - const int pipe_fd = run_command_pipe_fd.load(); - if (run_command_pipe_ready.load() && pipe_fd >= 0) { - parent_pipe_target = read_fd_target(getpid(), pipe_fd); - try { - long_handle = ProcessManager::start_process( - "/bin/sh", - {"-c", "printf ready > " + std::string(sentinel_path) + - "; exec sleep 3"}, - "", false, false, {}, 64); - child_exec_observed = - wait_for_process_name(long_handle.pid, "sleep", - std::chrono::milliseconds(1000)) && - access(sentinel_path, F_OK) == 0; - if (child_exec_observed) { - child_pipe_target = read_fd_target(long_handle.pid, pipe_fd); - } - } catch (...) { - } - } - - result.close_on_exec = - (run_command_pipe_flags.load() & FD_CLOEXEC) != 0; - result.descriptor_inherited = !parent_pipe_target.empty() && - child_pipe_target == parent_pipe_target; - result.setup_succeeded = run_command_pipe_ready.load() && - parent_pipe_target.rfind("pipe:[", 0) == 0 && - long_handle.pid > 0 && child_exec_observed; - - release_run_command_pipe.store(true); - if (long_handle.pid > 0) { - ProcessManager::kill_process(long_handle); - } - result.command_status = command.get(); - unlink(sentinel_path); - return result; -} - -ConcurrentCaptureResult run_overlapping_captured_spawns() { - ConcurrentCaptureResult result; - if (pthread_atfork(before_coordinated_fork, - after_coordinated_fork_in_parent, nullptr) != 0) { - return result; - } - - coordinate_forks.store(true); - first_fork_ready.store(false); - release_first_fork.store(false); - second_fork_completed.store(false); - fork_prepare_count.store(0); - fork_parent_count.store(0); - - ProcessHandle short_handle{nullptr, 0, nullptr}; - ProcessHandle long_handle{nullptr, 0, nullptr}; - std::exception_ptr short_error; - std::exception_ptr long_error; - std::atomic short_spawn_finished{false}; - std::atomic long_spawn_finished{false}; - std::atomic workers_can_exit{false}; - - std::thread short_thread([&]() { - try { - short_handle = ProcessManager::start_process( - "/bin/sh", {"-c", "printf 'short-capture' >&2"}, "", false, - false, {}, 64); - } catch (...) { - short_error = std::current_exception(); - second_fork_completed.store(true); - } - short_spawn_finished.store(true); - while (!workers_can_exit.load()) { - std::this_thread::yield(); - } - }); - - const auto coordination_deadline = - std::chrono::steady_clock::now() + std::chrono::seconds(5); - while (!first_fork_ready.load() && - std::chrono::steady_clock::now() < coordination_deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - std::thread long_thread; - if (first_fork_ready.load()) { - long_thread = std::thread([&]() { - try { - long_handle = ProcessManager::start_process( - "/bin/sh", {"-c", "exec sleep 3"}, "", false, false, {}, - 64); - } catch (...) { - long_error = std::current_exception(); - second_fork_completed.store(true); - } - long_spawn_finished.store(true); - while (!workers_can_exit.load()) { - std::this_thread::yield(); - } - }); - } else { - second_fork_completed.store(true); - long_spawn_finished.store(true); - } - - while (first_fork_ready.load() && !second_fork_completed.load() && - std::chrono::steady_clock::now() < coordination_deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - const bool forks_overlapped = second_fork_completed.load(); - release_first_fork.store(true); - if (!forks_overlapped) { - second_fork_completed.store(true); - } - - while ((!short_spawn_finished.load() || !long_spawn_finished.load()) && - std::chrono::steady_clock::now() < coordination_deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - coordinate_forks.store(false); - - if (!forks_overlapped || !short_spawn_finished.load() || - !long_spawn_finished.load() || short_error || long_error || - short_handle.pid <= 0 || long_handle.pid <= 0) { - ProcessManager::kill_process(short_handle); - ProcessManager::kill_process(long_handle); - workers_can_exit.store(true); - short_thread.join(); - if (long_thread.joinable()) { - long_thread.join(); - } - return result; - } - - const auto exit_deadline = - std::chrono::steady_clock::now() + std::chrono::seconds(5); - while (result.short_exit_code < 0 && - std::chrono::steady_clock::now() < exit_deadline) { - result.short_exit_code = ProcessManager::get_exit_code(short_handle); - if (result.short_exit_code < 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - } - - const auto read_started = std::chrono::steady_clock::now(); - result.short_output = ProcessManager::read_output(short_handle, 64, 1000); - result.short_read_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - read_started) - .count(); - if (result.short_exit_code < 0) { - ProcessManager::kill_process(short_handle); - } - ProcessManager::kill_process(long_handle); - workers_can_exit.store(true); - short_thread.join(); - long_thread.join(); - result.setup_succeeded = true; - return result; -} -#endif - -#ifdef __APPLE__ -struct ConcurrentRunWithOutputResult { - bool setup_succeeded = false; - bool short_completed_promptly = false; - int short_exit_code = -1; - int long_exit_code = -1; - long short_wait_ms = -1; - std::string short_output; -}; - -ConcurrentRunWithOutputResult run_overlapping_output_commands() { - ConcurrentRunWithOutputResult result; - char sentinel_path[] = "/tmp/lemonade-process-overlap-XXXXXX"; - const int sentinel_fd = mkstemp(sentinel_path); - if (sentinel_fd < 0) { - return result; - } - close(sentinel_fd); - unlink(sentinel_path); - - coordinate_forks.store(true); - first_output_pipe_ready.store(false); - release_first_output_command.store(false); - output_pipe_hook_count.store(0); - - std::string short_output; - auto short_command = std::async(std::launch::async, [&]() { - return ProcessManager::run_process_with_output( - "/bin/sh", {"-c", "printf 'short-capture\\n'"}, - [&short_output](const std::string& line) { - short_output += line; - return true; - }, - "", 5); - }); - - const auto first_pipe_deadline = - std::chrono::steady_clock::now() + std::chrono::seconds(1); - while (!first_output_pipe_ready.load() && - std::chrono::steady_clock::now() < first_pipe_deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - std::string long_output; - const std::string long_command_text = - "printf ready > " + std::string(sentinel_path) + "; sleep 3"; - auto long_command = std::async(std::launch::async, [&]() { - return ProcessManager::run_command(long_command_text, long_output, 5); - }); - - const auto sentinel_deadline = - std::chrono::steady_clock::now() + std::chrono::seconds(1); - while (access(sentinel_path, F_OK) != 0 && - std::chrono::steady_clock::now() < sentinel_deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - const bool setup_succeeded = first_output_pipe_ready.load() && - output_pipe_hook_count.load() == 2 && - access(sentinel_path, F_OK) == 0; - release_first_output_command.store(true); - coordinate_forks.store(false); - - const auto wait_started = std::chrono::steady_clock::now(); - result.short_completed_promptly = - short_command.wait_for(std::chrono::milliseconds(1000)) == - std::future_status::ready; - result.short_wait_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - wait_started) - .count(); - result.long_exit_code = long_command.get(); - result.short_exit_code = short_command.get(); - result.short_output = std::move(short_output); - result.setup_succeeded = setup_succeeded; - unlink(sentinel_path); - return result; -} - -bool wait_for_process_gone(pid_t pid, std::chrono::milliseconds timeout) { - const auto deadline = std::chrono::steady_clock::now() + timeout; - do { - errno = 0; - if (::kill(pid, 0) != 0 && errno == ESRCH) { - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } while (std::chrono::steady_clock::now() < deadline); - - errno = 0; - return ::kill(pid, 0) != 0 && errno == ESRCH; -} - -struct RunCommandTimeoutResult { - bool setup_succeeded = false; - bool descendant_gone = false; - int command_status = 0; -}; - -RunCommandTimeoutResult run_command_with_descendant_timeout() { - RunCommandTimeoutResult result; - char sentinel_path[] = "/tmp/lemonade-command-descendant-XXXXXX"; - const int sentinel_fd = mkstemp(sentinel_path); - if (sentinel_fd < 0) { - return result; - } - close(sentinel_fd); - unlink(sentinel_path); - - std::string output; - result.command_status = ProcessManager::run_command( - "sleep 30 & echo $! > " + std::string(sentinel_path) + "; wait", - output, 1); - - std::ifstream sentinel(sentinel_path); - pid_t descendant = -1; - sentinel >> descendant; - result.setup_succeeded = descendant > 0; - if (result.setup_succeeded) { - result.descendant_gone = - wait_for_process_gone(descendant, std::chrono::seconds(2)); - if (!result.descendant_gone) { - ::kill(descendant, SIGKILL); - } - } - unlink(sentinel_path); - return result; -} - -struct RunCommandBackgroundResult { - bool setup_succeeded = false; - bool descendant_gone = false; - int command_status = -1; -}; - -RunCommandBackgroundResult run_command_with_background_descendant() { - RunCommandBackgroundResult result; - char sentinel_path[] = "/tmp/lemonade-command-background-XXXXXX"; - const int sentinel_fd = mkstemp(sentinel_path); - if (sentinel_fd < 0) { - return result; - } - close(sentinel_fd); - unlink(sentinel_path); - - std::string output; - result.command_status = ProcessManager::run_command( - "sleep 30 /dev/null 2>&1 & echo $! > " + - std::string(sentinel_path), - output, 5); - - std::ifstream sentinel(sentinel_path); - pid_t descendant = -1; - sentinel >> descendant; - result.setup_succeeded = descendant > 0; - if (result.setup_succeeded) { - result.descendant_gone = - wait_for_process_gone(descendant, std::chrono::seconds(2)); - if (!result.descendant_gone) { - ::kill(descendant, SIGKILL); - } - } - unlink(sentinel_path); - return result; -} - -struct RunCommandReapOrderResult { - bool leader_reserved = false; - bool leader_reserved_after_cleanup = false; - bool leader_absent_after_reap = false; - int exit_observed_order = -1; - int group_cleanup_order = -1; - int final_reap_order = -1; - int command_status = -1; - std::string command_output; -}; - -RunCommandReapOrderResult run_command_without_descendants() { - inspect_run_command_reap_order.store(true); - run_command_reap_sequence.store(0); - run_command_exit_observed_order.store(-1); - run_command_group_cleanup_order.store(-1); - run_command_final_reap_order.store(-1); - run_command_leader_reserved.store(false); - run_command_leader_reserved_after_cleanup.store(false); - run_command_leader_absent_after_reap.store(false); - - RunCommandReapOrderResult result; - result.command_status = - ProcessManager::run_command("printf no-descendant; exit 9", - result.command_output, 5); - inspect_run_command_reap_order.store(false); - result.leader_reserved = run_command_leader_reserved.load(); - result.leader_reserved_after_cleanup = - run_command_leader_reserved_after_cleanup.load(); - result.leader_absent_after_reap = - run_command_leader_absent_after_reap.load(); - result.exit_observed_order = run_command_exit_observed_order.load(); - result.group_cleanup_order = run_command_group_cleanup_order.load(); - result.final_reap_order = run_command_final_reap_order.load(); - return result; -} -#endif - } // namespace -#ifdef __linux__ -extern "C" void lemonade_test_run_command_pipe_created(int fd) { - int flags; - do { - flags = fcntl(fd, F_GETFD); - } while (flags < 0 && errno == EINTR); - run_command_pipe_fd.store(fd); - run_command_pipe_flags.store(flags); - run_command_pipe_ready.store(true); - while (!release_run_command_pipe.load()) { - std::this_thread::yield(); - } -} -#endif - -#ifdef __APPLE__ -extern "C" void lemonade_test_run_with_output_pipe_created() { - if (!coordinate_forks.load()) { - return; - } - - const int index = output_pipe_hook_count.fetch_add(1); - if (index == 0) { - first_output_pipe_ready.store(true); - while (coordinate_forks.load() && - !release_first_output_command.load()) { - std::this_thread::yield(); - } - } else if (index == 1) { - return; - } -} - -bool exited_child_is_reserved(pid_t pid) { - siginfo_t child_info{}; - int result; - do { - result = waitid(P_PID, static_cast(pid), &child_info, - WEXITED | WNOHANG | WNOWAIT); - } while (result < 0 && errno == EINTR); - return result == 0 && child_info.si_pid == pid; -} - -extern "C" void lemonade_test_run_command_exit_observed(pid_t pid) { - if (!inspect_run_command_reap_order.load()) { - return; - } - run_command_leader_reserved.store(exited_child_is_reserved(pid)); - run_command_exit_observed_order.store( - run_command_reap_sequence.fetch_add(1)); -} - -extern "C" void lemonade_test_run_command_group_cleanup_complete(pid_t pid) { - if (!inspect_run_command_reap_order.load()) { - return; - } - run_command_leader_reserved_after_cleanup.store( - exited_child_is_reserved(pid)); - run_command_group_cleanup_order.store( - run_command_reap_sequence.fetch_add(1)); -} - -extern "C" void lemonade_test_run_command_final_reap_complete(pid_t pid) { - if (!inspect_run_command_reap_order.load()) { - return; - } - siginfo_t child_info{}; - errno = 0; - int result; - do { - result = waitid(P_PID, static_cast(pid), &child_info, - WEXITED | WNOHANG | WNOWAIT); - } while (result < 0 && errno == EINTR); - run_command_leader_absent_after_reap.store(result < 0 && errno == ECHILD); - run_command_final_reap_order.store(run_command_reap_sequence.fetch_add(1)); -} -#endif - int main() { - { - const ProcessHandle handle = ProcessManager::start_process( - "/bin/sh", - {"-c", "printf 'discarded-prefix:original-startup-error' >&2"}, - "", - false, - false, - {}, - 22); - check("start_process returns a child handle for captured output", - handle.pid > 0); - - if (handle.pid > 0) { - check("captured-output child exits successfully", - ProcessManager::wait_for_exit(handle, 5) == 0); - const std::string output = ProcessManager::read_output(handle, 22); - check("read_output returns a bounded tail from the original child", - output == "original-startup-error"); - } - } - - check("process output capture survives closed stdout and stderr", - capture_with_closed_standard_output() == 0); - -#ifdef __linux__ - { - const ConcurrentPopenResult result = run_overlapping_popen_and_exec(); - check("overlapping popen and exec complete setup", - result.setup_succeeded); - check("popen stream is close-on-exec", result.close_on_exec); - check("exec child cannot inherit the popen stream", - !result.descriptor_inherited); - check("popen command preserves output and wait status", - WIFEXITED(result.command_status) && - WEXITSTATUS(result.command_status) == 0 && - result.command_output == "popen-output"); - } - - { - const ConcurrentCaptureResult result = run_overlapping_captured_spawns(); - check("overlapping captured spawns complete setup", - result.setup_succeeded); - if (result.setup_succeeded) { - std::printf("[INFO] short exit=%d read=%ldms output='%s'\n", - result.short_exit_code, result.short_read_ms, - result.short_output.c_str()); - check("short captured process exits successfully", - result.short_exit_code == 0); - check("long process cannot retain the short process capture pipes", - result.short_read_ms < 750); - check("short process output remains isolated", - result.short_output == "short-capture"); - } - } -#endif - -#ifdef __APPLE__ - { - const RunCommandReapOrderResult result = - run_command_without_descendants(); - check("run_command no-descendant probe preserves output and status", - result.command_output == "no-descendant" && - WIFEXITED(result.command_status) && - WEXITSTATUS(result.command_status) == 9); - check("run_command reserves the exited leader through group cleanup", - result.leader_reserved && - result.leader_reserved_after_cleanup); - check("run_command reaps only after process-group cleanup", - result.exit_observed_order == 0 && - result.group_cleanup_order == 1 && - result.final_reap_order == 2 && - result.leader_absent_after_reap); - } - - { - const ConcurrentRunWithOutputResult result = - run_overlapping_output_commands(); - check("overlapping output commands complete setup", - result.setup_succeeded); - std::printf("[INFO] short exit=%d wait=%ldms output='%s'\n", - result.short_exit_code, result.short_wait_ms, - result.short_output.c_str()); - check("short output command exits successfully", - result.short_exit_code == 0); - check("long output command exits successfully", - result.long_exit_code == 0); - check("long command cannot retain the short command capture pipe", - result.short_completed_promptly); - check("short command output remains isolated", - result.short_output == "short-capture"); - } - - { - std::string output; - const int status = ProcessManager::run_command( - "printf 'line-one\\nunterminated'; exit 7", output, 5); - check("run_command preserves raw output bytes", - output == "line-one\nunterminated"); - check("run_command preserves wait-status semantics", - WIFEXITED(status) && WEXITSTATUS(status) == 7); - } - - { - const RunCommandTimeoutResult result = - run_command_with_descendant_timeout(); - check("run_command timeout descendant test completes setup", - result.setup_succeeded); - check("run_command reports timeout", result.command_status == -1); - check("run_command timeout terminates descendant processes", - result.descendant_gone); - } - - { - std::string output; - const auto started = std::chrono::steady_clock::now(); - const int status = ProcessManager::run_command( - "exec 1>&-; sleep 5", output, 1); - const auto elapsed = std::chrono::steady_clock::now() - started; - check("run_command enforces timeout after output closes", - status == -1); - check("run_command returns promptly after output closes", - elapsed < std::chrono::seconds(3)); - } - - { - const RunCommandBackgroundResult result = - run_command_with_background_descendant(); - check("run_command background descendant test completes setup", - result.setup_succeeded); - check("run_command preserves successful shell status", - WIFEXITED(result.command_status) && - WEXITSTATUS(result.command_status) == 0); - check("run_command success terminates residual process-group members", - result.descendant_gone); - } - - { - std::string output; - const int status = ProcessManager::run_command( - "sleep 1; printf 'no-deadline'", output, 0); - check("run_command zero timeout waits without a deadline", - WIFEXITED(status) && WEXITSTATUS(status) == 0 && - output == "no-deadline"); - } - - { - std::string output; - const int exit_code = ProcessManager::run_process_with_output( - "/bin/pwd", {}, - [&output](const std::string& line) { - output += line; - return true; - }, - "/private/tmp", 5); - check("output command preserves working-directory semantics", - exit_code == 0 && output == "/private/tmp"); - } - - { - const auto started = std::chrono::steady_clock::now(); - const int exit_code = ProcessManager::run_process_with_output( - "/bin/sh", {"-c", "printf 'cancel\\n'; sleep 3"}, - [](const std::string&) { return false; }, "", 5); - const auto elapsed = std::chrono::steady_clock::now() - started; - check("output callback cancellation kills and reaps promptly", - exit_code == -1 && elapsed < std::chrono::seconds(2)); - } -#endif - - { - const ProcessHandle handle = ProcessManager::start_process( - "/bin/sh", {"-c", "printf 'GET /health\\n'"}, "", true, true); - check("filtered output creates a reader-completion tracker", - handle.output_capture != nullptr); - check("filtered-output child exits successfully", - ProcessManager::wait_for_exit(handle, 5) == 0); - } - -#ifdef __linux__ { const pid_t child = spawn_exiting_child(42); check("fork exited child", child > 0); @@ -924,7 +128,6 @@ int main() { check("is_running() rejects non-existent PID", !ProcessManager::is_running( make_handle(std::numeric_limits::max()))); -#endif if (failures == 0) { std::printf("\nAll process_manager tests passed\n"); diff --git a/test/cpp/test_process_manager_windows.cpp b/test/cpp/test_process_manager_windows.cpp deleted file mode 100644 index 67d7b66105..0000000000 --- a/test/cpp/test_process_manager_windows.cpp +++ /dev/null @@ -1,549 +0,0 @@ -#ifndef _WINSOCKAPI_ -#define _WINSOCKAPI_ -#endif -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using lemon::utils::ProcessHandle; -using lemon::utils::ProcessManager; - -namespace { - -int failures = 0; - -void check(const char* name, bool condition) { - std::printf("[%s] %s\n", condition ? "PASS" : "FAIL", name); - if (!condition) { - ++failures; - } -} - -bool standard_handle_is_valid(DWORD stream) { - const HANDLE handle = GetStdHandle(stream); - if (handle == nullptr || handle == INVALID_HANDLE_VALUE) { - return false; - } - - SetLastError(ERROR_SUCCESS); - const DWORD type = GetFileType(handle); - return type != FILE_TYPE_UNKNOWN || GetLastError() == ERROR_SUCCESS; -} - -int probe_standard_handles() { - int result = 0; - if (!standard_handle_is_valid(STD_INPUT_HANDLE)) { - result |= 1; - } - if (!standard_handle_is_valid(STD_OUTPUT_HANDLE)) { - result |= 2; - } - if (!standard_handle_is_valid(STD_ERROR_HANDLE)) { - result |= 4; - } - - if ((result & 2) == 0) { - constexpr char message[] = "stdout-ready\n"; - DWORD written = 0; - WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), message, - static_cast(sizeof(message) - 1), &written, nullptr); - } - if ((result & 4) == 0) { - constexpr char message[] = "stderr-ready\n"; - DWORD written = 0; - WriteFile(GetStdHandle(STD_ERROR_HANDLE), message, - static_cast(sizeof(message) - 1), &written, nullptr); - } - return result; -} - -std::string current_executable() { - std::vector path(32768); - const DWORD length = GetModuleFileNameA( - nullptr, path.data(), static_cast(path.size())); - if (length == 0 || - static_cast(length) >= path.size()) { - return {}; - } - return std::string(path.data(), length); -} - -std::string unique_sentinel_path() { - char temp_directory[MAX_PATH + 1] = {}; - const DWORD directory_length = GetTempPathA( - static_cast(sizeof(temp_directory)), temp_directory); - if (directory_length == 0 || directory_length >= sizeof(temp_directory)) { - return {}; - } - - char path[MAX_PATH + 1] = {}; - if (GetTempFileNameA(temp_directory, "lpm", 0, path) == 0) { - return {}; - } - if (!DeleteFileA(path)) { - return {}; - } - return path; -} - -bool create_sentinel(const std::string& path) { - const HANDLE file = CreateFileA( - path.c_str(), GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, - CREATE_ALWAYS, FILE_ATTRIBUTE_TEMPORARY, nullptr); - if (file == INVALID_HANDLE_VALUE) { - return false; - } - return CloseHandle(file) != FALSE; -} - -bool sentinel_exists(const std::string& path) { - const DWORD attributes = GetFileAttributesA(path.c_str()); - return attributes != INVALID_FILE_ATTRIBUTES && - (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; -} - -bool wait_for_sentinel( - const std::string& path, - std::chrono::milliseconds timeout) { - const auto deadline = std::chrono::steady_clock::now() + timeout; - do { - if (sentinel_exists(path)) { - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } while (std::chrono::steady_clock::now() < deadline); - return sentinel_exists(path); -} - -bool wait_for_flag( - const std::atomic& flag, - std::chrono::milliseconds timeout) { - const auto deadline = std::chrono::steady_clock::now() + timeout; - do { - if (flag.load(std::memory_order_acquire)) { - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } while (std::chrono::steady_clock::now() < deadline); - return flag.load(std::memory_order_acquire); -} - -bool pipe_closed_without_marker(HANDLE pipe) { - char marker = '\0'; - DWORD bytes_read = 0; - if (ReadFile(pipe, &marker, 1, &bytes_read, nullptr)) { - return bytes_read == 0; - } - return GetLastError() == ERROR_BROKEN_PIPE; -} - -class TemporarySentinel { -public: - TemporarySentinel() : path_(unique_sentinel_path()) { - } - - ~TemporarySentinel() { - if (!path_.empty()) { - DeleteFileA(path_.c_str()); - } - } - - TemporarySentinel(const TemporarySentinel&) = delete; - TemporarySentinel& operator=(const TemporarySentinel&) = delete; - - const std::string& path() const { - return path_; - } - - bool valid() const { - return !path_.empty(); - } - - bool signal() const { - return valid() && create_sentinel(path_); - } - -private: - std::string path_; -}; - -class ScopedHandle { -public: - ScopedHandle() = default; - - ~ScopedHandle() { - reset(); - } - - ScopedHandle(const ScopedHandle&) = delete; - ScopedHandle& operator=(const ScopedHandle&) = delete; - - HANDLE get() const { - return handle_; - } - - HANDLE* receive() { - reset(); - return &handle_; - } - - void reset(HANDLE handle = nullptr) { - if (handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE) { - CloseHandle(handle_); - } - handle_ = handle; - } - -private: - HANDLE handle_ = nullptr; -}; - -int capture_wait_child(const std::string& release_path) { - constexpr char message[] = "capture-ready\n"; - DWORD written = 0; - if (!WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), message, - static_cast(sizeof(message) - 1), &written, - nullptr) || - written != sizeof(message) - 1) { - return 10; - } - return wait_for_sentinel(release_path, std::chrono::seconds(30)) ? 0 : 11; -} - -int long_wait_child( - const std::string& ready_path, - const std::string& release_path, - const std::string& canary_handle_value) { - std::uintptr_t canary_value = 0; - try { - std::size_t parsed = 0; - canary_value = static_cast( - std::stoull(canary_handle_value, &parsed)); - if (parsed != canary_handle_value.size()) { - return 20; - } - } catch (const std::exception&) { - return 20; - } - - constexpr char marker = 'I'; - DWORD written = 0; - WriteFile(reinterpret_cast(canary_value), &marker, 1, &written, - nullptr); - if (!create_sentinel(ready_path)) { - return 21; - } - return wait_for_sentinel(release_path, std::chrono::seconds(30)) ? 0 : 22; -} - -class InvalidStandardHandles { -public: - explicit InvalidStandardHandles( - HANDLE replacement = INVALID_HANDLE_VALUE) - : input_(GetStdHandle(STD_INPUT_HANDLE)), - output_(GetStdHandle(STD_OUTPUT_HANDLE)), - error_(GetStdHandle(STD_ERROR_HANDLE)) { - const BOOL input_set = - SetStdHandle(STD_INPUT_HANDLE, replacement); - const BOOL output_set = - SetStdHandle(STD_OUTPUT_HANDLE, replacement); - const BOOL error_set = - SetStdHandle(STD_ERROR_HANDLE, replacement); - installed_ = input_set && output_set && error_set; - } - - ~InvalidStandardHandles() { - SetStdHandle(STD_INPUT_HANDLE, input_); - SetStdHandle(STD_OUTPUT_HANDLE, output_); - SetStdHandle(STD_ERROR_HANDLE, error_); - } - - bool installed() const { - return installed_; - } - -private: - HANDLE input_; - HANDLE output_; - HANDLE error_; - bool installed_ = false; -}; - -void test_concurrent_capture_handle_isolation(const std::string& executable) { - TemporarySentinel capture_release; - TemporarySentinel long_ready; - TemporarySentinel long_release; - const bool sentinels_ready = capture_release.valid() && - long_ready.valid() && long_release.valid(); - check("create process-overlap sentinels", sentinels_ready); - if (!sentinels_ready) { - return; - } - - std::atomic capture_ready{false}; - std::string captured_output; - std::future capture_future; - bool capture_launched = false; - try { - capture_future = std::async( - std::launch::async, - [&]() { - return ProcessManager::run_process_with_output( - executable, {"--capture-wait", capture_release.path()}, - [&](const std::string& line) { - captured_output += line; - captured_output.push_back('\n'); - if (line == "capture-ready") { - capture_ready.store(true, std::memory_order_release); - } - return true; - }, - "", 20); - }); - capture_launched = true; - } catch (const std::exception&) { - } - check("launch capture child A", capture_launched); - if (!capture_launched) { - return; - } - - const bool capture_announced = - wait_for_flag(capture_ready, std::chrono::seconds(5)); - check("capture child A announces readiness", capture_announced); - - ScopedHandle canary_read; - ScopedHandle canary_write; - bool canary_created = false; - if (capture_announced) { - SECURITY_ATTRIBUTES attributes{}; - attributes.nLength = sizeof(attributes); - attributes.bInheritHandle = TRUE; - if (CreatePipe( - canary_read.receive(), canary_write.receive(), &attributes, - 0) && - SetHandleInformation( - canary_read.get(), HANDLE_FLAG_INHERIT, 0)) { - canary_created = true; - } - } - check("create inheritable overlap canary", canary_created); - - ProcessHandle long_handle{}; - bool long_started = false; - if (canary_created) { - try { - long_handle = ProcessManager::start_process( - executable, - {"--long-wait", long_ready.path(), long_release.path(), - std::to_string(reinterpret_cast( - canary_write.get()))}, "", - false, false); - long_started = long_handle.handle != nullptr; - } catch (const std::exception&) { - } - } - check("start long-lived child B during capture", long_started); - - bool long_announced = false; - if (long_started) { - long_announced = - wait_for_sentinel(long_ready.path(), std::chrono::seconds(5)); - } - check("long-lived child B announces readiness", long_announced); - - const bool long_running_before_release = - long_announced && ProcessManager::is_running(long_handle); - check("long-lived child B is running", long_running_before_release); - - canary_write.reset(); - const bool canary_excluded = - canary_created && long_running_before_release && - pipe_closed_without_marker(canary_read.get()); - check("child B excludes unrelated inheritable pipe writers", - canary_excluded); - - const bool capture_released = capture_release.signal(); - check("release capture child A", capture_released); - - bool capture_completed_promptly = false; - int capture_exit = -1; - bool capture_result_collected = false; - if (capture_released && long_running_before_release) { - capture_completed_promptly = - capture_future.wait_for(std::chrono::seconds(3)) == - std::future_status::ready; - if (capture_completed_promptly) { - try { - capture_exit = capture_future.get(); - capture_result_collected = true; - } catch (const std::exception&) { - } - } - } - - const bool long_running_after_capture = - capture_result_collected && ProcessManager::is_running(long_handle); - check("capture returns promptly while child B remains running", - capture_completed_promptly && long_running_after_capture); - - capture_release.signal(); - const bool long_released = long_release.signal(); - check("release long-lived child B", long_released); - - int long_exit = -1; - if (long_started) { - long_exit = ProcessManager::wait_for_exit(long_handle, 10); - if (long_exit >= 0) { - ProcessManager::reap_process(long_handle); - } else { - ProcessManager::kill_process(long_handle); - } - } - check("long-lived child B exits after release", long_exit == 0); - - if (!capture_result_collected) { - try { - capture_exit = capture_future.get(); - capture_result_collected = true; - } catch (const std::exception&) { - } - } - check("capture child A exits after release", - capture_result_collected && capture_exit == 0); - check("capture child A output is complete", - captured_output.find("capture-ready\n") != std::string::npos); -} - -} // namespace - -int main(int argc, char** argv) { - if (argc == 2 && std::string(argv[1]) == "--probe-stdio") { - return probe_standard_handles(); - } - if (argc == 3 && std::string(argv[1]) == "--capture-wait") { - return capture_wait_child(argv[2]); - } - if (argc == 5 && std::string(argv[1]) == "--long-wait") { - return long_wait_child(argv[2], argv[3], argv[4]); - } - - const std::string executable = current_executable(); - check("resolve test executable", !executable.empty()); - if (executable.empty()) { - return failures; - } - - ProcessHandle quiet_handle{}; - bool quiet_override_installed = false; - bool quiet_started = false; - { - InvalidStandardHandles invalid_stdio; - quiet_override_installed = invalid_stdio.installed(); - if (quiet_override_installed) { - try { - quiet_handle = ProcessManager::start_process( - executable, {"--probe-stdio"}, "", false, false); - quiet_started = quiet_handle.handle != nullptr; - } catch (const std::exception&) { - } - } - } - check("replace parent standard handles", quiet_override_installed); - check("start quiet child without parent standard handles", quiet_started); - - int quiet_exit = -1; - if (quiet_started) { - quiet_exit = ProcessManager::wait_for_exit(quiet_handle, 10); - if (quiet_exit >= 0) { - ProcessManager::reap_process(quiet_handle); - } else { - ProcessManager::kill_process(quiet_handle); - } - } - check("quiet child receives valid standard handles", quiet_exit == 0); - - ProcessHandle stale_handle{}; - bool stale_override_installed = false; - bool stale_started = false; - const HANDLE stale_value = reinterpret_cast( - static_cast(0x1234567)); - { - InvalidStandardHandles stale_stdio(stale_value); - stale_override_installed = stale_stdio.installed(); - if (stale_override_installed) { - try { - stale_handle = ProcessManager::start_process( - executable, {"--probe-stdio"}, "", false, false); - stale_started = stale_handle.handle != nullptr; - } catch (const std::exception&) { - } - } - } - check("replace parent standard handles with stale values", - stale_override_installed); - check("start child with stale parent standard handles", stale_started); - - int stale_exit = -1; - if (stale_started) { - stale_exit = ProcessManager::wait_for_exit(stale_handle, 10); - if (stale_exit >= 0) { - ProcessManager::reap_process(stale_handle); - } else { - ProcessManager::kill_process(stale_handle); - } - } - check("stale standard handles are replaced with valid handles", - stale_exit == 0); - - bool capture_override_installed = false; - int capture_exit = -1; - std::string captured_output; - { - InvalidStandardHandles invalid_stdio; - capture_override_installed = invalid_stdio.installed(); - if (capture_override_installed) { - try { - capture_exit = ProcessManager::run_process_with_output( - executable, {"--probe-stdio"}, - [&captured_output](const std::string& line) { - captured_output += line; - captured_output.push_back('\n'); - return true; - }, - "", 10); - } catch (const std::exception&) { - } - } - } - check("replace standard handles for captured child", - capture_override_installed); - check("captured child receives valid standard handles", capture_exit == 0); - check("captured stdout remains connected", - captured_output.find("stdout-ready\n") != std::string::npos); - check("captured stderr remains connected", - captured_output.find("stderr-ready\n") != std::string::npos); - - test_concurrent_capture_handle_isolation(executable); - - if (failures == 0) { - std::printf("\nAll Windows process_manager tests passed\n"); - return 0; - } - std::printf("\n%d Windows process_manager test(s) failed\n", failures); - return 1; -} diff --git a/test/cpp/test_streaming_heartbeat.cpp b/test/cpp/test_streaming_heartbeat.cpp index 34f74362d3..f414644db4 100644 --- a/test/cpp/test_streaming_heartbeat.cpp +++ b/test/cpp/test_streaming_heartbeat.cpp @@ -214,105 +214,6 @@ static void test_heartbeat_emission_during_pause_between_tokens() { check(telemetry_result.error_message.empty(), "stream completed without error"); } -static void test_heartbeat_emission_during_partial_frame_progress() { - httplib::Server backend; - std::atomic partial_frame_in_progress{false}; - std::atomic backend_progress_count{0}; - std::atomic heartbeat_during_partial_frame{0}; - - backend.Post("/v1/chat/completions", - [&](const httplib::Request&, httplib::Response& res) { - res.set_chunked_content_provider( - "text/event-stream", - [&](size_t, httplib::DataSink& sink) { - const std::string prefix = - "data: {\"choices\":[{\"delta\":{\"content\":\""; - partial_frame_in_progress.store( - true, std::memory_order_release); - sink.write(prefix.data(), prefix.size()); - for (int fragment = 0; fragment < 40; ++fragment) { - static constexpr char payload[] = "x"; - if (!sink.write(payload, sizeof(payload) - 1)) { - partial_frame_in_progress.store( - false, std::memory_order_release); - return false; - } - std::this_thread::sleep_for(25ms); - } - partial_frame_in_progress.store( - false, std::memory_order_release); - - const std::string suffix = "\"}}]}\n\n"; - sink.write(suffix.data(), suffix.size()); - const std::string done = "data: [DONE]\n\n"; - sink.write(done.data(), done.size()); - sink.done(); - return false; - }); - }); - - const int port = backend.bind_to_any_port("127.0.0.1"); - if (port <= 0) { - std::printf("[FAIL] failed to bind mock backend\n"); - ++g_failures; - return; - } - - std::thread backend_thread([&backend]() { - backend.listen_after_bind(); - }); - backend.wait_until_ready(); - - int data_chunk_count = 0; - int done_count = 0; - httplib::DataSink downstream; - downstream.write = [&](const char* data, size_t len) { - const std::string write(data, len); - if (write == ": ping\n\n" && - partial_frame_in_progress.load(std::memory_order_acquire)) { - heartbeat_during_partial_frame.fetch_add( - 1, std::memory_order_release); - } - if (write.find("\"content\":\"") != std::string::npos) { - ++data_chunk_count; - } - if (write.find("[DONE]") != std::string::npos) { - ++done_count; - } - return true; - }; - downstream.done = []() {}; - downstream.is_writable = []() { return true; }; - - lemon::StreamingProxy::TelemetryData telemetry_result; - lemon::StreamingProxy::forward_sse_stream( - "http://127.0.0.1:" + std::to_string(port) + - "/v1/chat/completions", - R"({"model":"test-model","stream":true})", - downstream, - [&](const lemon::StreamingProxy::TelemetryData& tel) { - telemetry_result = tel; - }, - 10, - [&]() { - backend_progress_count.fetch_add(1, std::memory_order_release); - }, - 100); - - backend.stop(); - backend_thread.join(); - - check(heartbeat_during_partial_frame.load(std::memory_order_acquire) >= 1, - "heartbeat reaches the client while an SSE frame is incomplete"); - check(backend_progress_count.load(std::memory_order_acquire) > 1, - "partial SSE fragments retain backend-progress signaling"); - check(data_chunk_count == 1, - "partial SSE fragments are forwarded as one complete frame"); - check(done_count == 1, "partial-frame stream preserves one done marker"); - check(telemetry_result.error_message.empty(), - "partial-frame stream completes without error"); -} - static void test_heartbeat_client_disconnect_aborts_upstream() { httplib::Server backend; std::atomic release_backend{false}; @@ -392,7 +293,6 @@ static void test_heartbeat_client_disconnect_aborts_upstream() { int main() { test_heartbeat_emission_during_prefill(); test_heartbeat_emission_during_pause_between_tokens(); - test_heartbeat_emission_during_partial_frame_progress(); test_heartbeat_client_disconnect_aborts_upstream(); if (g_failures == 0) { diff --git a/test/fixtures/llamacpp_validation_artifacts.json b/test/fixtures/llamacpp_validation_artifacts.json deleted file mode 100644 index e690ce564f..0000000000 --- a/test/fixtures/llamacpp_validation_artifacts.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "K2-Horizon-0.9B-GGUF": { - "revision": "8496259ac62d33192d0fffe708303ed2ca29a384", - "filename": "K2-Horizon-1B-BF16.gguf", - "size_bytes": 2159424896, - "sha256": "371010db1807bb07b62e738422ee0de26c1e15a347f31108ed2c6e219095a8b8", - "gguf_architecture": "k2-horizon", - "max_context_window": 131072 - }, - "K2-Horizon-3.7B-GGUF": { - "revision": "67b824b8079635b77e19a2a4cd2682a9716478c1", - "filename": "K2-Horizon-4B-BF16.gguf", - "size_bytes": 10128343424, - "sha256": "372ecf9977eb22c92739fec478f2c887daef659d7c55a609ac97b905d1a243b1", - "gguf_architecture": "k2-horizon", - "max_context_window": 524288 - }, - "K2-Horizon-7B-GGUF": { - "revision": "bcb8c25b76112ce96a962f5b8ab624435d1ee0c9", - "filename": "K2-Horizon-7B-BF16.gguf", - "size_bytes": 18010413440, - "sha256": "088c5d0814ef955d137fd1073ee3a68f6a53411fca44b75dc58a320640000444", - "gguf_architecture": "k2-horizon", - "max_context_window": 524288 - } -} diff --git a/test/fixtures/llamacpp_validation_models.json b/test/fixtures/llamacpp_validation_models.json deleted file mode 100644 index 45cfefc156..0000000000 --- a/test/fixtures/llamacpp_validation_models.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "K2-Horizon-0.9B-GGUF": { - "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf", - "recipe": "llamacpp", - "suggested": true, - "labels": [ - "chat", - "reasoning", - "tool-calling", - "hot" - ], - "size": 2.16 - }, - "K2-Horizon-3.7B-GGUF": { - "checkpoint": "IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf", - "recipe": "llamacpp", - "suggested": true, - "labels": [ - "chat", - "reasoning", - "tool-calling" - ], - "size": 10.13 - }, - "K2-Horizon-7B-GGUF": { - "checkpoint": "IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf", - "recipe": "llamacpp", - "suggested": true, - "labels": [ - "chat", - "reasoning", - "tool-calling" - ], - "size": 18.01 - } -} diff --git a/test/requirements.txt b/test/requirements.txt index d85c9d3551..6b5687e991 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -28,6 +28,3 @@ ollama # JSON Schema validator (required for routing policy / decision fixture tests) jsonschema - -# YAML parser (required for workflow supply-chain tests) -pyyaml diff --git a/test/test_llamacpp_capabilities.py b/test/test_llamacpp_capabilities.py deleted file mode 100644 index 80e12aeed5..0000000000 --- a/test/test_llamacpp_capabilities.py +++ /dev/null @@ -1,960 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for the llama.cpp capability validation profile.""" - -from __future__ import annotations - -import copy -import json -import types -import unittest -from unittest import mock - -from test.utils import llamacpp_capability_validation as capabilities - -MODEL = "builtin.K2-Horizon-0.9B-GGUF" - - -class CapabilityHarness: - def __init__(self) -> None: - self.requests = [] - self.reasoning_marker = None - self.omit_stream_done = False - self.early_stream_done = False - self.response_model = MODEL - self.stream_response_models = [MODEL, MODEL] - self.stream_reasoning = "high reasoning" - self.default_tool_response_is_text = False - self.stop_tool_call = False - self.tool_reasoning = False - self.reasoning_by_effort = { - "high": "high reasoning", - "medium": "medium reasoning", - "low": None, - } - self.tool_arguments = json.dumps( - capabilities.EXPECTED_TOOL_ARGUMENTS, - separators=(",", ":"), - ) - - @staticmethod - def response(status_code=200): - return types.SimpleNamespace(status_code=status_code) - - def openai(self, message, finish_reason="stop"): - return { - "id": "chatcmpl-test", - "model": self.response_model, - "choices": [ - { - "index": 0, - "finish_reason": finish_reason, - "message": {"role": "assistant", **message}, - } - ], - } - - def tool_call(self): - return { - "id": "call_weather", - "type": "function", - "function": { - "name": capabilities.TOOL_NAME, - "arguments": self.tool_arguments, - }, - } - - def stream_chunk(self, index, choices): - chunk = {"choices": choices} - model = self.stream_response_models[index] - if model is not None: - chunk["model"] = model - return chunk - - def request_json(self, method, url, timeout, **kwargs): - self.requests.append((method, url, kwargs["json"])) - payload = kwargs["json"] - if url.endswith("/api/chat"): - message = { - "role": "assistant", - "thinking": "Seventeen plus twenty-five is forty-two.", - "content": "42", - } - if self.stop_tool_call: - message["tool_calls"] = [self.tool_call()] - return self.response(), { - "model": self.response_model, - "done": True, - "done_reason": "stop", - "message": message, - } - if url.endswith("/v1/messages?beta=true"): - content = [{"type": "text", "text": "ANTHROPIC-OK"}] - if self.stop_tool_call: - content.append( - { - "type": "tool_use", - "id": "call_weather", - "name": capabilities.TOOL_NAME, - "input": capabilities.EXPECTED_TOOL_ARGUMENTS, - } - ) - return self.response(), { - "id": "msg_test", - "type": "message", - "role": "assistant", - "model": self.response_model, - "stop_reason": "end_turn", - "content": content, - "usage": {"input_tokens": 8, "output_tokens": 3}, - } - - kwargs_value = payload.get("chat_template_kwargs", {}) - if payload.get("tool_choice") == "none": - return self.response(), self.openai({"content": "K2-SUNNY-731"}) - if payload.get("tools"): - if ( - "tool_call_format" not in kwargs_value - and self.default_tool_response_is_text - ): - return self.response(), self.openai({"content": "Not a tool call."}) - message = {"content": None, "tool_calls": [self.tool_call()]} - if self.tool_reasoning: - message["reasoning_content"] = "thinking was supposed to be disabled" - return self.response(), self.openai( - message, - "tool_calls", - ) - if "reasoning_effort" in kwargs_value: - effort = kwargs_value["reasoning_effort"] - reasoning = self.reasoning_by_effort[effort] - if reasoning is not None and self.reasoning_marker == effort: - reasoning += " " - message = {"content": "42"} - if reasoning is not None: - message["reasoning_content"] = reasoning - if self.stop_tool_call: - message["tool_calls"] = [self.tool_call()] - return self.response(), self.openai(message) - message = {"content": "42"} - if self.stop_tool_call: - message["tool_calls"] = [self.tool_call()] - return self.response(), self.openai(message) - - def request_stream(self, method, url, timeout, **kwargs): - self.requests.append((method, url, kwargs["json"])) - payload = kwargs["json"] - if payload.get("tools"): - events = [ - "data: " - + json.dumps( - self.stream_chunk( - 0, - [ - { - "index": 0, - "delta": { - "tool_calls": [ - { - "index": 0, - "id": "call_stream", - "type": "function", - "function": { - "name": "lookup_", - "arguments": '{"city":"Paris","unit":"', - }, - } - ] - }, - } - ], - ), - separators=(",", ":"), - ), - "data: " - + json.dumps( - self.stream_chunk( - 1, - [ - { - "index": 0, - "delta": { - "tool_calls": [ - { - "index": 0, - "function": { - "name": "weather", - "arguments": ( - 'celsius","days":2,' - '"include_hourly":false}' - ), - }, - } - ] - }, - "finish_reason": "tool_calls", - } - ], - ), - separators=(",", ":"), - ), - ] - elif "reasoning_effort" in payload.get("chat_template_kwargs", {}): - events = [ - "data: " - + json.dumps( - self.stream_chunk( - 0, - [ - { - "index": 0, - "delta": {"reasoning_content": self.stream_reasoning}, - } - ], - ), - separators=(",", ":"), - ), - "data: " - + json.dumps( - self.stream_chunk( - 1, - [ - { - "index": 0, - "delta": {"content": "42"}, - "finish_reason": "stop", - } - ], - ), - separators=(",", ":"), - ), - ] - else: - events = [ - ": keep-alive", - "data: " - + json.dumps( - self.stream_chunk( - 0, - [{"index": 0, "delta": {"content": "4"}}], - ), - separators=(",", ":"), - ), - "data: " - + json.dumps( - self.stream_chunk( - 1, - [ - { - "index": 0, - "delta": {"content": "2"}, - "finish_reason": "stop", - } - ], - ), - separators=(",", ":"), - ), - 'data: {"choices":[],"usage":{"completion_tokens":2}}', - ] - if self.early_stream_done: - events.insert(len(events) - 1, "data: [DONE]") - elif not self.omit_stream_done: - events.append("data: [DONE]") - return self.response(), events - - -class LlamaCppCapabilityTests(unittest.TestCase): - def run_profile(self, harness=None): - active = harness or CapabilityHarness() - matrix, passed, summary = capabilities.validate_capabilities( - capabilities.K2_HORIZON_PROFILE, - base_url="http://localhost:13305/api/v1", - model=MODEL, - request_json=active.request_json, - request_stream=active.request_stream, - timeout=60, - ) - return active, matrix, passed, summary - - def test_marker_scan_lazily_visits_a_wide_array(self) -> None: - marker = "" - - class GuardedWideList(list): - def __len__(self): - return 200_000 - - def __getitem__(self, index): - if index == 0: - return marker - raise AssertionError("marker scan eagerly accessed a later sibling") - - def __iter__(self): - yield marker - raise AssertionError("marker scan continued after the first marker") - - found = capabilities.find_raw_ifm_control_marker( - GuardedWideList(), - "response.items", - ) - - self.assertEqual(found, (marker, "response.items[0]")) - - def test_marker_scan_preserves_depth_first_path_order(self) -> None: - found = capabilities.find_raw_ifm_control_marker( - { - "first": ["safe", {"text": ""}], - "second": "", - } - ) - - self.assertEqual(found, ("", "response.first[1].text")) - - def test_complete_profile_exercises_the_exact_contract(self) -> None: - harness, matrix, passed, summary = self.run_profile() - - self.assertTrue(passed, summary) - self.assertTrue(matrix["pass"]) - self.assertEqual(matrix["profile"], capabilities.K2_HORIZON_PROFILE) - self.assertEqual(matrix["model"], MODEL) - self.assertEqual( - [case["id"] for case in matrix["cases"]], - list(capabilities.K2_HORIZON_CASE_IDS), - ) - self.assertEqual(len(capabilities.K2_HORIZON_CASE_IDS), 14) - self.assertIn("openai_tool_xml_typed", capabilities.K2_HORIZON_CASE_IDS) - capabilities.validate_capability_matrix_evidence( - matrix, - capabilities.K2_HORIZON_PROFILE, - MODEL, - ) - - openai_payloads = [ - payload for _method, url, payload in harness.requests if "/api/v1/" in url - ] - self.assertEqual( - [ - payload["chat_template_kwargs"]["reasoning_effort"] - for payload in openai_payloads - if "reasoning_effort" in payload.get("chat_template_kwargs", {}) - ], - ["high", "high", "medium", "low"], - ) - self.assertTrue( - all( - payload["max_completion_tokens"] == 3072 - for payload in openai_payloads - if "reasoning_effort" in payload.get("chat_template_kwargs", {}) - ) - ) - self.assertEqual( - [ - payload["chat_template_kwargs"]["tool_call_format"] - for payload in openai_payloads - if payload.get("tools") - and payload.get("tool_choice") == "required" - and "tool_call_format" in payload.get("chat_template_kwargs", {}) - ], - ["json", "xml", "xml_typed", "xml"], - ) - default_tool_payloads = [ - payload - for payload in openai_payloads - if payload.get("tools") - and payload.get("tool_choice") == "required" - and "tool_call_format" not in payload.get("chat_template_kwargs", {}) - ] - self.assertEqual(len(default_tool_payloads), 1) - self.assertEqual( - default_tool_payloads[0]["chat_template_kwargs"], - {"enable_thinking": False}, - ) - followup = next( - payload - for payload in openai_payloads - if payload.get("tool_choice") == "none" - ) - assistant = followup["messages"][1] - tool_result = followup["messages"][2] - self.assertEqual(assistant["tool_calls"][0]["id"], "call_weather") - self.assertEqual(tool_result["tool_call_id"], "call_weather") - self.assertIn("K2-SUNNY-731", tool_result["content"]) - ollama_payload = next( - payload - for _method, url, payload in harness.requests - if url.endswith("/api/chat") - ) - self.assertEqual(ollama_payload["options"]["num_predict"], 3072) - - def test_reasoning_marker_fails_the_named_case_and_matrix(self) -> None: - harness = CapabilityHarness() - harness.reasoning_marker = "medium" - - _harness, matrix, passed, summary = self.run_profile(harness) - - self.assertFalse(passed) - self.assertFalse(matrix["pass"]) - failed = [case for case in matrix["cases"] if not case["pass"]] - self.assertEqual([case["id"] for case in failed], ["openai_reasoning_medium"]) - self.assertRegex(summary, "medium.*IFM|IFM.*medium") - with self.assertRaises(capabilities.CapabilityValidationError): - capabilities.validate_capability_matrix_evidence( - matrix, - capabilities.K2_HORIZON_PROFILE, - MODEL, - ) - - def test_low_reasoning_requires_the_deterministic_blank_think_faster_result( - self, - ) -> None: - harness = CapabilityHarness() - harness.reasoning_by_effort["low"] = "unexpected low reasoning" - - _harness, matrix, passed, summary = self.run_profile(harness) - - self.assertFalse(passed) - self.assertEqual( - [case["id"] for case in matrix["cases"] if not case["pass"]], - ["openai_reasoning_low"], - ) - self.assertIn("reasoning_content", summary) - - def test_stream_requires_a_terminal_done_frame(self) -> None: - harness = CapabilityHarness() - harness.omit_stream_done = True - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} - self.assertEqual( - failed_ids, - { - "openai_plain_off_stream", - "openai_reasoning_high_stream", - "openai_tool_stream_xml", - }, - ) - - def test_stream_rejects_done_before_the_final_data_frame(self) -> None: - harness = CapabilityHarness() - harness.early_stream_done = True - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} - self.assertEqual( - failed_ids, - { - "openai_plain_off_stream", - "openai_reasoning_high_stream", - "openai_tool_stream_xml", - }, - ) - - def test_nonstream_responses_are_bound_to_the_requested_model(self) -> None: - harness = CapabilityHarness() - harness.response_model = "builtin.Not-K2" - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} - self.assertTrue( - {case["id"] for case in matrix["cases"] if not case["stream"]}.issubset( - failed_ids - ) - ) - self.assertNotIn("openai_plain_off_stream", failed_ids) - - def test_stream_responses_are_bound_to_the_requested_model(self) -> None: - harness = CapabilityHarness() - harness.stream_response_models = ["builtin.Not-K2", "builtin.Not-K2"] - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - self.assertEqual( - {case["id"] for case in matrix["cases"] if not case["pass"]}, - { - "openai_plain_off_stream", - "openai_reasoning_high_stream", - "openai_tool_stream_xml", - }, - ) - - def test_stream_reasoning_requires_separate_marker_free_content(self) -> None: - for stream_reasoning in ("", "high reasoning"): - with self.subTest(stream_reasoning=stream_reasoning): - harness = CapabilityHarness() - harness.stream_reasoning = stream_reasoning - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - self.assertEqual( - {case["id"] for case in matrix["cases"] if not case["pass"]}, - {"openai_reasoning_high_stream"}, - ) - - def test_stream_accepts_the_canonical_requested_model_identity(self) -> None: - harness = CapabilityHarness() - harness.stream_response_models = [ - MODEL.removeprefix("builtin."), - MODEL.removeprefix("builtin."), - ] - - _harness, _matrix, passed, summary = self.run_profile(harness) - - self.assertTrue(passed, summary) - - def test_stream_rejects_changed_or_missing_backend_model_identity(self) -> None: - canonical_model = MODEL.removeprefix("builtin.") - cases = ( - [canonical_model, "different.gguf"], - [None, canonical_model], - [canonical_model, None], - [canonical_model, " "], - ) - for stream_models in cases: - with self.subTest(stream_models=stream_models): - harness = CapabilityHarness() - harness.stream_response_models = stream_models - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - self.assertEqual( - {case["id"] for case in matrix["cases"] if not case["pass"]}, - { - "openai_plain_off_stream", - "openai_reasoning_high_stream", - "openai_tool_stream_xml", - }, - ) - - def test_stream_rejects_duplicate_json_members(self) -> None: - invalid_events = ( - ( - 'data: {"model":"wrong","model":"backend",' - '"choices":[{"index":0,"delta":{"content":"42"},' - '"finish_reason":"stop"}]}', - "model", - ), - ( - 'data: {"model":"backend","choices":[],"choices":' - '[{"index":0,"delta":{"content":"42"},' - '"finish_reason":"stop"}]}', - "choices", - ), - ) - for event, duplicate_key in invalid_events: - with self.subTest(duplicate_key=duplicate_key): - with self.assertRaisesRegex( - capabilities.CapabilityValidationError, - f"repeats key '{duplicate_key}'", - ): - capabilities._decode_openai_stream([event, "data: [DONE]"]) - - def test_stream_enforces_an_independent_wall_clock_deadline(self) -> None: - events = [ - "data: " - + json.dumps( - { - "model": MODEL, - "choices": [ - { - "index": 0, - "delta": {"content": "42"}, - "finish_reason": "stop", - } - ], - } - ), - "data: [DONE]", - ] - - with ( - mock.patch("time.monotonic", side_effect=[0.0, 10**9]), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "wall-clock deadline", - ), - ): - capabilities._decode_openai_stream(events) - - def test_stream_rejects_an_event_count_above_the_bound(self) -> None: - terminal_events = [ - "data: " - + json.dumps( - { - "model": MODEL, - "choices": [ - { - "index": 0, - "delta": {"content": "42"}, - "finish_reason": "stop", - } - ], - } - ), - "data: [DONE]", - ] - events = [": keep-alive"] + terminal_events - - with ( - mock.patch.object(capabilities, "MAX_STREAM_EVENTS", 1), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "event count", - ), - ): - capabilities._decode_openai_stream(events) - - def test_stream_event_bound_ignores_blank_lines_and_heartbeats(self) -> None: - terminal_events = [ - "data: " - + json.dumps( - { - "model": MODEL, - "choices": [ - { - "index": 0, - "delta": {"content": "42"}, - "finish_reason": "stop", - } - ], - } - ), - "data: [DONE]", - ] - events = [item for _index in range(2500) for item in ("", ": heartbeat")] - events.extend(terminal_events) - - with mock.patch.object(capabilities, "MAX_STREAM_EVENTS", 2): - message, finish_reason, saw_done, model = ( - capabilities._decode_openai_stream(events) - ) - - self.assertEqual(message["content"], "42") - self.assertEqual(finish_reason, "stop") - self.assertTrue(saw_done) - self.assertEqual(model, MODEL) - - def test_stream_rejects_total_sse_bytes_above_the_bound(self) -> None: - events = [ - ":xxxx", - "data: " - + json.dumps( - { - "model": MODEL, - "choices": [ - { - "index": 0, - "delta": {"content": "42"}, - "finish_reason": "stop", - } - ], - } - ), - "data: [DONE]", - ] - - with ( - mock.patch.object(capabilities, "MAX_STREAM_SSE_BYTES", 5), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "SSE byte count", - ), - ): - capabilities._decode_openai_stream(events) - - def test_stream_rejects_accumulated_content_above_the_bound(self) -> None: - events = [ - "data: " - + json.dumps( - { - "model": MODEL, - "choices": [ - { - "index": 0, - "delta": {"content": "xxx"}, - "finish_reason": "stop", - } - ], - } - ), - "data: [DONE]", - ] - - with ( - mock.patch.object(capabilities, "MAX_STREAM_TEXT_BYTES", 2), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "content and reasoning", - ), - ): - capabilities._decode_openai_stream(events) - - def test_stream_rejects_tool_arguments_above_the_bound(self) -> None: - events = [ - "data: " - + json.dumps( - { - "model": MODEL, - "choices": [ - { - "index": 0, - "delta": { - "tool_calls": [ - { - "index": 0, - "id": "call_large", - "type": "function", - "function": { - "name": capabilities.TOOL_NAME, - "arguments": "xxx", - }, - } - ] - }, - "finish_reason": "tool_calls", - } - ], - } - ), - "data: [DONE]", - ] - - with ( - mock.patch.object(capabilities, "MAX_STREAM_TOOL_ARGUMENT_BYTES", 2), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "tool arguments", - ), - ): - capabilities._decode_openai_stream(events) - - def test_stream_rejects_choice_chunks_after_finish_reason(self) -> None: - later_choices = ( - {"index": 0, "delta": {"content": "2"}}, - {"index": 0, "delta": {"reasoning_content": "later"}}, - { - "index": 0, - "delta": { - "tool_calls": [ - { - "index": 0, - "id": "call_late", - "type": "function", - "function": { - "name": capabilities.TOOL_NAME, - "arguments": "{}", - }, - } - ] - }, - }, - {"index": 0, "delta": {}, "finish_reason": "stop"}, - ) - first = { - "model": "backend", - "choices": [ - { - "index": 0, - "delta": {"content": "4"}, - "finish_reason": "stop", - } - ], - } - for later_choice in later_choices: - with self.subTest(later_choice=later_choice): - later = {"model": "backend", "choices": [later_choice]} - events = [ - f"data: {json.dumps(first)}", - f"data: {json.dumps(later)}", - "data: [DONE]", - ] - - with self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "after a terminal finish_reason", - ): - capabilities._decode_openai_stream(events) - - def test_tool_arguments_require_exact_json_keys_and_types(self) -> None: - arguments = ( - '{"city":"London","city":"Paris","unit":"celsius",' - '"days":2,"include_hourly":false}', - '{"city":"Paris","unit":"celsius","days":2.0,"include_hourly":false}', - '{"city":"Paris","unit":"celsius","days":2,"include_hourly":0}', - ) - - for tool_arguments in arguments: - with self.subTest(tool_arguments=tool_arguments): - harness = CapabilityHarness() - harness.tool_arguments = tool_arguments - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - failed_ids = { - case["id"] for case in matrix["cases"] if not case["pass"] - } - self.assertTrue( - { - "openai_tool_json", - "openai_tool_xml", - "openai_tool_xml_typed", - }.issubset(failed_ids) - ) - - def test_malformed_xml_typed_tool_response_is_rejected_hard(self) -> None: - harness = CapabilityHarness() - payload = capabilities._tool_payload(MODEL, "xml_typed", stream=False) - - for finish_reason, arguments in ( - ("stop", harness.tool_arguments), - ("tool_calls", '{"city":'), - ): - with self.subTest(finish_reason=finish_reason, arguments=arguments): - harness.tool_arguments = arguments - - def request_json(_method, _url, timeout, **_kwargs): - self.assertEqual(timeout, 60) - return harness.response(), harness.openai( - {"content": None, "tool_calls": [harness.tool_call()]}, - finish_reason, - ) - - with self.assertRaises(capabilities.CapabilityValidationError): - capabilities._openai_tool_case( - request_json, - "http://localhost:13305/api/v1/chat/completions", - 60, - payload, - ) - - def test_omitted_tool_format_requires_the_xml_default(self) -> None: - harness = CapabilityHarness() - harness.default_tool_response_is_text = True - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - self.assertEqual( - {case["id"] for case in matrix["cases"] if not case["pass"]}, - {"openai_tool_default_xml"}, - ) - - def test_ordinary_stop_responses_reject_tool_calls(self) -> None: - harness = CapabilityHarness() - harness.stop_tool_call = True - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - self.assertEqual( - {case["id"] for case in matrix["cases"] if not case["pass"]}, - { - "openai_plain_off_nonstream", - "openai_reasoning_high", - "openai_reasoning_medium", - "openai_reasoning_low", - "ollama_thinking", - "anthropic_translation", - }, - ) - - def test_tool_requests_reject_reasoning_when_thinking_is_disabled(self) -> None: - harness = CapabilityHarness() - harness.tool_reasoning = True - - _harness, matrix, passed, _summary = self.run_profile(harness) - - self.assertFalse(passed) - failed_ids = {case["id"] for case in matrix["cases"] if not case["pass"]} - self.assertTrue( - { - "openai_tool_json", - "openai_tool_xml", - "openai_tool_xml_typed", - }.issubset(failed_ids) - ) - - def test_evidence_rejects_missing_unknown_and_non_boolean_cases(self) -> None: - _harness, matrix, passed, _summary = self.run_profile() - self.assertTrue(passed) - - mutations = [] - missing = copy.deepcopy(matrix) - missing["cases"].pop() - mutations.append(missing) - unknown = copy.deepcopy(matrix) - unknown["cases"][0]["id"] = "unknown" - mutations.append(unknown) - non_boolean = copy.deepcopy(matrix) - non_boolean["cases"][0]["pass"] = 1 - mutations.append(non_boolean) - extra = copy.deepcopy(matrix) - extra["unexpected"] = True - mutations.append(extra) - wrong_protocol = copy.deepcopy(matrix) - wrong_protocol["cases"][0]["protocol"] = "ollama" - mutations.append(wrong_protocol) - wrong_stream = copy.deepcopy(matrix) - wrong_stream["cases"][0]["stream"] = True - mutations.append(wrong_stream) - wrong_finish = copy.deepcopy(matrix) - wrong_finish["cases"][0]["finish_reason"] = "tool_calls" - mutations.append(wrong_finish) - stray_reasoning = copy.deepcopy(matrix) - stray_reasoning["cases"][0]["reasoning_chars"] = 1 - mutations.append(stray_reasoning) - stray_tool = copy.deepcopy(matrix) - stray_tool["cases"][0]["tool_name"] = capabilities.TOOL_NAME - mutations.append(stray_tool) - float_integer = copy.deepcopy(matrix) - float_tool_case = next( - case for case in float_integer["cases"] if case["id"] == "openai_tool_json" - ) - float_tool_case["tool_arguments"]["days"] = 2.0 - mutations.append(float_integer) - integer_boolean = copy.deepcopy(matrix) - boolean_tool_case = next( - case - for case in integer_boolean["cases"] - if case["id"] == "openai_tool_json" - ) - boolean_tool_case["tool_arguments"]["include_hourly"] = 0 - mutations.append(integer_boolean) - - for mutation in mutations: - with self.subTest(mutation=mutation): - with self.assertRaises(capabilities.CapabilityValidationError): - capabilities.validate_capability_matrix_evidence( - mutation, - capabilities.K2_HORIZON_PROFILE, - MODEL, - ) - - def test_unknown_profile_fails_without_requests(self) -> None: - harness = CapabilityHarness() - - with self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "Unsupported capability profile", - ): - capabilities.validate_capabilities( - "unknown", - base_url="http://localhost:13305/api/v1", - model=MODEL, - request_json=harness.request_json, - request_stream=harness.request_stream, - timeout=60, - ) - - self.assertEqual(harness.requests, []) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_llamacpp_check_reporting.py b/test/test_llamacpp_check_reporting.py deleted file mode 100644 index 35bb8f317e..0000000000 --- a/test/test_llamacpp_check_reporting.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for llama.cpp validation check reporting.""" - -from __future__ import annotations - -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -CORE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" -PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" -MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" -SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" - - -def workflow_job(workflow: Path, job_name: str, next_job_name: str | None) -> str: - text = workflow.read_text(encoding="utf-8") - job = text.split(f" {job_name}:\n", 1)[1] - if next_job_name is not None: - job = job.split(f" {next_job_name}:\n", 1)[0] - return job - - -class LlamaCppCheckReportingTests(unittest.TestCase): - def test_reusable_validation_callers_grant_check_write_permission(self) -> None: - callers = ( - (PR_WORKFLOW, "validate", "reconcile"), - (MANUAL_WORKFLOW, "validate", None), - (SCHEDULE_WORKFLOW, "validate", "publish"), - ) - - for workflow, job_name, next_job_name in callers: - with self.subTest(workflow=workflow.name): - job = workflow_job(workflow, job_name, next_job_name) - self.assertIn( - " checks: write\n", - job, - "The caller must grant each permission requested by the " - "reusable validation workflow.", - ) - - def test_reusable_workflow_starts_or_resets_check_before_validation(self) -> None: - core = CORE_WORKFLOW.read_text(encoding="utf-8") - invocation = workflow_job(CORE_WORKFLOW, "validate-invocation", "plan") - existing_branch = ' if [ -z "$check_run_id" ]; then\n' - invalid_id_guard = ' if ! [[ "$check_run_id" =~ ^[0-9]+$ ]]; then\n' - self.assertIn(existing_branch, invocation) - self.assertIn(invalid_id_guard, invocation) - existing_check = invocation.split(existing_branch, 1)[1].split( - invalid_id_guard, 1 - )[0] - - self.assertIn( - " check_run_id:\n" - " value: ${{ jobs.validate-invocation.outputs.check_run_id }}\n", - core.split("permissions:\n", 1)[0], - ) - self.assertIn(" checks: write\n", invocation) - self.assertIn( - " check_run_id: ${{ steps.start-check.outputs.check_run_id }}\n", - invocation, - ) - self.assertIn(" id: start-check\n", invocation) - self.assertIn("github.event_name == 'pull_request_target'", invocation) - self.assertIn("github.event_name == 'merge_group'", invocation) - self.assertIn( - "${AUTHORIZATION_EPOCH}-${RUN_NUMBER}-${RUN_ATTEMPT}-${RUN_ID}", - invocation, - ) - self.assertIn( - "AUTHORIZATION_EPOCH: ${{ inputs.authorization_epoch }}", invocation - ) - self.assertIn("RUN_NUMBER: ${{ github.run_number }}", invocation) - self.assertIn("-f status=in_progress", invocation) - self.assertIn( - 'echo "check_run_id=$check_run_id" >> "$GITHUB_OUTPUT"', invocation - ) - self.assertIn(" else\n", existing_check) - reset = existing_check.split(" else\n", 1)[1] - self.assertIn('"repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}"', reset) - self.assertIn("-f status=in_progress", reset) - self.assertNotIn("-f conclusion=", reset) - self.assertGreaterEqual(invocation.count("--paginate --slurp"), 2) - - def test_pr_wrapper_reconciles_the_check_started_by_reusable_validation( - self, - ) -> None: - wrapper = PR_WORKFLOW.read_text(encoding="utf-8") - policy = workflow_job(PR_WORKFLOW, "policy", "validate") - validate = workflow_job(PR_WORKFLOW, "validate", "reconcile") - reconcile = workflow_job(PR_WORKFLOW, "reconcile", None) - - self.assertNotIn("\n start-check:\n", wrapper) - self.assertNotIn("\n authorize-invocation:\n", wrapper) - self.assertIn( - " run_validation: ${{ steps.decision.outputs.run_validation }}", policy - ) - self.assertIn(" needs: policy\n", validate) - self.assertNotIn("start-check", validate) - self.assertIn(" needs: [policy, validate]\n", reconcile) - self.assertNotIn("needs.start-check", reconcile) - self.assertIn( - "CHECK_RUN_ID: ${{ needs.validate.outputs.check_run_id }}", reconcile - ) - - exact_check_lookup = reconcile.split( - ' if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then\n', 1 - )[1].split(' if [[ "$CHECK_RUN_ID" =~ ^[0-9]+$ ]]; then\n', 1)[0] - self.assertIn("commits/${MERGE_SHA}/check-runs", reconcile) - self.assertIn('--arg external_id "$external_id"', exact_check_lookup) - self.assertIn('.app.slug == "github-actions"', exact_check_lookup) - self.assertIn(".external_id == $external_id", exact_check_lookup) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_llamacpp_pr_file_classification.py b/test/test_llamacpp_pr_file_classification.py deleted file mode 100644 index 083240fd5a..0000000000 --- a/test/test_llamacpp_pr_file_classification.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for exact-commit llama.cpp pull-request file classification.""" - -from __future__ import annotations - -import importlib.util -import json -import os -import shutil -import subprocess -import tempfile -import textwrap -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -POLICY_HELPER = ROOT / ".github" / "scripts" / "llamacpp_required_check_policy.py" -PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" -BASE_SHA = "a" * 40 -CANDIDATE_A_SHA = "b" * 40 -CANDIDATE_B_SHA = "c" * 40 - - -def load_policy_helper(): - spec = importlib.util.spec_from_file_location( - "llamacpp_required_check_policy", POLICY_HELPER - ) - if spec is None or spec.loader is None: - raise RuntimeError("could not load llama.cpp required-check policy") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def compare_response( - *, - target_sha: str, - files: list[dict[str, object]], - base_sha: str = BASE_SHA, - merge_base_sha: str = BASE_SHA, -) -> dict[str, object]: - return { - "ahead_by": 1, - "base_commit": {"sha": base_sha}, - "behind_by": 0, - "commits": [{"sha": target_sha}], - "files": files, - "merge_base_commit": {"sha": merge_base_sha}, - "status": "ahead", - "total_commits": 1, - } - - -def extract_classify_step_script() -> str: - workflow = PR_WORKFLOW.read_text(encoding="utf-8") - step = workflow.split( - " - name: Classify the complete pull request file set\n", 1 - )[1].split(" - name:", 1)[0] - return textwrap.dedent(step.split(" run: |\n", 1)[1]) - - -class TestPullRequestFileClassification(unittest.TestCase): - def test_exact_compare_identity_can_be_classified_as_neutral(self) -> None: - policy = load_policy_helper() - - self.assertFalse( - policy.classify_compare_response( - compare_response( - target_sha=CANDIDATE_B_SHA, - files=[{"filename": "docs/dev/testing.md", "status": "modified"}], - ), - expected_base_sha=BASE_SHA, - expected_target_sha=CANDIDATE_B_SHA, - expected_count=1, - ) - ) - - def test_same_count_neutral_payload_for_other_candidate_is_rejected( - self, - ) -> None: - policy = load_policy_helper() - - with self.assertRaises(ValueError): - policy.classify_compare_response( - compare_response( - target_sha=CANDIDATE_A_SHA, - files=[{"filename": "docs/dev/testing.md", "status": "modified"}], - ), - expected_base_sha=BASE_SHA, - expected_target_sha=CANDIDATE_B_SHA, - expected_count=1, - ) - - def test_compare_file_cap_is_conservatively_relevant(self) -> None: - policy = load_policy_helper() - neutral_files = [ - {"filename": f"docs/file-{index}.md", "status": "modified"} - for index in range(300) - ] - - self.assertFalse( - policy.classify_compare_response( - compare_response( - target_sha=CANDIDATE_B_SHA, - files=neutral_files, - ), - expected_base_sha=BASE_SHA, - expected_target_sha=CANDIDATE_B_SHA, - expected_count=300, - ) - ) - self.assertTrue( - policy.classify_compare_response( - {}, - expected_base_sha=BASE_SHA, - expected_target_sha=CANDIDATE_B_SHA, - expected_count=301, - ) - ) - - def test_incomplete_compare_file_list_is_conservatively_relevant(self) -> None: - policy = load_policy_helper() - - self.assertTrue( - policy.classify_compare_response( - compare_response( - target_sha=CANDIDATE_B_SHA, - files=[{"filename": "docs/dev/testing.md", "status": "modified"}], - ), - expected_base_sha=BASE_SHA, - expected_target_sha=CANDIDATE_B_SHA, - expected_count=2, - ) - ) - - def test_compare_identity_and_file_schema_errors_fail_closed(self) -> None: - policy = load_policy_helper() - valid = compare_response( - target_sha=CANDIDATE_B_SHA, - files=[{"filename": "docs/dev/testing.md", "status": "modified"}], - ) - invalid_responses = { - "base": {**valid, "base_commit": {"sha": CANDIDATE_A_SHA}}, - "merge-base": { - **valid, - "merge_base_commit": {"sha": CANDIDATE_A_SHA}, - }, - "commit-count": {**valid, "total_commits": 2}, - "commit-list": {**valid, "commits": []}, - "file-path": { - **valid, - "files": [{"filename": "/docs/testing.md", "status": "modified"}], - }, - "file-status": { - **valid, - "files": [{"filename": "docs/testing.md", "status": "moved"}], - }, - } - - for name, response in invalid_responses.items(): - with self.subTest(name=name), self.assertRaises(ValueError): - policy.classify_compare_response( - response, - expected_base_sha=BASE_SHA, - expected_target_sha=CANDIDATE_B_SHA, - expected_count=1, - ) - - def test_compare_rename_from_protected_path_is_relevant(self) -> None: - policy = load_policy_helper() - - self.assertTrue( - policy.classify_compare_response( - compare_response( - target_sha=CANDIDATE_B_SHA, - files=[ - { - "filename": "docs/old-build-input.md", - "previous_filename": "CMakeLists.txt", - "status": "renamed", - } - ], - ), - expected_base_sha=BASE_SHA, - expected_target_sha=CANDIDATE_B_SHA, - expected_count=1, - ) - ) - - def test_identical_comparison_cannot_contain_changed_files(self) -> None: - policy = load_policy_helper() - invalid = { - "ahead_by": 0, - "base_commit": {"sha": BASE_SHA}, - "behind_by": 0, - "commits": [], - "files": [{"filename": "docs/testing.md", "status": "modified"}], - "merge_base_commit": {"sha": BASE_SHA}, - "status": "identical", - "total_commits": 0, - } - - with self.assertRaises(ValueError): - policy.classify_compare_response( - invalid, - expected_base_sha=BASE_SHA, - expected_target_sha=BASE_SHA, - expected_count=1, - ) - - def test_workflow_does_not_accept_stale_same_count_neutral_payload( - self, - ) -> None: - with tempfile.TemporaryDirectory() as directory: - temporary = Path(directory) - trusted_helper = ( - temporary - / "trusted" - / ".github" - / "scripts" - / "llamacpp_required_check_policy.py" - ) - trusted_helper.parent.mkdir(parents=True) - shutil.copyfile(POLICY_HELPER, trusted_helper) - response_path = temporary / "response.json" - response_path.write_text( - json.dumps( - compare_response( - target_sha=CANDIDATE_A_SHA, - files=[ - { - "filename": "docs/dev/testing.md", - "status": "modified", - } - ], - ) - ), - encoding="utf-8", - ) - calls_path = temporary / "gh-calls.txt" - output_path = temporary / "github-output.txt" - - fake_git = temporary / "git" - fake_git.write_text( - "#!/bin/sh\nprintf '%s\\n' \"$BASE_SHA\"\n", encoding="utf-8" - ) - fake_git.chmod(0o755) - fake_gh = temporary / "gh" - fake_gh.write_text( - """#!/usr/bin/env python3 -import json -import os -import sys -from pathlib import Path - -Path(os.environ["FAKE_GH_CALLS"]).write_text("\\n".join(sys.argv[1:])) -response = json.loads(Path(os.environ["FAKE_GH_RESPONSE"]).read_text()) -if any("/pulls/7/files" in argument for argument in sys.argv): - response = [response["files"]] -print(json.dumps(response)) -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - environment = os.environ.copy() - environment.update( - { - "BASE_SHA": BASE_SHA, - "EXPECTED_COUNT": "1", - "FAKE_GH_CALLS": str(calls_path), - "FAKE_GH_RESPONSE": str(response_path), - "GH_TOKEN": "test-token", - "GITHUB_OUTPUT": str(output_path), - "GITHUB_REPOSITORY": "LLM360/lemonade", - "MERGE_SHA": CANDIDATE_B_SHA, - "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", - "PR_NUMBER": "7", - "RUNNER_TEMP": str(temporary), - } - ) - - completed = subprocess.run( - ["bash", "-c", extract_classify_step_script()], - cwd=temporary, - env=environment, - check=False, - capture_output=True, - text=True, - ) - outputs = ( - output_path.read_text(encoding="utf-8") if output_path.exists() else "" - ) - calls = calls_path.read_text(encoding="utf-8") - - self.assertNotEqual(completed.returncode, 0) - self.assertNotIn("relevant=false", outputs) - self.assertIn(f"compare/{BASE_SHA}...{CANDIDATE_B_SHA}", calls) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_llamacpp_release_assets.py b/test/test_llamacpp_release_assets.py deleted file mode 100644 index bc630764bd..0000000000 --- a/test/test_llamacpp_release_assets.py +++ /dev/null @@ -1,300 +0,0 @@ -#!/usr/bin/env python3 -"""Unit tests for the managed llama.cpp release-asset contract.""" - -from __future__ import annotations - -import unittest - -from test.utils import llamacpp_release_assets as assets - -ROCM_ASSET_FAMILIES = { - "gfx1200": "gfx120X", - "gfx1201": "gfx120X", - "gfx1100": "gfx110X", - "gfx1101": "gfx110X", - "gfx1102": "gfx110X", - "gfx1103": "gfx110X", - "gfx1030": "gfx103X", - "gfx1031": "gfx103X", - "gfx1032": "gfx103X", - "gfx1033": "gfx103X", - "gfx1034": "gfx103X", - "gfx1035": "gfx103X", - "gfx1036": "gfx103X", -} - - -class LlamaCppReleaseAssetTests(unittest.TestCase): - def setUp(self) -> None: - self.requirements = assets.build_asset_requirements( - ggml_release="b100", - rocm_release="b200", - lemonade_release="b300", - backend_versions={ - "therock": {"version": "7.14.0"}, - "rocm_asset_families": ROCM_ASSET_FAMILIES, - }, - ) - - def test_cuda_covers_every_published_compute_capability_and_platform(self) -> None: - required = set(self.requirements["lemonade"]["cuda"]) - - for compute_capability in ( - "sm_75", - "sm_80", - "sm_86", - "sm_89", - "sm_90", - "sm_100", - "sm_120", - "sm_121", - ): - with self.subTest(compute_capability=compute_capability): - self.assertIn( - f"llama-b300-windows-cuda-{compute_capability}-x64.7z", - required, - ) - self.assertIn( - f"llama-b300-ubuntu-cuda-{compute_capability}-x64.tar.xz", - required, - ) - self.assertIn( - f"llama-b300-ubuntu-cuda-{compute_capability}-arm64.tar.xz", - required, - ) - - self.assertEqual(len(required), 24) - - def test_rocm_nightly_covers_advertised_platform_targets(self) -> None: - required = set(self.requirements["rocm"]["rocm-nightly"]) - both_platforms = ( - "gfx1151", - "gfx1150", - "gfx120X", - "gfx110X", - "gfx103X", - "gfx1152", - ) - - for target in both_platforms: - with self.subTest(target=target): - self.assertIn(f"llama-b200-windows-rocm-{target}-x64.zip", required) - self.assertIn(f"llama-b200-ubuntu-rocm-{target}-x64.zip", required) - - for target in ("gfx908", "gfx90a", "gfx942"): - with self.subTest(target=target): - self.assertIn(f"llama-b200-ubuntu-rocm-{target}-x64.zip", required) - self.assertNotIn(f"llama-b200-windows-rocm-{target}-x64.zip", required) - self.assertFalse(any("gfx950" in name for name in required)) - self.assertEqual(len(required), 15) - - def test_rocm_requirements_bind_each_asset_to_concrete_build_targets(self) -> None: - required = assets.build_rocm_asset_target_requirements( - rocm_release="b200", - backend_versions={"rocm_asset_families": ROCM_ASSET_FAMILIES}, - ) - - expected_gfx103x = { - "gfx1030", - "gfx1031", - "gfx1032", - "gfx1033", - "gfx1034", - "gfx1035", - "gfx1036", - } - self.assertEqual( - set(required["llama-b200-windows-rocm-gfx103X-x64.zip"]), - expected_gfx103x, - ) - self.assertEqual( - set(required["llama-b200-ubuntu-rocm-gfx103X-x64.zip"]), - expected_gfx103x, - ) - self.assertEqual( - required["llama-b200-ubuntu-rocm-gfx90a-x64.zip"], - ("gfx90a",), - ) - self.assertNotIn("llama-b200-windows-rocm-gfx90a-x64.zip", required) - self.assertFalse( - any( - "gfx950" in target - for targets in required.values() - for target in targets - ) - ) - self.assertEqual(set(required), set(self.requirements["rocm"]["rocm-nightly"])) - - def test_rocm_target_grouping_tracks_asset_family_mapping_drift(self) -> None: - changed_families = dict(ROCM_ASSET_FAMILIES) - changed_families["gfx1033"] = "gfx1033" - - required = assets.build_rocm_asset_target_requirements( - rocm_release="b200", - backend_versions={"rocm_asset_families": changed_families}, - ) - - self.assertEqual( - required["llama-b200-windows-rocm-gfx1033-x64.zip"], - ("gfx1033",), - ) - self.assertNotIn( - "gfx1033", - required["llama-b200-windows-rocm-gfx103X-x64.zip"], - ) - - def test_rocm_targets_use_resource_family_mapping_without_duplicates(self) -> None: - requirements = assets.build_asset_requirements( - ggml_release="b100", - rocm_release="b200", - lemonade_release="b300", - backend_versions={ - "therock": {"version": "7.14.0"}, - "rocm_asset_families": { - **ROCM_ASSET_FAMILIES, - "gfx1151": "gfx115X", - "gfx1152": "gfx115X", - }, - }, - ) - required = requirements["rocm"]["rocm-nightly"] - - self.assertEqual(required.count("llama-b200-windows-rocm-gfx115X-x64.zip"), 1) - self.assertEqual(required.count("llama-b200-ubuntu-rocm-gfx115X-x64.zip"), 1) - - def test_rocm_canonical_family_names_cannot_be_remapped(self) -> None: - collapsing_mapping = { - target: "gfx1151" - for target in ( - "gfx1150", - "gfx120X", - "gfx110X", - "gfx103X", - "gfx90a", - "gfx908", - "gfx1152", - "gfx942", - ) - } - - with self.assertRaisesRegex(ValueError, "concrete ROCm ISA"): - assets.build_asset_requirements( - ggml_release="b100", - rocm_release="b200", - lemonade_release="b300", - backend_versions={ - "therock": {"version": "7.14.0"}, - "rocm_asset_families": collapsing_mapping, - }, - ) - - def test_missing_new_target_keeps_rocm_nightly_pin_ineligible(self) -> None: - required = self.requirements["rocm"]["rocm-nightly"] - available = set(required) - missing_asset = "llama-b200-ubuntu-rocm-gfx942-x64.zip" - available.remove(missing_asset) - - eligible, missing = assets.evaluate_asset_group( - available, - self.requirements["rocm"], - ) - - self.assertEqual(eligible, []) - self.assertEqual(missing, {"rocm-nightly": [missing_asset]}) - - def test_rocm_eligibility_requires_attested_targets_but_allows_extras( - self, - ) -> None: - required_assets = self.requirements["rocm"]["rocm-nightly"] - required_targets = assets.build_rocm_asset_target_requirements( - rocm_release="b200", - backend_versions={"rocm_asset_families": ROCM_ASSET_FAMILIES}, - ) - available = {name: 1 for name in required_assets} - attested = { - name: (*targets, "gfx999") for name, targets in required_targets.items() - } - - eligible, missing = assets.evaluate_asset_group( - available, - self.requirements["rocm"], - required_asset_targets=required_targets, - attested_asset_targets=attested, - ) - - self.assertEqual(eligible, ["rocm-nightly"]) - self.assertEqual(missing, {"rocm-nightly": []}) - - incomplete = dict(attested) - target_asset = "llama-b200-windows-rocm-gfx103X-x64.zip" - incomplete[target_asset] = tuple( - target for target in incomplete[target_asset] if target != "gfx1035" - ) - eligible, missing = assets.evaluate_asset_group( - available, - self.requirements["rocm"], - required_asset_targets=required_targets, - attested_asset_targets=incomplete, - ) - self.assertEqual(eligible, []) - self.assertEqual(missing, {"rocm-nightly": [target_asset]}) - - def test_missing_rocm_attestation_does_not_affect_independent_groups(self) -> None: - ggml_available = { - name: 1 - for required in self.requirements["ggml"].values() - for name in required - } - rocm_available = {name: 1 for name in self.requirements["rocm"]["rocm-nightly"]} - required_targets = assets.build_rocm_asset_target_requirements( - rocm_release="b200", - backend_versions={"rocm_asset_families": ROCM_ASSET_FAMILIES}, - ) - - ggml_eligible, _ = assets.evaluate_asset_group( - ggml_available, - self.requirements["ggml"], - ) - rocm_eligible, missing = assets.evaluate_asset_group( - rocm_available, - self.requirements["rocm"], - required_asset_targets=required_targets, - attested_asset_targets={}, - ) - - self.assertEqual(set(ggml_eligible), {"cpu", "metal", "vulkan"}) - self.assertEqual(rocm_eligible, []) - self.assertEqual( - missing["rocm-nightly"], - self.requirements["rocm"]["rocm-nightly"], - ) - - def test_zero_sized_required_asset_keeps_backend_ineligible(self) -> None: - required = self.requirements["ggml"]["cpu"] - available = {name: 1 for name in required} - zero_sized_asset = "llama-b100-bin-win-cpu-x64.zip" - available[zero_sized_asset] = 0 - - eligible, missing = assets.evaluate_asset_group( - available, - self.requirements["ggml"], - ) - - self.assertNotIn("cpu", eligible) - self.assertEqual(missing["cpu"], [zero_sized_asset]) - - def test_invalid_release_tag_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "Invalid GGML_RELEASE"): - assets.build_asset_requirements( - ggml_release="main", - rocm_release="b200", - lemonade_release="b300", - backend_versions={ - "therock": {"version": "7.14.0"}, - "rocm_asset_families": {}, - }, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_llamacpp_release_manifest.py b/test/test_llamacpp_release_manifest.py deleted file mode 100644 index e0450f6f03..0000000000 --- a/test/test_llamacpp_release_manifest.py +++ /dev/null @@ -1,1397 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for canonical llama.cpp release-asset manifests.""" - -from __future__ import annotations - -import copy -import hashlib -import json -import tempfile -import unittest -from pathlib import Path - -from test.utils import llamacpp_release_assets as release_assets -from test.utils import llamacpp_release_manifest as manifest - - -def release_payload( - tag: str, - release_id: int, - asset_seed: int, - *, - repository: str = "ggml-org/llama.cpp", - publisher_claim_type: str = "source-release-tag", - publisher_claimed_source_commit: str | None = None, -) -> dict: - release_tag_commit = f"{release_id:040x}" - claimed_commit = publisher_claimed_source_commit or release_tag_commit - assets = [ - { - "name": "z-last.zip", - "id": asset_seed + 1, - "size": 200, - "digest": "sha256:" + "B" * 64, - "state": "uploaded", - "updated_at": "2026-09-05T12:34:56Z", - }, - { - "name": "a-first.zip", - "id": asset_seed, - "size": 100, - "digest": "sha256:" + "a" * 64, - "state": "uploaded", - "updated_at": "2026-09-05T12:30:00Z", - }, - ] - if publisher_claim_type == "immutable-source-manifest": - assets.append( - { - "name": ".llamacpp-source.json", - "id": asset_seed + 2, - "size": 512, - "digest": "sha256:" + "c" * 64, - "state": "uploaded", - "updated_at": "2026-09-05T12:36:00Z", - } - ) - return { - "body": "release notes", - "draft": False, - "id": release_id, - "immutable": True, - "publisher_claim_type": publisher_claim_type, - "publisher_claimed_source_commit": claimed_commit, - "release_tag_commit": release_tag_commit, - "source_repository": repository, - "tag_name": tag, - "upstream_reference": "refs/heads/master", - "upstream_reference_head": "d" * 40, - "assets": assets, - } - - -class LlamaCppReleaseManifestTests(unittest.TestCase): - def setUp(self) -> None: - self.ggml = release_payload("b1234", 10, 100) - self.rocm = release_payload( - "b1235", - 20, - 200, - publisher_claim_type="immutable-source-manifest", - ) - - def _immutable_source_manifest_fixture(self) -> tuple[dict, bytes, str]: - release = copy.deepcopy(self.ggml) - release["draft"] = False - release["tag_name"] = "b4321" - release["release_tag_commit"] = "b" * 40 - for asset in release["assets"]: - asset["state"] = "uploaded" - asset["digest"] = asset["digest"].lower() - source_commit = "c" * 40 - document = { - "assets": sorted( - ( - { - "digest": asset["digest"].lower(), - "name": asset["name"], - "size": asset["size"], - } - for asset in release["assets"] - ), - key=lambda asset: asset["name"], - ), - "release_repository": "lemonade-sdk/llama.cpp", - "release_tag": "b4321", - "release_tag_commit": "b" * 40, - "schema_version": 1, - "source_commit": source_commit, - "source_repository": "ggml-org/llama.cpp", - } - evidence = json.dumps( - document, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - release["assets"].append( - { - "digest": "sha256:" + hashlib.sha256(evidence).hexdigest(), - "id": 999, - "name": ".llamacpp-source.json", - "size": len(evidence), - "state": "uploaded", - "updated_at": "2026-09-05T12:40:00Z", - } - ) - return release, evidence, source_commit - - def _rocm_source_manifest_fixture( - self, - ) -> tuple[dict, bytes, str, list[dict]]: - release, evidence, source_commit = self._immutable_source_manifest_fixture() - document = json.loads(evidence) - document["release_repository"] = "lemonade-sdk/llamacpp-rocm" - document["schema_version"] = 2 - targets_by_name = { - "a-first.zip": ["gfx1036", "gfx1033", "gfx1035"], - "z-last.zip": ["gfx942"], - } - for binding in document["assets"]: - binding["build_targets"] = targets_by_name[binding["name"]] - evidence = json.dumps( - document, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - self._bind_source_evidence(release, evidence) - expected = [ - { - "build_targets": ["gfx1033", "gfx1035", "gfx1036"], - "digest": document["assets"][0]["digest"], - "name": "a-first.zip", - "size": 100, - }, - { - "build_targets": ["gfx942"], - "digest": document["assets"][1]["digest"], - "name": "z-last.zip", - "size": 200, - }, - ] - return release, evidence, source_commit, expected - - @staticmethod - def _bind_source_evidence(release: dict, evidence: bytes) -> None: - source_asset = next( - asset - for asset in release["assets"] - if asset["name"] == manifest.SOURCE_MANIFEST_ASSET_NAME - ) - source_asset["size"] = len(evidence) - source_asset["digest"] = "sha256:" + hashlib.sha256(evidence).hexdigest() - - @staticmethod - def _complete_managed_manifest( - versions: dict, - *, - ggml_tag: str = "b1", - rocm_tag: str = "b3", - lemonade_tag: str = "b2", - extra_rocm_target: str | None = None, - ) -> str: - requirements = release_assets.build_asset_requirements( - ggml_release=ggml_tag, - rocm_release=rocm_tag, - lemonade_release=lemonade_tag, - backend_versions=versions, - ) - rocm_targets = release_assets.build_rocm_asset_target_requirements( - rocm_release=rocm_tag, - backend_versions=versions, - ) - releases = [] - for release_index, (repository, tag, group) in enumerate( - ( - ("ggml-org/llama.cpp", ggml_tag, "ggml"), - ("lemonade-sdk/llamacpp-rocm", rocm_tag, "rocm"), - ("lemonade-sdk/llama.cpp", lemonade_tag, "lemonade"), - ), - start=1, - ): - payload = release_payload( - tag, - release_index, - release_index * 100, - repository="ggml-org/llama.cpp", - publisher_claim_type=( - "source-release-tag" - if repository == "ggml-org/llama.cpp" - else "immutable-source-manifest" - ), - ) - source_assets = [ - asset - for asset in payload["assets"] - if asset["name"] == manifest.SOURCE_MANIFEST_ASSET_NAME - ] - names = [ - name - for backend_names in requirements[group].values() - for name in backend_names - ] - binary_assets = [ - { - "digest": "sha256:" + f"{release_index * 1000 + index:064x}", - "id": release_index * 1000 + index, - "name": name, - "size": index, - "state": "uploaded", - "updated_at": "2026-09-05T12:40:00Z", - } - for index, name in enumerate(names, start=1) - ] - payload["assets"] = binary_assets + source_assets - if repository == "lemonade-sdk/llamacpp-rocm": - assets_by_name = {asset["name"]: asset for asset in binary_assets} - payload["build_target_attestations"] = [] - for name, required_targets in rocm_targets.items(): - target_asset = assets_by_name[name] - build_targets = list(required_targets) - if extra_rocm_target is not None: - build_targets.append(extra_rocm_target) - payload["build_target_attestations"].append( - { - "build_targets": build_targets, - "digest": target_asset["digest"], - "name": name, - "size": target_asset["size"], - } - ) - releases.append((repository, tag, payload)) - return manifest.build_release_asset_manifest(releases) - - def test_immutable_source_manifest_binds_release_source_and_assets(self) -> None: - release, evidence, source_commit = self._immutable_source_manifest_fixture() - - asset = manifest.locate_immutable_source_manifest_asset( - release, - "lemonade-sdk/llama.cpp", - ) - self.assertEqual(asset["id"], 999) - self.assertEqual( - manifest.validate_immutable_source_manifest( - evidence, - release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ), - source_commit, - ) - - def test_rocm_source_manifest_normalizes_asset_bound_build_targets(self) -> None: - release, evidence, source_commit, expected = ( - self._rocm_source_manifest_fixture() - ) - - actual_commit, actual_attestations = ( - manifest.validate_immutable_source_manifest_with_attestation( - evidence, - release, - "lemonade-sdk/llamacpp-rocm", - "b4321", - "b" * 40, - ) - ) - - self.assertEqual(actual_commit, source_commit) - self.assertEqual(actual_attestations, expected) - - def test_legacy_rocm_source_manifest_has_no_build_target_attestation( - self, - ) -> None: - release, evidence, source_commit, _expected = ( - self._rocm_source_manifest_fixture() - ) - document = json.loads(evidence) - document["schema_version"] = 1 - for binding in document["assets"]: - binding.pop("build_targets") - legacy = json.dumps( - document, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - self._bind_source_evidence(release, legacy) - - actual_commit, actual_attestations = ( - manifest.validate_immutable_source_manifest_with_attestation( - legacy, - release, - "lemonade-sdk/llamacpp-rocm", - "b4321", - "b" * 40, - ) - ) - - self.assertEqual(actual_commit, source_commit) - self.assertEqual(actual_attestations, []) - - def test_rocm_build_target_attestation_rejects_malformed_or_duplicate_claims( - self, - ) -> None: - release, evidence, _source_commit, _expected = ( - self._rocm_source_manifest_fixture() - ) - document = json.loads(evidence) - cases = ( - ( - "concrete ROCm ISA", - lambda value: value["assets"][0]["build_targets"].append("gfx103X"), - ), - ( - "duplicate build target", - lambda value: value["assets"][0]["build_targets"].append("gfx1033"), - ), - ( - "build_targets must be an array", - lambda value: value["assets"][0].update(build_targets="gfx1033"), - ), - ( - "unexpected fields", - lambda value: value["assets"][0].update(unexpected=True), - ), - ) - for expected_error, mutate in cases: - with self.subTest(expected_error=expected_error): - changed = copy.deepcopy(document) - mutate(changed) - changed_bytes = json.dumps( - changed, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - changed_release = copy.deepcopy(release) - self._bind_source_evidence(changed_release, changed_bytes) - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.validate_immutable_source_manifest_with_attestation( - changed_bytes, - changed_release, - "lemonade-sdk/llamacpp-rocm", - "b4321", - "b" * 40, - ) - - def test_source_manifest_schema_two_is_reserved_for_rocm(self) -> None: - release, evidence, _source_commit, _expected = ( - self._rocm_source_manifest_fixture() - ) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "schema_version 2 is only valid", - ): - manifest.validate_immutable_source_manifest_with_attestation( - evidence, - release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ) - - def test_immutable_source_manifest_rejects_unbound_or_ambiguous_input(self) -> None: - release, evidence, _source_commit = self._immutable_source_manifest_fixture() - document = json.loads(evidence) - - document_cases = ( - ("release repository", "release_repository", "attacker/llama.cpp"), - ("release tag", "release_tag", "b9999"), - ("release tag commit", "release_tag_commit", "d" * 40), - ("source repository", "source_repository", "attacker/llama.cpp"), - ("source commit", "source_commit", "abc123"), - ("asset metadata", "assets", document["assets"][:-1]), - ) - for expected_error, field, value in document_cases: - with self.subTest(field=field): - changed = copy.deepcopy(document) - changed[field] = value - changed_bytes = json.dumps( - changed, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - changed_release = copy.deepcopy(release) - self._bind_source_evidence(changed_release, changed_bytes) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.validate_immutable_source_manifest( - changed_bytes, - changed_release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ) - - duplicate_key_evidence = ( - evidence[:-1] + b',"source_commit":"' + b"d" * 40 + b'"}' - ) - duplicate_release = copy.deepcopy(release) - self._bind_source_evidence(duplicate_release, duplicate_key_evidence) - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "duplicate JSON member", - ): - manifest.validate_immutable_source_manifest( - duplicate_key_evidence, - duplicate_release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ) - - for mutate, expected_error in ( - (lambda payload: payload["assets"].pop(), "exactly one"), - ( - lambda payload: payload["assets"].append( - copy.deepcopy(payload["assets"][-1]) - ), - "exactly one", - ), - ( - lambda payload: payload["assets"][-1].update(size=0), - "size", - ), - ( - lambda payload: payload["assets"][-1].update(size=65_537), - "size", - ), - ( - lambda payload: payload["assets"][-1].update( - digest="sha256:" + "A" * 64 - ), - "sha256 digest", - ), - ( - lambda payload: payload["assets"][-1].update(state="new"), - "uploaded", - ), - ): - with self.subTest(expected_error=expected_error): - changed_release = copy.deepcopy(release) - mutate(changed_release) - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.locate_immutable_source_manifest_asset( - changed_release, - "lemonade-sdk/llama.cpp", - ) - - def test_immutable_source_manifest_schema_is_strict(self) -> None: - release, evidence, _source_commit = self._immutable_source_manifest_fixture() - document = json.loads(evidence) - cases = [] - - extra_top_level = copy.deepcopy(document) - extra_top_level["unexpected"] = True - cases.append(("unexpected fields", extra_top_level)) - for schema_version in (True, 1.0, 2, "1"): - changed = copy.deepcopy(document) - changed["schema_version"] = schema_version - cases.append(("schema_version", changed)) - extra_asset_field = copy.deepcopy(document) - extra_asset_field["assets"][0]["unexpected"] = True - cases.append(("unexpected fields", extra_asset_field)) - duplicate_asset = copy.deepcopy(document) - duplicate_asset["assets"].append(copy.deepcopy(duplicate_asset["assets"][0])) - cases.append(("duplicate asset", duplicate_asset)) - self_reference = copy.deepcopy(document) - self_reference["assets"][0]["name"] = manifest.SOURCE_MANIFEST_ASSET_NAME - cases.append(("exclude itself", self_reference)) - for expected_error, changed in cases: - with self.subTest(expected_error=expected_error): - changed_bytes = json.dumps( - changed, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - changed_release = copy.deepcopy(release) - self._bind_source_evidence(changed_release, changed_bytes) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.validate_immutable_source_manifest( - changed_bytes, - changed_release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ) - - reversed_document = copy.deepcopy(document) - reversed_document["assets"].reverse() - reversed_evidence = json.dumps( - reversed_document, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - reversed_release = copy.deepcopy(release) - self._bind_source_evidence(reversed_release, reversed_evidence) - self.assertEqual( - manifest.validate_immutable_source_manifest( - reversed_evidence, - reversed_release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ), - document["source_commit"], - ) - - def test_immutable_source_manifest_measures_raw_evidence_before_parsing( - self, - ) -> None: - release, evidence, _source_commit = self._immutable_source_manifest_fixture() - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, "size does not match" - ): - manifest.validate_immutable_source_manifest( - evidence + b" ", - release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ) - - same_size_corruption = b"[" + evidence[1:] - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "sha256 digest does not match", - ): - manifest.validate_immutable_source_manifest( - same_size_corruption, - release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ) - - invalid_utf8 = b"\xff" - invalid_utf8_release = copy.deepcopy(release) - self._bind_source_evidence(invalid_utf8_release, invalid_utf8) - with self.assertRaisesRegex(manifest.ReleaseManifestError, "UTF-8 JSON"): - manifest.validate_immutable_source_manifest( - invalid_utf8, - invalid_utf8_release, - "lemonade-sdk/llama.cpp", - "b4321", - "b" * 40, - ) - - def test_immutable_source_manifest_release_metadata_is_strict(self) -> None: - release, _evidence, _source_commit = self._immutable_source_manifest_fixture() - source_index = next( - index - for index, asset in enumerate(release["assets"]) - if asset["name"] == manifest.SOURCE_MANIFEST_ASSET_NAME - ) - cases = ( - ("release draft", lambda payload: payload.update(draft=True)), - ( - "positive integer", - lambda payload: payload["assets"][source_index].update(id=True), - ), - ( - "duplicate release asset name", - lambda payload: payload["assets"][1].update( - name=payload["assets"][0]["name"] - ), - ), - ( - "must be uploaded", - lambda payload: payload["assets"][0].update(state="new"), - ), - ( - "lowercase sha256 digest", - lambda payload: payload["assets"][0].update( - digest="sha256:" + "A" * 64 - ), - ), - ) - for expected_error, mutate in cases: - with self.subTest(expected_error=expected_error): - changed_release = copy.deepcopy(release) - mutate(changed_release) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.locate_immutable_source_manifest_asset( - changed_release, - "lemonade-sdk/llama.cpp", - ) - - def test_manifest_is_canonical_across_release_and_asset_order(self) -> None: - first = manifest.build_release_asset_manifest( - [ - ("lemonade-sdk/llamacpp-rocm", "b1235", self.rocm), - ("ggml-org/llama.cpp", "b1234", self.ggml), - ] - ) - reversed_assets = copy.deepcopy(self.ggml) - reversed_assets["assets"].reverse() - second = manifest.build_release_asset_manifest( - [ - ("ggml-org/llama.cpp", "b1234", reversed_assets), - ("lemonade-sdk/llamacpp-rocm", "b1235", self.rocm), - ] - ) - - self.assertEqual(first, second) - parsed = json.loads(first) - self.assertEqual(parsed["schema_version"], 5) - self.assertEqual( - [release["repository"] for release in parsed["releases"]], - ["ggml-org/llama.cpp", "lemonade-sdk/llamacpp-rocm"], - ) - self.assertEqual( - [asset["name"] for asset in parsed["releases"][0]["assets"]], - ["a-first.zip", "z-last.zip"], - ) - self.assertEqual( - parsed["releases"][0]["assets"][1]["digest"], - "sha256:" + "b" * 64, - ) - - def test_release_manifest_carries_normalized_rocm_build_targets(self) -> None: - payload = copy.deepcopy(self.rocm) - target_asset = payload["assets"][0] - payload["build_target_attestations"] = [ - { - "build_targets": ["gfx1036", "gfx1033", "gfx1035"], - "digest": target_asset["digest"].lower(), - "name": target_asset["name"], - "size": target_asset["size"], - } - ] - - canonical = manifest.build_release_asset_manifest( - [("lemonade-sdk/llamacpp-rocm", "b1235", payload)] - ) - release = json.loads(canonical)["releases"][0] - - self.assertEqual( - release["build_target_attestations"], - [ - { - "build_targets": ["gfx1033", "gfx1035", "gfx1036"], - "digest": target_asset["digest"].lower(), - "name": target_asset["name"], - "size": target_asset["size"], - } - ], - ) - self.assertEqual( - manifest.canonicalize_release_asset_manifest(canonical), canonical - ) - - def test_release_manifest_rejects_tampered_rocm_build_target_binding(self) -> None: - payload = copy.deepcopy(self.rocm) - target_asset = payload["assets"][0] - base_attestation = { - "build_targets": ["gfx1033"], - "digest": target_asset["digest"].lower(), - "name": target_asset["name"], - "size": target_asset["size"], - } - cases = ( - ("does not match release asset", {**base_attestation, "size": 999}), - ( - "duplicate build target", - {**base_attestation, "build_targets": ["gfx1033", "gfx1033"]}, - ), - ( - "concrete ROCm ISA", - {**base_attestation, "build_targets": ["gfx103X"]}, - ), - ) - for expected_error, attestation in cases: - with self.subTest(expected_error=expected_error): - changed = copy.deepcopy(payload) - changed["build_target_attestations"] = [attestation] - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.build_release_asset_manifest( - [("lemonade-sdk/llamacpp-rocm", "b1235", changed)] - ) - - def test_every_asset_requires_a_sha256_digest(self) -> None: - for digest in (None, "", "md5:" + "a" * 32, "sha256:abc"): - with self.subTest(digest=digest): - payload = copy.deepcopy(self.ggml) - payload["assets"][0]["digest"] = digest - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "sha256 digest", - ): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", payload)] - ) - - def test_release_and_asset_identity_fields_are_required(self) -> None: - cases = ( - ("release id", lambda payload: payload.pop("id")), - ("release immutable", lambda payload: payload.pop("immutable")), - ("release draft", lambda payload: payload.pop("draft")), - ( - "release tag commit", - lambda payload: payload.pop("release_tag_commit"), - ), - ( - "source repository", - lambda payload: payload.pop("source_repository"), - ), - ( - "publisher claim type", - lambda payload: payload.pop("publisher_claim_type"), - ), - ( - "publisher-claimed source commit", - lambda payload: payload.pop("publisher_claimed_source_commit"), - ), - ( - "upstream reference", - lambda payload: payload.pop("upstream_reference"), - ), - ( - "upstream reference head", - lambda payload: payload.pop("upstream_reference_head"), - ), - ("asset name", lambda payload: payload["assets"][0].pop("name")), - ("asset id", lambda payload: payload["assets"][0].pop("id")), - ("asset size", lambda payload: payload["assets"][0].pop("size")), - ( - "asset updated_at", - lambda payload: payload["assets"][0].pop("updated_at"), - ), - ( - "asset must be uploaded", - lambda payload: payload["assets"][0].pop("state"), - ), - ) - for expected_error, mutate in cases: - with self.subTest(expected_error=expected_error): - payload = copy.deepcopy(self.ggml) - mutate(payload) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", payload)] - ) - - def test_mutable_releases_fail_closed(self) -> None: - for immutable in (False, None, 1, "true"): - with self.subTest(immutable=immutable): - payload = copy.deepcopy(self.ggml) - payload["immutable"] = immutable - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "release immutable must be true", - ): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", payload)] - ) - - def test_draft_releases_fail_closed(self) -> None: - for draft in (True, None, 0, "false"): - with self.subTest(draft=draft): - payload = copy.deepcopy(self.ggml) - payload["draft"] = draft - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "release draft must be false", - ): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", payload)] - ) - - def test_publisher_claim_commits_must_be_full_git_oids(self) -> None: - for field in ("release_tag_commit", "publisher_claimed_source_commit"): - for commit in (None, "", "abc123", "g" * 40): - with self.subTest(field=field, commit=commit): - payload = copy.deepcopy(self.ggml) - payload[field] = commit - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - ( - "publisher-claimed source commit" - if field == "publisher_claimed_source_commit" - else field.replace("_", " ") - ), - ): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", payload)] - ) - - def test_packaging_commit_is_distinct_from_binary_source_commit(self) -> None: - payload = release_payload( - "b4321", - 30, - 300, - repository="ggml-org/llama.cpp", - publisher_claim_type="immutable-source-manifest", - publisher_claimed_source_commit="f" * 40, - ) - - parsed = json.loads( - manifest.build_release_asset_manifest( - [("lemonade-sdk/llama.cpp", "b4321", payload)] - ) - )["releases"][0] - - self.assertEqual(parsed["release_tag_commit"], f"{30:040x}") - self.assertEqual(parsed["source_repository"], "ggml-org/llama.cpp") - self.assertEqual(parsed["publisher_claimed_source_commit"], "f" * 40) - self.assertNotEqual( - parsed["release_tag_commit"], - parsed["publisher_claimed_source_commit"], - ) - - def test_upstream_release_tag_commit_must_equal_source_commit(self) -> None: - payload = copy.deepcopy(self.ggml) - payload["publisher_claimed_source_commit"] = "f" * 40 - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "release tag commit must equal publisher-claimed source commit", - ): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", payload)] - ) - - def test_release_note_source_claims_are_rejected(self) -> None: - payload = release_payload( - "b4321", - 30, - 300, - repository="ggml-org/llama.cpp", - publisher_claim_type="immutable-source-manifest", - publisher_claimed_source_commit="f" * 40, - ) - payload["publisher_claim_type"] = "release-note-marker" - payload["body"] = f"**Llama.cpp Source Commit**: {'f' * 40}" - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "publisher claim type", - ): - manifest.build_release_asset_manifest( - [("lemonade-sdk/llama.cpp", "b4321", payload)] - ) - - def test_fork_network_commit_is_not_upstream_ancestry(self) -> None: - claimed_commit = "3" * 40 - upstream_head = "7" * 40 - comparison = { - "base_commit": {"sha": claimed_commit}, - "merge_base_commit": {"sha": "4" * 40}, - "status": "diverged", - "url": ( - "https://api.github.com/repos/ggml-org/llama.cpp/compare/" - f"{claimed_commit}...{upstream_head}" - ), - } - - self.assertTrue( - hasattr(manifest, "require_upstream_ancestry"), - "upstream ancestry validator is required", - ) - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "not reachable", - ): - manifest.require_upstream_ancestry( - comparison, - "ggml-org/llama.cpp", - claimed_commit, - upstream_head, - ) - - def test_untrusted_manifest_anchor_must_reach_live_upstream_head(self) -> None: - manifest_anchor = "3" * 40 - live_upstream_head = "7" * 40 - comparison = { - "base_commit": {"sha": manifest_anchor}, - "merge_base_commit": {"sha": "4" * 40}, - "status": "diverged", - "url": ( - "https://api.github.com/repos/ggml-org/llama.cpp/compare/" - f"{manifest_anchor}...{live_upstream_head}" - ), - } - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "not reachable", - ): - manifest.require_upstream_ancestry( - comparison, - "ggml-org/llama.cpp", - manifest_anchor, - live_upstream_head, - ) - - def test_unchanged_managed_pins_do_not_require_a_manifest(self) -> None: - versions = { - "llamacpp": { - "cpu": "b1", - "cuda": "b2", - "metal": "b1", - "rocm-nightly": "b3", - "rocm-stable": "b2", - "vulkan": "b1", - } - } - - self.assertFalse( - manifest.require_release_manifest_for_managed_pin_changes( - versions, - copy.deepcopy(versions), - None, - ) - ) - - def test_rocm_selector_changes_require_their_managed_manifest_groups(self) -> None: - base = { - "llamacpp": { - "cpu": "b1", - "cuda": "b2", - "metal": "b1", - "rocm-nightly": "b3", - "rocm-stable": "b2", - "vulkan": "b1", - }, - "rocm_asset_families": {"gfx1033": "gfx103X"}, - "therock": {"version": "7.14.0"}, - } - - mapping_change = copy.deepcopy(base) - mapping_change["rocm_asset_families"]["gfx1033"] = "gfx1033" - self.assertEqual( - manifest.managed_llamacpp_pin_changes(base, mapping_change), - {"rocm-nightly": ("b3", "b3")}, - ) - - therock_change = copy.deepcopy(base) - therock_change["therock"]["version"] = "7.15.0" - self.assertEqual( - manifest.managed_llamacpp_pin_changes(base, therock_change), - {"rocm-stable": ("b2", "b2")}, - ) - - for field, value in ( - ("architectures", ["gfx103X", "gfx110X"]), - ("url_mapping", {"windows": "runtime.zip"}), - ): - with self.subTest(field=field): - changed = copy.deepcopy(base) - changed["therock"][field] = value - self.assertEqual( - manifest.managed_llamacpp_pin_changes(base, changed), - {"rocm-stable": ("b2", "b2")}, - ) - - comment_only = copy.deepcopy(base) - comment_only["therock"]["comment"] = "documentation only" - self.assertEqual( - manifest.managed_llamacpp_pin_changes(base, comment_only), - {}, - ) - - def test_rocm_pin_manifest_requires_complete_candidate_target_attestation( - self, - ) -> None: - base = { - "llamacpp": { - "cpu": "b1", - "cuda": "b2", - "metal": "b1", - "rocm-nightly": "b3", - "rocm-stable": "b2", - "vulkan": "b1", - }, - "rocm_asset_families": { - "gfx1033": "gfx103X", - "gfx1035": "gfx103X", - "gfx1036": "gfx103X", - }, - "therock": {"version": "7.14.0"}, - } - candidate = copy.deepcopy(base) - candidate["llamacpp"]["rocm-nightly"] = "b9" - complete = self._complete_managed_manifest( - candidate, - rocm_tag="b9", - extra_rocm_target="gfx999", - ) - self.assertTrue( - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - complete, - ) - ) - - incomplete = json.loads(complete) - rocm_release = next( - release - for release in incomplete["releases"] - if release["repository"] == "lemonade-sdk/llamacpp-rocm" - ) - target_attestation = next( - attestation - for attestation in rocm_release["build_target_attestations"] - if attestation["name"] == "llama-b9-windows-rocm-gfx103X-x64.zip" - ) - target_attestation["build_targets"].remove("gfx1035") - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "incomplete build-target attestation", - ): - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - json.dumps(incomplete), - ) - - rocm_release["build_target_attestations"] = [] - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "incomplete build-target attestation", - ): - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - json.dumps(incomplete), - ) - - def test_rocm_mapping_only_change_rejects_manifest_for_old_mapping(self) -> None: - base = { - "llamacpp": { - "cpu": "b1", - "cuda": "b2", - "metal": "b1", - "rocm-nightly": "b3", - "rocm-stable": "b2", - "vulkan": "b1", - }, - "rocm_asset_families": {"gfx1033": "gfx103X"}, - "therock": {"version": "7.14.0"}, - } - candidate = copy.deepcopy(base) - candidate["rocm_asset_families"]["gfx1033"] = "gfx1033" - old_mapping_manifest = self._complete_managed_manifest(base) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "llamacpp.rocm-nightly", - ): - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - old_mapping_manifest, - ) - - def test_changed_managed_pin_requires_a_matching_release_manifest(self) -> None: - base = { - "llamacpp": { - "cpu": "b1", - "cuda": "b2", - "metal": "b1", - "rocm-nightly": "b3", - "rocm-stable": "b2", - "vulkan": "b1", - }, - "rocm_asset_families": {}, - "therock": {"version": "7.14.0"}, - } - candidate = copy.deepcopy(base) - candidate["llamacpp"]["vulkan"] = "b1234" - requirements = release_assets.build_asset_requirements( - ggml_release="b1234", - rocm_release="b3", - lemonade_release="b2", - backend_versions=candidate, - ) - ggml = copy.deepcopy(self.ggml) - for offset, asset_name in enumerate( - requirements["ggml"]["vulkan"], - start=1, - ): - ggml["assets"].append( - { - "digest": "sha256:" + f"{offset:064x}", - "id": 1000 + offset, - "name": asset_name, - "size": offset, - "state": "uploaded", - "updated_at": "2026-09-05T12:40:00Z", - } - ) - release_manifest = manifest.build_release_asset_manifest( - [ - ("ggml-org/llama.cpp", "b1234", ggml), - ( - "lemonade-sdk/llamacpp-rocm", - "b3", - release_payload( - "b3", - 20, - 200, - publisher_claim_type="immutable-source-manifest", - ), - ), - ( - "lemonade-sdk/llama.cpp", - "b2", - release_payload( - "b2", - 30, - 300, - publisher_claim_type="immutable-source-manifest", - ), - ), - ] - ) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "changed managed llama.cpp pins", - ): - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - None, - ) - - self.assertTrue( - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - release_manifest, - ) - ) - - required_asset = requirements["ggml"]["vulkan"][-1] - for defect in ("missing", "zero-size"): - with self.subTest(defect=defect): - incomplete = json.loads(release_manifest) - ggml_release = next( - release - for release in incomplete["releases"] - if release["repository"] == "ggml-org/llama.cpp" - ) - if defect == "missing": - ggml_release["assets"] = [ - asset - for asset in ggml_release["assets"] - if asset["name"] != required_asset - ] - expected_error = "missing required asset" - else: - target_asset = next( - asset - for asset in ggml_release["assets"] - if asset["name"] == required_asset - ) - target_asset["size"] = 0 - expected_error = "positive size" - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - expected_error, - ): - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - json.dumps(incomplete), - ) - - mismatched = json.loads(release_manifest) - mismatched["releases"][0]["tag_name"] = "b9999" - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "does not match changed managed llama.cpp pin", - ): - manifest.require_release_manifest_for_managed_pin_changes( - base, - candidate, - json.dumps(mismatched), - ) - - def test_manifest_binds_publisher_claim_and_upstream_reference(self) -> None: - payload = copy.deepcopy(self.rocm) - payload.update( - { - "publisher_claim_type": "immutable-source-manifest", - "upstream_reference": "refs/heads/master", - "upstream_reference_head": "d" * 40, - } - ) - - parsed = json.loads( - manifest.build_release_asset_manifest( - [("lemonade-sdk/llamacpp-rocm", "b1235", payload)] - ) - )["releases"][0] - - self.assertEqual( - parsed.get("publisher_claimed_source_commit"), - payload["publisher_claimed_source_commit"], - ) - self.assertEqual( - parsed.get("publisher_claim_type"), "immutable-source-manifest" - ) - self.assertIs(parsed.get("release_immutable"), True) - self.assertEqual(parsed.get("upstream_reference"), "refs/heads/master") - self.assertEqual(parsed.get("upstream_reference_head"), "d" * 40) - self.assertNotIn("release_body_digest", parsed) - self.assertNotIn("source_commit", parsed) - - def test_tag_mismatch_and_duplicate_assets_fail_closed(self) -> None: - with self.assertRaisesRegex(manifest.ReleaseManifestError, "tag mismatch"): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b9999", self.ggml)] - ) - - payload = copy.deepcopy(self.ggml) - payload["assets"][1]["id"] = payload["assets"][0]["id"] - with self.assertRaisesRegex( - manifest.ReleaseManifestError, "duplicate asset id" - ): - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", payload)] - ) - - def test_comparison_rejects_any_asset_metadata_change(self) -> None: - expected = manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", self.ggml)] - ) - changed = copy.deepcopy(self.ggml) - changed["assets"][0]["updated_at"] = "2026-09-05T12:35:00Z" - actual = manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", changed)] - ) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "changed since validation", - ): - manifest.require_exact_manifest_match(expected, actual) - - manifest.require_exact_manifest_match(expected, expected) - - def test_canonical_manifest_rejects_duplicate_json_members(self) -> None: - canonical = manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", self.ggml)] - ) - duplicate = canonical[:-1] + ',"schema_version":5}' - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "duplicate JSON member", - ): - manifest.canonicalize_release_asset_manifest(duplicate) - - def test_canonical_manifest_requires_integer_schema_version(self) -> None: - canonical = json.loads( - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", self.ggml)] - ) - ) - for schema_version in (True, 5.0, "5"): - with self.subTest(schema_version=schema_version): - changed = copy.deepcopy(canonical) - changed["schema_version"] = schema_version - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "schema_version must be 5", - ): - manifest.canonicalize_release_asset_manifest(json.dumps(changed)) - - def test_comparison_rejects_any_publisher_claim_change(self) -> None: - release = release_payload( - "b1234", - 10, - 100, - publisher_claim_type="immutable-source-manifest", - publisher_claimed_source_commit="f" * 40, - ) - expected = manifest.build_release_asset_manifest( - [("lemonade-sdk/llama.cpp", "b1234", release)] - ) - cases = ( - ("release_tag_commit", "e" * 40), - ("publisher_claimed_source_commit", "e" * 40), - ) - for field, value in cases: - with self.subTest(field=field): - changed = copy.deepcopy(release) - changed[field] = value - actual = manifest.build_release_asset_manifest( - [("lemonade-sdk/llama.cpp", "b1234", changed)] - ) - - with self.assertRaisesRegex( - manifest.ReleaseManifestError, - "changed since validation", - ): - manifest.require_exact_manifest_match(expected, actual) - - def test_release_note_edits_do_not_change_manifest_identity(self) -> None: - expected = manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", self.ggml)] - ) - changed = copy.deepcopy(self.ggml) - changed["body"] = "updated release notes" - - self.assertEqual( - manifest.build_release_asset_manifest( - [("ggml-org/llama.cpp", "b1234", changed)] - ), - expected, - ) - - def test_materialized_manifest_binds_digest_and_publisher_claims(self) -> None: - expected = manifest.build_release_asset_manifest( - [ - ("ggml-org/llama.cpp", "b1234", self.ggml), - ("lemonade-sdk/llamacpp-rocm", "b1235", self.rocm), - ] - ) - expected_digest = hashlib.sha256(expected.encode("utf-8")).hexdigest() - - self.assertTrue( - hasattr(manifest, "materialize_release_asset_manifest"), - "manifest materializer is required", - ) - with tempfile.TemporaryDirectory() as directory: - output_path = Path(directory) / "release-manifest.json" - digest, summary = manifest.materialize_release_asset_manifest( - expected, - output_path, - ) - - self.assertEqual(digest, expected_digest) - self.assertEqual(output_path.read_text(encoding="utf-8"), expected + "\n") - self.assertIn(expected_digest, summary) - self.assertIn("Only immutable GitHub releases are accepted", summary) - self.assertIn("immutable publisher manifest", summary) - self.assertIn("not cryptographic build provenance", summary) - self.assertIn(self.ggml["publisher_claimed_source_commit"], summary) - self.assertIn(self.rocm["publisher_claimed_source_commit"], summary) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_llamacpp_system_backend.py b/test/test_llamacpp_system_backend.py index 04b69c4457..86a0cf3cff 100644 --- a/test/test_llamacpp_system_backend.py +++ b/test/test_llamacpp_system_backend.py @@ -55,8 +55,8 @@ def get_arg(flag, default): # lemond detects the system llama-server version by running -# `llama-server --version` and reading bounded output from the llamacpp backend -# resolver in src/cpp/server/backends/llamacpp/llamacpp_server.cpp. +# `llama-server --version` and reading one line of stdout from the llamacpp +# backend resolver in src/cpp/server/backends/llamacpp/llamacpp_server.cpp. # The real binary prints a version line and exits immediately. We must mirror # that: otherwise the probe blocks forever on our long-lived HTTP server and # /internal/set hangs until the client times out. @@ -168,69 +168,6 @@ def log_message(self, format, *args): ReusableHTTPServer(("127.0.0.1", port), Handler).serve_forever() """ -MOCK_INCOMPATIBLE_LLAMA_SERVER_PYTHON = """#!/usr/bin/env python3 -import os -import sys - -if "--version" in sys.argv or "--help" in sys.argv: - print("version: 0.3.0-dev (build 4242, commit incompatible)") - sys.exit(0) - -invocation_path = os.environ.get("MOCK_LLAMA_INVOCATION_PATH", "") -if invocation_path: - with open(invocation_path, "a", encoding="utf-8") as handle: - handle.write("model\\n") - -if "--log-file" in sys.argv: - print("error: unrecognized argument: --log-file", file=sys.stderr) - sys.exit(2) - -message = "error loading model: unknown model architecture: 'k2-horizon'" -print(message, file=sys.stderr) -sys.exit(1) -""" - -MOCK_RESOURCE_FAILURE_LLAMA_SERVER_PYTHON = """#!/usr/bin/env python3 -import os -import sys - -if "--version" in sys.argv or "--help" in sys.argv: - print("version: 0.3.0-dev (build 4242, commit incompatible)") - sys.exit(0) - -invocation_path = os.environ.get("MOCK_LLAMA_INVOCATION_PATH", "") -if invocation_path: - with open(invocation_path, "a", encoding="utf-8") as handle: - handle.write("model\\n") - -message = "error loading model: failed to allocate compute buffers" -print(message, file=sys.stderr) -sys.exit(1) -""" - -MOCK_CANCELLABLE_LLAMA_SERVER_PYTHON = """#!/usr/bin/env python3 -import os -import sys -import time - -if "--version" in sys.argv or "--help" in sys.argv: - print("version: 0.3.0-dev (build 4242, commit incompatible)") - sys.exit(0) - -invocation_path = os.environ.get("MOCK_LLAMA_INVOCATION_PATH", "") -if invocation_path: - with open(invocation_path, "a", encoding="utf-8") as handle: - handle.write("model\\n") - -print( - "error loading model: unknown model architecture: 'k2-horizon'", - file=sys.stderr, - flush=True, -) -while True: - time.sleep(1) -""" - def _is_server_running(port=PORT): """Check if the server is running on the given port.""" @@ -396,24 +333,15 @@ def setUpClass(cls): cls.temp_bin_dir = tempfile.mkdtemp(prefix="lemonade_llamacpp_mock_bin_") cls.cache_dir = tempfile.mkdtemp(prefix="lemonade_llamacpp_test_") cls.dummy_llama_server_path = os.path.join(cls.temp_bin_dir, "llama-server") - cls.k2_fixture_path = os.path.join(cls.temp_bin_dir, "k2-system-fixture.bin") cls.capture_path = os.path.join(cls.temp_bin_dir, "captured_chat_request.json") cls.control_path = os.path.join(cls.temp_bin_dir, "mock_control.json") - cls.invocation_path = os.path.join(cls.temp_bin_dir, "model_invocations.txt") cls.original_env = { name: os.environ.get(name) - for name in ( - "PATH", - "MOCK_LLAMA_REQUEST_PATH", - "MOCK_LLAMA_CONTROL_PATH", - "MOCK_LLAMA_INVOCATION_PATH", - ) + for name in ("PATH", "MOCK_LLAMA_REQUEST_PATH", "MOCK_LLAMA_CONTROL_PATH") } try: cls._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) - with open(cls.k2_fixture_path, "wb") as handle: - handle.write(b"GGUF") # Keep the external-system integration deterministic on AMD hosts as # well: production accepts the plugin beside a PATH llama-server. @@ -431,7 +359,6 @@ def setUpClass(cls): os.environ["PATH"] = cls.temp_bin_dir + os.pathsep + original_path os.environ["MOCK_LLAMA_REQUEST_PATH"] = cls.capture_path os.environ["MOCK_LLAMA_CONTROL_PATH"] = cls.control_path - os.environ["MOCK_LLAMA_INVOCATION_PATH"] = cls.invocation_path cls._write_mock_control({}) # One lemond and one spawned mock llama-server are enough for all @@ -490,64 +417,10 @@ def _write_mock_control(cls, control): with open(cls.control_path, "w", encoding="utf-8") as handle: json.dump(control, handle) - @classmethod - def _model_invocation_count(cls): - try: - with open(cls.invocation_path, "r", encoding="utf-8") as handle: - return sum(1 for line in handle if line.strip() == "model") - except FileNotFoundError: - return 0 - - @classmethod - def _wait_for_model_invocations(cls, expected, timeout=10): - deadline = time.time() + timeout - while time.time() < deadline: - if cls._model_invocation_count() >= expected: - return True - time.sleep(0.05) - return cls._model_invocation_count() >= expected - - @staticmethod - def _wait_for_job_status(job_id, expected, timeout=20): - deadline = time.time() + timeout - last_status = None - while time.time() < deadline: - response = requests.get( - f"http://localhost:{PORT}/api/v1/jobs/{job_id}", - timeout=TIMEOUT_DEFAULT, - ) - if response.status_code == 200: - last_status = response.json().get("status") - if last_status == expected: - return response.json() - time.sleep(0.1) - raise AssertionError( - f"job {job_id} did not reach {expected}; last status={last_status}" - ) - - def _register_k2_system_fixture(self): - k2_fixture = "user.K2-Horizon-System-Binary-Fixture" - registration = requests.post( - f"http://localhost:{PORT}/api/v1/models/register", - json={ - "model_name": k2_fixture, - "recipe": "llamacpp", - "checkpoint": self.k2_fixture_path, - "source": "local_path", - "labels": ["chat", "reasoning", "tool-calling"], - }, - timeout=TIMEOUT_DEFAULT, - ) - self.assertEqual(registration.status_code, 200, registration.text) - self.assertTrue(registration.json()["model"]["downloaded"]) - return k2_fixture - def setUp(self): print(f"\n=== Starting test: {self._testMethodName} ===") if os.path.exists(self.capture_path): os.remove(self.capture_path) - if os.path.exists(self.invocation_path): - os.remove(self.invocation_path) self._write_mock_control({}) @unittest.skipUnless( @@ -679,160 +552,6 @@ def test_008_backend_context_error_preserves_http_status(self): self.assertEqual(error["status_code"], 400) self.assertIn("exceeds the available context size", error["message"]) - @unittest.skipUnless( - sys.platform.startswith("linux"), "System backend only supported on Linux" - ) - def test_009_incompatible_system_binary_has_actionable_k2_diagnostic(self): - k2_fixture = self._register_k2_system_fixture() - - unload_response = requests.post( - f"http://localhost:{PORT}/api/v1/unload", - json={}, - timeout=TIMEOUT_DEFAULT, - ) - self.assertEqual(unload_response.status_code, 200, unload_response.text) - - self._write_llama_server(MOCK_INCOMPATIBLE_LLAMA_SERVER_PYTHON) - try: - ordinary_response = requests.post( - f"http://localhost:{PORT}/api/v1/load", - json={ - "model_name": ENDPOINT_TEST_MODEL, - "llamacpp_backend": "system", - }, - timeout=TIMEOUT_MODEL_OPERATION, - ) - self.assertNotEqual(ordinary_response.status_code, 200) - ordinary_message = ordinary_response.json()["error"]["message"] - self.assertTrue( - ordinary_message.endswith(": llama-server failed to start"), - ordinary_message, - ) - self.assertNotIn("K2-Horizon-capable", ordinary_message) - self.assertEqual(self._model_invocation_count(), 2) - - os.remove(self.invocation_path) - - k2_response = requests.post( - f"http://localhost:{PORT}/api/v1/load", - json={"model_name": k2_fixture, "llamacpp_backend": "system"}, - timeout=TIMEOUT_MODEL_OPERATION, - ) - self.assertNotEqual(k2_response.status_code, 200) - message = k2_response.json()["error"]["message"] - self.assertIn(k2_fixture, message) - self.assertIn(self.dummy_llama_server_path, message) - self.assertIn("b4242", message) - self.assertIn( - "Original error: error loading model: unknown model architecture: " - "'k2-horizon'", - message, - ) - self.assertIn("K2-Horizon-capable", message) - self.assertIn("model/K2Horizon", message) - self.assertIn("PATH", message) - self.assertEqual( - self._model_invocation_count(), - 2, - "diagnostic collection must not re-launch the model command", - ) - finally: - self._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) - - @unittest.skipUnless( - sys.platform.startswith("linux"), "System backend only supported on Linux" - ) - def test_010_k2_resource_failure_keeps_ordinary_startup_diagnostic(self): - k2_fixture = self._register_k2_system_fixture() - unload_response = requests.post( - f"http://localhost:{PORT}/api/v1/unload", - json={}, - timeout=TIMEOUT_DEFAULT, - ) - self.assertEqual(unload_response.status_code, 200, unload_response.text) - - self._write_llama_server(MOCK_RESOURCE_FAILURE_LLAMA_SERVER_PYTHON) - try: - response = requests.post( - f"http://localhost:{PORT}/api/v1/load", - json={"model_name": k2_fixture, "llamacpp_backend": "system"}, - timeout=TIMEOUT_MODEL_OPERATION, - ) - self.assertNotEqual(response.status_code, 200) - message = response.json()["error"]["message"] - self.assertTrue( - message.endswith(": llama-server failed to start"), - message, - ) - self.assertNotIn("K2-Horizon-capable", message) - self.assertNotIn("model/K2Horizon", message) - self.assertEqual( - self._model_invocation_count(), - 2, - "resource failure diagnosis must not repeat the model command", - ) - finally: - self._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) - - @unittest.skipUnless( - sys.platform.startswith("linux"), "System backend only supported on Linux" - ) - def test_011_cancelled_k2_load_does_not_relaunch_model_command(self): - k2_fixture = self._register_k2_system_fixture() - unload_response = requests.post( - f"http://localhost:{PORT}/api/v1/unload", - json={}, - timeout=TIMEOUT_DEFAULT, - ) - self.assertEqual(unload_response.status_code, 200, unload_response.text) - - job_id = None - self._write_llama_server(MOCK_CANCELLABLE_LLAMA_SERVER_PYTHON) - try: - response = requests.post( - f"http://localhost:{PORT}/api/v1/jobs", - json={ - "name": "cancel-k2-system-load", - "definition": { - "steps": [ - { - "id": "load", - "op": "load", - "params": { - "model": k2_fixture, - "llamacpp_backend": "system", - }, - } - ] - }, - }, - timeout=TIMEOUT_DEFAULT, - ) - self.assertEqual(response.status_code, 202, response.text) - job_id = response.json()["id"] - self.assertTrue(self._wait_for_model_invocations(1)) - - interrupt_response = requests.post( - f"http://localhost:{PORT}/api/v1/jobs/{job_id}/interrupt", - timeout=TIMEOUT_DEFAULT, - ) - self.assertEqual( - interrupt_response.status_code, 200, interrupt_response.text - ) - self._wait_for_job_status(job_id, "interrupted") - self.assertEqual( - self._model_invocation_count(), - 1, - "a cancelled load must not retry or replay the model command", - ) - finally: - if job_id is not None: - requests.delete( - f"http://localhost:{PORT}/api/v1/jobs/{job_id}", - timeout=TIMEOUT_DEFAULT, - ) - self._write_llama_server(MOCK_LLAMA_SERVER_PYTHON) - def _run_tests(): """Run llamacpp system backend tests.""" diff --git a/test/test_llamacpp_validation_artifacts.py b/test/test_llamacpp_validation_artifacts.py deleted file mode 100644 index 7e6062ea2e..0000000000 --- a/test/test_llamacpp_validation_artifacts.py +++ /dev/null @@ -1,314 +0,0 @@ -import hashlib -import json -import os -import struct -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path -from unittest import mock - -from test.utils import llamacpp_validation_artifacts as artifacts -from test.utils.llamacpp_validation_artifacts import ( - ValidationArtifactError, - verify_validation_artifacts, -) - -MODEL_ID = "K2-Horizon-Test-GGUF" -REPOSITORY = "IFM/K2-Horizon-Test-GGUF" -REVISION = "a" * 40 -FILENAME = "model.gguf" - - -def gguf_string(value): - encoded = value.encode("utf-8") - return struct.pack(" dict: - matrix = planning.create_validation_plan("schedule") - for row in matrix["include"]: - if models_by_target is None or row["target"] not in models_by_target: - continue - planned_models = models_by_target[row["target"]] - row["models"] = list(planned_models) - row["expected_models"] = list(planned_models) - return matrix - - -def passing_capability_matrix(model: str = K2_SMALL) -> dict: - cases = [] - for case_id in capabilities.K2_HORIZON_CASE_IDS: - protocol = "openai" - if case_id == "ollama_thinking": - protocol = "ollama" - elif case_id == "anthropic_translation": - protocol = "anthropic" - stream = case_id in { - "openai_plain_off_stream", - "openai_reasoning_high_stream", - "openai_tool_stream_xml", - } - tool_case = case_id in { - "openai_tool_json", - "openai_tool_xml", - "openai_tool_xml_typed", - "openai_tool_default_xml", - "openai_tool_stream_xml", - } - reasoning_case = ( - case_id.startswith("openai_reasoning_") - and case_id != "openai_reasoning_low" - ) or case_id == "ollama_thinking" - content_case = case_id in { - "openai_plain_off_nonstream", - "openai_plain_off_stream", - "openai_tool_result_followup", - "anthropic_translation", - "ollama_thinking", - } or case_id.startswith("openai_reasoning_") - finish_reason = "stop" - if tool_case: - finish_reason = "tool_calls" - elif case_id == "anthropic_translation": - finish_reason = "end_turn" - cases.append( - { - "id": case_id, - "protocol": protocol, - "stream": stream, - "pass": True, - "status_code": 200, - "finish_reason": finish_reason, - "content_chars": 2 if content_case else 0, - "reasoning_chars": 8 if reasoning_case else 0, - "tool_call_count": 1 if tool_case else 0, - "terminal_frame": True, - "tool_name": capabilities.TOOL_NAME if tool_case else None, - "tool_arguments": ( - capabilities.EXPECTED_TOOL_ARGUMENTS if tool_case else None - ), - "parity_with": ( - "openai_tool_xml" - if case_id in {"openai_tool_default_xml", "openai_tool_stream_xml"} - else None - ), - "raw_ifm_marker": None, - "error": None, - } - ) - return { - "profile": capabilities.K2_HORIZON_PROFILE, - "model": f"builtin.{model.removeprefix('builtin.')}", - "pass": True, - "cases": cases, - } - - -def passing_result(model: str = K2_SMALL, *, capability: bool | None = None) -> dict: - result = { - "model": model, - "pass": True, - "response": "Four.", - "input_tokens": 10, - "output_tokens": 2, - "time_to_first_token": 0.5, - "tokens_per_second": 4.0, - } - if capability if capability is not None else model == K2_SMALL: - result["capability_matrix"] = passing_capability_matrix(model) - return result - - -class LlamaCppValidationEvidenceTests(unittest.TestCase): - def write_expected_results(self, root: Path, matrix: dict | None = None) -> None: - selected_matrix = matrix or expected_matrix() - for row in selected_matrix["include"]: - filename = f"llamacpp_validation_{row['target']}.json" - records = [ - passing_result(model, capability=model == K2_SMALL) - for model in row["expected_models"] - ] - (root / filename).write_text( - json.dumps(records), - encoding="utf-8", - ) - restart_filename = f"llamacpp_restart_validation_{row['target']}.json" - restart_model = row["capability_models"][0] - (root / restart_filename).write_text( - json.dumps([passing_result(restart_model, capability=False)]), - encoding="utf-8", - ) - - def test_all_expected_nonempty_passing_results_are_accepted(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - - results = evidence.load_and_validate_results(root, expected_matrix()) - - self.assertEqual( - set(results), - set(evidence.EXPECTED_RESULT_FILES) - | set(evidence.EXPECTED_RESTART_RESULT_FILES), - ) - self.assertGreater( - len(results["llamacpp_validation_windows-vulkan.json"]), - 1, - ) - self.assertTrue( - all( - len(records) == 1 - for filename, records in results.items() - if filename != "llamacpp_validation_windows-vulkan.json" - ) - ) - - def test_missing_empty_and_invalid_json_files_fail_closed(self) -> None: - cases = { - "missing": None, - "empty": "", - "invalid": "not json", - "empty list": "[]", - } - for name, replacement in cases.items(): - with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - target = root / evidence.EXPECTED_RESULT_FILES[0] - if replacement is None: - target.unlink() - else: - target.write_text(replacement, encoding="utf-8") - - with self.assertRaises(evidence.ValidationEvidenceError): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_restart_evidence_must_be_present_exact_and_passing(self) -> None: - mutations = { - "missing": None, - "empty": "", - "invalid": "not json", - "empty list": "[]", - "failed": json.dumps([{**passing_result(capability=False), "pass": False}]), - "wrong model": json.dumps( - [passing_result("Other-Llama", capability=False)] - ), - "extra model": json.dumps( - [ - passing_result(capability=False), - passing_result("Other-Llama", capability=False), - ] - ), - "unexpected capabilities": json.dumps([passing_result()]), - } - target_name = "llamacpp_restart_validation_linux-vulkan.json" - for name, replacement in mutations.items(): - with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - target = root / target_name - if replacement is None: - target.unlink() - else: - target.write_text(replacement, encoding="utf-8") - - with self.assertRaises(evidence.ValidationEvidenceError): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_failed_or_malformed_records_fail_closed(self) -> None: - cases = { - "failed": [{**passing_result(), "pass": False}], - "non_boolean pass": [{**passing_result(), "pass": 1}], - "blank response": [{**passing_result(), "response": " \t"}], - "invalid metric": [{**passing_result(), "tokens_per_second": "unexpected"}], - "negative token count": [{**passing_result(), "input_tokens": -1}], - "missing field": [ - { - key: value - for key, value in passing_result().items() - if key != "response" - } - ], - "non_object": ["PASS"], - "duplicate model": [passing_result(), passing_result()], - } - for name, replacement in cases.items(): - with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - (root / evidence.EXPECTED_RESULT_FILES[0]).write_text( - json.dumps(replacement), - encoding="utf-8", - ) - - with self.assertRaises(evidence.ValidationEvidenceError): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_exact_producer_record_schema_is_accepted(self) -> None: - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "result.json" - expected = passing_result() - path.write_text(json.dumps([expected]), encoding="utf-8") - - records = evidence.load_and_validate_result_file( - path, - [K2_SMALL], - capability_profile=capabilities.K2_HORIZON_PROFILE, - capability_models=[K2_SMALL], - ) - - self.assertEqual(records, [expected]) - - def test_result_file_above_byte_limit_fails_closed(self) -> None: - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "result.json" - path.write_text( - json.dumps([passing_result(capability=False)]), - encoding="utf-8", - ) - - with ( - mock.patch.object( - evidence, - "MAX_VALIDATION_EVIDENCE_BYTES", - path.stat().st_size - 1, - ), - self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "byte limit", - ), - ): - evidence.load_and_validate_result_file(path, [K2_SMALL]) - - def test_result_file_at_byte_limit_is_accepted(self) -> None: - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "result.json" - expected = passing_result(capability=False) - path.write_text(json.dumps([expected]), encoding="utf-8") - - with mock.patch.object( - evidence, - "MAX_VALIDATION_EVIDENCE_BYTES", - path.stat().st_size, - ): - records = evidence.load_and_validate_result_file(path, [K2_SMALL]) - - self.assertEqual(records, [expected]) - - def test_result_file_symlink_fails_closed(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - source = root / "source.json" - source.write_text( - json.dumps([passing_result(capability=False)]), - encoding="utf-8", - ) - path = root / "result.json" - try: - path.symlink_to(source.name) - except OSError as exc: - self.skipTest(f"symlinks unavailable: {exc}") - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "regular file", - ): - evidence.load_and_validate_result_file(path, [K2_SMALL]) - - def test_result_and_capability_schemas_reject_unknown_fields(self) -> None: - def add_result_field(records: list[dict]) -> None: - records[0]["schema_version"] = 999 - - def add_matrix_field(records: list[dict]) -> None: - capability_record = next( - record for record in records if "capability_matrix" in record - ) - capability_record["capability_matrix"]["verification"] = "failed" - - def add_case_field(records: list[dict]) -> None: - capability_record = next( - record for record in records if "capability_matrix" in record - ) - capability_record["capability_matrix"]["cases"][0][ - "verification" - ] = "failed" - - mutations = { - "result": add_result_field, - "capability matrix": add_matrix_field, - "capability case": add_case_field, - } - for name, mutate in mutations.items(): - with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - target = root / evidence.EXPECTED_RESULT_FILES[0] - records = json.loads(target.read_text(encoding="utf-8")) - mutate(records) - target.write_text(json.dumps(records), encoding="utf-8") - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "unexpected.*fields", - ): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_runtime_models_must_exactly_match_each_planned_lane(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - target = root / "llamacpp_validation_windows-rocm-stable.json" - target.write_text( - json.dumps([passing_result("some-other-model")]), - encoding="utf-8", - ) - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "do not match planned models", - ): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_selected_models_must_exactly_match_expected_models(self) -> None: - matrix = expected_matrix() - matrix["include"][0]["models"] = [] - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root, matrix) - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "selected models", - ): - evidence.load_and_validate_results(root, matrix) - - def test_every_planned_lane_must_include_k2_small(self) -> None: - matrix = expected_matrix() - for row in matrix["include"]: - row["models"] = ["some-other-model"] - row["expected_models"] = ["some-other-model"] - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - for filename in evidence.EXPECTED_RESULT_FILES: - (root / filename).write_text( - json.dumps([passing_result("some-other-model")]), - encoding="utf-8", - ) - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - K2_SMALL, - ): - evidence.load_and_validate_results( - root, - matrix, - ) - - def test_exact_per_lane_planned_model_sets_are_accepted(self) -> None: - matrix = expected_matrix() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root, matrix) - - results = evidence.load_and_validate_results(root, matrix) - - self.assertEqual( - len(results["llamacpp_validation_windows-vulkan.json"]), - len(matrix["include"][0]["expected_models"]), - ) - - def test_exact_lane_mapping_is_required(self) -> None: - mutations = {} - missing = expected_matrix() - missing["include"].pop() - mutations["missing"] = missing - wrong_backend = expected_matrix() - wrong_backend["include"][2]["backend"] = "cuda" - mutations["backend"] = wrong_backend - wrong_pin = expected_matrix() - wrong_pin["include"][3]["managed_pin"] = "cpu" - mutations["pin"] = wrong_pin - wrong_platform = expected_matrix() - wrong_platform["include"][1]["build_platform"] = "windows" - mutations["platform"] = wrong_platform - wrong_runner = expected_matrix() - wrong_runner["include"][0]["runner"] = ["windows-latest"] - mutations["runner"] = wrong_runner - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - for name, matrix in mutations.items(): - with self.subTest(name=name): - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "scheduled validation lanes|mapping", - ): - evidence.load_and_validate_results(root, matrix) - - def test_large_k2_models_require_the_128gb_vulkan_lane(self) -> None: - for model in ( - planning.K2_MEDIUM, - planning.K2_LARGE, - f"builtin.{planning.K2_MEDIUM}", - f"builtin.{planning.K2_LARGE}", - ): - matrix = expected_matrix({"linux-vulkan": [K2_SMALL, model]}) - with self.subTest(model=model), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root, matrix) - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "128gb", - ): - evidence.load_and_validate_results(root, matrix) - - def test_capability_evidence_is_required_and_validated(self) -> None: - mutations = {} - missing = passing_result() - missing.pop("capability_matrix") - mutations["missing"] = missing - failed = passing_result() - failed["capability_matrix"]["pass"] = False - mutations["failed"] = failed - marker = passing_result() - marker["capability_matrix"]["cases"][0]["raw_ifm_marker"] = "" - mutations["marker"] = marker - wrong_model = passing_result() - wrong_model["capability_matrix"]["model"] = "builtin.Other-Llama" - mutations["wrong model"] = wrong_model - - for name, result in mutations.items(): - with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - (root / evidence.EXPECTED_RESULT_FILES[0]).write_text( - json.dumps([result]), - encoding="utf-8", - ) - - with self.assertRaises(evidence.ValidationEvidenceError): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_duplicate_json_keys_in_evidence_fail_closed(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - target = root / evidence.EXPECTED_RESULT_FILES[0] - raw = json.dumps([passing_result()]) - raw = raw.replace( - '"city": "Paris"', - '"city": "London", "city": "Paris"', - 1, - ) - target.write_text(raw, encoding="utf-8") - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "duplicate JSON key", - ): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_nonstandard_json_numeric_constants_fail_closed(self) -> None: - for constant in ("NaN", "Infinity", "-Infinity"): - with ( - self.subTest(constant=constant), - tempfile.TemporaryDirectory() as directory, - ): - root = Path(directory) - self.write_expected_results(root) - target = root / evidence.EXPECTED_RESULT_FILES[0] - raw = target.read_text(encoding="utf-8").replace( - '"tokens_per_second": 4.0', - f'"tokens_per_second": {constant}', - 1, - ) - target.write_text(raw, encoding="utf-8") - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "nonstandard JSON numeric constant", - ): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_overflowed_numeric_metrics_fail_closed(self) -> None: - for field, value in ( - ("time_to_first_token", "1e400"), - ("tokens_per_second", "1e400"), - ): - with self.subTest(field=field), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - target = root / evidence.EXPECTED_RESULT_FILES[0] - raw = target.read_text(encoding="utf-8") - raw = raw.replace( - f'"{field}": {passing_result()[field]}', - f'"{field}": {value}', - 1, - ) - target.write_text(raw, encoding="utf-8") - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "non-finite JSON number", - ): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_overflowed_numeric_extra_field_fails_closed(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - target = root / evidence.EXPECTED_RESULT_FILES[0] - raw = target.read_text(encoding="utf-8").replace( - "{", '{"untrusted_extra": 1e400,', 1 - ) - target.write_text(raw, encoding="utf-8") - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "non-finite JSON number", - ): - evidence.load_and_validate_results(root, expected_matrix()) - - def test_overflowed_numeric_extra_matrix_field_fails_closed(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - raw_matrix = json.dumps(expected_matrix()).replace( - "{", '{"untrusted_extra": 1e400,', 1 - ) - - with ( - mock.patch( - "sys.argv", - [ - "llamacpp_validation_evidence.py", - "--directory", - str(root), - "--expected-matrix-json", - raw_matrix, - ], - ), - self.assertRaisesRegex(SystemExit, "1"), - ): - evidence.main() - - def test_non_designated_model_cannot_attach_capability_evidence(self) -> None: - matrix = expected_matrix() - planned_models = matrix["include"][0]["expected_models"] - other_model = next(model for model in planned_models if model != K2_SMALL) - records = [ - passing_result( - model, - capability=model in {K2_SMALL, other_model}, - ) - for model in planned_models - ] - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root, matrix) - (root / "llamacpp_validation_windows-vulkan.json").write_text( - json.dumps(records), encoding="utf-8" - ) - - with self.assertRaisesRegex( - evidence.ValidationEvidenceError, - "unexpected capability matrix", - ): - evidence.load_and_validate_results(root, matrix) - - def test_capability_plan_fields_are_paired_and_model_bound(self) -> None: - mutations = {} - missing_profile = expected_matrix() - missing_profile["include"][0].pop("capability_profile") - mutations["missing profile"] = missing_profile - missing_models = expected_matrix() - missing_models["include"][0].pop("capability_models") - mutations["missing models"] = missing_models - unselected = expected_matrix() - unselected["include"][0]["capability_models"] = ["Other-Llama"] - mutations["unselected model"] = unselected - unknown_profile = expected_matrix() - unknown_profile["include"][0]["capability_profile"] = "unknown" - mutations["unknown profile"] = unknown_profile - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.write_expected_results(root) - for name, matrix in mutations.items(): - with self.subTest(name=name): - with self.assertRaises(evidence.ValidationEvidenceError): - evidence.load_and_validate_results(root, matrix) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_process_inheritance_contract.py b/test/test_process_inheritance_contract.py deleted file mode 100644 index 2ae049f2bf..0000000000 --- a/test/test_process_inheritance_contract.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python3 -"""Source contracts for subprocess handle inheritance.""" - -from __future__ import annotations - -import re -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -WINDOWS_PROCESS_SOURCE = ( - ROOT / "src" / "cpp" / "server" / "utils" / "platform" / "process_windows.cpp" -) -WINDOWS_PROCESS_TEST_SOURCE = ROOT / "test" / "cpp" / "test_process_manager_windows.cpp" -CMAKE_SOURCE = ROOT / "CMakeLists.txt" -CPP_WORKFLOW_SOURCE = ( - ROOT / ".github" / "workflows" / "cpp_server_build_test_release.yml" -) -LINUX_PROCESS_SOURCE = ( - ROOT / "src" / "cpp" / "server" / "utils" / "platform" / "process_linux.cpp" -) -MACOS_PROCESS_SOURCE = ( - ROOT / "src" / "cpp" / "server" / "utils" / "platform" / "process_macos.cpp" -) -MCP_CLIENT_SOURCE = ROOT / "src" / "cpp" / "server" / "mcp_client.cpp" -SYSTEM_INFO_SOURCE = ROOT / "src" / "cpp" / "server" / "system_info.cpp" - - -class ProcessInheritanceContractTests(unittest.TestCase): - def test_windows_process_creation_uses_one_restricted_handle_gateway(self): - source = WINDOWS_PROCESS_SOURCE.read_text(encoding="utf-8") - - self.assertEqual(source.count("CreateProcessA("), 1) - self.assertEqual( - source.count("create_process_with_restricted_handles("), - 4, - ) - for required in ( - "STARTUPINFOEXA", - "PROC_THREAD_ATTRIBUTE_HANDLE_LIST", - "EXTENDED_STARTUPINFO_PRESENT", - "UpdateProcThreadAttribute", - "DuplicateHandle", - "DUPLICATE_SAME_ACCESS", - ): - with self.subTest(required=required): - self.assertIn(required, source) - - def test_windows_gateway_normalizes_missing_standard_handles(self): - source = WINDOWS_PROCESS_SOURCE.read_text(encoding="utf-8") - helper_start = source.index( - "static BOOL create_process_with_restricted_handles(" - ) - helper_end = source.index("\n}\n\n// Helper function", helper_start) + 2 - helper = source[helper_start:helper_end] - - for required in ( - "STARTF_USESTDHANDLES", - "fallback_handles", - '"NUL", access', - "GENERIC_READ", - "GENERIC_WRITE", - "GetFileType(source_handles[i])", - "source_handles[i] = fallback", - "CloseHandle(fallback_handles[i])", - ): - with self.subTest(required=required): - self.assertIn(required, helper) - - self.assertLess(helper.index('"NUL", access'), helper.index("DuplicateHandle(")) - - def test_windows_missing_standard_handle_runtime_test_is_wired_into_ci(self): - runtime_test = WINDOWS_PROCESS_TEST_SOURCE.read_text(encoding="utf-8") - cmake_source = CMAKE_SOURCE.read_text(encoding="utf-8") - workflow = CPP_WORKFLOW_SOURCE.read_text(encoding="utf-8") - - for required in ( - "InvalidStandardHandles", - "INVALID_HANDLE_VALUE", - "stale_value", - "GetFileType(handle)", - "--probe-stdio", - "quiet child receives valid standard handles", - "captured child receives valid standard handles", - ): - with self.subTest(required=required): - self.assertIn(required, runtime_test) - - self.assertIn("BUILD_TESTING AND WIN32", cmake_source) - self.assertIn("add_cpp_ci_test(ProcessManagerWindowsTest CI ON", cmake_source) - self.assertIn( - "--target test_mcp_client_config test_process_manager_windows", workflow - ) - self.assertIn("mcp_client_config|ProcessManagerWindowsTest", workflow) - - def test_other_windows_process_creation_cannot_inherit_unlisted_handles(self): - mcp_source = MCP_CLIENT_SOURCE.read_text(encoding="utf-8") - system_info_source = SYSTEM_INFO_SOURCE.read_text(encoding="utf-8") - - create_process_sites = {} - for source_path in (ROOT / "src" / "cpp").rglob("*.cpp"): - source = source_path.read_text(encoding="utf-8") - call_count = source.count("CreateProcessA(") + source.count( - "CreateProcessW(" - ) - if call_count: - create_process_sites[source_path.relative_to(ROOT).as_posix()] = ( - call_count - ) - self.assertEqual( - create_process_sites, - { - "src/cpp/server/mcp_client.cpp": 1, - "src/cpp/server/system_info.cpp": 1, - "src/cpp/server/utils/platform/process_windows.cpp": 1, - }, - ) - - self.assertEqual(mcp_source.count("CreateProcessW("), 1) - for required in ( - "STARTUPINFOEXW", - "PROC_THREAD_ATTRIBUTE_HANDLE_LIST", - "EXTENDED_STARTUPINFO_PRESENT", - ): - with self.subTest(required=required): - self.assertIn(required, mcp_source) - - self.assertEqual(system_info_source.count("CreateProcessA("), 1) - self.assertIn( - "nullptr, nullptr, FALSE, CREATE_NO_WINDOW", - system_info_source, - ) - - def test_linux_capture_pipes_are_atomically_close_on_exec(self): - source = LINUX_PROCESS_SOURCE.read_text(encoding="utf-8") - system_info_source = SYSTEM_INFO_SOURCE.read_text(encoding="utf-8") - helper_start = source.index("static bool create_pipe_above_standard_streams") - helper_end = source.index("\n}\n", helper_start) + 2 - helper = source[helper_start:helper_end] - - self.assertIn("pipe2(pipe_fds, O_CLOEXEC)", helper) - self.assertNotRegex(helper, r"(?", run_command_start - ) - run_command = process_source[run_command_start:run_command_end] - self.assertIn("spawn_capturing_output(", run_command) - self.assertNotIn("popen(", run_command) - reserved_options = run_command.index("int wait_options = WEXITED | WNOWAIT;") - reserved_exit = run_command.index("waitid(", reserved_options) - group_cleanup = run_command.index( - "kill_process_group(pid, false);", reserved_exit - ) - final_reap = run_command.index("waitpid(pid, &status, 0)", group_cleanup) - self.assertNotIn("waitpid(pid, &status, WNOHANG)", run_command) - self.assertLess(reserved_options, reserved_exit) - self.assertLess(reserved_exit, group_cleanup) - self.assertLess(group_cleanup, final_reap) - - spawn_start = process_source.index("ProcessHandle MacOSProcessPlatform::spawn(") - spawn_end = process_source.index( - "void MacOSProcessPlatform::terminate(", spawn_start - ) - spawn = process_source[spawn_start:spawn_end] - for standard_stream in ("STDIN_FILENO", "STDOUT_FILENO", "STDERR_FILENO"): - with self.subTest(standard_stream=standard_stream): - self.assertIn( - f"file_actions.add_inherit_if_open({standard_stream})", spawn - ) - - apple_branch_start = mcp_source.index( - "#ifdef __APPLE__", mcp_source.index("if (pipe_creation_failed)") - ) - apple_branch_end = mcp_source.index("#else", apple_branch_start) - apple_branch = mcp_source[apple_branch_start:apple_branch_end] - self.assertNotIn("::fork()", apple_branch) - for required in ( - "posix_spawn(", - "posix_spawn_file_actions_adddup2", - "posix_spawn_file_actions_addclose", - "posix_spawn_file_actions_addchdir_np", - "posix_spawnattr_setpgroup(&attributes, 0)", - "POSIX_SPAWN_CLOEXEC_DEFAULT", - "POSIX_SPAWN_SETPGROUP", - ): - with self.subTest(required=required): - self.assertIn(required, apple_branch) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_publish_llamacpp_update.py b/test/test_publish_llamacpp_update.py deleted file mode 100644 index ca218cf7a0..0000000000 --- a/test/test_publish_llamacpp_update.py +++ /dev/null @@ -1,1982 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for llama.cpp update publication recovery.""" - -from __future__ import annotations - -import hashlib -import json -import os -import shutil -import stat -import subprocess -import tempfile -import unittest -from pathlib import Path - -from test.utils import llamacpp_release_assets as release_assets -from test.utils import llamacpp_release_manifest as release_manifest - -ROOT = Path(__file__).resolve().parents[1] -PUBLISHER = ROOT / ".github" / "scripts" / "publish_llamacpp_update.sh" -SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" -CHECKOUT_SHA = "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09" -DOWNLOAD_ARTIFACT_SHA = "37930b1c2abaa49bbe596cd826c3c89aef350131" -VERSIONS_PATH = Path("src/cpp/resources/backend_versions.json") -MANIFEST_PATH = Path(".github/llamacpp_release_manifest.json") - - -class PublishLlamaCppUpdateTests(unittest.TestCase): - def setUp(self) -> None: - self.temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary_directory.cleanup) - self.temp_root = Path(self.temporary_directory.name) - self.repo = self.temp_root / "repo" - self.remote = self.temp_root / "remote.git" - self.fake_bin = self.temp_root / "fake-bin" - self.event_log = self.temp_root / "events.log" - self.snapshot_count = self.temp_root / "snapshot-count.txt" - self.closed_prs = self.temp_root / "closed-prs.txt" - self.dynamic_open_prs = self.temp_root / "dynamic-open-prs.txt" - self.pr_create_failed = self.temp_root / "pr-create-failed.txt" - self.pr_draft_state = self.temp_root / "pr-draft-state.txt" - self.rocm_latest_query_count = self.temp_root / "rocm-latest-count.txt" - self.real_git = shutil.which("git") - if not self.real_git: - self.skipTest("git is required") - - self.run_command([self.real_git, "init", "--quiet", "--bare", self.remote]) - self.repo.mkdir() - self.run_git("init", "--quiet") - self.run_git("config", "user.name", "Test Author") - self.run_git("config", "user.email", "test@example.com") - self.base_versions = { - "llamacpp": { - "cpu": "b1", - "cuda": "b1", - "metal": "b1", - "rocm-nightly": "b1", - "rocm-stable": "b1", - "vulkan": "b1", - }, - "rocm_asset_families": {}, - "therock": {"version": "7.14.0"}, - } - self.candidate_versions = { - **self.base_versions, - "llamacpp": { - "cpu": "b100", - "cuda": "b200", - "metal": "b100", - "rocm-nightly": "b300", - "rocm-stable": "b200", - "vulkan": "b100", - }, - } - versions = self.repo / VERSIONS_PATH - versions.parent.mkdir(parents=True) - versions.write_text( - json.dumps(self.base_versions, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - self.upstream_head = "f" * 40 - self.live_upstream_head = "9" * 40 - requirements = release_assets.build_asset_requirements( - ggml_release="b100", - rocm_release="b300", - lemonade_release="b200", - backend_versions=self.candidate_versions, - ) - rocm_target_requirements = release_assets.build_rocm_asset_target_requirements( - rocm_release="b300", - backend_versions=self.candidate_versions, - ) - self.release_payloads = { - "ggml": self.release_payload( - repository="ggml-org/llama.cpp", - tag="b100", - release_id=100, - release_tag_commit="a" * 40, - claimed_source_commit="a" * 40, - asset_names=self.asset_names(requirements["ggml"]), - ), - "rocm": self.release_payload( - repository="lemonade-sdk/llamacpp-rocm", - tag="b300", - release_id=300, - release_tag_commit="b" * 40, - claimed_source_commit="c" * 40, - asset_names=self.asset_names(requirements["rocm"]), - build_targets_by_asset=rocm_target_requirements, - ), - "lemonade": self.release_payload( - repository="lemonade-sdk/llama.cpp", - tag="b200", - release_id=200, - release_tag_commit="d" * 40, - claimed_source_commit="e" * 40, - asset_names=self.asset_names(requirements["lemonade"]), - ), - } - self.expected_manifest = self.build_expected_manifest() - manifest_path = self.repo / MANIFEST_PATH - manifest_path.parent.mkdir(parents=True, exist_ok=True) - manifest_path.write_text(self.expected_manifest + "\n", encoding="utf-8") - (self.repo / "pr_body.md").write_text("initial body\n", encoding="utf-8") - self.run_git("add", "--all") - self.run_git("commit", "--quiet", "--message", "base") - self.run_git("branch", "-M", "main") - self.run_git("remote", "add", "origin", str(self.remote)) - self.run_git("push", "--quiet", "--set-upstream", "origin", "main") - self.install_command_wrappers() - - @property - def branch_name(self) -> str: - base_sha = self.run_git("rev-parse", "HEAD") - versions_blob = self.run_git("hash-object", str(VERSIONS_PATH)) - manifest_digest = hashlib.sha256(self.expected_manifest.encode()).hexdigest() - return ( - f"auto/llamacpp-update-b100-b200-b300-{base_sha}-{versions_blob}-" - f"{manifest_digest}" - ) - - def release_payload( - self, - *, - repository: str, - tag: str, - release_id: int, - release_tag_commit: str, - claimed_source_commit: str, - asset_names: list[str], - build_targets_by_asset: dict[str, tuple[str, ...]] | None = None, - ) -> dict: - assets = [ - { - "digest": "sha256:" - + hashlib.sha256( - f"{repository}@{tag}:{asset_name}".encode("utf-8") - ).hexdigest(), - "id": release_id * 1000 + index, - "name": asset_name, - "size": release_id + index, - "state": "uploaded", - "updated_at": "2026-09-06T12:00:00Z", - } - for index, asset_name in enumerate(asset_names, start=1) - ] - source_manifest = None - build_target_attestations = [] - if repository != "ggml-org/llama.cpp": - source_bindings = [] - for asset in assets: - binding = { - "digest": asset["digest"], - "name": asset["name"], - "size": asset["size"], - } - if build_targets_by_asset is not None: - binding["build_targets"] = list( - build_targets_by_asset.get(asset["name"], ()) - ) - build_target_attestations.append(dict(binding)) - source_bindings.append(binding) - source_manifest = json.dumps( - { - "assets": source_bindings, - "release_repository": repository, - "release_tag": tag, - "release_tag_commit": release_tag_commit, - "schema_version": 2 if build_targets_by_asset is not None else 1, - "source_commit": claimed_source_commit, - "source_repository": "ggml-org/llama.cpp", - }, - separators=(",", ":"), - sort_keys=True, - ) - source_bytes = source_manifest.encode("utf-8") - assets.append( - { - "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), - "id": release_id * 10 + 1, - "name": ".llamacpp-source.json", - "size": len(source_bytes), - "state": "uploaded", - "updated_at": "2026-09-06T12:01:00Z", - } - ) - return { - "assets": assets, - "body": "release notes", - "draft": False, - "id": release_id, - "immutable": True, - "tag_name": tag, - "_claimed_source_commit": claimed_source_commit, - "_build_target_attestations": build_target_attestations, - "_release_tag_commit": release_tag_commit, - "_source_manifest": source_manifest, - } - - @staticmethod - def asset_names(requirements: dict[str, list[str]]) -> list[str]: - return [ - asset_name - for backend_assets in requirements.values() - for asset_name in backend_assets - ] - - def build_expected_manifest(self) -> str: - releases = [] - for key, repository, tag in ( - ("ggml", "ggml-org/llama.cpp", "b100"), - ("rocm", "lemonade-sdk/llamacpp-rocm", "b300"), - ("lemonade", "lemonade-sdk/llama.cpp", "b200"), - ): - payload = dict(self.release_payloads[key]) - claimed_commit = payload.pop("_claimed_source_commit") - build_target_attestations = payload.pop("_build_target_attestations") - release_tag_commit = payload.pop("_release_tag_commit") - payload.pop("_source_manifest") - payload.update( - { - "publisher_claim_type": ( - "source-release-tag" - if repository == "ggml-org/llama.cpp" - else "immutable-source-manifest" - ), - "publisher_claimed_source_commit": claimed_commit, - "build_target_attestations": build_target_attestations, - "release_tag_commit": release_tag_commit, - "source_repository": "ggml-org/llama.cpp", - "upstream_reference": "refs/heads/master", - "upstream_reference_head": self.upstream_head, - } - ) - releases.append((repository, tag, payload)) - return release_manifest.build_release_asset_manifest(releases) - - def run_command( - self, - command: list[str], - *, - cwd: Path | None = None, - env: dict[str, str] | None = None, - check: bool = True, - ) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - cwd=cwd, - env=env, - check=check, - capture_output=True, - text=True, - ) - - def run_git(self, *args: str) -> str: - return self.run_command( - [self.real_git, *args], - cwd=self.repo, - ).stdout.strip() - - def install_command_wrappers(self) -> None: - self.fake_bin.mkdir() - git_wrapper = self.fake_bin / "git" - git_wrapper.write_text( - "#!/bin/sh\n" - "printf 'git' >> \"$EVENT_LOG\"\n" - 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' - "printf '\\n' >> \"$EVENT_LOG\"\n" - 'if [ "$1" = push ]; then\n' - ' if [ "${GIT_TERMINAL_PROMPT:-}" != 0 ] || ' - '[ "${GIT_CONFIG_KEY_0:-}" != credential.helper ] || ' - '[ "${GIT_CONFIG_VALUE_0+x}" != x ] || ' - '[ "${GIT_CONFIG_VALUE_0}" != "" ] || ' - '[ ! -x "${GIT_ASKPASS:-}" ]; then\n' - " printf 'missing ephemeral authenticated push configuration\\n' >&2\n" - " exit 96\n" - " fi\n" - ' username=$("$GIT_ASKPASS" "Username for https://github.com")\n' - ' password=$("$GIT_ASKPASS" "Password for https://github.com")\n' - ' if [ "$username" != x-access-token ] || ' - '[ "$password" != "$GH_TOKEN" ]; then\n' - " printf 'invalid ephemeral authenticated push configuration\\n' >&2\n" - " exit 97\n" - " fi\n" - " printf 'authenticated-push\\n' >> \"$EVENT_LOG\"\n" - "fi\n" - 'if [ "$1" = push ] && ' - '[ "$MUTATE_BRANCH_AFTER_PUSH" = true ]; then\n' - ' "$REAL_GIT" "$@" || exit $?\n' - ' "$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" update-ref ' - '"refs/heads/$EXPECTED_BRANCH" "$VALIDATED_BASE_SHA"\n' - " exit $?\n" - "fi\n" - 'exec "$REAL_GIT" "$@"\n', - encoding="utf-8", - ) - git_wrapper.chmod(git_wrapper.stat().st_mode | stat.S_IXUSR) - - gh_wrapper = self.fake_bin / "gh" - gh_wrapper.write_text( - "#!/bin/sh\n" - "printf 'gh' >> \"$EVENT_LOG\"\n" - 'printf \' <%s>\' "$@" >> "$EVENT_LOG"\n' - "printf '\\n' >> \"$EVENT_LOG\"\n" - 'if [ "$1" = api ] && [ "$2" = graphql ]; then\n' - " printf '%s\\n' \"$AUTHENTICATED_ACTOR\"\n" - " exit 0\n" - "fi\n" - 'if [ "$1" = pr ] && [ "$2" = create ]; then\n' - " requested_draft=false\n" - ' case " $* " in *" --draft "*) requested_draft=true ;; esac\n' - ' if [ "$MUTATE_BRANCH_BEFORE_PR_CREATE_FAILURE" = true ]; then\n' - ' "$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" update-ref ' - '"refs/heads/$EXPECTED_BRANCH" "$VALIDATED_BASE_SHA"\n' - " fi\n" - ' if [ "$PR_CREATE_EXIT_CODE" -ne 0 ]; then\n' - ' if [ "$CREATE_PR_BEFORE_FAILURE" = true ]; then\n' - ' head_sha=$("$REAL_GIT" ' - '--git-dir="$REMOTE_REPOSITORY_PATH" show-ref --verify --hash ' - '"refs/heads/$EXPECTED_BRANCH")\n' - " printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' " - '"$CREATED_PR_NUMBER" "$EXPECTED_BRANCH" "LLM360/lemonade" ' - '"$EXPECTED_PUBLICATION_ACTOR" "main" "$head_sha" ' - '> "$DYNAMIC_OPEN_PRS_FILE"\n' - ' if [ "$requested_draft" = true ]; then\n' - " printf 'draft\\n' > \"$PR_DRAFT_STATE_FILE\"\n" - " fi\n" - ' if [ "$DUPLICATE_PR_BEFORE_FAILURE" = true ]; then\n' - " printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' " - '"42" "$EXPECTED_BRANCH" "LLM360/lemonade" ' - '"$EXPECTED_PUBLICATION_ACTOR" "main" "$head_sha" ' - '>> "$DYNAMIC_OPEN_PRS_FILE"\n' - " fi\n" - " fi\n" - " printf 'failed\\n' > \"$PR_CREATE_FAILED_FILE\"\n" - " printf 'simulated pull request creation failure\\n' >&2\n" - ' exit "$PR_CREATE_EXIT_CODE"\n' - " fi\n" - ' if [ "$MUTATE_BRANCH_AFTER_PR_CREATE" = true ]; then\n' - ' "$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" update-ref ' - '"refs/heads/$EXPECTED_BRANCH" "$VALIDATED_BASE_SHA"\n' - " fi\n" - ' if [ "$requested_draft" = true ]; then\n' - " printf 'draft\\n' > \"$PR_DRAFT_STATE_FILE\"\n" - " fi\n" - " printf 'https://github.com/LLM360/lemonade/pull/%s\\n' " - '"$CREATED_PR_NUMBER"\n' - " exit 0\n" - "fi\n" - 'if [ "$1" = pr ] && [ "$2" = ready ]; then\n' - ' if [ "$PR_READY_EXIT_CODE" -ne 0 ]; then\n' - " printf 'simulated pull request ready failure\\n' >&2\n" - ' exit "$PR_READY_EXIT_CODE"\n' - " fi\n" - ' if [ "$PR_READY_CLOSES_CURRENT" = true ]; then\n' - ' printf \'%s\\n\' "$3" >> "$CLOSED_PRS_FILE"\n' - " fi\n" - ' if [ "$PR_READY_NOOP" != true ]; then\n' - ' rm -f "$PR_DRAFT_STATE_FILE"\n' - " fi\n" - " exit 0\n" - "fi\n" - 'if [ "$1" = api ]; then\n' - ' case "$2" in\n' - " repos/ggml-org/llama.cpp) printf '%s\\n' master ;;\n" - " repos/ggml-org/llama.cpp/releases) " - "printf '%s\\n' \"$FRESH_GGML_RELEASE\" ;;\n" - " repos/lemonade-sdk/llamacpp-rocm/releases/latest)\n" - " count=0\n" - ' if [ -f "$ROCM_LATEST_QUERY_COUNT_FILE" ]; then ' - 'count=$(cat "$ROCM_LATEST_QUERY_COUNT_FILE"); fi\n' - " count=$((count + 1))\n" - " printf '%s\\n' \"$count\" > " - '"$ROCM_LATEST_QUERY_COUNT_FILE"\n' - ' if [ "$ROCM_RELEASE_DRIFT_AT" -gt 0 ] && ' - '[ "$count" -ge "$ROCM_RELEASE_DRIFT_AT" ]; then\n' - " printf 'b301\\n'\n" - " else\n" - " printf '%s\\n' \"$FRESH_ROCM_RELEASE\"\n" - " fi ;;\n" - " repos/lemonade-sdk/llama.cpp/releases/latest) " - "printf '%s\\n' \"$FRESH_LEMONADE_RELEASE\" ;;\n" - " repos/ggml-org/llama.cpp/releases/tags/b100)\n" - " snapshot=0\n" - ' if [ -f "$SNAPSHOT_COUNT_FILE" ]; then ' - 'snapshot=$(cat "$SNAPSHOT_COUNT_FILE"); fi\n' - " snapshot=$((snapshot + 1))\n" - ' printf \'%s\\n\' "$snapshot" > "$SNAPSHOT_COUNT_FILE"\n' - " printf '%s\\n' \"$GGML_RELEASE_PAYLOAD\" ;;\n" - " repos/lemonade-sdk/llamacpp-rocm/releases/tags/b300)\n" - ' if [ "$MUTATE_ON_SNAPSHOT" -gt 0 ] && ' - '[ "$(cat "$SNAPSHOT_COUNT_FILE")" -ge "$MUTATE_ON_SNAPSHOT" ]; then\n' - " printf '%s\\n' \"$MUTATED_ROCM_RELEASE_PAYLOAD\"\n" - " else\n" - " printf '%s\\n' \"$ROCM_RELEASE_PAYLOAD\"\n" - " fi ;;\n" - " repos/lemonade-sdk/llama.cpp/releases/tags/b200) " - "printf '%s\\n' \"$LEMONADE_RELEASE_PAYLOAD\" ;;\n" - " repos/lemonade-sdk/llamacpp-rocm/releases/assets/3001) " - "printf '%s' \"$ROCM_SOURCE_MANIFEST\" ;;\n" - " repos/lemonade-sdk/llama.cpp/releases/assets/2001) " - "printf '%s' \"$LEMONADE_SOURCE_MANIFEST\" ;;\n" - " repos/ggml-org/llama.cpp/commits/tags/b100) " - "printf '%s\\n' \"$GGML_RELEASE_TAG_COMMIT\" ;;\n" - " repos/lemonade-sdk/llamacpp-rocm/commits/tags/b300) " - "printf '%s\\n' \"$ROCM_RELEASE_TAG_COMMIT\" ;;\n" - " repos/lemonade-sdk/llama.cpp/commits/tags/b200) " - "printf '%s\\n' \"$LEMONADE_RELEASE_TAG_COMMIT\" ;;\n" - " repos/ggml-org/llama.cpp/commits/heads/master) " - "printf '%s\\n' \"$LIVE_UPSTREAM_HEAD\" ;;\n" - " repos/*/commits/b*|repos/*/commits/master) " - "printf '%040d\\n' 7 ;;\n" - " repos/ggml-org/llama.cpp/compare/*)\n" - " comparison=${2##*/}; base=${comparison%%...*}; " - "head=${comparison#*...}\n" - " status=ahead; merge_base=$base\n" - ' if [ -n "$DIVERGENT_COMMIT" ] && ' - '[ "$base" = "$DIVERGENT_COMMIT" ]; then\n' - " status=diverged; merge_base=$LIVE_UPSTREAM_HEAD\n" - " fi\n" - " printf " - '\'{"status":"%s","base_commit":{"sha":"%s"},' - '"merge_base_commit":{"sha":"%s"},' - '"url":"https://api.github.com/repos/ggml-org/llama.cpp/compare/%s...%s"}\\n\' ' - '"$status" "$base" "$merge_base" "$base" "$head" ;;\n' - " repos/LLM360/lemonade/pulls)\n" - ' if [ -f "$PR_CREATE_FAILED_FILE" ] && ' - '[ "$FAIL_POST_CREATE_PULLS_QUERY" = true ]; then exit 92; fi\n' - " printf '%s\\n' \"$OPEN_PR_RECORDS\"\n" - ' if [ -f "$DYNAMIC_OPEN_PRS_FILE" ]; then ' - "sed -n '1,$p' \"$DYNAMIC_OPEN_PRS_FILE\"; fi ;;\n" - " repos/LLM360/lemonade/actions/permissions/workflow) exit 91 ;;\n" - " repos/LLM360/lemonade/pulls/[0-9]*)\n" - " number=${2##*/}\n" - ' case " $* " in\n' - ' *" --method PATCH "*" state=closed "*)\n' - ' printf \'%s\\n\' "$number" >> "$CLOSED_PRS_FILE"\n' - " printf '{}\\n'\n" - " exit 0 ;;\n" - " esac\n" - " records=$(printf '%s\\n' \"$OPEN_PR_RECORDS\")\n" - ' if [ -f "$DYNAMIC_OPEN_PRS_FILE" ]; then\n' - " records=$(printf '%s\\n%s\\n' \"$records\" " - '"$(sed -n \'1,$p\' "$DYNAMIC_OPEN_PRS_FILE")")\n' - " fi\n" - " record=$(printf '%s\\n' \"$records\" | " - "awk -F '\\t' -v number=\"$number\" '$1 == number { print; exit }')\n" - " head_ref=$(printf '%s' \"$record\" | cut -f2)\n" - " head_repository=$(printf '%s' \"$record\" | cut -f3)\n" - " author=$(printf '%s' \"$record\" | cut -f4)\n" - " base_ref=$(printf '%s' \"$record\" | cut -f5)\n" - " head_ref=${head_ref:-$EXPECTED_BRANCH}\n" - " head_repository=${head_repository:-LLM360/lemonade}\n" - " author=${author:-$EXPECTED_PUBLICATION_ACTOR}\n" - " base_ref=${base_ref:-main}\n" - " head_ref=${PR_DETAIL_HEAD_REF:-$head_ref}\n" - " head_repository=${PR_DETAIL_HEAD_REPOSITORY:-$head_repository}\n" - " author=${PR_DETAIL_AUTHOR:-$author}\n" - " base_ref=${PR_DETAIL_BASE_REF:-$base_ref}\n" - " state=open\n" - " draft=false\n" - ' if [ "$number" = "$CREATED_PR_NUMBER" ] && ' - '[ -f "$PR_DRAFT_STATE_FILE" ]; then draft=true; fi\n' - " draft=${PR_DETAIL_DRAFT:-$draft}\n" - ' if [ -f "$CLOSED_PRS_FILE" ] && ' - 'grep -qx "$number" "$CLOSED_PRS_FILE"; then state=closed; fi\n' - ' head_sha=$("$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" ' - 'show-ref --verify --hash "refs/heads/$head_ref" 2>/dev/null || ' - "printf '%040d' 0)\n" - " head_sha=${PR_DETAIL_HEAD_SHA:-$head_sha}\n" - ' base_sha=$("$REAL_GIT" --git-dir="$REMOTE_REPOSITORY_PATH" ' - 'show-ref --verify --hash "refs/heads/$base_ref" 2>/dev/null || ' - "printf '%040d' 0)\n" - " printf " - '\'{"state":"%s","draft":%s,"head":{"ref":"%s",' - '"sha":"%s","repo":{"full_name":"%s"}},' - '"user":{"login":"%s"},"base":{"ref":"%s",' - '"sha":"%s"}}\\n\' ' - '"$state" "$draft" "$head_ref" "$head_sha" "$head_repository" ' - '"$author" "$base_ref" "$base_sha" ;;\n' - " esac\n" - "fi\n" - "exit 0\n", - encoding="utf-8", - ) - gh_wrapper.chmod(gh_wrapper.stat().st_mode | stat.S_IXUSR) - - def publisher_env( - self, - *, - authenticated_actor: str = "github-actions[bot]", - create_pr_before_failure: bool = False, - divergent_commit: str = "", - duplicate_pr_before_failure: bool = False, - fail_post_create_pulls_query: bool = False, - include_expected_manifest: bool = True, - include_publication_token: bool = True, - mutate_on_snapshot: int = 0, - open_pr_records: str = "", - fresh_ggml_release: str = "b100", - mutate_branch_after_push: bool = False, - mutate_branch_after_pr_create: bool = False, - mutate_branch_before_pr_create_failure: bool = False, - pr_create_exit_code: int = 0, - pr_detail_base_ref: str = "", - pr_detail_head_ref: str = "", - pr_detail_draft: str = "", - pr_ready_closes_current: bool = False, - pr_ready_exit_code: int = 0, - pr_ready_noop: bool = False, - rocm_release_drift_at: int = 0, - expected_publication_actor: str | None = "github-actions[bot]", - validated_base_sha: str | None = None, - ) -> dict[str, str]: - api_payloads = {} - for name, payload in self.release_payloads.items(): - api_payloads[name] = { - key: value for key, value in payload.items() if not key.startswith("_") - } - mutated_rocm = dict(api_payloads["rocm"]) - mutated_rocm["assets"] = [dict(asset) for asset in mutated_rocm["assets"]] - mutated_rocm["assets"][0]["updated_at"] = "2026-09-06T12:02:00Z" - env = os.environ.copy() - env.update( - { - "AUTHENTICATED_ACTOR": authenticated_actor, - "CREATE_PR_BEFORE_FAILURE": ( - "true" if create_pr_before_failure else "false" - ), - "DIVERGENT_COMMIT": divergent_commit, - "DYNAMIC_OPEN_PRS_FILE": str(self.dynamic_open_prs), - "DUPLICATE_PR_BEFORE_FAILURE": ( - "true" if duplicate_pr_before_failure else "false" - ), - "FAIL_POST_CREATE_PULLS_QUERY": ( - "true" if fail_post_create_pulls_query else "false" - ), - "CLOSED_PRS_FILE": str(self.closed_prs), - "CREATED_PR_NUMBER": "41", - "EVENT_LOG": str(self.event_log), - "EXPECTED_BRANCH": self.branch_name, - "FRESH_GGML_RELEASE": fresh_ggml_release, - "FRESH_LEMONADE_RELEASE": "b200", - "FRESH_ROCM_RELEASE": "b300", - "GGML_RELEASE_PAYLOAD": json.dumps(api_payloads["ggml"]), - "GGML_RELEASE_TAG_COMMIT": self.release_payloads["ggml"][ - "_release_tag_commit" - ], - "GITHUB_REPOSITORY": "LLM360/lemonade", - "GITHUB_REPOSITORY_OWNER": "LLM360", - "LEMONADE_RELEASE_PAYLOAD": json.dumps(api_payloads["lemonade"]), - "LEMONADE_SOURCE_MANIFEST": self.release_payloads["lemonade"][ - "_source_manifest" - ], - "LEMONADE_RELEASE_TAG_COMMIT": self.release_payloads["lemonade"][ - "_release_tag_commit" - ], - "LLAMACPP_LEMONADE_RELEASE": "b200", - "LLAMACPP_RELEASE": "b100", - "LLAMACPP_ROCM_RELEASE": "b300", - "LIVE_UPSTREAM_HEAD": self.live_upstream_head, - "MUTATED_ROCM_RELEASE_PAYLOAD": json.dumps(mutated_rocm), - "MUTATE_ON_SNAPSHOT": str(mutate_on_snapshot), - "MUTATE_BRANCH_AFTER_PUSH": ( - "true" if mutate_branch_after_push else "false" - ), - "MUTATE_BRANCH_AFTER_PR_CREATE": ( - "true" if mutate_branch_after_pr_create else "false" - ), - "MUTATE_BRANCH_BEFORE_PR_CREATE_FAILURE": ( - "true" if mutate_branch_before_pr_create_failure else "false" - ), - "OPEN_PR_RECORDS": open_pr_records, - "PATH": f"{self.fake_bin}{os.pathsep}{env['PATH']}", - "PR_BODY_FILE": "pr_body.md", - "PR_CREATE_EXIT_CODE": str(pr_create_exit_code), - "PR_CREATE_FAILED_FILE": str(self.pr_create_failed), - "PR_DRAFT_STATE_FILE": str(self.pr_draft_state), - "PR_DETAIL_AUTHOR": "", - "PR_DETAIL_BASE_REF": pr_detail_base_ref, - "PR_DETAIL_HEAD_REF": pr_detail_head_ref, - "PR_DETAIL_DRAFT": pr_detail_draft, - "PR_DETAIL_HEAD_REPOSITORY": "", - "PR_DETAIL_HEAD_SHA": "", - "PR_READY_CLOSES_CURRENT": ( - "true" if pr_ready_closes_current else "false" - ), - "PR_READY_EXIT_CODE": str(pr_ready_exit_code), - "PR_READY_NOOP": "true" if pr_ready_noop else "false", - "REAL_GIT": self.real_git, - "REMOTE_REPOSITORY_PATH": str(self.remote), - "ROCM_RELEASE_PAYLOAD": json.dumps(api_payloads["rocm"]), - "ROCM_LATEST_QUERY_COUNT_FILE": str(self.rocm_latest_query_count), - "ROCM_RELEASE_DRIFT_AT": str(rocm_release_drift_at), - "ROCM_SOURCE_MANIFEST": self.release_payloads["rocm"][ - "_source_manifest" - ], - "ROCM_RELEASE_TAG_COMMIT": self.release_payloads["rocm"][ - "_release_tag_commit" - ], - "SNAPSHOT_COUNT_FILE": str(self.snapshot_count), - "VALIDATED_BASE_SHA": validated_base_sha - or self.run_git("rev-parse", "HEAD"), - } - ) - if expected_publication_actor is None: - env.pop("EXPECTED_PUBLICATION_ACTOR", None) - else: - env["EXPECTED_PUBLICATION_ACTOR"] = expected_publication_actor - if include_publication_token: - env["GH_TOKEN"] = "dedicated-publication-token" - else: - env.pop("GH_TOKEN", None) - if include_expected_manifest: - env["EXPECTED_RELEASE_ASSET_MANIFEST"] = self.expected_manifest - return env - - def run_publisher( - self, - *, - authenticated_actor: str = "github-actions[bot]", - create_pr_before_failure: bool = False, - divergent_commit: str = "", - duplicate_pr_before_failure: bool = False, - fail_post_create_pulls_query: bool = False, - include_expected_manifest: bool = True, - include_publication_token: bool = True, - mutate_on_snapshot: int = 0, - open_pr_records: str = "", - fresh_ggml_release: str = "b100", - mutate_branch_after_push: bool = False, - mutate_branch_after_pr_create: bool = False, - mutate_branch_before_pr_create_failure: bool = False, - pr_create_exit_code: int = 0, - pr_detail_base_ref: str = "", - pr_detail_head_ref: str = "", - pr_detail_draft: str = "", - pr_ready_closes_current: bool = False, - pr_ready_exit_code: int = 0, - pr_ready_noop: bool = False, - rocm_release_drift_at: int = 0, - expected_publication_actor: str | None = "github-actions[bot]", - validated_base_sha: str | None = None, - ) -> subprocess.CompletedProcess[str]: - return self.run_command( - ["bash", str(PUBLISHER)], - cwd=self.repo, - env=self.publisher_env( - authenticated_actor=authenticated_actor, - create_pr_before_failure=create_pr_before_failure, - divergent_commit=divergent_commit, - duplicate_pr_before_failure=duplicate_pr_before_failure, - fail_post_create_pulls_query=fail_post_create_pulls_query, - include_expected_manifest=include_expected_manifest, - include_publication_token=include_publication_token, - mutate_on_snapshot=mutate_on_snapshot, - open_pr_records=open_pr_records, - fresh_ggml_release=fresh_ggml_release, - mutate_branch_after_push=mutate_branch_after_push, - mutate_branch_after_pr_create=mutate_branch_after_pr_create, - mutate_branch_before_pr_create_failure=( - mutate_branch_before_pr_create_failure - ), - pr_create_exit_code=pr_create_exit_code, - pr_detail_base_ref=pr_detail_base_ref, - pr_detail_head_ref=pr_detail_head_ref, - pr_detail_draft=pr_detail_draft, - pr_ready_closes_current=pr_ready_closes_current, - pr_ready_exit_code=pr_ready_exit_code, - pr_ready_noop=pr_ready_noop, - rocm_release_drift_at=rocm_release_drift_at, - expected_publication_actor=expected_publication_actor, - validated_base_sha=validated_base_sha, - ), - check=False, - ) - - def change_versions(self) -> None: - (self.repo / VERSIONS_PATH).write_text( - json.dumps(self.candidate_versions, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - - def advance_base_versions(self, versions: dict[str, str]) -> None: - document = { - **self.base_versions, - "llamacpp": {**self.base_versions["llamacpp"], **versions}, - } - (self.repo / VERSIONS_PATH).write_text( - json.dumps(document, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - self.run_git("add", str(VERSIONS_PATH)) - self.run_git("commit", "--quiet", "--message", "advance backend pins") - self.run_git("push", "--quiet", "origin", "main") - - def trusted_open_pr_record(self, number: int = 17) -> str: - return ( - f"{number}\t{self.branch_name}\tLLM360/lemonade\t" - "github-actions[bot]\tmain" - ) - - def create_remote_update_branch( - self, - versions_contents: str, - *, - unrelated_contents: str | None = None, - ) -> tuple[str, str, str]: - base_oid = self.run_git("rev-parse", "HEAD") - overrides = json.loads(versions_contents) - document = { - **self.candidate_versions, - "llamacpp": { - **self.candidate_versions["llamacpp"], - **overrides["llamacpp"], - }, - } - (self.repo / VERSIONS_PATH).write_text( - json.dumps(document, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - if unrelated_contents is not None: - (self.repo / "unrelated.txt").write_text( - unrelated_contents, encoding="utf-8" - ) - branch_name = self.branch_name - self.run_git("add", "--all") - self.run_git("commit", "--quiet", "--message", "automated update") - branch_oid = self.run_git("rev-parse", "HEAD") - self.run_git( - "push", - "--quiet", - "origin", - f"HEAD:refs/heads/{branch_name}", - ) - self.run_git("reset", "--quiet", "--hard", base_oid) - return base_oid, branch_name, branch_oid - - def events(self) -> list[str]: - if not self.event_log.exists(): - return [] - return self.event_log.read_text(encoding="utf-8").splitlines() - - def assert_no_remote_mutation(self) -> None: - events = self.events() - self.assertFalse(any(event.startswith("git ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertFalse( - any( - event.startswith("gh ") - and any( - marker in event - for marker in ( - "<--method> ", - "<--method> ", - "<--method> ", - "<-f> ", - ) - ) - for event in events - ) - ) - - def test_missing_publication_token_fails_before_any_mutation(self) -> None: - self.change_versions() - body_before = (self.repo / "pr_body.md").read_text(encoding="utf-8") - manifest_before = (self.repo / MANIFEST_PATH).read_text(encoding="utf-8") - - result = self.run_publisher(include_publication_token=False) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("GH_TOKEN", result.stderr) - self.assertEqual(self.events(), []) - self.assertEqual( - (self.repo / "pr_body.md").read_text(encoding="utf-8"), body_before - ) - self.assertEqual( - (self.repo / MANIFEST_PATH).read_text(encoding="utf-8"), manifest_before - ) - - def test_missing_expected_publication_actor_fails_before_any_mutation( - self, - ) -> None: - self.change_versions() - body_before = (self.repo / "pr_body.md").read_text(encoding="utf-8") - - result = self.run_publisher(expected_publication_actor=None) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("EXPECTED_PUBLICATION_ACTOR", result.stderr) - self.assertEqual(self.events(), []) - self.assertEqual( - (self.repo / "pr_body.md").read_text(encoding="utf-8"), body_before - ) - - def test_authenticated_actor_mismatch_fails_before_any_mutation(self) -> None: - self.change_versions() - body_before = (self.repo / "pr_body.md").read_text(encoding="utf-8") - - result = self.run_publisher( - authenticated_actor="octocat", - expected_publication_actor="lemonade-release[bot]", - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("authenticated publication actor", result.stderr) - self.assertEqual( - self.events(), - [ - "gh <-f> " - "<--jq> <.data.viewer.login>" - ], - ) - self.assert_no_remote_mutation() - self.assertEqual( - (self.repo / "pr_body.md").read_text(encoding="utf-8"), body_before - ) - - def test_configured_publication_actor_owns_commit_and_pull_request( - self, - ) -> None: - actor = "lemonade-release[bot]" - self.change_versions() - - result = self.run_publisher( - authenticated_actor=actor, - expected_publication_actor=actor, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(self.run_git("show", "-s", "--format=%an", "HEAD"), actor) - self.assertEqual( - self.run_git("show", "-s", "--format=%ae", "HEAD"), - f"{actor}@users.noreply.github.com", - ) - self.assertTrue( - any(event.startswith("gh ") for event in self.events()) - ) - self.assertTrue( - any(event.startswith("gh ") for event in self.events()) - ) - - def test_push_uses_ephemeral_askpass_without_persisting_the_token(self) -> None: - self.change_versions() - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("authenticated-push", self.events()) - - def test_unchanged_versions_report_stale_pr_for_manual_cleanup(self) -> None: - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher(open_pr_records=stale_record) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("nothing to update", result.stdout) - self.assertIn( - "Superseded pull request #8 " - "(auto/llamacpp-update-b90-b190-b290-deadbeef1234) " - "requires manual cleanup.", - result.stdout, - ) - events = self.events() - self.assertFalse( - any("" in event for event in events) - ) - self.assertFalse(any("<-f> " in event for event in events)) - self.assertFalse(any(event.startswith("git ") for event in events)) - - def test_unchanged_versions_preserve_stale_pr_when_release_tags_drift( - self, - ) -> None: - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - open_pr_records=stale_record, - rocm_release_drift_at=2, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("candidate releases changed", result.stderr) - self.assertFalse(any("<-f> " in event for event in self.events())) - - def test_untrusted_stale_pull_request_is_not_closed(self) -> None: - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\toctocat\tmain" - ) - - result = self.run_publisher(open_pr_records=stale_record) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse( - any("" in event for event in self.events()) - ) - - def test_wrong_base_stale_pull_request_is_not_closed(self) -> None: - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\trelease" - ) - - result = self.run_publisher(open_pr_records=stale_record) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse( - any("" in event for event in self.events()) - ) - - def test_stale_candidate_fails_before_query_or_push(self) -> None: - self.change_versions() - - result = self.run_publisher(fresh_ggml_release="b101") - - self.assertNotEqual(result.returncode, 0) - self.assertIn("candidate releases changed", result.stderr) - events = self.events() - self.assertFalse( - any("" in event for event in events) - ) - self.assertFalse(any(event.startswith("git ") for event in events)) - - def test_scheduled_publication_rejects_all_managed_pin_downgrades(self) -> None: - self.advance_base_versions({"vulkan": "b6000", "cpu": "b6000"}) - self.change_versions() - - result = self.run_publisher() - - self.assertNotEqual(result.returncode, 0) - self.assertIn("would downgrade llamacpp.cpu from b6000 to b100", result.stderr) - events = self.events() - self.assertFalse(any(event.startswith("git ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - - def test_scheduled_publication_rejects_a_mixed_backend_downgrade(self) -> None: - self.advance_base_versions({"vulkan": "b50", "cpu": "b150"}) - self.change_versions() - - result = self.run_publisher() - - self.assertNotEqual(result.returncode, 0) - self.assertIn("would downgrade llamacpp.cpu from b150 to b100", result.stderr) - events = self.events() - self.assertFalse(any(event.startswith("git ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - - def test_scheduled_publication_rejects_arbitrary_precision_downgrade( - self, - ) -> None: - self.advance_base_versions({"cpu": "b9223372036854775808"}) - candidate = { - **self.candidate_versions, - "llamacpp": { - **self.candidate_versions["llamacpp"], - "cpu": "b9223372036854775807", - }, - } - (self.repo / VERSIONS_PATH).write_text( - json.dumps(candidate, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - - result = self.run_publisher() - - self.assertNotEqual(result.returncode, 0) - self.assertIn( - "would downgrade llamacpp.cpu from b9223372036854775808 " - "to b9223372036854775807", - result.stderr, - ) - self.assert_no_remote_mutation() - - def test_missing_release_manifest_fails_before_external_queries(self) -> None: - self.change_versions() - - result = self.run_publisher(include_expected_manifest=False) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("EXPECTED_RELEASE_ASSET_MANIFEST", result.stderr) - self.assertEqual(self.events(), []) - - def test_release_mutation_at_final_recheck_prevents_remote_mutation(self) -> None: - self.change_versions() - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - mutate_on_snapshot=3, - open_pr_records=stale_record, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("changed since validation", result.stderr) - events = self.events() - self.assertFalse(any(event.startswith("git ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertFalse(any("<-f> " in event for event in events)) - - def test_missing_required_asset_prevents_remote_mutation(self) -> None: - self.change_versions() - missing_asset = "llama-b100-bin-win-vulkan-x64.zip" - self.release_payloads["ggml"]["assets"] = [ - asset - for asset in self.release_payloads["ggml"]["assets"] - if asset["name"] != missing_asset - ] - self.expected_manifest = self.build_expected_manifest() - - result = self.run_publisher() - - self.assertNotEqual(result.returncode, 0) - self.assertIn("llamacpp.vulkan", result.stderr) - self.assertIn(missing_asset, result.stderr) - self.assert_no_remote_mutation() - - def test_zero_size_required_asset_prevents_remote_mutation(self) -> None: - self.change_versions() - zero_size_asset = "llama-b100-bin-ubuntu-arm64.tar.gz" - for asset in self.release_payloads["ggml"]["assets"]: - if asset["name"] == zero_size_asset: - asset["size"] = 0 - break - else: - self.fail(f"fixture is missing required asset {zero_size_asset}") - self.expected_manifest = self.build_expected_manifest() - - result = self.run_publisher() - - self.assertNotEqual(result.returncode, 0) - self.assertIn("llamacpp.cpu", result.stderr) - self.assertIn("positive size", result.stderr) - self.assertIn(zero_size_asset, result.stderr) - self.assert_no_remote_mutation() - - def test_capture_uses_namespaced_refs_when_branch_and_tag_names_collide( - self, - ) -> None: - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - events = self.events() - self.assertTrue( - any( - "" in event - for event in events - ) - ) - for repository, tag in ( - ("ggml-org/llama.cpp", "b100"), - ("lemonade-sdk/llamacpp-rocm", "b300"), - ("lemonade-sdk/llama.cpp", "b200"), - ): - self.assertTrue( - any( - f"" in event - for event in events - ) - ) - self.assertFalse( - any(f"" in event for event in events) - ) - self.assertFalse( - any( - "" in event for event in events - ) - ) - - def test_mutable_release_prevents_remote_mutation(self) -> None: - self.change_versions() - self.release_payloads["rocm"]["immutable"] = False - - result = self.run_publisher() - - self.assertNotEqual(result.returncode, 0) - self.assertIn("release immutable must be true", result.stderr) - events = self.events() - self.assertFalse(any(event.startswith("git ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - - def test_corrupt_source_manifest_prevents_remote_mutation(self) -> None: - self.change_versions() - self.release_payloads["rocm"]["_source_manifest"] += " " - - result = self.run_publisher() - - self.assertNotEqual(result.returncode, 0) - self.assertIn("source manifest size does not match", result.stderr) - events = self.events() - self.assertFalse(any(event.startswith("git ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - - def test_fork_network_manifest_anchor_fails_against_live_upstream(self) -> None: - self.change_versions() - - result = self.run_publisher(divergent_commit=self.upstream_head) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("not reachable", result.stderr) - events = self.events() - self.assertTrue( - any( - f"{self.upstream_head}...{self.live_upstream_head}>" in event - for event in events - ) - ) - self.assertFalse(any(event.startswith("git ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - - def test_new_pull_request_does_not_depend_on_actions_admin_permission( - self, - ) -> None: - self.change_versions() - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - events = self.events() - self.assertFalse( - any("actions/permissions/workflow" in event for event in events) - ) - self.assertTrue(any(event.startswith("gh ") for event in events)) - - def test_new_pull_request_is_promoted_only_after_final_identity_checks( - self, - ) -> None: - self.change_versions() - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - events = self.events() - create_index = next( - index - for index, event in enumerate(events) - if event.startswith("gh ") - ) - ready_index = next( - index - for index, event in enumerate(events) - if event.startswith("gh ") - ) - self.assertIn("<--draft>", events[create_index]) - self.assertGreater(ready_index, create_index) - self.assertTrue( - any( - create_index < index < ready_index - and "" in event - for index, event in enumerate(events) - ) - ) - self.assertFalse(self.pr_draft_state.exists()) - - def test_branch_move_after_draft_creation_leaves_no_ready_pull_request( - self, - ) -> None: - self.change_versions() - - result = self.run_publisher(mutate_branch_after_pr_create=True) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("identity changed before publication", result.stderr) - events = self.events() - create = next(event for event in events if event.startswith("gh ")) - self.assertIn("<--draft>", create) - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertTrue(self.pr_draft_state.exists()) - - def test_latest_release_drift_before_push_prevents_remote_mutation(self) -> None: - self.change_versions() - - result = self.run_publisher(rocm_release_drift_at=2) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("candidate releases changed", result.stderr) - self.assert_no_remote_mutation() - - def test_latest_release_drift_after_draft_creation_prevents_promotion( - self, - ) -> None: - self.change_versions() - - result = self.run_publisher(rocm_release_drift_at=3) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("candidate releases changed", result.stderr) - events = self.events() - self.assertTrue(any(event.startswith("gh ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertTrue(self.pr_draft_state.exists()) - - def test_late_release_tag_drift_preserves_superseded_pull_request(self) -> None: - self.change_versions() - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - open_pr_records=stale_record, - rocm_release_drift_at=3, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("candidate releases changed", result.stderr) - events = self.events() - self.assertFalse(any("<-f> " in event for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertTrue(self.pr_draft_state.exists()) - - def test_late_release_manifest_drift_preserves_superseded_pull_request( - self, - ) -> None: - self.change_versions() - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - mutate_on_snapshot=4, - open_pr_records=stale_record, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("changed since validation", result.stderr) - events = self.events() - self.assertFalse(any("<-f> " in event for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertTrue(self.pr_draft_state.exists()) - - def test_ready_api_failure_preserves_superseded_pull_request(self) -> None: - self.change_versions() - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - open_pr_records=stale_record, - pr_ready_exit_code=92, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("simulated pull request ready failure", result.stderr) - events = self.events() - self.assertTrue(any(event.startswith("gh ") for event in events)) - self.assertFalse(any("<-f> " in event for event in events)) - self.assertTrue(self.pr_draft_state.exists()) - - def test_ready_noop_preserves_superseded_pull_request(self) -> None: - self.change_versions() - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - open_pr_records=stale_record, - pr_ready_noop=True, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("remained a draft", result.stderr) - self.assertFalse(any("<-f> " in event for event in self.events())) - self.assertTrue(self.pr_draft_state.exists()) - - def test_concurrent_current_pr_closure_preserves_superseded_pull_request( - self, - ) -> None: - self.change_versions() - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - open_pr_records=stale_record, - pr_ready_closes_current=True, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("identity changed before publication", result.stderr) - close_events = [ - event for event in self.events() if "<-f> " in event - ] - self.assertFalse( - any("" in event for event in close_events) - ) - - def test_latest_release_drift_before_failed_create_reconciliation_is_safe( - self, - ) -> None: - self.change_versions() - - result = self.run_publisher( - create_pr_before_failure=True, - pr_create_exit_code=42, - rocm_release_drift_at=3, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("candidate releases changed", result.stderr) - events = self.events() - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertTrue(self.pr_draft_state.exists()) - - def test_failed_pull_request_creation_preserves_new_branch_after_zero_recheck( - self, - ) -> None: - self.change_versions() - branch_name = self.branch_name - - result = self.run_publisher(pr_create_exit_code=42) - - self.assertNotEqual(result.returncode, 0) - published_oid = self.run_git("rev-parse", "HEAD") - branch_lookup = self.run_command( - [ - self.real_git, - "--git-dir", - str(self.remote), - "show-ref", - "--verify", - f"refs/heads/{branch_name}", - ], - check=False, - ) - self.assertEqual(branch_lookup.returncode, 0) - self.assertEqual(branch_lookup.stdout.split()[0], published_oid) - events = self.events() - create_index = next( - index - for index, event in enumerate(events) - if event.startswith("gh ") - ) - recheck_index = next( - index - for index, event in enumerate(events) - if index > create_index - and event.startswith("gh ") - ) - self.assertIn(f"<-f> ", events[recheck_index]) - self.assertIn("<-f> ", events[recheck_index]) - self.assertLess(create_index, recheck_index) - self.assertFalse( - any( - event.startswith("git ") - and f"<:refs/heads/{branch_name}>" in event - for event in events - ) - ) - self.assertIn("preserving", result.stderr) - - def test_failed_pull_request_creation_preserves_preexisting_branch(self) -> None: - _base_oid, branch_name, branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - self.change_versions() - - result = self.run_publisher(pr_create_exit_code=42) - - self.assertNotEqual(result.returncode, 0) - remote_oid = self.run_command( - [ - self.real_git, - "--git-dir", - str(self.remote), - "show-ref", - "--verify", - "--hash", - f"refs/heads/{branch_name}", - ] - ).stdout.strip() - self.assertEqual(remote_oid, branch_oid) - self.assertFalse( - any( - event.startswith("git ") - and f"<:refs/heads/{branch_name}>" in event - for event in self.events() - ) - ) - - def test_failed_pull_request_creation_adopts_trusted_concurrent_pr(self) -> None: - self.change_versions() - branch_name = self.branch_name - - result = self.run_publisher( - create_pr_before_failure=True, - pr_create_exit_code=42, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - self.run_command( - [ - self.real_git, - "--git-dir", - str(self.remote), - "show-ref", - "--verify", - "--hash", - f"refs/heads/{branch_name}", - ] - ).stdout.strip(), - self.run_git("rev-parse", "HEAD"), - ) - events = self.events() - self.assertFalse( - any( - event.startswith("git ") - and f"<:refs/heads/{branch_name}>" in event - for event in events - ) - ) - self.assertTrue( - any("" in event for event in events) - ) - create = next(event for event in events if event.startswith("gh ")) - self.assertIn("<--draft>", create) - self.assertTrue(any(event.startswith("gh ") for event in events)) - self.assertFalse(self.pr_draft_state.exists()) - - def test_failed_pull_request_creation_preserves_branch_when_recheck_fails( - self, - ) -> None: - self.change_versions() - branch_name = self.branch_name - - result = self.run_publisher( - fail_post_create_pulls_query=True, - pr_create_exit_code=42, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertEqual( - self.run_command( - [ - self.real_git, - "--git-dir", - str(self.remote), - "show-ref", - "--verify", - f"refs/heads/{branch_name}", - ], - check=False, - ).returncode, - 0, - ) - self.assertFalse( - any( - event.startswith("git ") - and f"<:refs/heads/{branch_name}>" in event - for event in self.events() - ) - ) - - def test_failed_pull_request_creation_preserves_branch_when_recheck_is_ambiguous( - self, - ) -> None: - self.change_versions() - branch_name = self.branch_name - - result = self.run_publisher( - create_pr_before_failure=True, - duplicate_pr_before_failure=True, - pr_create_exit_code=42, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("multiple matching pull requests", result.stderr) - self.assertEqual( - self.run_command( - [ - self.real_git, - "--git-dir", - str(self.remote), - "show-ref", - "--verify", - "--hash", - f"refs/heads/{branch_name}", - ] - ).stdout.strip(), - self.run_git("rev-parse", "HEAD"), - ) - self.assertFalse( - any( - event.startswith("git ") - and f"<:refs/heads/{branch_name}>" in event - for event in self.events() - ) - ) - - def test_failed_pull_request_creation_preserves_concurrently_moved_branch( - self, - ) -> None: - validated_base = self.run_git("rev-parse", "HEAD") - self.change_versions() - branch_name = self.branch_name - - result = self.run_publisher( - mutate_branch_before_pr_create_failure=True, - pr_create_exit_code=42, - ) - - self.assertNotEqual(result.returncode, 0) - remote_oid = self.run_command( - [ - self.real_git, - "--git-dir", - str(self.remote), - "show-ref", - "--verify", - "--hash", - f"refs/heads/{branch_name}", - ] - ).stdout.strip() - self.assertEqual(remote_oid, validated_base) - self.assertFalse( - any( - event.startswith("git ") - and f"<:refs/heads/{branch_name}>" in event - for event in self.events() - ) - ) - - def test_publication_binds_manifest_to_branch_commit_and_pr_body(self) -> None: - self.change_versions() - (self.repo / MANIFEST_PATH).write_text("tampered\n", encoding="utf-8") - expected_branch = self.branch_name - expected_digest = hashlib.sha256( - self.expected_manifest.encode("utf-8") - ).hexdigest() - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(self.run_git("branch", "--show-current"), expected_branch) - self.assertEqual( - self.run_git("show", f"HEAD:{MANIFEST_PATH}"), - self.expected_manifest, - ) - self.assertIn( - f"LlamaCpp-Release-Manifest-SHA256: {expected_digest}", - self.run_git("log", "-1", "--pretty=%B"), - ) - body = (self.repo / "pr_body.md").read_text(encoding="utf-8") - self.assertIn(f"Canonical manifest SHA-256: `{expected_digest}`", body) - self.assertIn(self.upstream_head, body) - self.assertIn("`true`", body) - self.assertIn("Only immutable GitHub releases are accepted", body) - self.assertIn("immutable publisher manifest", body) - self.assertIn("not cryptographic build provenance", body) - for payload in self.release_payloads.values(): - self.assertIn(payload["_release_tag_commit"], body) - self.assertIn(payload["_claimed_source_commit"], body) - self.assertGreaterEqual(int(self.snapshot_count.read_text()), 3) - - def test_mismatched_validated_base_is_rejected_before_publication(self) -> None: - self.change_versions() - - result = self.run_publisher(validated_base_sha="0" * 40) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("validated base", result.stderr.lower()) - self.assertFalse( - any("" in event for event in self.events()) - ) - - def test_remote_base_advance_is_rejected_before_publication(self) -> None: - validated_base = self.run_git("rev-parse", "HEAD") - (self.repo / "main.txt").write_text("new main content\n", encoding="utf-8") - self.run_git("add", "main.txt") - self.run_git("commit", "--quiet", "--message", "advance main") - self.run_git("push", "--quiet", "origin", "main") - self.run_git("reset", "--quiet", "--hard", validated_base) - self.change_versions() - - result = self.run_publisher(validated_base_sha=validated_base) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("advanced after validation", result.stderr) - self.assertFalse( - any("" in event for event in self.events()) - ) - - def test_trusted_open_pr_is_refreshed_on_the_same_validated_base(self) -> None: - _base_oid, branch_name, observed_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - self.change_versions() - - result = self.run_publisher( - open_pr_records=self.trusted_open_pr_record(), - ) - - self.assertEqual(result.returncode, 0, result.stderr) - events = self.events() - query_index = next( - index - for index, event in enumerate(events) - if "" in event - ) - push_index = next( - index - for index, event in enumerate(events) - if event.startswith("git ") - ) - edit_index = next( - index - for index, event in enumerate(events) - if event.startswith("gh ") - ) - self.assertLess(query_index, push_index) - self.assertLess(push_index, edit_index) - self.assertIn("", events[query_index]) - self.assertNotIn("", events[query_index]) - self.assertIn(".base.ref", events[query_index]) - self.assertIn( - f"<--force-with-lease=refs/heads/{branch_name}:{observed_oid}>", - events[push_index], - ) - self.assertIn( - f"<{observed_oid}:refs/heads/{branch_name}>", - events[push_index], - ) - self.assertNotIn(f"", events[push_index]) - self.assertIn("<17>", events[edit_index]) - self.assertIn("<--repo> ", events[edit_index]) - self.assertIn("<--body-file> ", events[edit_index]) - self.assertFalse(any(event.startswith("gh ") for event in events)) - self.assertFalse( - any("actions/permissions/workflow" in event for event in events) - ) - - def test_retargeted_open_pr_fails_before_refresh(self) -> None: - self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - self.change_versions() - - result = self.run_publisher( - open_pr_records=self.trusted_open_pr_record(), - pr_detail_base_ref="release", - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("identity changed before publication", result.stderr) - self.assertFalse( - any(event.startswith("gh ") for event in self.events()) - ) - - def test_remote_branch_move_after_push_fails_before_pr_mutation(self) -> None: - self.change_versions() - - result = self.run_publisher(mutate_branch_after_push=True) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("publication branch changed after push", result.stderr) - self.assertFalse( - any( - event.startswith("gh ") - or event.startswith("gh ") - for event in self.events() - ) - ) - - def test_concurrently_retargeted_stale_pr_is_only_reported(self) -> None: - stale_record = ( - "8\tauto/llamacpp-update-b90-b190-b290-deadbeef1234\t" - "LLM360/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher( - open_pr_records=stale_record, - pr_detail_head_ref="retargeted-branch", - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - "Superseded pull request #8 " - "(auto/llamacpp-update-b90-b190-b290-deadbeef1234) " - "requires manual cleanup.", - result.stdout, - ) - self.assertFalse( - any("" in event for event in self.events()) - ) - self.assertFalse(any("<-f> " in event for event in self.events())) - - def test_trusted_open_pr_with_unrelated_changes_is_not_overwritten(self) -> None: - _base_oid, branch_name, _branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n', - unrelated_contents="do not overwrite\n", - ) - self.change_versions() - - record = f"17\t{branch_name}\tLLM360/lemonade\tgithub-actions[bot]\tmain" - result = self.run_publisher(open_pr_records=record) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("changes unexpected path unrelated.txt", result.stderr) - self.assertFalse(any(event.startswith("git ") for event in self.events())) - - def test_untrusted_open_pr_does_not_authorize_branch_overwrite(self) -> None: - _base_oid, branch_name, branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - self.change_versions() - untrusted_record = f"17\t{branch_name}\tLLM360/lemonade\toctocat\tmain" - - result = self.run_publisher(open_pr_records=untrusted_record) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("untrusted open pull request", result.stderr) - self.assertFalse(any(event.startswith("git ") for event in self.events())) - - def test_cross_repository_same_named_pr_cannot_block_publication(self) -> None: - _base_oid, branch_name, branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - self.change_versions() - cross_repo_record = ( - f"17\t{branch_name}\tattacker/lemonade\tgithub-actions[bot]\tmain" - ) - - result = self.run_publisher(open_pr_records=cross_repo_record) - - self.assertEqual(result.returncode, 0, result.stderr) - push = next(event for event in self.events() if event.startswith("git ")) - self.assertIn( - f"<--force-with-lease=refs/heads/{branch_name}:{branch_oid}>", - push, - ) - self.assertIn(f"<{branch_oid}:refs/heads/{branch_name}>", push) - self.assertNotIn(f"", push) - create = next( - event for event in self.events() if event.startswith("gh ") - ) - self.assertIn("<--repo> ", create) - self.assertIn(f"<--head> <{branch_name}>", create) - self.assertNotIn(f"", create) - - def test_wrong_base_pull_request_is_not_trusted(self) -> None: - _base_oid, branch_name, branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - self.change_versions() - wrong_base_record = ( - f"17\t{branch_name}\tLLM360/lemonade\t" "github-actions[bot]\trelease" - ) - - result = self.run_publisher(open_pr_records=wrong_base_record) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("untrusted open pull request", result.stderr) - events = self.events() - self.assertFalse(any(event.startswith("git ") for event in events)) - - def test_evolved_orphan_uses_a_new_content_addressed_branch(self) -> None: - _base_oid, old_branch, _branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b1"}}\n' - ) - self.change_versions() - new_branch = self.branch_name - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertNotEqual(old_branch, new_branch) - push = next(event for event in self.events() if event.startswith("git ")) - self.assertIn(f"", push) - self.assertTrue( - any(event.startswith("gh ") for event in self.events()) - ) - - def test_base_advance_uses_a_new_branch_and_reports_the_old_pull_request( - self, - ) -> None: - _base_oid, old_branch, _branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - (self.repo / "main.txt").write_text("new main content\n", encoding="utf-8") - self.run_git("add", "main.txt") - self.run_git("commit", "--quiet", "--message", "advance main") - self.run_git("push", "--quiet", "origin", "main") - self.change_versions() - new_branch = self.branch_name - old_record = f"17\t{old_branch}\tLLM360/lemonade\tgithub-actions[bot]\tmain" - - result = self.run_publisher(open_pr_records=old_record) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertNotEqual(old_branch, new_branch) - events = self.events() - self.assertFalse( - any("" in event for event in events) - ) - self.assertFalse(any("<-f> " in event for event in events)) - ready_message = "Marked pull request #41 ready for review." - report_message = ( - f"Superseded pull request #17 ({old_branch}) requires manual cleanup." - ) - self.assertIn(ready_message, result.stdout) - self.assertIn(report_message, result.stdout) - self.assertLess( - result.stdout.index(ready_message), result.stdout.index(report_message) - ) - push = next(event for event in events if event.startswith("git ")) - self.assertIn(f"", push) - - def test_base_advance_does_not_leave_an_orphan_branch_blocker(self) -> None: - _base_oid, old_branch, _branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - (self.repo / "main.txt").write_text("new main content\n", encoding="utf-8") - self.run_git("add", "main.txt") - self.run_git("commit", "--quiet", "--message", "advance main") - self.run_git("push", "--quiet", "origin", "main") - self.change_versions() - new_branch = self.branch_name - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertNotEqual(old_branch, new_branch) - push = next(event for event in self.events() if event.startswith("git ")) - self.assertIn(f"", push) - - def test_matching_historical_branch_is_reused_for_a_new_pr(self) -> None: - _base_oid, branch_name, branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - self.change_versions() - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - events = self.events() - query = next( - event for event in events if "" in event - ) - self.assertIn("", query) - self.assertNotIn("", query) - push = next(event for event in events if event.startswith("git ")) - self.assertIn( - f"<--force-with-lease=refs/heads/{branch_name}:{branch_oid}>", - push, - ) - self.assertTrue(any(event.startswith("gh ") for event in events)) - self.assertFalse(any(event.startswith("gh ") for event in events)) - - def test_absent_branch_uses_an_explicit_empty_lease(self) -> None: - self.change_versions() - branch_name = self.branch_name - - result = self.run_publisher() - - self.assertEqual(result.returncode, 0, result.stderr) - push = next(event for event in self.events() if event.startswith("git ")) - self.assertIn( - f"<--force-with-lease=refs/heads/{branch_name}:>", - push, - ) - self.assertTrue( - any(event.startswith("gh ") for event in self.events()) - ) - - def test_shallow_checkout_can_refresh_an_existing_pull_request(self) -> None: - _base_oid, branch_name, _branch_oid = self.create_remote_update_branch( - '{"llamacpp":{"vulkan":"b100","cpu":"b100"}}\n' - ) - - shallow_repo = self.temp_root / "shallow-repo" - self.run_command( - [ - self.real_git, - "clone", - "--quiet", - "--depth=1", - "--branch", - "main", - f"file://{self.remote}", - shallow_repo, - ] - ) - self.repo = shallow_repo - self.change_versions() - record = f"17\t{branch_name}\tLLM360/lemonade\tgithub-actions[bot]\tmain" - - result = self.run_publisher(open_pr_records=record) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(self.run_git("rev-parse", "--is-shallow-repository"), "false") - self.assertTrue( - any(event.startswith("gh ") for event in self.events()) - ) - - -class PublishLlamaCppWorkflowTests(unittest.TestCase): - def test_publication_uses_only_a_dedicated_token_and_expected_actor( - self, - ) -> None: - schedule = SCHEDULE_WORKFLOW.read_text(encoding="utf-8") - publication = schedule.split(" publish:\n", 1)[1] - preflight_marker = " - name: Require dedicated publication identity\n" - checkout_marker = f" - uses: actions/checkout@{CHECKOUT_SHA} # v5.1.0\n" - self.assertIn(preflight_marker, publication) - self.assertIn(checkout_marker, publication) - preflight = publication.split(preflight_marker, 1)[1].split(checkout_marker, 1)[ - 0 - ] - checkout = publication.split(checkout_marker, 1)[1].split("\n - name:", 1)[ - 0 - ] - manifest = publication.split( - " - name: Verify release asset manifest\n", 1 - )[1].split(" - name: Publish update pull request\n", 1)[0] - publisher = publication.split(" - name: Publish update pull request\n", 1)[ - 1 - ] - - self.assertLess( - publication.index(" - name: Require dedicated publication identity"), - publication.index(f" - uses: actions/checkout@{CHECKOUT_SHA}"), - ) - self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", preflight) - self.assertIn( - "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", - preflight, - ) - self.assertIn("gh api graphql", preflight) - self.assertIn(".data.viewer.login", preflight) - self.assertIn("token: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", checkout) - self.assertIn("persist-credentials: false", checkout) - self.assertEqual( - publication.count( - f"uses: actions/download-artifact@{DOWNLOAD_ARTIFACT_SHA} # v7.0.0" - ), - 2, - ) - self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", manifest) - self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", publisher) - self.assertIn( - "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", - publisher, - ) - self.assertNotIn("secrets.GITHUB_TOKEN", publication) - self.assertNotIn("contents: write", publication) - self.assertNotIn("pull-requests: write", publication) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_run_llamacpp_validation.py b/test/test_run_llamacpp_validation.py deleted file mode 100644 index 22e5f76175..0000000000 --- a/test/test_run_llamacpp_validation.py +++ /dev/null @@ -1,4466 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for the portable llama.cpp validation process runner.""" - -from __future__ import annotations - -import argparse -import ctypes -import importlib.util -import json -import os -import signal -import subprocess -import sys -import tempfile -import time -import unittest -from pathlib import Path -from unittest import mock - -from test.utils import llamacpp_capability_validation as capabilities -from test.utils.llamacpp_validation_plan import create_validation_plan - -ROOT = Path(__file__).resolve().parents[1] -RUNNER_PATH = ROOT / ".github" / "scripts" / "run_llamacpp_validation.py" -K2_SMALL = "K2-Horizon-0.9B-GGUF" -K2_MEDIUM = "K2-Horizon-3.7B-GGUF" -K2_SMALL_FILENAME = "K2-Horizon-1B-BF16.gguf" - - -def _passing_capability_matrix(model: str) -> dict: - cases = [] - for case_id in capabilities.K2_HORIZON_CASE_IDS: - protocol = "openai" - if case_id == "ollama_thinking": - protocol = "ollama" - elif case_id == "anthropic_translation": - protocol = "anthropic" - stream = case_id in { - "openai_plain_off_stream", - "openai_reasoning_high_stream", - "openai_tool_stream_xml", - } - tool_case = case_id in { - "openai_tool_json", - "openai_tool_xml", - "openai_tool_xml_typed", - "openai_tool_default_xml", - "openai_tool_stream_xml", - } - reasoning_case = ( - case_id.startswith("openai_reasoning_") - and case_id != "openai_reasoning_low" - ) or case_id == "ollama_thinking" - content_case = case_id in { - "openai_plain_off_nonstream", - "openai_plain_off_stream", - "openai_tool_result_followup", - "anthropic_translation", - "ollama_thinking", - } or case_id.startswith("openai_reasoning_") - finish_reason = "stop" - if tool_case: - finish_reason = "tool_calls" - elif case_id == "anthropic_translation": - finish_reason = "end_turn" - cases.append( - { - "id": case_id, - "protocol": protocol, - "stream": stream, - "pass": True, - "status_code": 200, - "finish_reason": finish_reason, - "content_chars": 2 if content_case else 0, - "reasoning_chars": 8 if reasoning_case else 0, - "tool_call_count": 1 if tool_case else 0, - "terminal_frame": True, - "tool_name": capabilities.TOOL_NAME if tool_case else None, - "tool_arguments": ( - capabilities.EXPECTED_TOOL_ARGUMENTS if tool_case else None - ), - "parity_with": ( - "openai_tool_xml" - if case_id in {"openai_tool_default_xml", "openai_tool_stream_xml"} - else None - ), - "raw_ifm_marker": None, - "error": None, - } - ) - return { - "profile": capabilities.K2_HORIZON_PROFILE, - "model": f"builtin.{model.removeprefix('builtin.')}", - "pass": True, - "cases": cases, - } - - -def log_line(tag: str, message: str) -> str: - if tag == "Process": - message = f"[49972] 0.13.484.059 I {message}" - return f"2026-09-06 12:00:00.000 [Info] ({tag}) {message}" - - -def backend_log_line(backend: str) -> str: - return log_line("LlamaCpp", f"Using LlamaCpp Backend: {backend}") - - -def verified_artifacts(runner, root: Path | None = None): - artifact_path = Path("/cache") / K2_SMALL_FILENAME - if root is not None: - artifact_path = root / "verified-artifacts" / K2_SMALL_FILENAME - artifact_path.parent.mkdir(parents=True, exist_ok=True) - artifact_path.touch() - artifact_path = artifact_path.resolve() - return { - K2_SMALL: runner.VerifiedValidationArtifact( - model_id=K2_SMALL, - resolved_path=artifact_path, - size_bytes=0, - sha256="0" * 64, - ) - } - - -def bind_verified_artifact_path(contents: str, expected_artifacts: dict) -> str: - expected_path = expected_artifacts[K2_SMALL].resolved_path - return contents.replace( - f"/cache/{K2_SMALL_FILENAME}", - str(expected_path), - ) - - -def load_runner(): - spec = importlib.util.spec_from_file_location( - "run_llamacpp_validation", RUNNER_PATH - ) - if spec is None or spec.loader is None: - raise RuntimeError("could not load llama.cpp validation runner") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -class FakeProcess: - next_pid = 4100 - - def __init__(self): - self.pid = FakeProcess.next_pid - FakeProcess.next_pid += 1 - - @staticmethod - def poll(): - return None - - -def _pid_is_active(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - return True - - -def _pid_is_running(pid: int) -> bool: - try: - import psutil - except ModuleNotFoundError: - return _pid_is_active(pid) - try: - return psutil.Process(pid).status() not in { - psutil.STATUS_DEAD, - psutil.STATUS_ZOMBIE, - } - except psutil.Error: - return False - - -def _process_tree_inspection_available() -> bool: - if sys.platform != "darwin": - return True - try: - import psutil - - psutil.Process(os.getpid()).children(recursive=True) - except ModuleNotFoundError: - return False - except (OSError, psutil.Error): - return False - return True - - -DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE = _process_tree_inspection_available() - - -class LlamaCppValidationRunnerTests(unittest.TestCase): - def test_direct_help_invocation_imports_repository_modules(self): - result = subprocess.run( - [sys.executable, str(RUNNER_PATH), "--help"], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - - def test_restart_command_is_bounded_cached_smoke_with_distinct_evidence(self): - runner = load_runner() - output_path = Path("isolated") / "restart.json" - - command = runner.build_restart_validation_command( - python_executable=Path("python"), - backend="vulkan", - channel="", - target="windows-vulkan", - model=K2_SMALL, - port=13305, - output_path=output_path, - ) - - self.assertEqual(command[0], "python") - self.assertEqual( - Path(command[1]), (ROOT / "test/validate_llamacpp.py").resolve() - ) - self.assertTrue(runner.VALIDATION_MODEL_CATALOG.is_absolute()) - self.assertTrue(runner.VALIDATION_ARTIFACT_LOCKS.is_absolute()) - runner.VALIDATION_MODEL_CATALOG.relative_to(ROOT.resolve()) - runner.VALIDATION_ARTIFACT_LOCKS.relative_to(ROOT.resolve()) - self.assertIn("--skip-install", command) - self.assertEqual(command.count("--model"), 1) - self.assertIn(K2_SMALL, command) - self.assertNotIn("--capability-profile", command) - self.assertNotIn("--capability-model", command) - self.assertNotIn("--logs-dir", command) - self.assertIn(str(output_path), command) - self.assertNotIn("server-restart-logs-windows-vulkan", command) - - def test_protected_validation_scrubs_ambient_download_credentials(self): - runner = load_runner() - - class FakeContainment: - @staticmethod - def popen_kwargs(): - return {} if os.name == "nt" else {"start_new_session": True} - - @staticmethod - def assign(_process): - return None - - @staticmethod - def close(): - return True - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - github_output = root / "github-output" - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="metal", - channel="", - target="macos-metal", - models=K2_SMALL, - lite=False, - capability_profile="k2-horizon-v1", - capability_models=K2_SMALL, - allow_huggingface_download_credentials=False, - port=13305, - ) - - old_directory = Path.cwd() - os.chdir(root) - try: - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object(runner.time, "sleep"), - mock.patch.object( - runner, - "_create_process_containment", - side_effect=lambda **_kwargs: FakeContainment(), - ), - mock.patch.object( - runner.subprocess, - "Popen", - side_effect=(FakeProcess(), FakeProcess(), FakeProcess()), - ) as popen, - mock.patch.object( - runner, - "_run_contained_command", - side_effect=lambda command, **_kwargs: ( - self._complete_command(command).returncode - ), - ) as run, - mock.patch.object( - runner, - "_shutdown_server", - side_effect=(True, True, True), - ) as shutdown, - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=verified_artifacts(runner), - ) as verify_artifacts, - mock.patch.object( - runner, - "require_accelerator_attestation", - ) as attest_accelerator, - mock.patch.object( - runner, - "revalidate_validation_cache", - wraps=runner.revalidate_validation_cache, - ) as revalidate_cache, - mock.patch.dict( - os.environ, - { - "RUNNER_TEMP": str(runner_temp), - "HF_ENDPOINT": "https://example.invalid", - "hF_ToKeN": "download-token", - "hUgGiNg_FaCe_HuB_tOkEn": "legacy-download-token", - "HuGgInGfAcE_ToKeN": "alternate-download-token", - "LEMONADE_API_KEY": "api-key", - "LEMONADE_DEFAULTS_PATH": "/host/defaults.json", - "LEMONADE_LLAMACPP_METAL_BIN": "/host/llama-server", - "lemonade_rocm_install_method": "wheel", - "LEMONADE_BACKEND_WATCHDOG_POLL_SECONDS": "1", - "LEMONADE_CACHE_DIR": "/host/lemonade-cache", - "LEMONADE_GGML_HIP_PATH": "/host/libggml-hip.so", - "HF_HOME": "/host/huggingface", - "LEMONADE_ALLOWED_ORIGINS": "https://example.invalid", - "ROCM_PATH": "/host/rocm", - "LLAMA_ARG_MODEL": "/host/model.gguf", - "LLAMA_ARG_CTX_SIZE": "1", - "HTTP_PROXY": "http://proxy.invalid:3128", - "all_proxy": "socks5://proxy.invalid:1080", - "NO_PROXY": "example.invalid", - "GITHUB_OUTPUT": str(github_output), - "GITHUB_ENV": "/runner/command/environment", - "ACTIONS_RUNTIME_TOKEN": "actions-token", - "LD_PRELOAD": "/attacker/inject.so", - "PYTHONPATH": "/attacker/python", - "UNKNOWN_DEPLOYMENT_SECRET": "secret", - }, - ), - ): - result = runner.run_validation(args) - finally: - os.chdir(old_directory) - github_output_contents = github_output.read_text(encoding="utf-8") - - self.assertEqual(result, 0) - self.assertEqual(popen.call_count, 3) - self.assertEqual(run.call_count, 3) - self.assertEqual(shutdown.call_count, 3) - self.assertEqual(verify_artifacts.call_count, 3) - self.assertEqual(attest_accelerator.call_count, 2) - self.assertEqual(revalidate_cache.call_count, 10) - self.assertEqual( - attest_accelerator.call_args_list[0].args[3], - verified_artifacts(runner), - ) - self.assertEqual( - attest_accelerator.call_args_list[1].args[3], - verified_artifacts(runner), - ) - self.assertEqual( - attest_accelerator.call_args_list[0].kwargs["selected_models"], - [K2_SMALL], - ) - self.assertEqual( - attest_accelerator.call_args_list[1].kwargs["selected_models"], - [K2_SMALL], - ) - - first_server = popen.call_args_list[0] - second_server = popen.call_args_list[1] - third_server = popen.call_args_list[2] - self.assertEqual(first_server.args[0][1], second_server.args[0][1]) - self.assertEqual(first_server.args[0][1], third_server.args[0][1]) - self.assertFalse(first_server.args[0][2].startswith("--")) - self.assertEqual(first_server.args[0][2], second_server.args[0][2]) - self.assertEqual(first_server.args[0][2], third_server.args[0][2]) - self.assertEqual(first_server.args[0][3:], second_server.args[0][3:]) - self.assertEqual(first_server.args[0][3:], third_server.args[0][3:]) - expected_hf_cache = first_server.kwargs["env"]["HF_HUB_CACHE"] - hf_cache_path = Path(expected_hf_cache) - hf_cache_path.relative_to(runner_temp.resolve()) - self.assertEqual(hf_cache_path.name, "hf-hub") - self.assertEqual(hf_cache_path.parent.name, "lemonade-cache") - self.assertTrue( - hf_cache_path.parents[1].name.startswith("llamacpp-validation-macos-metal-") - ) - private_temp = first_server.kwargs["env"]["TMPDIR"] - self.assertEqual(first_server.kwargs["env"]["TMP"], private_temp) - self.assertEqual(first_server.kwargs["env"]["TEMP"], private_temp) - Path(private_temp).relative_to(runner_temp.resolve()) - self.assertEqual(Path(private_temp).name, "private-temp") - download_credential_keys = { - "HF_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - "HUGGINGFACE_TOKEN", - } - for call in popen.call_args_list: - self.assertFalse( - download_credential_keys & {key.upper() for key in call.kwargs["env"]} - ) - for call in popen.call_args_list: - if os.name != "nt": - self.assertIs(call.kwargs.get("start_new_session"), True) - self.assertEqual(call.kwargs["env"]["HF_HUB_CACHE"], expected_hf_cache) - self.assertEqual(call.kwargs["env"]["TMPDIR"], private_temp) - self.assertEqual(call.kwargs["env"]["TMP"], private_temp) - self.assertEqual(call.kwargs["env"]["TEMP"], private_temp) - self.assertEqual(call.kwargs["env"]["HOME"], private_temp) - self.assertEqual(call.kwargs["env"]["USERPROFILE"], private_temp) - self.assertEqual(call.kwargs["env"]["APPDATA"], private_temp) - self.assertEqual(call.kwargs["env"]["LOCALAPPDATA"], private_temp) - self.assertNotIn("HF_ENDPOINT", call.kwargs["env"]) - self.assertNotIn("LEMONADE_API_KEY", call.kwargs["env"]) - self.assertNotIn("LEMONADE_LLAMACPP_METAL_BIN", call.kwargs["env"]) - self.assertNotIn("lemonade_rocm_install_method", call.kwargs["env"]) - self.assertNotIn( - "LEMONADE_BACKEND_WATCHDOG_POLL_SECONDS", call.kwargs["env"] - ) - self.assertNotIn("LEMONADE_CACHE_DIR", call.kwargs["env"]) - self.assertNotIn("LEMONADE_GGML_HIP_PATH", call.kwargs["env"]) - self.assertNotIn("HF_HOME", call.kwargs["env"]) - self.assertNotIn("LEMONADE_ALLOWED_ORIGINS", call.kwargs["env"]) - self.assertNotIn("ROCM_PATH", call.kwargs["env"]) - self.assertEqual( - { - key.upper(): value - for key, value in call.kwargs["env"].items() - if key.upper().startswith("LLAMA_ARG_") - }, - {"LLAMA_ARG_LOG_VERBOSITY": "4"}, - ) - self.assertNotIn("HTTP_PROXY", call.kwargs["env"]) - self.assertNotIn("all_proxy", call.kwargs["env"]) - self.assertNotIn("GITHUB_ENV", call.kwargs["env"]) - self.assertNotIn("GITHUB_OUTPUT", call.kwargs["env"]) - self.assertNotIn("ACTIONS_RUNTIME_TOKEN", call.kwargs["env"]) - self.assertNotIn("LD_PRELOAD", call.kwargs["env"]) - self.assertNotIn("PYTHONPATH", call.kwargs["env"]) - self.assertNotIn("UNKNOWN_DEPLOYMENT_SECRET", call.kwargs["env"]) - self.assertEqual(call.kwargs["env"]["NO_PROXY"], "127.0.0.1") - self.assertEqual(call.kwargs["env"]["no_proxy"], "127.0.0.1") - self.assertEqual(call.kwargs["env"]["LEMONADE_CI_MODE"], "True") - self.assertEqual( - call.kwargs["env"]["LEMONADE_DEFAULTS_PATH"], - str((resources / "defaults.json").resolve()), - ) - - prepare_command = run.call_args_list[0].args[0] - primary_command = run.call_args_list[1].args[0] - restart_command = run.call_args_list[2].args[0] - self.assertIn("--prepare-only", prepare_command) - self.assertNotIn("--skip-install", prepare_command) - self.assertNotIn("--prepare-only", primary_command) - self.assertIn("--skip-install", primary_command) - self.assertIn("--skip-install", restart_command) - for call in run.call_args_list: - self.assertFalse( - download_credential_keys & {key.upper() for key in call.kwargs["env"]} - ) - self.assertEqual( - Path(primary_command[primary_command.index("--output") + 1]).name, - "llamacpp_validation_macos-metal.json", - ) - self.assertEqual( - Path(restart_command[restart_command.index("--output") + 1]).name, - "llamacpp_restart_validation_macos-metal.json", - ) - for call in run.call_args_list: - self.assertEqual(call.kwargs["env"]["HF_HUB_CACHE"], expected_hf_cache) - self.assertEqual(call.kwargs["env"]["TMPDIR"], private_temp) - self.assertEqual(call.kwargs["env"]["TMP"], private_temp) - self.assertEqual(call.kwargs["env"]["TEMP"], private_temp) - self.assertEqual(call.kwargs["env"]["HOME"], private_temp) - self.assertEqual(call.kwargs["env"]["USERPROFILE"], private_temp) - self.assertEqual(call.kwargs["env"]["APPDATA"], private_temp) - self.assertEqual(call.kwargs["env"]["LOCALAPPDATA"], private_temp) - self.assertNotIn("HF_ENDPOINT", call.kwargs["env"]) - self.assertNotIn("LEMONADE_API_KEY", call.kwargs["env"]) - self.assertNotIn("LEMONADE_LLAMACPP_METAL_BIN", call.kwargs["env"]) - self.assertEqual( - { - key.upper(): value - for key, value in call.kwargs["env"].items() - if key.upper().startswith("LLAMA_ARG_") - }, - {"LLAMA_ARG_LOG_VERBOSITY": "4"}, - ) - self.assertNotIn("GITHUB_ENV", call.kwargs["env"]) - self.assertNotIn("GITHUB_OUTPUT", call.kwargs["env"]) - self.assertNotIn("ACTIONS_RUNTIME_TOKEN", call.kwargs["env"]) - self.assertNotIn("LD_PRELOAD", call.kwargs["env"]) - self.assertNotIn("PYTHONPATH", call.kwargs["env"]) - self.assertNotIn("UNKNOWN_DEPLOYMENT_SECRET", call.kwargs["env"]) - - server_log_names = { - Path(call.kwargs[stream].name).name - for call in popen.call_args_list - for stream in ("stdout", "stderr") - } - self.assertEqual( - server_log_names, - { - "lemond.prepare.stdout.log", - "lemond.prepare.stderr.log", - "lemond.stdout.log", - "lemond.stderr.log", - "lemond.restart.stdout.log", - "lemond.restart.stderr.log", - }, - ) - for call in popen.call_args_list: - Path(call.kwargs["stdout"].name).relative_to(runner_temp.resolve()) - Path(call.kwargs["stderr"].name).relative_to(runner_temp.resolve()) - - workflow_outputs = dict( - line.split("=", 1) for line in github_output_contents.splitlines() - ) - self.assertEqual( - Path(workflow_outputs["validation_result"]).name, - "llamacpp_validation_macos-metal.json", - ) - self.assertEqual( - Path(workflow_outputs["restart_result"]).name, - "llamacpp_restart_validation_macos-metal.json", - ) - self.assertEqual(len(workflow_outputs), 8) - for output_path in workflow_outputs.values(): - Path(output_path).relative_to(runner_temp.resolve()) - - @staticmethod - def _complete_command(command): - if "--prepare-only" not in command: - output_path = Path(command[command.index("--output") + 1]) - models = [ - command[index + 1] - for index, argument in enumerate(command) - if argument == "--model" - ] - capability_models = { - command[index + 1].removeprefix("builtin.") - for index, argument in enumerate(command) - if argument == "--capability-model" - } - records = [] - for model in models: - record = { - "model": model, - "pass": True, - "response": "Four.", - "input_tokens": 10, - "output_tokens": 2, - "time_to_first_token": 0.5, - "tokens_per_second": 4.0, - } - if model.removeprefix("builtin.") in capability_models: - record["capability_matrix"] = _passing_capability_matrix(model) - records.append(record) - output_path.write_text(json.dumps(records), encoding="utf-8") - return argparse.Namespace(returncode=0) - - def _complete_process(self, **kwargs): - kwargs["stdout_path"].touch() - kwargs["stderr_path"].touch() - return self._complete_command(kwargs["command"]).returncode, True - - def test_validation_attests_locked_model_with_selected_unlocked_schedule_model( - self, - ): - runner = load_runner() - schedule_models = create_validation_plan("schedule")["include"][0]["models"] - artifact_locks = json.loads( - (ROOT / "test/fixtures/llamacpp_validation_artifacts.json").read_text( - encoding="utf-8" - ) - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - expected_artifacts = {} - artifact_root = root / "verified-artifacts" - artifact_root.mkdir() - for model, lock in artifact_locks.items(): - artifact_path = artifact_root / lock["filename"] - artifact_path.touch() - expected_artifacts[model] = runner.VerifiedValidationArtifact( - model_id=model, - resolved_path=artifact_path.resolve(), - size_bytes=0, - sha256="0" * 64, - ) - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="windows-vulkan", - models=",".join(schedule_models), - lite=False, - capability_profile="k2-horizon-v1", - capability_models=K2_SMALL, - port=13305, - ) - - def completed_model_load(model: str, gguf_path: Path) -> str: - return "\n".join( - ( - log_line("LlamaCpp", f"Loading model: builtin.{model}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: {gguf_path}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line( - "Process", - "load_tensors: offloaded 12/24 layers to GPU", - ), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - - def complete_with_attestation_logs(**kwargs): - result = self._complete_process(**kwargs) - command = kwargs["command"] - if "--prepare-only" in command: - return result - models = [ - command[index + 1].removeprefix("builtin.") - for index, argument in enumerate(command) - if argument == "--model" - ] - sessions = [ - completed_model_load( - model, - ( - expected_artifacts[model].resolved_path - if model in expected_artifacts - else Path("/cache") / f"{model}.gguf" - ), - ) - for model in models - ] - kwargs["stderr_path"].write_text( - "\n".join(sessions) + "\n", encoding="utf-8" - ) - return result - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=complete_with_attestation_logs, - ), - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=expected_artifacts, - ), - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - ): - try: - result = runner.run_validation(args) - except RuntimeError as exc: - self.fail(f"selected unlocked schedule model was rejected: {exc}") - - self.assertEqual(result, 0) - - def test_validation_rejects_empty_primary_evidence_before_restart(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="linux-vulkan", - models=K2_SMALL, - lite=False, - capability_profile="k2-horizon-v1", - capability_models=K2_SMALL, - port=13305, - ) - phases = 0 - - def write_empty_evidence(**kwargs): - nonlocal phases - phases += 1 - kwargs["stdout_path"].touch() - kwargs["stderr_path"].touch() - command = kwargs["command"] - if "--prepare-only" not in command: - output_path = Path(command[command.index("--output") + 1]) - output_path.write_text("[]", encoding="utf-8") - return 0, True - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=write_empty_evidence, - ), - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value={}, - ), - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - self.assertRaisesRegex(RuntimeError, "nonempty array"), - ): - runner.run_validation(args) - - self.assertEqual(phases, 2) - - def test_validation_rejects_restart_evidence_for_another_model(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="linux-vulkan", - models=K2_SMALL, - lite=False, - capability_profile="k2-horizon-v1", - capability_models=K2_SMALL, - port=13305, - ) - - def write_wrong_restart_evidence(**kwargs): - result = self._complete_process(**kwargs) - command = kwargs["command"] - output_path = Path(command[command.index("--output") + 1]) - if output_path.name.startswith("llamacpp_restart_validation_"): - output_path.write_text( - json.dumps( - [ - { - "model": "Other-Llama", - "pass": True, - "response": "Four.", - "input_tokens": 10, - "output_tokens": 2, - "time_to_first_token": 0.5, - "tokens_per_second": 4.0, - } - ] - ), - encoding="utf-8", - ) - return result - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=write_wrong_restart_evidence, - ) as run_process, - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value={}, - ), - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - self.assertRaisesRegex(RuntimeError, "do not match planned models"), - ): - runner.run_validation(args) - - self.assertEqual(run_process.call_count, 3) - - def test_accelerator_attestation_requires_ordered_device_and_offload_markers(self): - runner = load_runner() - positive = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - log = root / "lemond.stderr.log" - empty_companion = root / "lemond.stdout.log" - empty_companion.touch() - log.write_text( - bind_verified_artifact_path(positive, expected_artifacts), - encoding="utf-8", - ) - identities = runner.require_accelerator_attestation( - [empty_companion, log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - self.assertEqual( - [identity.path for identity in identities], [empty_companion, log] - ) - - failures = { - "missing backend": positive.replace( - backend_log_line("vulkan") + "\n", "" - ), - "wrong backend": positive.replace( - backend_log_line("vulkan"), backend_log_line("cuda") - ), - "duplicate backend": positive.replace( - backend_log_line("vulkan"), - backend_log_line("vulkan") + "\n" + backend_log_line("vulkan"), - ), - "malformed backend": positive.replace( - backend_log_line("vulkan"), - log_line("LlamaCpp", "Using LlamaCpp Backend: vulkan extra"), - ), - "backend after artifact identity": positive.replace( - backend_log_line("vulkan") - + "\n" - + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}") - + "\n" - + backend_log_line("vulkan"), - ), - "backend after completed load": positive - + "\n" - + backend_log_line("vulkan"), - "zero": positive.replace("12/24", "0/24"), - "greater than total": positive.replace("12/24", "25/24"), - "malformed offload": positive.replace("12/24", "many/all"), - "missing offload": positive.replace( - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU") - + "\n", - "", - ), - "missing child success": positive.replace( - log_line("Process", "srv llama_server: model loaded"), "" - ), - "offload before device": positive.replace( - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ) - + "\n" - + log_line( - "Process", "load_tensors: offloaded 12/24 layers to GPU" - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU") - + "\n" - + log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - ), - "missing device": positive.replace( - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ) - + "\n", - "", - ), - "missing artifact identity": positive.replace( - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}") - + "\n", - "", - ), - "artifact suffix is not exact": positive.replace( - K2_SMALL_FILENAME, f"{K2_SMALL_FILENAME}.evil" - ), - "device before artifact identity": positive.replace( - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}") - + "\n" - + log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ) - + "\n" - + log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - ), - "other model cannot mask target": positive.replace( - K2_SMALL, "Other-Llama" - ), - "later draft masks target": positive.replace( - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ) - + "\n" - + log_line( - "Process", "load_tensors: offloaded 12/24 layers to GPU" - ), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ) - + "\n" - + log_line("Process", "load_tensors: offloaded 0/24 layers to GPU") - + "\n" - + log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan1 (GPU)", - ) - + "\n" - + log_line( - "Process", "load_tensors: offloaded 24/24 layers to GPU" - ), - ), - "deceptive child success text": positive.replace( - log_line("Process", "srv llama_server: model loaded"), - log_line("Process", "did not see srv llama_server: model loaded"), - ), - "second zero offload on same line": positive.replace( - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line( - "Process", - "load_tensors: offloaded 12/24 layers to GPU " - "load_tensors: offloaded 0/24 layers to GPU", - ), - ), - "malformed then valid offload on same line": positive.replace( - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line( - "Process", - "load_tensors: offloaded many/all layers to GPU " - "load_tensors: offloaded 12/24 layers to GPU", - ), - ), - } - for name, contents in failures.items(): - with self.subTest(name=name): - log.write_text( - bind_verified_artifact_path(contents, expected_artifacts), - encoding="utf-8", - ) - with self.assertRaisesRegex(RuntimeError, "accelerator"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - - unexpected_model = ( - positive - + "\n" - + log_line("LlamaCpp", "Loading model: builtin.Unexpected-Llama") - ) - log.write_text( - bind_verified_artifact_path(unexpected_model, expected_artifacts), - encoding="utf-8", - ) - with self.assertRaisesRegex(RuntimeError, "unexpected model"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - - def test_selected_unlocked_load_cannot_replace_locked_model_attestation(self): - runner = load_runner() - schedule_models = create_validation_plan("schedule")["include"][0]["models"] - unlocked_model = next( - model - for model in schedule_models - if model not in {K2_SMALL, "K2-Horizon-3.7B-GGUF", "K2-Horizon-7B-GGUF"} - ) - unlocked_load = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: builtin.{unlocked_model}"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - log = root / "lemond.stderr.log" - log.write_text(unlocked_load, encoding="utf-8") - with self.assertRaisesRegex(RuntimeError, "missing models"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - verified_artifacts(runner, root), - "vulkan", - selected_models=[unlocked_model, K2_SMALL], - ) - - def test_accelerator_attestation_rejects_same_basename_from_another_path(self): - runner = load_runner() - positive = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /unverified/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - unverified_artifact = root / "unverified" / K2_SMALL_FILENAME - unverified_artifact.parent.mkdir() - unverified_artifact.touch() - log = root / "lemond.stderr.log" - log.write_text( - positive.replace( - f"/unverified/{K2_SMALL_FILENAME}", - str(unverified_artifact.resolve()), - ), - encoding="utf-8", - ) - - with self.assertRaisesRegex(RuntimeError, "wrong artifact"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - - def test_accelerator_attestation_binds_device_to_requested_backend(self): - runner = load_runner() - backend_devices = { - "cuda": "CUDA0", - "metal": "MTL0", - "rocm-stable": "ROCm0", - "vulkan": "Vulkan0", - } - - def attestation(backend, device): - return "\n".join( - ( - log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), - backend_log_line(backend), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - f"llama_prepare_model_devices: using device {device} (GPU)", - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - log = root / "lemond.stderr.log" - for backend, device in backend_devices.items(): - with self.subTest(backend=backend, device=device): - log.write_text( - bind_verified_artifact_path( - attestation(backend, device), expected_artifacts - ), - encoding="utf-8", - ) - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - expected_backend=backend, - ) - - wrong_device = next( - candidate - for other_backend, candidate in backend_devices.items() - if other_backend != backend - ) - with self.subTest(backend=backend, wrong_device=wrong_device): - log.write_text( - bind_verified_artifact_path( - attestation(backend, wrong_device), expected_artifacts - ), - encoding="utf-8", - ) - with self.assertRaisesRegex(RuntimeError, "backend"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - expected_backend=backend, - ) - - def test_rocm_attestation_binds_exact_release_channel(self): - runner = load_runner() - - def attestation(channel): - return "\n".join( - ( - log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), - log_line("LlamaCpp", f"Using LlamaCpp Backend: rocm-{channel}"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device ROCm0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - log = root / "lemond.stderr.log" - for channel in ("stable", "nightly"): - expected_backend = f"rocm-{channel}" - log.write_text( - bind_verified_artifact_path( - attestation(channel), expected_artifacts - ), - encoding="utf-8", - ) - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - expected_backend=expected_backend, - ) - - swapped_channel = "nightly" if channel == "stable" else "stable" - with self.assertRaisesRegex(RuntimeError, "backend"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - expected_backend=f"rocm-{swapped_channel}", - ) - - def test_attestation_backend_resolution_rejects_channel_ambiguity(self): - runner = load_runner() - self.assertEqual( - runner.resolve_attestation_backend("rocm", "stable"), "rocm-stable" - ) - self.assertEqual( - runner.resolve_attestation_backend("rocm", "nightly"), "rocm-nightly" - ) - self.assertEqual(runner.resolve_attestation_backend("metal", ""), "metal") - - for backend, channel in ( - ("rocm", ""), - ("rocm", "candidate"), - ("metal", "stable"), - ): - with self.subTest(backend=backend, channel=channel): - with self.assertRaisesRegex(RuntimeError, "channel"): - runner.resolve_attestation_backend(backend, channel) - - def test_cpu_attestation_requires_cpu_only_build_markers(self): - runner = load_runner() - positive = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: builtin.{K2_SMALL}"), - backend_log_line("cpu"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "load_tensors: CPU_Mapped model buffer size = 2056.83 MiB", - ), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - accelerator_markers = ( - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 0/24 layers to GPU"), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - log = root / "lemond.stderr.log" - log.write_text( - bind_verified_artifact_path(positive, expected_artifacts), - encoding="utf-8", - ) - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - expected_backend="cpu", - ) - - missing_cpu_buffer = positive.replace( - log_line( - "Process", - "load_tensors: CPU_Mapped model buffer size = 2056.83 MiB", - ) - + "\n", - "", - ) - log.write_text( - bind_verified_artifact_path(missing_cpu_buffer, expected_artifacts), - encoding="utf-8", - ) - with self.assertRaisesRegex(RuntimeError, "CPU|cpu"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - expected_backend="cpu", - ) - - for marker in accelerator_markers: - with self.subTest(marker=marker): - invalid_cpu_log = positive.replace( - log_line("Process", "srv llama_server: model loaded"), - marker - + "\n" - + log_line("Process", "srv llama_server: model loaded"), - ) - log.write_text( - bind_verified_artifact_path( - invalid_cpu_log, expected_artifacts - ), - encoding="utf-8", - ) - with self.assertRaisesRegex(RuntimeError, "CPU|cpu"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - expected_backend="cpu", - ) - - def test_accelerator_attestation_does_not_carry_state_across_log_streams(self): - runner = load_runner() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - stdout_log = root / "stdout.log" - stderr_log = root / "stderr.log" - partial_load = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - ) - ) - stdout_log.write_text( - bind_verified_artifact_path(partial_load, expected_artifacts), - encoding="utf-8", - ) - stderr_log.write_text( - "\n".join( - ( - log_line( - "Process", "load_tensors: offloaded 12/24 layers to GPU" - ), - log_line("Process", "srv llama_server: model loaded"), - ) - ), - encoding="utf-8", - ) - - with self.assertRaisesRegex(RuntimeError, "incomplete"): - runner.require_accelerator_attestation( - [stdout_log, stderr_log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - - def test_accelerator_attestation_rejects_every_bad_duplicate_or_late_offload(self): - runner = load_runner() - positive = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - bad_second_loads = { - "zero": "\n".join( - ( - log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 0/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ), - "missing": "\n".join( - ( - log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "srv llama_server: model loaded"), - ) - ), - "incomplete": log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), - "late zero outside a load": log_line( - "Process", "load_tensors: offloaded 0/24 layers to GPU" - ), - "late malformed outside a load": ( - log_line("Process", "load_tensors: offloaded many/all layers to GPU") - ), - } - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - log = root / "lemond.stdout.log" - for name, second_load in bad_second_loads.items(): - with self.subTest(name=name): - contents = bind_verified_artifact_path( - f"{positive}\n{second_load}\n", expected_artifacts - ) - log.write_text(contents, encoding="utf-8") - with self.assertRaisesRegex(RuntimeError, "accelerator|incomplete"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - - def test_accelerator_attestation_rejects_oversized_log_lines(self): - runner = load_runner() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - log = root / "lemond.stdout.log" - log.write_bytes(b"x" * (runner.MAX_ATTESTATION_LINE_BYTES + 1)) - - with self.assertRaisesRegex(RuntimeError, "line is too long"): - runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - - def test_accelerator_attestation_enforces_aggregate_log_budget(self): - runner = load_runner() - positive = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - positive = bind_verified_artifact_path(positive, expected_artifacts) - first_log = root / "lemond.stdout.log" - second_log = root / "lemond.stderr.log" - first_log.write_text(positive, encoding="utf-8") - second_log.write_text("12345", encoding="utf-8") - budget = len(positive.encode("utf-8")) + 4 - - with ( - mock.patch.object(runner, "MAX_ATTESTATION_LOG_BYTES", budget), - self.assertRaisesRegex(RuntimeError, "byte budget"), - ): - runner.require_accelerator_attestation( - [first_log, second_log], - root, - "validation", - expected_artifacts, - "vulkan", - ) - - def test_attested_log_replacement_fails_final_identity_check(self): - runner = load_runner() - contents = "\n".join( - ( - log_line("LlamaCpp", f"Loading model: {K2_SMALL}"), - backend_log_line("vulkan"), - log_line("LlamaCpp", f"Using GGUF: /cache/{K2_SMALL_FILENAME}"), - log_line( - "Process", - "llama_prepare_model_devices: using device Vulkan0 (GPU)", - ), - log_line("Process", "load_tensors: offloaded 12/24 layers to GPU"), - log_line("Process", "srv llama_server: model loaded"), - ) - ) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - expected_artifacts = verified_artifacts(runner, root) - contents = bind_verified_artifact_path(contents, expected_artifacts) - log = root / "lemond.stdout.log" - log.write_text(contents, encoding="utf-8") - identity = runner.require_accelerator_attestation( - [log], - root, - "validation", - expected_artifacts, - "vulkan", - )[0] - log.rename(root / "original.log") - log.write_text(contents, encoding="utf-8") - - with self.assertRaisesRegex(RuntimeError, "changed"): - runner._revalidate_regular_file( - identity, - root, - require_nonempty=False, - ) - - def test_restart_is_not_attempted_when_primary_validation_fails(self): - runner = load_runner() - - class FakeContainment: - @staticmethod - def popen_kwargs(): - return {} - - @staticmethod - def assign(_process): - return None - - @staticmethod - def close(): - return True - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="linux-vulkan", - models=K2_SMALL, - lite=False, - capability_profile="k2-horizon-v1", - capability_models=K2_SMALL, - port=13305, - ) - - old_directory = Path.cwd() - os.chdir(root) - try: - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object(runner.time, "sleep"), - mock.patch.object( - runner, - "_create_process_containment", - side_effect=lambda **_kwargs: FakeContainment(), - ), - mock.patch.object( - runner.subprocess, - "Popen", - side_effect=(FakeProcess(), FakeProcess()), - ) as popen, - mock.patch.object( - runner, - "_run_contained_command", - side_effect=(0, 7), - ) as run, - mock.patch.object( - runner, - "_shutdown_server", - side_effect=(True, True), - ) as shutdown, - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=verified_artifacts(runner), - ) as verify, - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - ): - result = runner.run_validation(args) - finally: - os.chdir(old_directory) - - self.assertEqual(result, 7) - self.assertEqual(popen.call_count, 2) - self.assertEqual(run.call_count, 2) - self.assertEqual(shutdown.call_count, 2) - verify.assert_called_once() - - def test_direct_locked_model_is_verified_without_a_capability_profile(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="linux-vulkan", - models=f"builtin.{K2_SMALL}", - lite=False, - capability_profile="", - capability_models="", - port=13305, - ) - - old_directory = Path.cwd() - os.chdir(root) - try: - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=self._complete_process, - ) as run_process, - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=verified_artifacts(runner), - ) as verify, - mock.patch.object( - runner, - "require_accelerator_attestation", - return_value=[], - ) as attest, - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - ): - result = runner.run_validation(args) - finally: - os.chdir(old_directory) - - self.assertEqual(result, 0) - self.assertEqual(run_process.call_count, 3) - self.assertEqual(verify.call_count, 3) - self.assertEqual(attest.call_count, 2) - self.assertEqual( - attest.call_args_list[0].args[3], - verified_artifacts(runner), - ) - self.assertEqual( - attest.call_args_list[1].args[3], - verified_artifacts(runner), - ) - for call in verify.call_args_list: - self.assertEqual(call.args[3], [f"builtin.{K2_SMALL}"]) - - def test_direct_mixed_models_restart_the_selected_locked_candidate(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="linux-vulkan", - models=f"Other-Model,builtin.{K2_SMALL}", - lite=False, - capability_profile="", - capability_models="", - port=13305, - ) - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=self._complete_process, - ) as run_process, - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=verified_artifacts(runner), - ), - mock.patch.object( - runner, - "require_accelerator_attestation", - return_value=[], - ) as attest, - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - ): - result = runner.run_validation(args) - - self.assertEqual(result, 0) - restart_command = run_process.call_args_list[2].kwargs["command"] - restart_model_index = restart_command.index("--model") + 1 - self.assertEqual(restart_command[restart_model_index], f"builtin.{K2_SMALL}") - self.assertEqual(attest.call_count, 2) - self.assertEqual( - attest.call_args_list[0].kwargs["selected_models"], - ["Other-Model", f"builtin.{K2_SMALL}"], - ) - self.assertEqual( - attest.call_args_list[1].kwargs["selected_models"], - [f"builtin.{K2_SMALL}"], - ) - - def test_capability_model_is_the_restart_target_when_other_models_are_locked(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="windows-vulkan", - models=f"{K2_MEDIUM},{K2_SMALL}", - lite=False, - capability_profile="k2-horizon-v1", - capability_models=K2_SMALL, - port=13305, - ) - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=self._complete_process, - ) as run_process, - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value={}, - ), - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - ): - result = runner.run_validation(args) - - self.assertEqual(result, 0) - restart_command = run_process.call_args_list[2].kwargs["command"] - restart_model_index = restart_command.index("--model") + 1 - self.assertEqual(restart_command[restart_model_index], K2_SMALL) - - def test_cpu_lane_verifies_locked_artifacts_with_cpu_attestation(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="cpu", - channel="", - target="linux-cpu", - models=K2_SMALL, - lite=False, - capability_profile="", - capability_models="", - port=13305, - ) - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=self._complete_process, - ), - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=verified_artifacts(runner), - ) as verify, - mock.patch.object( - runner, - "require_accelerator_attestation", - return_value=[], - ) as attest, - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - ): - result = runner.run_validation(args) - - self.assertEqual(result, 0) - self.assertEqual(verify.call_count, 3) - self.assertEqual(attest.call_count, 2) - for call in attest.call_args_list: - self.assertEqual(call.args[4], "cpu") - - def test_rocm_lane_attests_exact_channel_for_primary_and_restart(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - - for channel in ("stable", "nightly"): - with self.subTest(channel=channel): - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="rocm", - channel=channel, - target=f"linux-rocm-{channel}", - models=K2_SMALL, - lite=False, - capability_profile="", - capability_models="", - port=13305, - ) - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=self._complete_process, - ), - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=verified_artifacts(runner), - ), - mock.patch.object( - runner, - "require_accelerator_attestation", - return_value=[], - ) as attest, - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - ): - result = runner.run_validation(args) - - self.assertEqual(result, 0) - self.assertEqual(attest.call_count, 2) - for call in attest.call_args_list: - self.assertEqual(call.args[4], f"rocm-{channel}") - - def test_artifact_change_during_full_validation_stops_before_restart(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - state = root / "artifact-state" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - state.write_text("trusted", encoding="ascii") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="metal", - channel="", - target="macos-metal", - models=K2_SMALL, - lite=False, - capability_profile="", - capability_models="", - port=13305, - ) - - process_count = 0 - - def run_process(**kwargs): - nonlocal process_count - process_count += 1 - self._complete_process(**kwargs) - if process_count == 2: - state.write_text("swapped", encoding="ascii") - return 0, True - - def verify(*_args): - if state.read_text(encoding="ascii") != "trusted": - raise ValueError("artifact changed across inference") - return verified_artifacts(runner) - - old_directory = Path.cwd() - os.chdir(root) - try: - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=run_process, - ), - mock.patch.object( - runner, - "verify_validation_artifacts", - side_effect=verify, - ) as verify_artifacts, - mock.patch.dict(os.environ, {"RUNNER_TEMP": str(runner_temp)}), - self.assertRaisesRegex(ValueError, "across inference"), - ): - runner.run_validation(args) - finally: - os.chdir(old_directory) - - self.assertEqual(process_count, 2) - self.assertEqual(verify_artifacts.call_count, 2) - - def test_attested_log_replacement_stops_workflow_output_emission(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - lemond = root / "lemond" - python = root / "python" - runner_temp = root / "runner-temp" - github_output = root / "github-output" - lemond.touch() - python.touch() - runner_temp.mkdir() - resources = root / "resources" - resources.mkdir() - (resources / "defaults.json").write_text("{}", encoding="utf-8") - args = argparse.Namespace( - lemond=lemond, - python=python, - backend="vulkan", - channel="", - target="linux-vulkan", - models=K2_SMALL, - lite=False, - capability_profile="", - capability_models="", - port=13305, - ) - validation_identity = None - - def attest( - log_paths, - allowed_root, - _phase, - _expected_artifacts, - _expected_backend, - **_kwargs, - ): - nonlocal validation_identity - identities = [ - runner._capture_regular_file( - path, - allowed_root, - require_nonempty=False, - ) - for path in log_paths - ] - if validation_identity is None: - validation_identity = identities[0] - else: - attacked_path = validation_identity.path - attacked_path.rename(attacked_path.with_suffix(".original")) - attacked_path.touch() - return identities - - with ( - mock.patch.object(runner, "_require_idle_runner"), - mock.patch.object(runner, "_require_unused_port"), - mock.patch.object( - runner, - "_run_validation_process", - side_effect=self._complete_process, - ), - mock.patch.object( - runner, - "verify_validation_artifacts", - return_value=verified_artifacts(runner), - ), - mock.patch.object( - runner, - "require_accelerator_attestation", - side_effect=attest, - ), - mock.patch.dict( - os.environ, - { - "RUNNER_TEMP": str(runner_temp), - "GITHUB_OUTPUT": str(github_output), - }, - ), - self.assertRaisesRegex(RuntimeError, "changed"), - ): - runner.run_validation(args) - - self.assertFalse(github_output.exists()) - - def test_cache_root_rejects_a_preexisting_symlink(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - real_runner_temp = root / "real-runner-temp" - runner_temp = root / "runner-temp" - real_runner_temp.mkdir() - runner_temp.symlink_to(real_runner_temp, target_is_directory=True) - - create_cache = getattr(runner, "create_validation_cache", None) - self.assertIsNotNone(create_cache) - with self.assertRaisesRegex(RuntimeError, "symlink or junction"): - create_cache(runner_temp, "linux-vulkan") - - def test_validation_cache_is_unpredictable_and_has_a_revalidation_contract(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - runner_temp = Path(directory) - first = runner.create_validation_cache(runner_temp, "linux-vulkan") - runner.revalidate_validation_cache(first) - for identity in first: - self.assertTrue(identity.path.is_dir()) - try: - second = runner.create_validation_cache(runner_temp, "linux-vulkan") - except (OSError, RuntimeError) as exc: - self.fail(f"cache path was predictable or reusable: {exc}") - - self.assertNotEqual(first, second) - revalidate = getattr(runner, "revalidate_validation_cache", None) - self.assertIsNotNone(revalidate) - - def test_server_log_files_are_created_exclusively(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - cache = root / "cache" - cache.mkdir() - stdout_path = root / "stdout.log" - stdout_path.write_text("stale", encoding="utf-8") - with ( - mock.patch.object(runner.subprocess, "Popen") as popen, - self.assertRaises(FileExistsError), - ): - runner._run_validation_process( - lemond=root / "lemond", - cache_directory=cache, - config_directory=cache, - command=["validate"], - environment={}, - port=13305, - pid_path=root / "lemond.pid", - stdout_path=stdout_path, - stderr_path=root / "stderr.log", - ) - - popen.assert_not_called() - - def test_containment_assignment_precedes_pid_and_close_failure_keeps_pid(self): - runner = load_runner() - events = [] - - class FakeContainment: - @staticmethod - def popen_kwargs(): - return {} - - @staticmethod - def assign(_process): - events.append("assign") - - @staticmethod - def close(): - events.append("close") - return False - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - cache = root / "cache" - cache.mkdir() - pid_path = root / "lemond.pid" - - def run_client( - selected_command, - *, - env, - containment, - timeout_seconds, - ): - self.assertTrue(pid_path.is_file()) - self.assertEqual(selected_command, ["validate"]) - self.assertEqual(env, {}) - self.assertIsInstance(containment, FakeContainment) - self.assertEqual( - timeout_seconds, - runner.VALIDATION_PHASE_TIMEOUT_SECONDS, - ) - events.append("client") - return 0 - - def shutdown(*_args): - events.append("shutdown") - return True - - with ( - mock.patch.object( - runner, - "_create_process_containment", - return_value=FakeContainment(), - ), - mock.patch.object( - runner.subprocess, - "Popen", - side_effect=lambda *_args, **_kwargs: ( - events.append("popen") or FakeProcess() - ), - ), - mock.patch.object(runner.time, "sleep"), - mock.patch.object( - runner, - "_run_contained_command", - side_effect=run_client, - ), - mock.patch.object( - runner, - "_shutdown_server", - side_effect=shutdown, - ), - ): - exit_code, cleanup_succeeded = runner._run_validation_process( - lemond=root / "lemond", - cache_directory=cache, - config_directory=cache, - command=["validate"], - environment={}, - port=13305, - pid_path=pid_path, - stdout_path=root / "stdout.log", - stderr_path=root / "stderr.log", - ) - pid_evidence_retained = pid_path.is_file() - - self.assertEqual(exit_code, 0) - self.assertFalse(cleanup_succeeded) - self.assertEqual(events, ["popen", "assign", "client", "shutdown", "close"]) - self.assertTrue(pid_evidence_retained) - - def test_validation_cache_rejects_directory_replacement_and_symlink_swap(self): - runner = load_runner() - - for cache_name in ( - "hf_cache_directory", - "private_temp_directory", - "results_directory", - "logs_directory", - ): - for attack in ("directory", "symlink"): - self._assert_cache_directory_attack_is_rejected( - runner, cache_name, attack - ) - - def _assert_cache_directory_attack_is_rejected( - self, runner, cache_name: str, attack: str - ) -> None: - with ( - self.subTest(cache_name=cache_name, attack=attack), - tempfile.TemporaryDirectory() as directory, - ): - root = Path(directory) - cache = runner.create_validation_cache(root, "linux-vulkan") - attacked_path = getattr(cache, cache_name).path - original = attacked_path.with_name(f"{attacked_path.name}-original") - attacked_path.rename(original) - if attack == "directory": - attacked_path.mkdir() - else: - outside = root / "outside" - outside.mkdir() - try: - attacked_path.symlink_to(outside, target_is_directory=True) - except OSError as exc: - self.skipTest(f"directory symlinks are unavailable: {exc}") - - with self.assertRaisesRegex(RuntimeError, "changed|symlink or junction"): - runner.revalidate_validation_cache(cache) - - def test_pid_file_failure_still_stops_the_started_server(self): - runner = load_runner() - - class FakeContainment: - @staticmethod - def popen_kwargs(): - return {} - - @staticmethod - def assign(_process): - return None - - @staticmethod - def close(): - return True - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - cache = root / "cache" - cache.mkdir() - pid_path = root / "pid-as-directory" - pid_path.mkdir() - with ( - mock.patch.object( - runner, - "_create_process_containment", - return_value=FakeContainment(), - ), - mock.patch.object( - runner.subprocess, "Popen", return_value=FakeProcess() - ), - mock.patch.object(runner, "_run_contained_command") as run, - mock.patch.object( - runner, "_shutdown_server", return_value=True - ) as shutdown, - self.assertRaises(OSError), - ): - runner._run_validation_process( - lemond=root / "lemond", - cache_directory=cache, - config_directory=cache, - command=["validate"], - environment={}, - port=13305, - pid_path=pid_path, - stdout_path=root / "stdout.log", - stderr_path=root / "stderr.log", - ) - - run.assert_not_called() - shutdown.assert_called_once() - - def test_already_exited_server_is_an_unclean_validation_failure(self): - runner = load_runner() - - class ExitedProcess: - pid = 999 - - @staticmethod - def poll(): - return 17 - - with tempfile.TemporaryDirectory() as directory: - cache = Path(directory) / "cache" - cache.mkdir() - with mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([], True), - ): - clean = runner._shutdown_server(ExitedProcess(), 13305, cache) - - self.assertFalse(clean) - - def test_nonzero_clean_shutdown_exit_is_a_validation_failure(self): - runner = load_runner() - - class NonzeroShutdownProcess: - pid = 1001 - - def __init__(self): - self.returncode = None - - def poll(self): - return self.returncode - - def wait(self, timeout): - del timeout - self.returncode = 139 - return self.returncode - - with tempfile.TemporaryDirectory() as directory: - cache = Path(directory) / "cache" - cache.mkdir() - with ( - mock.patch.object(runner, "_open_local_url"), - mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([], True), - ), - ): - clean = runner._shutdown_server(NonzeroShutdownProcess(), 13305, cache) - - self.assertFalse(clean) - - def test_clean_shutdown_uses_http_and_returns_true(self): - runner = load_runner() - wait_timeouts = [] - - class CleanShutdownProcess: - pid = 1003 - - def __init__(self): - self.returncode = None - - def poll(self): - return self.returncode - - def wait(self, timeout): - wait_timeouts.append(timeout) - self.returncode = 0 - return self.returncode - - process = CleanShutdownProcess() - with tempfile.TemporaryDirectory() as directory: - cache = Path(directory) / "cache" - cache.mkdir() - with ( - mock.patch.object(runner, "_open_local_url") as open_url, - mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([], True), - ), - ): - clean = runner._shutdown_server(process, 13305, cache) - - self.assertTrue(clean) - self.assertTrue(wait_timeouts) - self.assertTrue(all(timeout > 0 for timeout in wait_timeouts)) - open_url.assert_called_once() - self.assertEqual(open_url.call_args.kwargs, {"timeout": 10}) - - def test_shutdown_cancellation_rescans_cache_before_bounded_abort(self): - runner = load_runner() - cancellation_deadline = time.monotonic() + 4 - cancellation = runner._ValidationCancelled( - signal.SIGINT, - cancellation_deadline, - ) - orphan = argparse.Namespace(pid=2002) - process = argparse.Namespace(pid=2001) - process.poll = mock.Mock(return_value=None) - containment = object() - - with ( - mock.patch.object( - runner, - "_open_local_url_with_cancellation", - side_effect=cancellation, - ), - mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([orphan], True), - ) as scan, - mock.patch.object(runner, "_force_kill_processes") as force_kill, - mock.patch.object(runner, "_abort_process_containment") as abort, - mock.patch.object(runner, "_reap_direct_process") as reap, - self.assertRaises(runner._ValidationCancelled) as raised, - ): - runner._shutdown_server( - process, - 13305, - Path("/validation-cache"), - containment, - ) - - self.assertIs(raised.exception, cancellation) - scan.assert_called_once_with(Path("/validation-cache"), process.pid) - force_kill.assert_called_once_with([orphan], cancellation_deadline) - abort.assert_called_once_with(containment, cancellation_deadline) - reap.assert_called_once_with(process, cancellation_deadline) - - def test_cleaned_orphan_process_still_invalidates_the_lane(self): - runner = load_runner() - - class CleanShutdownProcess: - pid = 1002 - - def __init__(self): - self.returncode = None - - def poll(self): - return self.returncode - - def wait(self, timeout): - del timeout - self.returncode = 0 - return self.returncode - - orphan = object() - with tempfile.TemporaryDirectory() as directory: - cache = Path(directory) / "cache" - cache.mkdir() - with ( - mock.patch.object(runner, "_open_local_url"), - mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([orphan], True), - ), - mock.patch.object(runner, "_stop_processes", return_value=True) as stop, - ): - clean = runner._shutdown_server(CleanShutdownProcess(), 13305, cache) - - self.assertFalse(clean) - stop.assert_called_once_with([orphan]) - - def test_orphan_cleanup_waits_for_term_exit_before_using_kill(self): - runner = load_runner() - events = [] - - class FakePsutilError(Exception): - pass - - class ExitsOnTermProcess: - @staticmethod - def terminate(): - events.append("terminate") - - @staticmethod - def kill(): - events.append("kill") - - def wait_procs(processes, timeout): - events.append(("wait", timeout)) - return processes, [] - - fake_psutil = argparse.Namespace( - Error=FakePsutilError, - wait_procs=wait_procs, - ) - with mock.patch.dict(sys.modules, {"psutil": fake_psutil}): - clean = runner._stop_processes([ExitsOnTermProcess()]) - - self.assertTrue(clean) - self.assertEqual(events[0], "terminate") - self.assertEqual(events[1][0], "wait") - self.assertGreater(events[1][1], 0) - self.assertLessEqual(events[1][1], runner.PROCESS_WAIT_POLL_SECONDS) - self.assertNotIn("kill", events) - - @unittest.skipIf(os.name == "nt", "POSIX process-group test") - def test_process_group_probe_falls_back_when_psutil_enumeration_fails(self): - runner = load_runner() - - for enumeration_error in ( - PermissionError("denied"), - OSError("enumeration failed"), - ): - fake_psutil = mock.Mock() - fake_psutil.STATUS_DEAD = "dead" - fake_psutil.STATUS_ZOMBIE = "zombie" - fake_psutil.Error = RuntimeError - fake_psutil.process_iter.side_effect = enumeration_error - with ( - self.subTest(error=type(enumeration_error).__name__), - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - mock.patch.object(runner.os, "killpg", return_value=None) as killpg, - ): - self.assertTrue( - runner._PosixProcessGroupContainment._has_live_members(4321) - ) - killpg.assert_called_once_with(4321, 0) - - fake_psutil = mock.Mock() - fake_psutil.STATUS_DEAD = "dead" - fake_psutil.STATUS_ZOMBIE = "zombie" - fake_psutil.Error = RuntimeError - fake_psutil.process_iter.side_effect = PermissionError("denied") - with ( - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - mock.patch.object( - runner.os, - "killpg", - side_effect=ProcessLookupError, - ), - ): - self.assertFalse( - runner._PosixProcessGroupContainment._has_live_members(4321) - ) - - dead_process = mock.Mock() - dead_process.pid = 9875 - dead_process.info = {"status": "dead"} - - def partial_enumeration(_attributes): - yield dead_process - raise OSError("enumeration failed") - - fake_psutil = mock.Mock() - fake_psutil.STATUS_DEAD = "dead" - fake_psutil.STATUS_ZOMBIE = "zombie" - fake_psutil.Error = RuntimeError - fake_psutil.process_iter.side_effect = partial_enumeration - with ( - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - mock.patch.object(runner.os, "killpg", return_value=None) as killpg, - ): - self.assertTrue( - runner._PosixProcessGroupContainment._has_live_members(4321) - ) - killpg.assert_called_once_with(4321, 0) - - @unittest.skipIf(os.name == "nt", "POSIX process-group test") - def test_normal_process_group_close_force_kills_when_cancellation_is_pending(self): - runner = load_runner() - containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" - ) - containment._process_groups = [4324] - cancellation = argparse.Namespace(signum=signal.SIGINT) - - with ( - mock.patch.object( - runner, - "_ACTIVE_CANCELLATION_STATE", - cancellation, - ), - mock.patch.object( - runner._PosixProcessGroupContainment, - "_has_live_members", - return_value=True, - ), - mock.patch.object(runner.os, "killpg") as killpg, - ): - clean = containment.close() - - self.assertFalse(clean) - self.assertEqual( - killpg.call_args_list, - [ - mock.call(4324, signal.SIGTERM), - mock.call(4324, signal.SIGKILL), - ], - ) - - def test_cache_process_scan_fails_closed_when_psutil_enumeration_fails(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - for enumeration_error in ( - PermissionError("denied"), - OSError("enumeration failed"), - ): - fake_psutil = mock.Mock() - fake_psutil.Error = RuntimeError - fake_psutil.process_iter.side_effect = enumeration_error - with ( - self.subTest(error=type(enumeration_error).__name__), - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - ): - scan = runner._cache_processes(Path(directory), os.getpid()) - - self.assertFalse(scan.complete) - self.assertEqual(scan.processes, []) - - def test_partial_cache_process_scan_retains_known_matches(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - cache = Path(directory).resolve() - known_process = mock.Mock() - known_process.pid = 9876 - known_process.info = {"exe": str(cache / "llama-server")} - - def partial_enumeration(_attributes): - yield known_process - raise PermissionError("enumeration failed") - - fake_psutil = mock.Mock() - fake_psutil.Error = RuntimeError - fake_psutil.process_iter.side_effect = partial_enumeration - with mock.patch.dict(sys.modules, {"psutil": fake_psutil}): - scan = runner._cache_processes(cache, os.getpid()) - - self.assertFalse(scan.complete) - self.assertEqual(scan.processes, [known_process]) - - def test_incomplete_cache_process_scan_invalidates_cleanup_and_stops_matches(self): - runner = load_runner() - - class ExitedProcess: - pid = 9877 - - @staticmethod - def poll(): - return 0 - - known_process = object() - with tempfile.TemporaryDirectory() as directory: - cache = Path(directory) - with ( - mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([known_process], False), - ), - mock.patch.object(runner, "_stop_processes", return_value=True) as stop, - ): - clean = runner._shutdown_server(ExitedProcess(), 13305, cache) - - self.assertFalse(clean) - stop.assert_called_once_with([known_process]) - - def test_empty_incomplete_cache_process_scan_invalidates_cleanup(self): - runner = load_runner() - - class ExitedProcess: - pid = 9878 - - @staticmethod - def poll(): - return 0 - - with tempfile.TemporaryDirectory() as directory: - with mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([], False), - ): - clean = runner._shutdown_server( - ExitedProcess(), - 13305, - Path(directory), - ) - - self.assertFalse(clean) - - def test_contained_command_timeout_closes_containment(self): - runner = load_runner() - events = [] - - class TimedOutProcess: - pid = 4322 - - def __init__(self): - self.returncode = None - - def poll(self): - return self.returncode - - def wait(self, timeout): - events.append(("wait", timeout)) - if self.returncode is None: - time.sleep(timeout) - raise subprocess.TimeoutExpired(["validate"], timeout) - return self.returncode - - def terminate(self): - events.append("terminate") - self.returncode = -signal.SIGTERM - - def kill(self): - events.append("kill") - self.returncode = -signal.SIGKILL - - class FakeContainment: - @staticmethod - def popen_kwargs(): - return {} - - @staticmethod - def assign(_process): - events.append("assign") - - @staticmethod - def abort(_deadline): - events.append("abort") - return False - - @staticmethod - def close(): - events.append("close") - return False - - process = TimedOutProcess() - with ( - mock.patch.object(runner.subprocess, "Popen", return_value=process), - self.assertRaisesRegex(RuntimeError, "timed out after 0.01 seconds"), - ): - runner._run_contained_command( - ["validate"], - env={}, - containment=FakeContainment(), - timeout_seconds=0.01, - ) - - self.assertIn("abort", events) - self.assertLess(events.index("abort"), events.index("kill")) - self.assertNotIn("terminate", events) - self.assertNotIn(("wait", 5), events) - - def test_contained_command_preserves_nonzero_exit_code(self): - runner = load_runner() - - class ExitedProcess: - pid = 4323 - - @staticmethod - def wait(timeout): - del timeout - return 23 - - class FakeContainment: - @staticmethod - def popen_kwargs(): - return {} - - @staticmethod - def assign(_process): - return None - - @staticmethod - def close(): - raise AssertionError("normal process exit must not close containment") - - with mock.patch.object( - runner.subprocess, - "Popen", - return_value=ExitedProcess(), - ): - exit_code = runner._run_contained_command( - ["validate"], - env={}, - containment=FakeContainment(), - timeout_seconds=30, - ) - - self.assertEqual(exit_code, 23) - - def test_darwin_containment_requires_the_trusted_ephemeral_runner_contract(self): - runner = load_runner() - trusted_environment = { - "GITHUB_ACTIONS": "true", - runner.DARWIN_EPHEMERAL_RUNNER_CONTEXT_KEY: ( - runner.DARWIN_EPHEMERAL_RUNNER_CONTEXT - ), - } - - with mock.patch.object(runner.sys, "platform", "darwin"): - with self.assertRaisesRegex( - RuntimeError, "Darwin cannot contain session-escaping descendants" - ): - runner._create_process_containment() - with ( - mock.patch.dict( - os.environ, - {"GITHUB_ACTIONS": "true"}, - clear=True, - ), - self.assertRaisesRegex(RuntimeError, "requires the trusted"), - ): - runner._create_process_containment( - allow_darwin_github_hosted_ephemeral_runner=True, - backend="metal", - target="macos-metal", - ) - with ( - mock.patch.dict(os.environ, trusted_environment, clear=True), - self.assertRaisesRegex(RuntimeError, "restricted to macos-metal"), - ): - runner._create_process_containment( - allow_darwin_github_hosted_ephemeral_runner=True, - backend="vulkan", - target="macos-metal", - ) - with mock.patch.dict(os.environ, trusted_environment, clear=True): - containment = runner._create_process_containment( - allow_darwin_github_hosted_ephemeral_runner=True, - backend="metal", - target="macos-metal", - ) - - self.assertTrue(containment.close()) - - def test_linux_subreaper_lease_restores_only_the_original_disabled_state(self): - runner = load_runner() - - class RunnerProcess: - @staticmethod - def children(recursive): - del recursive - return [] - - fake_psutil = argparse.Namespace( - DEAD_STATUS="dead", - Error=OSError, - NoSuchProcess=ProcessLookupError, - Process=lambda _pid: RunnerProcess(), - STATUS_DEAD="dead", - STATUS_ZOMBIE="zombie", - ZombieProcess=ProcessLookupError, - ) - with ( - mock.patch.object(runner.sys, "platform", "linux"), - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - mock.patch.object( - runner, "_linux_child_subreaper_state", return_value=False - ) as get_state, - mock.patch.object(runner, "_set_linux_child_subreaper") as set_state, - ): - first = runner._PosixProcessGroupContainment() - second = runner._PosixProcessGroupContainment() - self.assertTrue(first.close()) - self.assertTrue(second.close()) - - get_state.assert_called_once_with() - self.assertEqual(set_state.call_args_list, [mock.call(True), mock.call(False)]) - - def test_linux_subreaper_initialization_fails_before_process_launch(self): - runner = load_runner() - - class RunnerProcess: - @staticmethod - def children(recursive): - del recursive - return [] - - fake_psutil = argparse.Namespace( - Error=OSError, - Process=lambda _pid: RunnerProcess(), - ) - with ( - mock.patch.object(runner.sys, "platform", "linux"), - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - mock.patch.object( - runner, - "_acquire_linux_child_subreaper", - side_effect=OSError("prctl denied"), - ), - mock.patch.object(runner.subprocess, "Popen") as popen, - self.assertRaisesRegex(OSError, "prctl denied"), - ): - runner._PosixProcessGroupContainment() - - popen.assert_not_called() - - def test_linux_subreaper_rejects_unsafe_preexisting_process_state(self): - runner = load_runner() - - class RunnerProcess: - descendants = [] - - @classmethod - def children(cls, recursive): - del recursive - return cls.descendants - - fake_psutil = argparse.Namespace( - Error=OSError, - Process=lambda _pid: RunnerProcess(), - ) - with ( - mock.patch.object(runner.sys, "platform", "linux"), - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - mock.patch.object( - runner.signal, - "getsignal", - return_value=signal.SIG_IGN, - ), - mock.patch.object(runner, "_acquire_linux_child_subreaper") as acquire, - self.assertRaisesRegex(RuntimeError, "default SIGCHLD"), - ): - runner._PosixProcessGroupContainment() - acquire.assert_not_called() - - RunnerProcess.descendants = [object()] - with ( - mock.patch.object(runner.sys, "platform", "linux"), - mock.patch.dict(sys.modules, {"psutil": fake_psutil}), - mock.patch.object( - runner.signal, - "getsignal", - return_value=signal.SIG_DFL, - ), - mock.patch.object(runner, "_acquire_linux_child_subreaper") as acquire, - self.assertRaisesRegex(RuntimeError, "pre-existing runner descendants"), - ): - runner._PosixProcessGroupContainment() - acquire.assert_not_called() - - @unittest.skipIf(os.name == "nt", "POSIX descendant tracking test") - @unittest.skipIf( - not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, - "Darwin process-tree inspection is unavailable", - ) - def test_clean_contained_process_exit_remains_clean_with_tracking(self): - runner = load_runner() - containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" - ) - - exit_code = runner._run_contained_command( - [sys.executable, "-c", "pass"], - env=dict(os.environ), - containment=containment, - timeout_seconds=30, - ) - - self.assertEqual(exit_code, 0) - self.assertTrue(containment.close()) - - @unittest.skipIf(os.name == "nt", "POSIX gated process launch test") - @unittest.skipIf( - not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, - "Darwin process-tree inspection is unavailable", - ) - def test_posix_gate_assigns_containment_before_candidate_code_runs(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - marker = Path(directory) / "candidate-ran.txt" - containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" - ) - original_assign = containment.assign - - def assert_candidate_is_blocked(process): - self.assertFalse(marker.exists()) - original_assign(process) - - try: - with mock.patch.object( - containment, - "assign", - side_effect=assert_candidate_is_blocked, - ): - process = containment.start_process( - [ - sys.executable, - "-c", - ( - "import pathlib, sys; " - "pathlib.Path(sys.argv[1]).write_text(" - "'ran', encoding='ascii')" - ), - str(marker), - ], - env=dict(os.environ), - ) - self.assertEqual(process.wait(timeout=10), 0) - self.assertEqual(marker.read_text(encoding="ascii"), "ran") - finally: - self.assertTrue(containment.close()) - - @unittest.skipUnless( - sys.platform.startswith("linux"), "Linux child-subreaper containment test" - ) - def test_linux_subreaper_reaps_adopted_child_without_stealing_root_status(self): - runner = load_runner() - daemon_program = """ -import os -import pathlib -import sys - -first_child = os.fork() -if first_child: - os._exit(0) -os.setsid() -second_child = os.fork() -if second_child: - os._exit(0) -pathlib.Path(sys.argv[1]).write_text(str(os.getpid()), encoding="ascii") -""" - root_program = f""" -import pathlib -import subprocess -import sys -import time - -subprocess.Popen( - [sys.executable, "-c", {daemon_program!r}, sys.argv[1]], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, -) -while not pathlib.Path(sys.argv[1]).is_file(): - time.sleep(0.01) -time.sleep(0.3) -raise SystemExit(23) -""" - - with tempfile.TemporaryDirectory() as directory: - daemon_pid_path = Path(directory) / "daemon.pid" - containment = runner._PosixProcessGroupContainment() - try: - exit_code = runner._run_contained_command( - [ - sys.executable, - "-c", - root_program, - str(daemon_pid_path), - ], - env=dict(os.environ), - containment=containment, - timeout_seconds=30, - ) - daemon_pid = int(daemon_pid_path.read_text(encoding="ascii")) - self.assertEqual(exit_code, 23) - self.assertFalse(_pid_is_active(daemon_pid)) - finally: - self.assertTrue(containment.close()) - - def test_cancellation_reuses_one_cleanup_deadline(self): - runner = load_runner() - cancellation = argparse.Namespace(signum=signal.SIGINT) - - with ( - mock.patch.object( - runner, - "_ACTIVE_CANCELLATION_STATE", - cancellation, - ), - mock.patch.object(runner.time, "monotonic", return_value=100.0) as clock, - ): - with self.assertRaises(runner._ValidationCancelled) as first: - runner._raise_if_cancelled() - with self.assertRaises(runner._ValidationCancelled) as second: - runner._raise_if_cancelled() - - self.assertEqual(first.exception.cleanup_deadline, 104.0) - self.assertEqual( - second.exception.cleanup_deadline, - first.exception.cleanup_deadline, - ) - clock.assert_called_once_with() - - @unittest.skipIf(os.name == "nt", "POSIX signal containment test") - def test_sigint_and_sigterm_kill_contained_command_descendants(self): - for selected_signal in (signal.SIGINT, signal.SIGTERM): - with self.subTest(selected_signal=selected_signal): - self._assert_runner_signal_kills_descendants(selected_signal) - - @unittest.skipIf(os.name == "nt", "POSIX signal containment test") - def test_sigint_force_kills_term_ignoring_tree_before_escalation(self): - elapsed = self._assert_runner_signal_kills_descendants( - signal.SIGINT, - ignore_sigterm=True, - ) - - self.assertLess(elapsed, 6.0) - - @unittest.skipIf(os.name == "nt", "POSIX signal containment test") - @unittest.skipIf( - not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, - "Darwin process-tree inspection is unavailable", - ) - def test_sigint_and_sigterm_kill_session_escaping_descendants(self): - for selected_signal in (signal.SIGINT, signal.SIGTERM): - with self.subTest(selected_signal=selected_signal): - elapsed = self._assert_runner_signal_kills_descendants( - selected_signal, - ignore_sigterm=True, - escape_session=True, - ) - self.assertLess(elapsed, 6.0) - - @unittest.skipUnless( - sys.platform.startswith("linux"), "Linux child-subreaper containment test" - ) - def test_sigint_and_sigterm_kill_fast_double_fork_session_escape(self): - for selected_signal in (signal.SIGINT, signal.SIGTERM): - with self.subTest(selected_signal=selected_signal): - elapsed = self._assert_runner_signal_kills_descendants( - selected_signal, - ignore_sigterm=True, - double_fork=True, - ) - self.assertLess(elapsed, 6.0) - - def _assert_runner_signal_kills_descendants( - self, - selected_signal: int, - *, - ignore_sigterm: bool = False, - escape_session: bool = False, - double_fork: bool = False, - ) -> float: - descendant_setup = [] - if double_fork: - descendant_setup.extend( - [ - "first_child = os.fork()", - "if first_child:", - " os._exit(0)", - "os.setsid()", - "second_child = os.fork()", - "if second_child:", - " os._exit(0)", - ] - ) - elif escape_session: - descendant_setup.append("os.setsid()") - if ignore_sigterm: - descendant_setup.append("signal.signal(signal.SIGTERM, signal.SIG_IGN)") - descendant_program = f""" -import os -import pathlib -import signal -import sys -import time - -{chr(10).join(descendant_setup)} -pathlib.Path(sys.argv[1]).write_text(str(os.getpid()), encoding="ascii") -time.sleep(30) -""" - parent_signal_setup = "" - if ignore_sigterm: - parent_signal_setup = "signal.signal(signal.SIGTERM, signal.SIG_IGN); " - child_program = f""" -import os -import pathlib -import signal -import subprocess -import sys -import time - -{parent_signal_setup} -child = subprocess.Popen( - [sys.executable, "-c", {descendant_program!r}, sys.argv[2]], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, -) -while not pathlib.Path(sys.argv[2]).is_file(): - time.sleep(0.01) -descendant_pid = pathlib.Path(sys.argv[2]).read_text(encoding="ascii") -pathlib.Path(sys.argv[1]).write_text( - f"{{os.getpid()}},{{descendant_pid}}", - encoding="ascii", -) -time.sleep(30) -""" - with tempfile.TemporaryDirectory() as directory: - pid_path = Path(directory) / "contained-pids.txt" - descendant_ready_path = Path(directory) / "descendant-ready.txt" - helper_program = f""" -import importlib.util -import os -import sys - -spec = importlib.util.spec_from_file_location("signal_runner", {str(RUNNER_PATH)!r}) -runner = importlib.util.module_from_spec(spec) -spec.loader.exec_module(runner) -command = [ - sys.executable, - "-c", - {child_program!r}, - {str(pid_path)!r}, - {str(descendant_ready_path)!r}, -] - -def run_validation(_args): - containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" - ) - try: - return runner._run_contained_command( - command, - env=dict(os.environ), - containment=containment, - timeout_seconds=30, - ) - finally: - runner._close_process_containment(containment) - -runner.run_validation = run_validation -sys.argv = [ - "run_llamacpp_validation.py", - "--lemond", - sys.executable, - "--python", - sys.executable, - "--backend", - "cpu", - "--target", - "linux-cpu", -] -runner.main() -""" - helper = subprocess.Popen( - [sys.executable, "-c", helper_program], - cwd=ROOT, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - command_pid = None - descendant_pid = None - try: - deadline = time.monotonic() + 10 - pid_values = None - while pid_values is None and time.monotonic() < deadline: - if helper.poll() is not None: - break - try: - values = pid_path.read_text(encoding="ascii").split(",") - if len(values) == 2: - pid_values = tuple(int(value) for value in values) - except (FileNotFoundError, ValueError): - pass - time.sleep(0.05) - if pid_values is None: - stdout, stderr = helper.communicate(timeout=5) - self.fail( - "signal test helper did not start its descendants: " - f"stdout={stdout!r}, stderr={stderr!r}" - ) - command_pid, descendant_pid = pid_values - signal_started = time.monotonic() - helper.send_signal(selected_signal) - stdout, stderr = helper.communicate(timeout=15) - signal_elapsed = time.monotonic() - signal_started - deadline = time.monotonic() + 10 - while ( - _pid_is_running(command_pid) or _pid_is_running(descendant_pid) - ) and time.monotonic() < deadline: - time.sleep(0.05) - - self.assertEqual( - helper.returncode, - 128 + selected_signal, - f"stdout={stdout!r}, stderr={stderr!r}", - ) - self.assertFalse(_pid_is_running(command_pid)) - self.assertFalse(_pid_is_running(descendant_pid)) - finally: - if helper.poll() is None: - helper.kill() - helper.wait(timeout=5) - if command_pid is not None: - try: - if ( - os.getpgid(command_pid) == command_pid - and command_pid != os.getpgrp() - ): - os.killpg(command_pid, signal.SIGKILL) - except OSError: - pass - if descendant_pid is not None and _pid_is_active(descendant_pid): - os.kill(descendant_pid, signal.SIGKILL) - return signal_elapsed - - @unittest.skipIf(os.name == "nt", "POSIX signal containment test") - def test_sigint_during_blocked_shutdown_aborts_term_ignoring_server_tree(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - pid_path = root / "server-pids.txt" - descendant_ready_path = root / "server-descendant-ready.txt" - shutdown_blocked_path = root / "shutdown-blocked.txt" - descendant_program = ( - "import pathlib, signal, sys, time; " - "signal.signal(signal.SIGTERM, signal.SIG_IGN); " - "pathlib.Path(sys.argv[1]).write_text('ready', encoding='ascii'); " - "time.sleep(30)" - ) - server_program = f""" -import os -import pathlib -import signal -import subprocess -import sys -import time - -signal.signal(signal.SIGTERM, signal.SIG_IGN) -child = subprocess.Popen([sys.executable, "-c", {descendant_program!r}, sys.argv[2]]) -while not pathlib.Path(sys.argv[2]).is_file(): - time.sleep(0.01) -pathlib.Path(sys.argv[1]).write_text( - f"{{os.getpid()}},{{child.pid}}", - encoding="ascii", -) -time.sleep(30) -""" - helper_program = f""" -import contextlib -import importlib.util -import os -import pathlib -import sys -import time - -spec = importlib.util.spec_from_file_location("shutdown_runner", {str(RUNNER_PATH)!r}) -runner = importlib.util.module_from_spec(spec) -spec.loader.exec_module(runner) -containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" -) -server = runner.subprocess.Popen( - [ - sys.executable, - "-c", - {server_program!r}, - {str(pid_path)!r}, - {str(descendant_ready_path)!r}, - ], - **containment.popen_kwargs(), -) -containment.assign(server) - -def blocked_shutdown(_request, timeout): - del timeout - pathlib.Path({str(shutdown_blocked_path)!r}).write_text( - "blocked", - encoding="ascii", - ) - time.sleep(30) - return contextlib.nullcontext() - -runner._open_local_url = blocked_shutdown -try: - with runner._CancellationSignalGuard(): - try: - runner._shutdown_server( - server, - 13305, - pathlib.Path({str(root)!r}), - containment, - ) - runner._raise_if_cancelled() - finally: - runner._close_process_containment(containment) -except runner._ValidationCancelled as exc: - raise SystemExit(128 + exc.signum) from None -""" - helper = subprocess.Popen( - [sys.executable, "-c", helper_program], - cwd=ROOT, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - server_pid = None - descendant_pid = None - try: - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - try: - pid_values = tuple( - int(value) - for value in pid_path.read_text(encoding="ascii").split(",") - ) - except (FileNotFoundError, ValueError): - pid_values = () - if len(pid_values) == 2 and shutdown_blocked_path.is_file(): - server_pid, descendant_pid = pid_values - break - if helper.poll() is not None: - break - time.sleep(0.05) - if server_pid is None or descendant_pid is None: - stdout, stderr = helper.communicate(timeout=5) - self.fail( - "shutdown signal helper did not reach the blocked request: " - f"stdout={stdout!r}, stderr={stderr!r}" - ) - - signal_started = time.monotonic() - helper.send_signal(signal.SIGINT) - try: - stdout, stderr = helper.communicate(timeout=6) - except subprocess.TimeoutExpired: - self.fail("blocked shutdown ignored cancellation for 6 seconds") - signal_elapsed = time.monotonic() - signal_started - - self.assertEqual( - helper.returncode, - 128 + signal.SIGINT, - f"stdout={stdout!r}, stderr={stderr!r}", - ) - self.assertLess(signal_elapsed, 6.0) - self.assertFalse(_pid_is_running(server_pid)) - self.assertFalse(_pid_is_running(descendant_pid)) - finally: - if helper.poll() is None: - helper.kill() - if server_pid is not None: - try: - if ( - os.getpgid(server_pid) == server_pid - and server_pid != os.getpgrp() - ): - os.killpg(server_pid, signal.SIGKILL) - except OSError: - pass - try: - helper.communicate(timeout=5) - except subprocess.TimeoutExpired: - helper.kill() - helper.wait(timeout=5) - - @unittest.skipIf(os.name == "nt", "POSIX signal cleanup test") - @unittest.skipIf( - not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, - "Darwin process-tree inspection is unavailable", - ) - def test_sigint_during_orphan_cleanup_aborts_containment_and_orphan(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - cache = root / "cache" - cache.mkdir() - contained_pids_path = root / "contained-pids.txt" - contained_descendant_ready_path = root / "contained-ready.txt" - orphan_pid_path = root / "orphan.pid" - orphan_ready_path = root / "orphan-ready.txt" - orphan_term_path = root / "orphan-term.txt" - contained_descendant_program = ( - "import pathlib, signal, sys, time; " - "signal.signal(signal.SIGTERM, signal.SIG_IGN); " - "pathlib.Path(sys.argv[1]).write_text('ready', encoding='ascii'); " - "time.sleep(30)" - ) - contained_program = f""" -import os -import pathlib -import signal -import subprocess -import sys -import time - -signal.signal(signal.SIGTERM, signal.SIG_IGN) -child = subprocess.Popen( - [sys.executable, "-c", {contained_descendant_program!r}, sys.argv[2]], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, -) -while not pathlib.Path(sys.argv[2]).is_file(): - time.sleep(0.01) -pathlib.Path(sys.argv[1]).write_text( - f"{{os.getpid()}},{{child.pid}}", - encoding="ascii", -) -time.sleep(30) -""" - orphan_program = """ -import pathlib -import signal -import sys -import time - -term_path = pathlib.Path(sys.argv[2]) - -def ignore_term(_signum, _frame): - term_path.write_text("term", encoding="ascii") - -signal.signal(signal.SIGTERM, ignore_term) -pathlib.Path(sys.argv[1]).write_text("ready", encoding="ascii") -while True: - time.sleep(1) -""" - helper_program = f""" -import importlib.util -import os -import pathlib -import signal -import subprocess -import sys -import time -import psutil as real_psutil - -spec = importlib.util.spec_from_file_location("orphan_runner", {str(RUNNER_PATH)!r}) -runner = importlib.util.module_from_spec(spec) -spec.loader.exec_module(runner) -actual_containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" -) -contained = subprocess.Popen( - [ - sys.executable, - "-c", - {contained_program!r}, - {str(contained_pids_path)!r}, - {str(contained_descendant_ready_path)!r}, - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - **actual_containment.popen_kwargs(), -) -actual_containment.assign(contained) -orphan = subprocess.Popen( - [ - sys.executable, - "-c", - {orphan_program!r}, - {str(orphan_ready_path)!r}, - {str(orphan_term_path)!r}, - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, -) -while not pathlib.Path({str(orphan_ready_path)!r}).is_file(): - time.sleep(0.01) -pathlib.Path({str(orphan_pid_path)!r}).write_text(str(orphan.pid), encoding="ascii") - -class OrphanProcess: - pid = orphan.pid - - @staticmethod - def terminate(): - os.kill(orphan.pid, signal.SIGTERM) - - @staticmethod - def kill(): - os.kill(orphan.pid, signal.SIGKILL) - -def wait_procs(processes, timeout): - deadline = time.monotonic() + timeout - while True: - alive = [] - for process in processes: - try: - waited_pid, _status = os.waitpid(process.pid, os.WNOHANG) - except ChildProcessError: - waited_pid = process.pid - if waited_pid == 0: - alive.append(process) - if not alive: - return processes, [] - if time.monotonic() >= deadline: - return [], alive - time.sleep(0.01) - -real_psutil.wait_procs = wait_procs -sys.modules["psutil"] = real_psutil -runner._cache_processes = lambda _cache, _excluded: runner._CacheProcessScan( - [OrphanProcess()], - True, -) - -class DeferredContainment: - @staticmethod - def close(): - return False - - @staticmethod - def abort(deadline): - return actual_containment.abort(deadline) - -class ExitedProcess: - pid = os.getpid() - - @staticmethod - def poll(): - return 0 - - @staticmethod - def wait(timeout): - del timeout - return 0 - -try: - with runner._CancellationSignalGuard(): - runner._shutdown_server( - ExitedProcess(), - 13305, - pathlib.Path({str(cache)!r}), - DeferredContainment(), - ) - runner._raise_if_cancelled() -except runner._ValidationCancelled as exc: - raise SystemExit(128 + exc.signum) from None -""" - helper = subprocess.Popen( - [sys.executable, "-c", helper_program], - cwd=ROOT, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - orphan_pid = None - contained_pid = None - contained_descendant_pid = None - try: - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - try: - orphan_pid = int(orphan_pid_path.read_text(encoding="ascii")) - except (FileNotFoundError, ValueError): - orphan_pid = None - try: - contained_pids = tuple( - int(value) - for value in contained_pids_path.read_text( - encoding="ascii" - ).split(",") - ) - except (FileNotFoundError, ValueError): - contained_pids = () - if len(contained_pids) == 2: - contained_pid, contained_descendant_pid = contained_pids - if ( - orphan_pid is not None - and contained_pid is not None - and contained_descendant_pid is not None - and orphan_term_path.is_file() - ): - break - if helper.poll() is not None: - break - time.sleep(0.05) - if ( - orphan_pid is None - or contained_pid is None - or contained_descendant_pid is None - or not orphan_term_path.is_file() - ): - stdout, stderr = helper.communicate(timeout=5) - self.fail( - "shutdown helper did not reach orphan cleanup: " - f"stdout={stdout!r}, stderr={stderr!r}" - ) - - signal_started = time.monotonic() - helper.send_signal(signal.SIGINT) - try: - stdout, stderr = helper.communicate(timeout=3) - except subprocess.TimeoutExpired: - self.fail("orphan cleanup ignored cancellation for 3 seconds") - signal_elapsed = time.monotonic() - signal_started - - self.assertEqual( - helper.returncode, - 128 + signal.SIGINT, - f"stdout={stdout!r}, stderr={stderr!r}", - ) - self.assertLess(signal_elapsed, 3.0) - self.assertFalse( - _pid_is_running(orphan_pid), - f"stdout={stdout!r}, stderr={stderr!r}", - ) - self.assertFalse( - _pid_is_running(contained_pid), - f"stdout={stdout!r}, stderr={stderr!r}", - ) - self.assertFalse( - _pid_is_running(contained_descendant_pid), - f"stdout={stdout!r}, stderr={stderr!r}", - ) - finally: - if helper.poll() is None: - helper.kill() - if orphan_pid is not None and _pid_is_active(orphan_pid): - os.kill(orphan_pid, signal.SIGKILL) - if contained_pid is not None: - try: - if ( - os.getpgid(contained_pid) == contained_pid - and contained_pid != os.getpgrp() - ): - os.killpg(contained_pid, signal.SIGKILL) - except OSError: - pass - try: - helper.communicate(timeout=5) - except subprocess.TimeoutExpired: - helper.kill() - helper.wait(timeout=5) - - @unittest.skipIf(os.name == "nt", "POSIX process-group test") - def test_shutdown_terminates_an_external_descendant_in_the_server_session(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - child_pid_path = root / "child.pid" - containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" - ) - parent = subprocess.Popen( - [ - sys.executable, - "-c", - ( - "import pathlib, subprocess, sys; " - "child = subprocess.Popen(['/bin/sleep', '30']); " - "pathlib.Path(sys.argv[1]).write_text(str(child.pid), " - "encoding='ascii')" - ), - str(child_pid_path), - ], - **containment.popen_kwargs(), - ) - containment.assign(parent) - parent.wait(timeout=5) - child_pid = int(child_pid_path.read_text(encoding="ascii")) - try: - self.assertTrue(_pid_is_active(child_pid)) - with mock.patch.object( - runner, - "_cache_processes", - return_value=runner._CacheProcessScan([], True), - ): - clean = runner._shutdown_server( - parent, - 13305, - root, - containment, - ) - - self.assertFalse(clean) - deadline = time.monotonic() + 5 - while _pid_is_active(child_pid) and time.monotonic() < deadline: - time.sleep(0.05) - self.assertFalse(_pid_is_active(child_pid)) - self.assertFalse(containment.close()) - finally: - if _pid_is_active(child_pid): - os.kill(child_pid, signal.SIGKILL) - - @unittest.skipIf(os.name == "nt", "POSIX process-group test") - def test_validation_command_descendants_are_in_the_contained_session(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - child_pid_path = Path(directory) / "child.pid" - containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" - ) - exit_code = runner._run_contained_command( - [ - sys.executable, - "-c", - ( - "import pathlib, subprocess, sys; " - "child = subprocess.Popen(['/bin/sleep', '30']); " - "pathlib.Path(sys.argv[1]).write_text(str(child.pid), " - "encoding='ascii')" - ), - str(child_pid_path), - ], - env=dict(os.environ), - containment=containment, - timeout_seconds=30, - ) - child_pid = int(child_pid_path.read_text(encoding="ascii")) - try: - self.assertEqual(exit_code, 0) - self.assertTrue(_pid_is_active(child_pid)) - self.assertFalse(containment.close()) - deadline = time.monotonic() + 5 - while _pid_is_active(child_pid) and time.monotonic() < deadline: - time.sleep(0.05) - self.assertFalse(_pid_is_active(child_pid)) - finally: - if _pid_is_active(child_pid): - os.kill(child_pid, signal.SIGKILL) - - @unittest.skipIf(os.name == "nt", "POSIX descendant tracking test") - @unittest.skipIf( - not DARWIN_PROCESS_TREE_INSPECTION_AVAILABLE, - "Darwin process-tree inspection is unavailable", - ) - def test_close_kills_retained_session_escape_after_parent_exit(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - child_pid_path = root / "child.pid" - child_ready_path = root / "child-ready.txt" - descendant_program = ( - "import os, pathlib, sys, time; " - "os.setsid(); " - "pathlib.Path(sys.argv[1]).write_text('ready', encoding='ascii'); " - "time.sleep(30)" - ) - parent_program = f""" -import pathlib -import subprocess -import sys -import time - -child = subprocess.Popen( - [sys.executable, "-c", {descendant_program!r}, sys.argv[2]], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, -) -ready = pathlib.Path(sys.argv[2]) -deadline = time.monotonic() + 5 -while not ready.is_file() and time.monotonic() < deadline: - time.sleep(0.01) -pathlib.Path(sys.argv[1]).write_text(str(child.pid), encoding="ascii") -time.sleep(0.5) -""" - containment = runner._PosixProcessGroupContainment( - allow_untracked_descendants=sys.platform == "darwin" - ) - exit_code = runner._run_contained_command( - [ - sys.executable, - "-c", - parent_program, - str(child_pid_path), - str(child_ready_path), - ], - env=dict(os.environ), - containment=containment, - timeout_seconds=30, - ) - child_pid = int(child_pid_path.read_text(encoding="ascii")) - try: - self.assertEqual(exit_code, 0) - self.assertTrue(_pid_is_running(child_pid)) - self.assertFalse(containment.close()) - deadline = time.monotonic() + 5 - while _pid_is_running(child_pid) and time.monotonic() < deadline: - time.sleep(0.05) - self.assertFalse(_pid_is_running(child_pid)) - finally: - if _pid_is_active(child_pid): - os.kill(child_pid, signal.SIGKILL) - - def test_windows_job_object_sets_kill_on_close_and_closes_once(self): - runner = load_runner() - events = [] - - class FakeKernel32: - @staticmethod - def CreateJobObjectW(_attributes, _name): - events.append("create") - return 123 - - @staticmethod - def SetInformationJobObject(handle, info_class, pointer, size): - information = ctypes.cast( - pointer, - ctypes.POINTER(runner._JobObjectExtendedLimitInformation), - ).contents - events.append( - ( - "configure", - handle, - info_class, - information.basic_limit_information.limit_flags, - size, - ) - ) - return 1 - - @staticmethod - def AssignProcessToJobObject(handle, process_handle): - events.append(("assign", handle, process_handle)) - return 1 - - @staticmethod - def QueryInformationJobObject( - handle, info_class, pointer, size, _return_size - ): - information = ctypes.cast( - pointer, - ctypes.POINTER(runner._JobObjectBasicAccountingInformation), - ).contents - information.active_processes = 0 - events.append(("query", handle, info_class, size)) - return 1 - - @staticmethod - def CloseHandle(handle): - events.append(("close", handle)) - return 1 - - job = runner._WindowsKillOnCloseJob.create(FakeKernel32()) - process = argparse.Namespace(_handle=456) - job.assign(process) - - self.assertTrue(job.abort(time.monotonic() + 4)) - self.assertTrue(job.close()) - self.assertEqual(events[0], "create") - self.assertEqual( - events[1][0:4], - ( - "configure", - 123, - 9, - 0x00002000, - ), - ) - self.assertEqual( - events[1][4], ctypes.sizeof(runner._JobObjectExtendedLimitInformation) - ) - self.assertEqual(events[2], ("assign", 123, 456)) - self.assertEqual( - events[3:], - [ - ( - "query", - 123, - 1, - ctypes.sizeof(runner._JobObjectBasicAccountingInformation), - ), - ("close", 123), - ], - ) - - def test_windows_gate_assigns_before_fast_descendants_and_job_close_kills_tree( - self, - ): - runner = load_runner() - events = [] - assigned = False - descendants = {"child": False, "grandchild": False} - - class FakeStartupInfo: - pass - - class FakeProcess: - _handle = 456 - pid = 654 - - class FakeKernel32: - @staticmethod - def CreateEventW(_attributes, _manual_reset, _initial_state, _name): - events.append("create-gate") - return 789 - - @staticmethod - def SetHandleInformation(handle, mask, flags): - events.append(("inherit", handle, mask, flags)) - return 1 - - @staticmethod - def AssignProcessToJobObject(handle, process_handle): - nonlocal assigned - events.append(("assign", handle, process_handle)) - assigned = True - return 1 - - @staticmethod - def SetEvent(handle): - events.append(("release", handle)) - if not assigned: - raise AssertionError( - "candidate gate released before Job assignment" - ) - descendants["child"] = True - descendants["grandchild"] = True - return 1 - - @staticmethod - def QueryInformationJobObject( - _handle, _info_class, pointer, _size, _return_size - ): - information = ctypes.cast( - pointer, - ctypes.POINTER(runner._JobObjectBasicAccountingInformation), - ).contents - information.active_processes = sum(descendants.values()) + 1 - return 1 - - @staticmethod - def CloseHandle(handle): - events.append(("close", handle)) - if handle == 123: - descendants["child"] = False - descendants["grandchild"] = False - return 1 - - def popen(command, **kwargs): - events.append("popen-gate") - self.assertEqual( - command[0:5], - [sys.executable, "-I", "-S", "-c", runner._WINDOWS_GATE_PROGRAM], - ) - self.assertEqual(command[5:], ["789", "candidate", "spawn-now"]) - self.assertTrue(kwargs["close_fds"]) - self.assertEqual( - kwargs["startupinfo"].lpAttributeList, - {"handle_list": [789]}, - ) - self.assertFalse(any(descendants.values())) - return FakeProcess() - - job = runner._WindowsKillOnCloseJob(FakeKernel32(), 123) - with ( - mock.patch.object( - runner.subprocess, - "STARTUPINFO", - FakeStartupInfo, - create=True, - ), - mock.patch.object(runner.subprocess, "Popen", side_effect=popen), - ): - process = runner._start_contained_process( - job, - ["candidate", "spawn-now"], - env={}, - ) - - self.assertEqual(process.pid, 654) - self.assertTrue(all(descendants.values())) - self.assertLess( - events.index(("assign", 123, 456)), events.index(("release", 789)) - ) - self.assertLess( - events.index(("inherit", 789, 1, 0)), events.index(("release", 789)) - ) - self.assertFalse(job.close()) - self.assertFalse(any(descendants.values())) - - def test_windows_gate_assignment_failure_kills_wrapper_and_closes_raw_handles( - self, - ): - runner = load_runner() - events = [] - - class FakeStartupInfo: - pass - - class FakeProcess: - _handle = 456 - pid = 654 - - def __init__(self): - self.returncode = None - - def poll(self): - return self.returncode - - def kill(self): - events.append("kill-wrapper") - self.returncode = 1 - - def wait(self, timeout): - events.append(("wait-wrapper", timeout)) - return self.returncode - - class FakeKernel32: - @staticmethod - def CreateEventW(_attributes, _manual_reset, _initial_state, _name): - return 789 - - @staticmethod - def SetHandleInformation(_handle, _mask, _flags): - return 1 - - @staticmethod - def AssignProcessToJobObject(_handle, _process_handle): - events.append("assign-failed") - return 0 - - @staticmethod - def SetEvent(_handle): - events.append("released") - return 1 - - @staticmethod - def QueryInformationJobObject( - _handle, _info_class, pointer, _size, _return_size - ): - information = ctypes.cast( - pointer, - ctypes.POINTER(runner._JobObjectBasicAccountingInformation), - ).contents - information.active_processes = 0 - return 1 - - @staticmethod - def CloseHandle(handle): - events.append(("close", handle)) - return 1 - - process = FakeProcess() - job = runner._WindowsKillOnCloseJob(FakeKernel32(), 123) - with ( - mock.patch.object( - runner.subprocess, - "STARTUPINFO", - FakeStartupInfo, - create=True, - ), - mock.patch.object(runner.subprocess, "Popen", return_value=process), - mock.patch.object(runner, "_windows_last_error", return_value=5), - self.assertRaisesRegex(OSError, "AssignProcessToJobObject"), - ): - job.start_process(["candidate"], env={}) - - self.assertNotIn("released", events) - self.assertIn("kill-wrapper", events) - self.assertIn(("close", 123), events) - self.assertIn(("close", 789), events) - - @unittest.skipUnless(os.name == "nt", "Windows Job Object integration test") - def test_windows_job_close_kills_real_gated_child_and_grandchild(self): - runner = load_runner() - grandchild_program = "import time; time.sleep(60)" - child_program = f""" -import os -import pathlib -import subprocess -import sys -import time - -grandchild = subprocess.Popen([sys.executable, "-c", {grandchild_program!r}]) -pathlib.Path(sys.argv[1]).write_text( - f"{{os.getpid()}},{{grandchild.pid}}", encoding="ascii" -) -time.sleep(60) -""" - candidate_program = f""" -import subprocess -import sys -import time - -subprocess.Popen([sys.executable, "-c", {child_program!r}, sys.argv[1]]) -time.sleep(60) -""" - - with tempfile.TemporaryDirectory() as directory: - pid_path = Path(directory) / "descendants.txt" - job = runner._WindowsKillOnCloseJob.create() - process = None - descendant_pids = [] - try: - process = job.start_process( - [sys.executable, "-c", candidate_program, str(pid_path)], - env=dict(os.environ), - ) - deadline = time.monotonic() + 10 - while not pid_path.is_file() and time.monotonic() < deadline: - time.sleep(0.05) - self.assertTrue(pid_path.is_file()) - descendant_pids = [ - int(value) - for value in pid_path.read_text(encoding="ascii").split(",") - ] - self.assertTrue(all(_pid_is_running(pid) for pid in descendant_pids)) - self.assertFalse(job.close()) - deadline = time.monotonic() + 5 - while ( - any(_pid_is_running(pid) for pid in descendant_pids) - and time.monotonic() < deadline - ): - time.sleep(0.05) - self.assertFalse(any(_pid_is_running(pid) for pid in descendant_pids)) - finally: - job.close() - if process is not None: - runner._reap_direct_process(process, time.monotonic() + 5) - try: - import psutil - except ModuleNotFoundError: - psutil = None - if psutil is not None: - for pid in descendant_pids: - try: - psutil.Process(pid).kill() - except psutil.Error: - pass - - def test_windows_job_query_failure_closes_the_job_and_fails_closed(self): - runner = load_runner() - closed_handles = [] - - class FakeKernel32: - @staticmethod - def QueryInformationJobObject( - _handle, _info_class, _pointer, _size, _return_size - ): - return 0 - - @staticmethod - def CloseHandle(handle): - closed_handles.append(handle) - return 1 - - job = runner._WindowsKillOnCloseJob(FakeKernel32(), 123) - - self.assertFalse(job.close()) - self.assertFalse(job.close()) - self.assertEqual(closed_handles, [123]) - - def test_windows_job_configuration_failure_closes_the_raw_handle(self): - runner = load_runner() - closed_handles = [] - - class FakeKernel32: - @staticmethod - def CreateJobObjectW(_attributes, _name): - return 123 - - @staticmethod - def SetInformationJobObject(_handle, _info_class, _pointer, _size): - return 0 - - @staticmethod - def CloseHandle(handle): - closed_handles.append(handle) - return 1 - - with self.assertRaisesRegex(OSError, "SetInformationJobObject"): - runner._WindowsKillOnCloseJob.create(FakeKernel32()) - - self.assertEqual(closed_handles, [123]) - - def test_server_environment_inherits_only_minimal_runtime_values(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - defaults = Path(directory) / "defaults.json" - defaults.write_text("{}", encoding="utf-8") - environment = runner.build_server_environment( - Path(directory), - inherited={ - "LEMONADE_API_KEY": "user-key", - "LEMONADE_ADMIN_API_KEY": "admin-key", - "LEMONADE_DEFAULTS_PATH": "/host/defaults.json", - "LEMONADE_LLAMACPP_VULKAN_BIN": "/host/llama-server", - "lemonade_rocm_install_method": "wheel", - "LEMONADE_BACKEND_WATCHDOG": "0", - "lemonade_backend_watchdog_poll_seconds": "999", - "LEMONADE_CACHE_DIR": "/host/lemonade-cache", - "LEMONADE_GGML_HIP_PATH": "/host/libggml-hip.so", - "HF_HOME": "/host/huggingface", - "HF_TOKEN": "download-token", - "hugging_face_hub_token": "legacy-download-token", - "HUGGINGFACE_TOKEN": "alternate-download-token", - "LEMONADE_ALLOWED_ORIGINS": "https://example.invalid", - "rocm_path": "/host/rocm", - "HF_ENDPOINT": "https://example.invalid", - "HTTP_PROXY": "http://proxy.invalid:3128", - "https_proxy": "http://proxy.invalid:3128", - "ALL_PROXY": "socks5://proxy.invalid:1080", - "NO_PROXY": "example.invalid", - "LLAMA_ARG_MODEL": "/host/model.gguf", - "LLAMA_ARG_N_GPU_LAYERS": "999", - "llama_arg_ctx_size": "1", - "lemonade_ci_mode": "false", - "PATH": "/trusted/bin", - "LANG": "en_US.UTF-8", - "LC_ALL": "C.UTF-8", - "LC_CTYPE": "UTF-8", - "TZ": "UTC", - "SSL_CERT_FILE": "/trusted/ca.pem", - "SSL_CERT_DIR": "/trusted/ca", - "CURL_CA_BUNDLE": "/trusted/curl-ca.pem", - "REQUESTS_CA_BUNDLE": "/trusted/requests-ca.pem", - "SystemRoot": "C:\\Windows", - "windir": "C:\\Windows", - "COMSPEC": "C:\\Windows\\System32\\cmd.exe", - "PATHEXT": ".COM;.EXE;.BAT;.CMD", - "SYSTEMDRIVE": "C:", - "PROCESSOR_ARCHITECTURE": "AMD64", - "NUMBER_OF_PROCESSORS": "8", - "CUDA_VISIBLE_DEVICES": "0", - "NVIDIA_VISIBLE_DEVICES": "GPU-123", - "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", - "__NV_PRIME_RENDER_OFFLOAD": "1", - "HIP_VISIBLE_DEVICES": "0", - "ROCR_VISIBLE_DEVICES": "0", - "GPU_DEVICE_ORDINAL": "0", - "HSA_OVERRIDE_GFX_VERSION": "11.5.1", - "VK_ICD_FILENAMES": "/trusted/vulkan/icd.json", - "VK_DRIVER_FILES": "/trusted/vulkan/driver.json", - "VK_ADD_DRIVER_FILES": "/trusted/vulkan/additional.json", - "GGML_METAL_NO_RESIDENCY": "1", - "HOME": "/runner/profile", - "USERPROFILE": "C:\\Users\\runner", - "APPDATA": "C:\\Users\\runner\\AppData\\Roaming", - "LOCALAPPDATA": "C:\\Users\\runner\\AppData\\Local", - "GITHUB_OUTPUT": "/runner/command/output", - "GITHUB_ENV": "/runner/command/environment", - "GITHUB_PATH": "/runner/command/path", - "GITHUB_STEP_SUMMARY": "/runner/command/summary", - "ACTIONS_RUNTIME_TOKEN": "actions-token", - "LD_PRELOAD": "/attacker/inject.so", - "DYLD_INSERT_LIBRARIES": "/attacker/inject.dylib", - "DYLD_LIBRARY_PATH": "/attacker/dylibs", - "PYTHONPATH": "/attacker/python", - "PYTHONSTARTUP": "/attacker/startup.py", - "BASH_ENV": "/attacker/bashrc", - "UNKNOWN_DEPLOYMENT_SECRET": "secret", - "PRESERVED": "value", - }, - private_temp_directory=Path(directory) / "private-temp", - candidate_defaults_path=defaults, - ) - - expected_private_temp = str((Path(directory) / "private-temp").resolve()) - self.assertNotIn("LEMONADE_API_KEY", environment) - self.assertNotIn("LEMONADE_ADMIN_API_KEY", environment) - self.assertNotIn("LEMONADE_LLAMACPP_VULKAN_BIN", environment) - self.assertNotIn("lemonade_rocm_install_method", environment) - self.assertNotIn("LEMONADE_BACKEND_WATCHDOG", environment) - self.assertNotIn("lemonade_backend_watchdog_poll_seconds", environment) - self.assertNotIn("LEMONADE_CACHE_DIR", environment) - self.assertNotIn("LEMONADE_GGML_HIP_PATH", environment) - self.assertNotIn("HF_HOME", environment) - self.assertFalse( - { - "HF_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - "HUGGINGFACE_TOKEN", - } - & {key.upper() for key in environment} - ) - self.assertNotIn("LEMONADE_ALLOWED_ORIGINS", environment) - self.assertNotIn("rocm_path", environment) - self.assertNotIn("HF_ENDPOINT", environment) - self.assertEqual( - { - key.upper(): value - for key, value in environment.items() - if key.upper().startswith("LLAMA_ARG_") - }, - {"LLAMA_ARG_LOG_VERBOSITY": "4"}, - ) - self.assertNotIn("HTTP_PROXY", environment) - self.assertNotIn("https_proxy", environment) - self.assertNotIn("ALL_PROXY", environment) - self.assertEqual(environment["NO_PROXY"], "127.0.0.1") - self.assertEqual(environment["no_proxy"], "127.0.0.1") - self.assertEqual(environment["TMPDIR"], expected_private_temp) - self.assertEqual(environment["TMP"], expected_private_temp) - self.assertEqual(environment["TEMP"], expected_private_temp) - self.assertEqual(environment["LEMONADE_DEFAULTS_PATH"], str(defaults.resolve())) - self.assertEqual(environment["LEMONADE_CI_MODE"], "True") - self.assertNotIn("lemonade_ci_mode", environment) - self.assertEqual(environment["PATH"], "/trusted/bin") - for key, value in { - "LANG": "en_US.UTF-8", - "LC_ALL": "C.UTF-8", - "LC_CTYPE": "UTF-8", - "TZ": "UTC", - "SSL_CERT_FILE": "/trusted/ca.pem", - "SSL_CERT_DIR": "/trusted/ca", - "CURL_CA_BUNDLE": "/trusted/curl-ca.pem", - "REQUESTS_CA_BUNDLE": "/trusted/requests-ca.pem", - "COMSPEC": "C:\\Windows\\System32\\cmd.exe", - "PATHEXT": ".COM;.EXE;.BAT;.CMD", - "SYSTEMDRIVE": "C:", - "PROCESSOR_ARCHITECTURE": "AMD64", - "NUMBER_OF_PROCESSORS": "8", - "CUDA_VISIBLE_DEVICES": "0", - "NVIDIA_VISIBLE_DEVICES": "GPU-123", - "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", - "__NV_PRIME_RENDER_OFFLOAD": "1", - "HIP_VISIBLE_DEVICES": "0", - "ROCR_VISIBLE_DEVICES": "0", - "GPU_DEVICE_ORDINAL": "0", - "HSA_OVERRIDE_GFX_VERSION": "11.5.1", - "VK_ICD_FILENAMES": "/trusted/vulkan/icd.json", - "VK_DRIVER_FILES": "/trusted/vulkan/driver.json", - "VK_ADD_DRIVER_FILES": "/trusted/vulkan/additional.json", - "GGML_METAL_NO_RESIDENCY": "1", - }.items(): - self.assertEqual(environment.get(key), value) - self.assertEqual(environment.get("SYSTEMROOT"), "C:\\Windows") - self.assertEqual(environment.get("WINDIR"), "C:\\Windows") - for key in ("HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA"): - self.assertEqual(environment[key], expected_private_temp) - for key in ( - "ACTIONS_RUNTIME_TOKEN", - "BASH_ENV", - "DYLD_INSERT_LIBRARIES", - "DYLD_LIBRARY_PATH", - "GITHUB_ENV", - "GITHUB_OUTPUT", - "GITHUB_PATH", - "GITHUB_STEP_SUMMARY", - "LD_PRELOAD", - "PRESERVED", - "PYTHONPATH", - "PYTHONSTARTUP", - "UNKNOWN_DEPLOYMENT_SECRET", - ): - self.assertNotIn(key, environment) - - def test_server_environment_maps_only_explicit_canonical_download_token(self): - runner = load_runner() - inherited = { - "HF_TOKEN": "download-token", - "hf_token": "ambient-case-variant", - "hugging_face_hub_token": "legacy-download-token", - "HuGgInGfAcE_ToKeN": "alternate-download-token", - } - - with tempfile.TemporaryDirectory() as directory: - environment = runner.build_server_environment( - Path(directory), - inherited=inherited, - private_temp_directory=Path(directory) / "private-temp", - allow_download_credentials=True, - ) - - self.assertEqual( - { - key.upper(): value - for key, value in environment.items() - if key.upper() - in { - "HF_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - "HUGGINGFACE_TOKEN", - } - }, - {"HF_TOKEN": "download-token"}, - ) - - def test_candidate_defaults_must_be_a_real_adjacent_build_resource(self): - runner = load_runner() - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - build = root / "build" - outside = root / "outside-resources" - build.mkdir() - outside.mkdir() - (outside / "defaults.json").write_text("{}", encoding="utf-8") - (build / "lemond").touch() - try: - (build / "resources").symlink_to( - outside, - target_is_directory=True, - ) - except OSError as exc: - self.skipTest(f"directory symlinks are unavailable: {exc}") - - with self.assertRaisesRegex(RuntimeError, "build resource"): - runner.resolve_candidate_defaults(build / "lemond") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_validate_llamacpp_action_pins.py b/test/test_validate_llamacpp_action_pins.py deleted file mode 100644 index ce9bb2e57e..0000000000 --- a/test/test_validate_llamacpp_action_pins.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -"""Supply-chain contract for protected llama.cpp validation workflows.""" - -from __future__ import annotations - -import re -import tempfile -import unittest -from pathlib import Path - -import yaml -from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode - -ROOT = Path(__file__).resolve().parents[1] -CORE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" -DOCS_AND_STYLE_WORKFLOW = ROOT / ".github" / "workflows" / "docs_and_style.yml" -MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" -PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" -SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" -SERVER_BASE = ROOT / "test" / "utils" / "server_base.py" -PROTECTED_WORKFLOWS = ( - PR_WORKFLOW, - CORE_WORKFLOW, - MANUAL_WORKFLOW, - SCHEDULE_WORKFLOW, -) -PINNED_ACTION_PATTERN = re.compile( - r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*@[0-9a-f]{40}$" -) -REVIEWED_ACTION_PINS = { - "actions/checkout": "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09", - "actions/download-artifact": "37930b1c2abaa49bbe596cd826c3c89aef350131", - "actions/setup-python": "ece7cb06caefa5fff74198d8649806c4678c61a1", - "actions/upload-artifact": "bbbca2ddaa5d8feaa63e36b76fdaad77386f024f", -} - - -def read_uses_references(path: Path) -> list[str]: - references: list[str] = [] - visited_nodes: set[int] = set() - - def visit(node: Node | None) -> None: - if node is None or id(node) in visited_nodes: - return - visited_nodes.add(id(node)) - - if isinstance(node, MappingNode): - for key, value in node.value: - if isinstance(key, ScalarNode) and key.value == "uses": - if not isinstance(value, ScalarNode): - raise ValueError(f"uses value must be a scalar in {path}") - references.append(value.value) - else: - visit(key) - visit(value) - elif isinstance(node, SequenceNode): - for value in node.value: - visit(value) - - try: - for document in yaml.compose_all( - path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader - ): - visit(document) - except yaml.YAMLError as error: - raise ValueError(f"invalid YAML in {path}: {error}") from error - return references - - -def resolve_local_uses(repository_root: Path, reference: str) -> Path: - target = (repository_root / reference.removeprefix("./")).resolve() - try: - target.relative_to(repository_root.resolve()) - except ValueError as error: - raise ValueError( - f"local action escapes repository root: {reference}" - ) from error - if target.is_dir(): - manifests = [ - candidate - for candidate in (target / "action.yml", target / "action.yaml") - if candidate.is_file() - ] - if len(manifests) != 1: - raise ValueError(f"local action has no unique manifest: {reference}") - return manifests[0] - if not target.is_file(): - raise ValueError(f"local uses target does not exist: {reference}") - return target - - -def collect_external_actions( - repository_root: Path, entrypoints: tuple[Path, ...] -) -> list[tuple[Path, str]]: - active: set[Path] = set() - visited: set[Path] = set() - external: list[tuple[Path, str]] = [] - - def visit(path: Path) -> None: - path = path.resolve() - try: - path.relative_to(repository_root.resolve()) - except ValueError as error: - raise ValueError(f"uses target escapes repository root: {path}") from error - if path in active: - raise ValueError(f"local uses cycle detected at {path}") - if path in visited: - return - active.add(path) - for reference in read_uses_references(path): - if reference.startswith("./"): - visit(resolve_local_uses(repository_root, reference)) - else: - external.append((path, reference)) - active.remove(path) - visited.add(path) - - for entrypoint in entrypoints: - visit(entrypoint) - return external - - -class LlamaCppProtectedActionPinTests(unittest.TestCase): - def test_all_protected_entrypoints_are_scanned(self) -> None: - self.assertEqual( - set(PROTECTED_WORKFLOWS), - {PR_WORKFLOW, CORE_WORKFLOW, MANUAL_WORKFLOW, SCHEDULE_WORKFLOW}, - ) - - def test_uses_are_read_from_yaml_structure(self) -> None: - with tempfile.TemporaryDirectory() as directory: - repository_root = Path(directory) / "repository" - workflow = repository_root / ".github" / "workflows" / "entry.yml" - workflow.parent.mkdir(parents=True) - workflow.write_text( - """\ -metadata: !untrusted-tag ignored -jobs: - quoted: - "uses" : "actions/checkout@main" - flow: {name: flow, 'uses': 'actions/setup-python@main'} - spaced: - uses : actions/upload-artifact@main - script: - run: | - uses: actions/download-artifact@main -""", - encoding="utf-8", - ) - - self.assertEqual( - collect_external_actions(repository_root, (workflow,)), - [ - (workflow.resolve(), "actions/checkout@main"), - (workflow.resolve(), "actions/setup-python@main"), - (workflow.resolve(), "actions/upload-artifact@main"), - ], - ) - - def test_manual_and_schedule_entrypoints_find_mutable_actions(self) -> None: - with tempfile.TemporaryDirectory() as directory: - repository_root = Path(directory) / "repository" - workflows = repository_root / ".github" / "workflows" - action = repository_root / ".github" / "actions" / "nested" - workflows.mkdir(parents=True) - action.mkdir(parents=True) - manual = workflows / "validate_llamacpp_manual.yml" - schedule = workflows / "validate_llamacpp_schedule.yml" - core = workflows / "validate_llamacpp_core.yml" - manual.write_text( - "jobs: {direct: {uses: actions/checkout@main}}\n", - encoding="utf-8", - ) - schedule.write_text( - "jobs: {delegated: {'uses': './.github/workflows/validate_llamacpp_core.yml'}}\n", - encoding="utf-8", - ) - core.write_text( - "jobs:\n nested:\n uses : ./.github/actions/nested\n", - encoding="utf-8", - ) - (action / "action.yml").write_text( - "runs: {using: composite, steps: [{uses: actions/setup-python@v6}]}\n", - encoding="utf-8", - ) - - external_actions = collect_external_actions( - repository_root, (manual, schedule) - ) - - self.assertEqual( - {reference for _, reference in external_actions}, - {"actions/checkout@main", "actions/setup-python@v6"}, - ) - for _, reference in external_actions: - self.assertNotRegex(reference, PINNED_ACTION_PATTERN) - - def test_every_external_action_uses_an_immutable_commit(self) -> None: - external_actions = collect_external_actions(ROOT, PROTECTED_WORKFLOWS) - self.assertTrue(external_actions) - for source, reference in external_actions: - with self.subTest(source=source.relative_to(ROOT), reference=reference): - self.assertRegex(reference, PINNED_ACTION_PATTERN) - action, commit = reference.rsplit("@", 1) - if action in REVIEWED_ACTION_PINS: - self.assertEqual(commit, REVIEWED_ACTION_PINS[action]) - - def test_local_action_recursion_rejects_cycles_and_root_escapes(self) -> None: - with tempfile.TemporaryDirectory() as directory: - repository_root = Path(directory) / "repository" - first = repository_root / ".github" / "actions" / "first" - second = repository_root / ".github" / "actions" / "second" - first.mkdir(parents=True) - second.mkdir(parents=True) - first_manifest = first / "action.yml" - second_manifest = second / "action.yml" - first_manifest.write_text( - "runs:\n using: composite\n steps:\n - uses: " - "./.github/actions/second\n", - encoding="utf-8", - ) - second_manifest.write_text( - "runs:\n using: composite\n steps:\n - uses: " - "./.github/actions/first\n", - encoding="utf-8", - ) - with self.assertRaisesRegex(ValueError, "cycle"): - collect_external_actions(repository_root, (first_manifest,)) - - outside = Path(directory) / "outside.yml" - outside.write_text( - "runs:\n using: composite\n steps: []\n", encoding="utf-8" - ) - first_manifest.write_text( - "runs:\n using: composite\n steps:\n - uses: ./../outside.yml\n", - encoding="utf-8", - ) - with self.assertRaisesRegex(ValueError, "escapes repository root"): - collect_external_actions(repository_root, (first_manifest,)) - - def test_focused_test_dependencies_are_installed_explicitly(self) -> None: - workflow = DOCS_AND_STYLE_WORKFLOW.read_text(encoding="utf-8") - pre_commit_job = workflow.split(" pre-commit:\n", 1)[1].split( - " markdown-link-check:\n", 1 - )[0] - install_step = pre_commit_job.split( - " - name: Run focused Python unit tests\n", 1 - )[0] - self.assertIn( - "pip install pre-commit psutil requests openai pyyaml", install_step - ) - self.assertIn( - 'python -c "import test.utils.server_base"', - pre_commit_job, - ) - server_base = SERVER_BASE.read_text(encoding="utf-8") - self.assertIn("from openai import OpenAI, AsyncOpenAI", server_base) - self.assertIn("import requests", server_base) - - def test_cross_platform_validation_step_explicitly_uses_bash(self) -> None: - workflow = CORE_WORKFLOW.read_text(encoding="utf-8") - validate_job = workflow.split(" validate:\n", 1)[1].split( - " validation-gate:\n", 1 - )[0] - step_marker = " - name: Run validation with lemond\n" - self.assertEqual(validate_job.count(step_marker), 1) - step = validate_job.split(step_marker, 1)[1].split(" - name:", 1)[0] - - self.assertIn(" runs-on: ${{ matrix.runner }}\n", validate_job) - self.assertIn("CANDIDATE_LEMOND:", step) - self.assertIn("runner.os == 'Windows'", step) - self.assertIn( - "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py", step - ) - self.assertIn('--lemond "$CANDIDATE_LEMOND"', step) - self.assertIn(" shell: bash\n", step) - self.assertNotIn(" if:", step) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_validate_llamacpp_selection.py b/test/test_validate_llamacpp_selection.py deleted file mode 100644 index b97f6f2b0a..0000000000 --- a/test/test_validate_llamacpp_selection.py +++ /dev/null @@ -1,3211 +0,0 @@ -#!/usr/bin/env python3 -"""Unit tests for llama.cpp validation model selection.""" - -from __future__ import annotations - -import argparse -import contextlib -import importlib.util -import io -import json -import socket -import subprocess -import sys -import tempfile -import threading -import time -import types -import unittest -from pathlib import Path -from unittest import mock - -from test.utils import llamacpp_capability_validation as capabilities -from test.utils import validation_model_selection as selection - -MODEL_REGISTRY = ( - Path(__file__).resolve().parents[1] - / "src" - / "cpp" - / "resources" - / "server_models.json" -) -VALIDATION_MODEL_CATALOG = ( - Path(__file__).resolve().parent / "fixtures" / "llamacpp_validation_models.json" -) -VALIDATION_ARTIFACTS = ( - Path(__file__).resolve().parent / "fixtures" / "llamacpp_validation_artifacts.json" -) -LLAMACPP_GUIDE = ( - Path(__file__).resolve().parents[1] - / "docs" - / "guide" - / "configuration" - / "llamacpp.md" -) - - -def load_validation_module(): - requests_module = types.ModuleType("requests") - requests_module.RequestException = RuntimeError - - class StubSession: - trust_env = True - - @staticmethod - def request(*args, **kwargs): - return requests_module.request(*args, **kwargs) - - @staticmethod - def close(): - return None - - requests_module.Session = StubSession - - server_base_module = types.ModuleType("utils.server_base") - server_base_module._auth_headers = lambda: {} - server_base_module.pull_model_with_retry = lambda *_args, **_kwargs: None - server_base_module.unload_all_models = lambda **_kwargs: None - server_base_module.wait_for_server = lambda **_kwargs: None - - test_models_module = types.ModuleType("utils.test_models") - test_models_module.PORT = 13305 - test_models_module.TIMEOUT_DEFAULT = 60 - - utils_package = types.ModuleType("utils") - utils_package.__path__ = [] - module_path = Path(__file__).resolve().parent / "validate_llamacpp.py" - spec = importlib.util.spec_from_file_location( - "validate_llamacpp_under_test", module_path - ) - if spec is None or spec.loader is None: - raise RuntimeError(f"Cannot load {module_path}") - - module = importlib.util.module_from_spec(spec) - stubs = { - "requests": requests_module, - "utils": utils_package, - "utils.llamacpp_capability_validation": capabilities, - "utils.server_base": server_base_module, - "utils.test_models": test_models_module, - "utils.validation_model_selection": selection, - } - with mock.patch.dict(sys.modules, stubs): - spec.loader.exec_module(module) - return module - - -VALIDATION = load_validation_module() - - -@contextlib.contextmanager -def stalled_http_response(prefix: bytes, suffix: bytes, stall_seconds=10): - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - listener.settimeout(30) - address = listener.getsockname() - release = threading.Event() - prefix_sent = threading.Event() - stall_elapsed = threading.Event() - - def serve(): - try: - connection, _address = listener.accept() - with connection: - request = b"" - while b"\r\n\r\n" not in request: - chunk = connection.recv(4096) - if not chunk: - return - request += chunk - connection.sendall(prefix) - prefix_sent.set() - if not release.wait(stall_seconds): - stall_elapsed.set() - connection.sendall(suffix) - except OSError: - pass - - server = threading.Thread(target=serve) - server.start() - try: - yield ( - f"http://127.0.0.1:{address[1]}/stream", - prefix_sent, - stall_elapsed, - ) - finally: - release.set() - try: - socket.create_connection(address, timeout=0.1).close() - except OSError: - pass - listener.close() - server.join(timeout=5) - if server.is_alive(): - raise AssertionError("local test server did not stop") - - -@contextlib.contextmanager -def trickling_http_response(): - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - listener.settimeout(30) - address = listener.getsockname() - release = threading.Event() - body_started = threading.Event() - trickle_elapsed = threading.Event() - - def serve(): - try: - connection, _address = listener.accept() - with connection: - request = b"" - while b"\r\n\r\n" not in request: - chunk = connection.recv(4096) - if not chunk: - return - request += chunk - connection.sendall( - b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" - b"Connection: close\r\n\r\n" - ) - body_started.set() - automatic_deadline = time.monotonic() + 30 - while not release.wait(0.01): - if time.monotonic() >= automatic_deadline: - trickle_elapsed.set() - return - connection.sendall(b"x") - except OSError: - pass - - server = threading.Thread(target=serve) - server.start() - try: - yield ( - f"http://127.0.0.1:{address[1]}/stream", - body_started, - trickle_elapsed, - ) - finally: - release.set() - try: - socket.create_connection(address, timeout=0.1).close() - except OSError: - pass - listener.close() - server.join(timeout=5) - if server.is_alive(): - raise AssertionError("local trickle server did not stop") - - -@contextlib.contextmanager -def trickling_http_headers(): - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - listener.settimeout(30) - address = listener.getsockname() - release = threading.Event() - headers_started = threading.Event() - trickle_elapsed = threading.Event() - - def serve(): - try: - connection, _address = listener.accept() - with connection: - request = b"" - while b"\r\n\r\n" not in request: - chunk = connection.recv(4096) - if not chunk: - return - request += chunk - connection.sendall(b"HTTP/1.1 200 OK\r\nX-Padding: ") - headers_started.set() - automatic_deadline = time.monotonic() + 30 - while not release.wait(0.01): - if time.monotonic() >= automatic_deadline: - trickle_elapsed.set() - return - connection.sendall(b"x") - except OSError: - pass - - server = threading.Thread(target=serve) - server.start() - try: - yield ( - f"http://127.0.0.1:{address[1]}/json", - headers_started, - trickle_elapsed, - ) - finally: - release.set() - try: - socket.create_connection(address, timeout=0.1).close() - except OSError: - pass - listener.close() - server.join(timeout=5) - if server.is_alive(): - raise AssertionError("local header trickle server did not stop") - - -@contextlib.contextmanager -def fixed_http_response(body: bytes): - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - listener.settimeout(30) - address = listener.getsockname() - response_sent = threading.Event() - - def serve(): - try: - connection, _address = listener.accept() - with connection: - request = b"" - while b"\r\n\r\n" not in request: - chunk = connection.recv(4096) - if not chunk: - return - request += chunk - connection.sendall( - b"HTTP/1.1 200 OK\r\n" - b"Content-Type: text/plain; charset=utf-8\r\n" - + f"Content-Length: {len(body)}\r\n".encode("ascii") - + b"X-Test: bounded\r\nConnection: close\r\n\r\n" - + body - ) - response_sent.set() - except OSError: - pass - - server = threading.Thread(target=serve) - server.start() - try: - yield f"http://127.0.0.1:{address[1]}/json", response_sent - finally: - try: - socket.create_connection(address, timeout=0.1).close() - except OSError: - pass - listener.close() - server.join(timeout=5) - if server.is_alive(): - raise AssertionError("local fixed-response server did not stop") - - -@contextlib.contextmanager -def held_open_pull_response(body: bytes): - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - listener.settimeout(30) - address = listener.getsockname() - terminal_sent = threading.Event() - release = threading.Event() - - def serve(): - try: - connection, _address = listener.accept() - with connection: - request = b"" - while b"\r\n\r\n" not in request: - chunk = connection.recv(4096) - if not chunk: - return - request += chunk - connection.sendall( - b"HTTP/1.1 200 OK\r\n" - b"Content-Type: text/event-stream\r\n" - b"Connection: close\r\n\r\n" + body - ) - terminal_sent.set() - release.wait(30) - except OSError: - pass - - server = threading.Thread(target=serve) - server.start() - try: - yield ( - f"http://127.0.0.1:{address[1]}/api/v1/pull", - terminal_sent, - release, - ) - finally: - release.set() - try: - socket.create_connection(address, timeout=0.1).close() - except OSError: - pass - listener.close() - server.join(timeout=5) - if server.is_alive(): - raise AssertionError("held-open pull response server did not stop") - - -class ChunkedResponse: - def __init__(self, chunks, status_code=200, content_type="text/event-stream"): - self.chunks = chunks - self.status_code = status_code - self.headers = {"Content-Type": content_type} - self.encoding = "utf-8" - self.closed = False - - def iter_content(self, **_kwargs): - yield from self.chunks - - def close(self): - self.closed = True - - -class LlamaCppValidationSelectionTests(unittest.TestCase): - def setUp(self) -> None: - self.catalog = [ - { - "id": "Zulu-Hot", - "recipe": "llamacpp", - "labels": ["chat", "hot"], - "checkpoint": "example/zulu:Zulu.gguf", - "size": 4.0, - }, - { - "id": "Alpha-Hot", - "recipe": "llamacpp", - "labels": ["hot", "chat"], - "checkpoint": "example/alpha:Alpha.gguf", - "size": 1.0, - }, - { - "id": "Cold-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/cold:Cold.gguf", - "size": 2.0, - }, - { - "id": "Hot-Image", - "recipe": "sdcpp", - "labels": ["hot", "image"], - "checkpoint": "example/image:Image.gguf", - "size": 0.5, - }, - ] - - def select(self, requested_model_ids=None, lite=False): - def resolve_builtin_model(canonical_model_id): - model_id = canonical_model_id.removeprefix("builtin.") - return next( - (model for model in self.catalog if model["id"] == model_id), None - ) - - return selection.select_llamacpp_models( - self.catalog, - requested_model_ids=requested_model_ids, - lite=lite, - builtin_model_resolver=resolve_builtin_model, - ) - - def test_default_selection_filters_and_sorts_hot_llamacpp_models(self) -> None: - selected = self.select() - - self.assertEqual([model["id"] for model in selected], ["Alpha-Hot", "Zulu-Hot"]) - - def test_lite_selection_uses_smallest_hot_llamacpp_model(self) -> None: - selected = self.select(lite=True) - - self.assertEqual([model["id"] for model in selected], ["Alpha-Hot"]) - - def test_explicit_selection_preserves_order_and_accepts_non_hot_model(self) -> None: - selected = self.select(["Cold-Llama", "Alpha-Hot"]) - - self.assertEqual( - [model["id"] for model in selected], ["Cold-Llama", "Alpha-Hot"] - ) - - def test_explicit_selection_preserves_duplicate_ids(self) -> None: - selected = self.select(["Alpha-Hot", "Alpha-Hot"]) - - self.assertEqual( - [model["id"] for model in selected], ["Alpha-Hot", "Alpha-Hot"] - ) - - def test_explicit_selection_rejects_lite_mode(self) -> None: - with self.assertRaisesRegex(ValueError, "mutually exclusive"): - self.select(["Alpha-Hot"], lite=True) - - def test_explicit_selection_resolves_and_loads_canonical_builtin(self) -> None: - resolved_ids = [] - shadow = { - "id": "K2-Horizon-0.9B-GGUF", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "user/model:shadow.gguf", - } - built_in = { - "id": "K2-Horizon-0.9B-GGUF", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:model.gguf", - } - - def resolve_builtin_model(canonical_model_id): - resolved_ids.append(canonical_model_id) - return built_in - - selected = selection.select_llamacpp_models( - [shadow], - requested_model_ids=["K2-Horizon-0.9B-GGUF"], - builtin_model_resolver=resolve_builtin_model, - ) - - self.assertEqual(resolved_ids, ["builtin.K2-Horizon-0.9B-GGUF"]) - self.assertEqual(selected[0]["id"], "K2-Horizon-0.9B-GGUF") - self.assertEqual(selected[0]["load_id"], "builtin.K2-Horizon-0.9B-GGUF") - self.assertEqual( - selected[0]["checkpoint"], - "IFM/K2-Horizon-0.9B-GGUF:model.gguf", - ) - - def test_explicit_selection_rejects_resolver_response_for_another_model( - self, - ) -> None: - wrong_model = { - "id": "Different-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/different:model.gguf", - } - - with self.assertRaisesRegex( - selection.ModelSelectionError, - "requested.*K2-Horizon-0.9B-GGUF.*Different-Llama", - ): - selection.select_llamacpp_models( - [], - requested_model_ids=["K2-Horizon-0.9B-GGUF"], - builtin_model_resolver=lambda _model_id: wrong_model, - ) - - def test_shadowed_builtin_keeps_canonical_id_for_validation_evidence(self) -> None: - built_in = { - "id": "builtin.Shadowed-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/builtin:Shadowed-Llama.gguf", - } - - selected = selection.select_llamacpp_models( - [], - requested_model_ids=["Shadowed-Llama"], - builtin_model_resolver=lambda _model_id: built_in, - ) - - self.assertEqual(selected[0]["id"], "builtin.Shadowed-Llama") - self.assertEqual(selected[0]["load_id"], "builtin.Shadowed-Llama") - - def test_explicit_selection_rejects_unverified_bare_catalog_entry(self) -> None: - registered_model = [ - { - "id": "K2-Horizon-0.9B-GGUF", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "user/model:shadow.gguf", - } - ] - - with self.assertRaisesRegex(ValueError, "built-in"): - selection.select_llamacpp_models( - registered_model, - requested_model_ids=["K2-Horizon-0.9B-GGUF"], - ) - - def test_unknown_explicit_model_has_clear_error(self) -> None: - with self.assertRaisesRegex(ValueError, "Missing-Model"): - self.select(["Missing-Model"]) - - def test_non_llamacpp_explicit_model_has_clear_error(self) -> None: - with self.assertRaisesRegex(ValueError, "Hot-Image.*sdcpp"): - self.select(["Hot-Image"]) - - def test_non_chat_llamacpp_explicit_models_have_clear_errors(self) -> None: - for model_id, label in ( - ("Embed-Llama", "embeddings"), - ("Rerank-Llama", "reranking"), - ): - with self.subTest(label=label): - self.catalog.append( - { - "id": model_id, - "recipe": "llamacpp", - "labels": [label], - "checkpoint": f"example/{label}:model.gguf", - } - ) - - with self.assertRaisesRegex( - ValueError, f"{model_id}.*does not support chat" - ): - self.select([model_id]) - - def test_invalid_checkpoint_marks_explicit_model_unpullable(self) -> None: - records = [ - {"id": "Broken-Llama", "recipe": "llamacpp", "labels": ["chat"]}, - { - "id": "Broken-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "", - }, - { - "id": "Broken-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": 123, - }, - { - "id": "Broken-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoints": {"main": ""}, - }, - ] - for record in records: - with self.subTest(record=record): - with self.assertRaisesRegex(ValueError, "Broken-Llama.*checkpoint"): - selection.select_llamacpp_models( - [], - requested_model_ids=["Broken-Llama"], - builtin_model_resolver=lambda _model_id, value=record: value, - ) - - def test_explicit_selection_accepts_checkpoints_main(self) -> None: - model = { - "id": "Multi-Checkpoint-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoints": { - "main": "example/model:main.gguf", - "draft": "example/model:draft.gguf", - }, - } - - selected = selection.select_llamacpp_models( - [], - requested_model_ids=["Multi-Checkpoint-Llama"], - builtin_model_resolver=lambda _model_id: model, - ) - - self.assertEqual(selected[0]["id"], "Multi-Checkpoint-Llama") - - def test_lite_selection_does_not_validate_unselected_models(self) -> None: - self.catalog.append( - { - "id": "Broken-Large-Hot", - "recipe": "llamacpp", - "labels": ["chat", "hot"], - "size": 100.0, - } - ) - - selected = self.select(lite=True) - - self.assertEqual([model["id"] for model in selected], ["Alpha-Hot"]) - - def test_default_selection_keeps_hot_models_without_catalog_checkpoints( - self, - ) -> None: - self.catalog.append( - { - "id": "Local-Hot", - "recipe": "llamacpp", - "labels": ["chat", "hot"], - "size": 3.0, - } - ) - - selected = self.select() - - self.assertEqual( - [model["id"] for model in selected], - ["Alpha-Hot", "Local-Hot", "Zulu-Hot"], - ) - - def test_empty_explicit_list_uses_default_selection(self) -> None: - selected = self.select([]) - - self.assertEqual([model["id"] for model in selected], ["Alpha-Hot", "Zulu-Hot"]) - - def test_model_argument_is_repeatable(self) -> None: - parser = argparse.ArgumentParser() - selection.add_model_selection_arguments(parser) - - args = parser.parse_args(["--model", "Cold-Llama", "--model", "Alpha-Hot"]) - - self.assertEqual(args.model, ["Cold-Llama", "Alpha-Hot"]) - self.assertFalse(args.lite) - - def test_model_and_lite_arguments_are_mutually_exclusive(self) -> None: - parser = argparse.ArgumentParser() - selection.add_model_selection_arguments(parser) - - with contextlib.redirect_stderr(io.StringIO()): - with self.assertRaises(SystemExit) as raised: - parser.parse_args(["--model", "Alpha-Hot", "--lite"]) - - self.assertEqual(raised.exception.code, 2) - - -class K2HorizonCatalogTests(unittest.TestCase): - @staticmethod - def expected_models() -> dict: - return { - "K2-Horizon-0.9B-GGUF": { - "checkpoint": "IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf", - "recipe": "llamacpp", - "suggested": True, - "labels": ["chat", "reasoning", "tool-calling", "hot"], - "size": 2.16, - }, - "K2-Horizon-3.7B-GGUF": { - "checkpoint": "IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf", - "recipe": "llamacpp", - "suggested": True, - "labels": ["chat", "reasoning", "tool-calling"], - "size": 10.13, - }, - "K2-Horizon-7B-GGUF": { - "checkpoint": "IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf", - "recipe": "llamacpp", - "suggested": True, - "labels": ["chat", "reasoning", "tool-calling"], - "size": 18.01, - }, - } - - def test_product_catalog_does_not_advertise_k2_horizon(self) -> None: - product_models = json.loads(MODEL_REGISTRY.read_text(encoding="utf-8")) - - for model_id in self.expected_models(): - with self.subTest(model_id=model_id): - self.assertNotIn(model_id, product_models) - - def test_public_guide_does_not_advertise_validation_candidates(self) -> None: - guide = LLAMACPP_GUIDE.read_text(encoding="utf-8") - - self.assertNotIn("## K2-Horizon", guide) - for model_id in self.expected_models(): - with self.subTest(model_id=model_id): - self.assertNotIn(model_id, guide) - - def test_k2_horizon_validation_models_have_exact_catalog_contract(self) -> None: - models = json.loads(VALIDATION_MODEL_CATALOG.read_text(encoding="utf-8")) - self.assertEqual(models, self.expected_models()) - - def test_k2_horizon_validation_artifacts_are_immutably_bound(self) -> None: - artifacts = json.loads(VALIDATION_ARTIFACTS.read_text(encoding="utf-8")) - expected = { - "K2-Horizon-0.9B-GGUF": { - "revision": "8496259ac62d33192d0fffe708303ed2ca29a384", - "filename": "K2-Horizon-1B-BF16.gguf", - "size_bytes": 2159424896, - "sha256": "371010db1807bb07b62e738422ee0de26c1e15a347f31108ed2c6e219095a8b8", - "gguf_architecture": "k2-horizon", - "max_context_window": 131072, - }, - "K2-Horizon-3.7B-GGUF": { - "revision": "67b824b8079635b77e19a2a4cd2682a9716478c1", - "filename": "K2-Horizon-4B-BF16.gguf", - "size_bytes": 10128343424, - "sha256": "372ecf9977eb22c92739fec478f2c887daef659d7c55a609ac97b905d1a243b1", - "gguf_architecture": "k2-horizon", - "max_context_window": 524288, - }, - "K2-Horizon-7B-GGUF": { - "revision": "bcb8c25b76112ce96a962f5b8ab624435d1ee0c9", - "filename": "K2-Horizon-7B-BF16.gguf", - "size_bytes": 18010413440, - "sha256": "088c5d0814ef955d137fd1073ee3a68f6a53411fca44b75dc58a320640000444", - "gguf_architecture": "k2-horizon", - "max_context_window": 524288, - }, - } - - self.assertEqual(artifacts, expected) - - def test_model_catalogs_have_no_duplicate_ids(self) -> None: - for path in (MODEL_REGISTRY, VALIDATION_MODEL_CATALOG): - duplicate_ids = [] - - def unique_object(pairs): - parsed = {} - for key, value in pairs: - if key in parsed: - duplicate_ids.append(key) - parsed[key] = value - return parsed - - json.loads( - path.read_text(encoding="utf-8"), - object_pairs_hook=unique_object, - ) - - with self.subTest(path=path): - self.assertEqual(duplicate_ids, []) - - -class LlamaCppValidationRuntimeTests(unittest.TestCase): - @staticmethod - def response(status_code=200): - return types.SimpleNamespace(status_code=status_code) - - def test_request_json_rejects_duplicate_response_members(self) -> None: - response_text = '{"model":"wrong","model":"right"}' - with fixed_http_response(response_text.encode("utf-8")) as (url, _sent): - _response, body = VALIDATION.request_json( - "GET", - url, - timeout=5, - ) - - self.assertEqual(body, {"raw_text": response_text}) - - def test_json_transport_deadline_interrupts_unterminated_trickle(self) -> None: - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with trickling_http_response() as (url, body_started, trickle_elapsed): - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "wall-clock deadline", - ), - ): - VALIDATION.request_json("GET", url, timeout=10) - - self.assertTrue(body_started.is_set()) - self.assertFalse(trickle_elapsed.is_set()) - - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_json_transport_deadline_covers_response_header_trickle(self) -> None: - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with trickling_http_headers() as (url, headers_started, trickle_elapsed): - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "wall-clock deadline", - ), - ): - VALIDATION.request_json("GET", url, timeout=5) - - self.assertTrue(headers_started.is_set()) - self.assertFalse(trickle_elapsed.is_set()) - - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_json_transport_accepts_exact_response_byte_bound(self) -> None: - body = b"x" * VALIDATION.MAX_STREAM_SSE_BYTES - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with fixed_http_response(body) as (url, response_sent): - with mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ): - response, parsed = VALIDATION.request_json("GET", url, timeout=30) - - self.assertTrue(response_sent.is_set()) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content, body) - self.assertEqual(response.text, body.decode("utf-8")) - self.assertEqual(response.headers["X-Test"], "bounded") - self.assertEqual(parsed, {"raw_text": body.decode("utf-8")}) - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_json_transport_rejects_response_above_byte_bound(self) -> None: - body = b"x" * (VALIDATION.MAX_STREAM_SSE_BYTES + 1) - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with fixed_http_response(body) as (url, _response_sent): - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "response byte count exceeds bounds", - ), - ): - VALIDATION.request_json("GET", url, timeout=30) - - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_json_transport_rejects_malformed_worker_framing_after_reap(self) -> None: - class CompletedProcess: - returncode = 0 - - def __init__(self): - self.communicate_calls = 0 - - def communicate(self, input=None, timeout=None): - del input, timeout - self.communicate_calls += 1 - return b"invalid framing", b"" - - def poll(self): - return self.returncode - - process = CompletedProcess() - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - return_value=process, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "invalid framing", - ), - ): - VALIDATION.request_json( - "GET", - "http://127.0.0.1:13305/json", - timeout=10, - ) - - self.assertEqual(process.communicate_calls, 1) - self.assertIsNotNone(process.poll()) - - def test_http_worker_is_reaped_when_post_spawn_clock_read_is_interrupted( - self, - ) -> None: - class StartedProcess: - def __init__(self): - self.returncode = None - self.killed = False - self.communicate_calls = [] - - def communicate(self, input=None, timeout=None): - self.communicate_calls.append((input, timeout)) - self.returncode = -9 - return b"", b"" - - def kill(self): - self.killed = True - - def poll(self): - return self.returncode - - interruption = KeyboardInterrupt() - process = StartedProcess() - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - return_value=process, - ), - mock.patch.object( - VALIDATION.time, - "monotonic", - side_effect=(0.0, interruption), - ), - self.assertRaises(KeyboardInterrupt) as raised, - ): - VALIDATION._execute_http_worker( - "GET", - "http://127.0.0.1:13305/json", - 60, - {}, - None, - "json", - 60, - "HTTP response", - ) - - self.assertIs(raised.exception, interruption) - self.assertTrue(process.killed) - self.assertEqual(process.communicate_calls, [(None, None)]) - self.assertIsNotNone(process.poll()) - - def test_http_worker_preserves_control_exception_during_spawn(self) -> None: - interruption = KeyboardInterrupt() - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=interruption, - ), - mock.patch.object( - VALIDATION, - "_kill_and_reap_stream_worker", - ) as cleanup, - mock.patch.object(VALIDATION.time, "monotonic", return_value=0.0), - self.assertRaises(KeyboardInterrupt) as raised, - ): - VALIDATION._execute_http_worker( - "GET", - "http://127.0.0.1:13305/json", - 60, - {}, - None, - "json", - 60, - "HTTP response", - ) - - self.assertIs(raised.exception, interruption) - cleanup.assert_not_called() - - def test_stream_transport_deadline_covers_slow_response_headers(self) -> None: - prefix = b"HTTP/1.1 " - suffix = ( - b"200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n" - ) - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with stalled_http_response(prefix, suffix) as ( - url, - prefix_sent, - stall_elapsed, - ): - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "wall-clock deadline", - ), - ): - _response, lines = VALIDATION.request_stream( - "POST", - url, - timeout=5, - wall_clock_deadline=time.monotonic() + 5, - json={"stream": True}, - ) - list(lines) - - self.assertTrue(prefix_sent.is_set()) - self.assertFalse(stall_elapsed.is_set()) - - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_stream_transport_deadline_covers_blocked_body_cleanup(self) -> None: - prefix = ( - b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" - b"Connection: close\r\n\r\n" - ) - suffix = b"data: [DONE]\n" - with stalled_http_response(prefix, suffix) as ( - url, - prefix_sent, - stall_elapsed, - ): - with self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "wall-clock deadline", - ): - _response, lines = VALIDATION.request_stream( - "POST", - url, - timeout=5, - wall_clock_deadline=time.monotonic() + 5, - json={"stream": True}, - ) - list(lines) - - self.assertTrue(prefix_sent.is_set()) - self.assertFalse(stall_elapsed.is_set()) - - def test_stream_transport_timeout_kills_and_reaps_worker(self) -> None: - class TimedOutProcess: - def __init__(self): - self.returncode = None - self.killed = False - self.communicate_calls = [] - - def communicate(self, input=None, timeout=None): - self.communicate_calls.append((input, timeout)) - if timeout is not None: - raise subprocess.TimeoutExpired(["stream-worker"], timeout) - self.returncode = -9 - return b"", b"" - - def kill(self): - self.killed = True - - def poll(self): - return self.returncode - - process = TimedOutProcess() - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - return_value=process, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "wall-clock deadline", - ), - ): - _response, lines = VALIDATION.request_stream( - "POST", - "http://127.0.0.1:13305/stream", - timeout=60, - wall_clock_deadline=time.monotonic() + 60, - json={"stream": True}, - ) - list(lines) - - self.assertTrue(process.killed) - self.assertEqual(len(process.communicate_calls), 2) - self.assertIsNotNone(process.poll()) - - def test_stream_transport_deadline_interrupts_unterminated_trickle(self) -> None: - with trickling_http_response() as (url, body_started, trickle_elapsed): - with self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "wall-clock deadline", - ): - _response, lines = VALIDATION.request_stream( - "POST", - url, - timeout=2, - wall_clock_deadline=time.monotonic() + 5, - json={"stream": True}, - ) - list(lines) - - self.assertTrue(body_started.is_set()) - self.assertFalse(trickle_elapsed.is_set()) - - def test_stream_transport_reassembles_lines_and_bounds_worker_input(self) -> None: - class CompletedProcess: - def __init__(self, output): - self.output = output - self.returncode = 0 - self.communicate_calls = [] - - def communicate(self, input=None, timeout=None): - self.communicate_calls.append((input, timeout)) - return self.output, b"" - - def poll(self): - return self.returncode - - output = VALIDATION._encode_stream_worker_output( - 200, - b"data: one\r\ndata: two\n\n", - ) - process = CompletedProcess(output) - authorization = "Bearer test-secret" - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - return_value=process, - ) as popen, - mock.patch.object( - VALIDATION, - "_auth_headers", - return_value={"Authorization": authorization}, - ), - ): - returned, lines = VALIDATION.request_stream( - "POST", - "http://localhost:13305/api/v1/chat/completions", - timeout=60, - json={"stream": True}, - ) - - self.assertEqual( - list(lines), - [b"data: one\r", b"data: two"], - ) - - self.assertEqual(returned.status_code, 200) - request_spec = json.loads(process.communicate_calls[0][0]) - self.assertNotIn("wall_clock_deadline", request_spec) - self.assertEqual( - request_spec["max_response_bytes"], - VALIDATION.MAX_STREAM_SSE_BYTES, - ) - self.assertEqual(request_spec["headers"]["Authorization"], authorization) - self.assertNotIn(authorization, popen.call_args.args[0]) - - def test_stream_transport_worker_success_is_reaped(self) -> None: - prefix = ( - b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" - b"Connection: close\r\n\r\n" - ) - suffix = b"data: one\ndata: [DONE]\n" - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with stalled_http_response(prefix, suffix, stall_seconds=0) as ( - url, - prefix_sent, - _stall_elapsed, - ): - with mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ): - response, lines = VALIDATION.request_stream( - "POST", - url, - timeout=5, - wall_clock_deadline=time.monotonic() + 10, - json={"stream": True}, - ) - - self.assertTrue(prefix_sent.is_set()) - - self.assertEqual(response.status_code, 200) - self.assertEqual(list(lines), [b"data: one", b"data: [DONE]"]) - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_stream_transport_rejects_worker_event_count_above_bound(self) -> None: - output = VALIDATION._encode_stream_worker_output( - 200, - b"data: {}\n\ndata: {}\n\ndata: {}\n\n", - ) - - with ( - mock.patch.object(VALIDATION, "MAX_STREAM_EVENTS", 2), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "event count", - ), - ): - VALIDATION._decode_stream_worker_output(output) - - def test_stream_transport_rejects_a_line_above_its_independent_bound( - self, - ) -> None: - output = VALIDATION._encode_stream_worker_output(200, b"data: 123456\n\n") - - with ( - mock.patch.object(VALIDATION, "MAX_STREAM_SSE_LINE_BYTES", 8), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "SSE line exceeds bounds", - ), - ): - VALIDATION._decode_stream_worker_output(output) - - def test_stream_transport_counts_semantic_frames_not_blank_lines(self) -> None: - chunk_count = 2500 - chunk = json.dumps( - { - "model": "backend-model", - "choices": [ - { - "index": 0, - "delta": {"content": "x"}, - "finish_reason": None, - } - ], - }, - separators=(",", ":"), - ).encode("utf-8") - terminal = json.dumps( - { - "model": "backend-model", - "choices": [ - { - "index": 0, - "delta": {}, - "finish_reason": "stop", - } - ], - }, - separators=(",", ":"), - ).encode("utf-8") - body = (b"data: " + chunk + b"\n\n") * chunk_count - body += b"data: " + terminal + b"\n\ndata: [DONE]\n\n" - - lines = VALIDATION._split_stream_lines(body) - message, finish_reason, saw_done, model = capabilities._decode_openai_stream( - lines - ) - - self.assertEqual(len(lines), chunk_count + 2) - self.assertEqual(message["content"], "x" * chunk_count) - self.assertEqual(finish_reason, "stop") - self.assertTrue(saw_done) - self.assertEqual(model, "backend-model") - self.assertGreaterEqual(capabilities.MAX_STREAM_EVENTS, 3072 + 2) - - def test_stream_transport_rejects_unterminated_body_above_byte_bound(self) -> None: - class OversizedResponse: - def __init__(self): - self.closed = False - - def iter_content(self, **_kwargs): - yield b"abc" - yield b"def" - - def close(self): - self.closed = True - - response = OversizedResponse() - with ( - mock.patch.object( - VALIDATION.requests, - "request", - return_value=response, - create=True, - ), - mock.patch.object( - VALIDATION, - "_load_stream_worker_request", - return_value=( - "stream", - "POST", - "http://127.0.0.1:13305/stream", - 60, - {}, - {"stream": True}, - 5, - ), - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "SSE byte count", - ), - ): - VALIDATION._run_stream_worker() - - self.assertTrue(response.closed) - - def test_json_worker_closes_response_above_byte_bound(self) -> None: - class OversizedResponse: - def __init__(self): - self.closed = False - - def iter_content(self, **_kwargs): - yield b"abc" - yield b"def" - - def close(self): - self.closed = True - - response = OversizedResponse() - with ( - mock.patch.object( - VALIDATION.requests, - "request", - return_value=response, - create=True, - ), - mock.patch.object( - VALIDATION, - "_load_stream_worker_request", - return_value=( - "json", - "GET", - "http://127.0.0.1:13305/json", - 60, - {}, - None, - 5, - ), - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "HTTP response byte count", - ), - ): - VALIDATION._run_stream_worker() - - self.assertTrue(response.closed) - - def test_json_worker_accepts_exact_custom_response_bound(self) -> None: - class BoundedResponse: - status_code = 200 - headers = {"Content-Type": "application/json"} - encoding = "utf-8" - - def __init__(self): - self.closed = False - - def iter_content(self, **_kwargs): - yield b"abc" - yield b"de" - - def close(self): - self.closed = True - - response = BoundedResponse() - captured_bodies = [] - stdout = types.SimpleNamespace(buffer=io.BytesIO()) - - def encode(_status, body, _headers, _encoding): - captured_bodies.append(body) - return b"encoded" - - with ( - mock.patch.object( - VALIDATION.requests, - "request", - return_value=response, - create=True, - ), - mock.patch.object( - VALIDATION, - "_load_stream_worker_request", - return_value=( - "json", - "GET", - "http://127.0.0.1:13305/json", - 60, - {}, - None, - 5, - ), - ), - mock.patch.object( - VALIDATION, - "_encode_stream_worker_output", - side_effect=encode, - ), - mock.patch.object(VALIDATION.sys, "stdout", stdout), - ): - VALIDATION._run_stream_worker() - - self.assertEqual(captured_bodies, [b"abcde"]) - self.assertEqual(stdout.buffer.getvalue(), b"encoded") - self.assertTrue(response.closed) - - def request_json_for_chat( - self, - chat_message, - operations=None, - load_payloads=None, - response_model="builtin.Test-Llama", - unload_status_code=200, - ): - loaded = False - - def request_json(method, url, timeout, **kwargs): - nonlocal loaded - del method, timeout - operation = url.rsplit("/", maxsplit=1)[-1] - if operations is not None: - operations.append(operation) - if operation == "load": - loaded = True - if load_payloads is not None: - load_payloads.append(kwargs["json"]) - return self.response(), {} - if operation == "unload": - loaded = False - return self.response(unload_status_code), {} - if operation == "health": - return self.response(), { - "all_models_loaded": ( - [ - { - "model_name": "builtin.Test-Llama", - "recipe_options": {"llamacpp_backend": "vulkan"}, - } - ] - if loaded - else [] - ) - } - if operation == "completions": - message = ( - chat_message(kwargs["json"]) - if callable(chat_message) - else chat_message - ) - return self.response(), { - "model": response_model, - "choices": [{"message": message}], - } - if operation == "stats": - return self.response(), {"output_tokens": 4} - return self.response(), {} - - return request_json - - def test_validation_uses_ipv4_loopback_for_server_readiness(self) -> None: - with ( - mock.patch.object(VALIDATION, "wait_for_server") as wait, - mock.patch.object( - VALIDATION, - "request_json", - return_value=(self.response(), {"status": "ok"}), - ) as request, - contextlib.redirect_stdout(io.StringIO()), - ): - VALIDATION.require_running_server("http://127.0.0.1:13305/api/v1", 13305) - - wait.assert_called_once_with( - port=13305, - timeout=VALIDATION.TIMEOUT_HEALTH, - host="127.0.0.1", - ) - self.assertTrue(request.call_args.args[1].startswith("http://127.0.0.1:")) - - def test_server_readiness_bounds_each_connect_attempt(self) -> None: - clock = types.SimpleNamespace(now=0.0) - connect_timeouts = [] - - def connect(_address, timeout): - connect_timeouts.append(timeout) - clock.now += timeout - raise OSError("not ready") - - def sleep(delay): - clock.now += delay - - with ( - mock.patch.object(VALIDATION.time, "monotonic", lambda: clock.now), - mock.patch.object(VALIDATION.time, "sleep", side_effect=sleep), - mock.patch.object( - VALIDATION.socket, - "create_connection", - side_effect=connect, - ), - self.assertRaisesRegex(TimeoutError, "2.5 seconds"), - ): - VALIDATION.wait_for_server(port=13305, timeout=2.5, host="127.0.0.1") - - self.assertGreater(len(connect_timeouts), 1) - self.assertTrue( - all( - 0 < timeout <= VALIDATION.SERVER_CONNECT_ATTEMPT_TIMEOUT_SECONDS - for timeout in connect_timeouts - ) - ) - - def test_unload_all_models_uses_bounded_worker_request_and_retries(self) -> None: - failed = self.response(500) - succeeded = self.response(404) - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=[(failed, {}), (succeeded, {})], - ) as request, - mock.patch.object(VALIDATION.time, "sleep") as sleep, - ): - response = VALIDATION.unload_all_models( - port=13305, - attempts=3, - host="127.0.0.1", - ) - - self.assertIs(response, succeeded) - self.assertEqual(request.call_count, 2) - request.assert_called_with( - "POST", - "http://127.0.0.1:13305/api/v1/unload", - timeout=VALIDATION.TIMEOUT_DEFAULT, - json={}, - ) - sleep.assert_called_once_with(1) - - def test_http_worker_ignores_ambient_proxy_configuration(self) -> None: - response = ChunkedResponse( - (b"ok",), - status_code=503, - content_type="text/plain", - ) - session = mock.Mock() - session.request.return_value = response - stdout = types.SimpleNamespace(buffer=io.BytesIO()) - with ( - mock.patch.object( - VALIDATION.requests, - "Session", - return_value=session, - ), - mock.patch.object( - VALIDATION, - "_load_stream_worker_request", - return_value=( - "pull_sse", - "POST", - "http://127.0.0.1:13305/api/v1/pull", - 60, - {}, - {"model_name": "builtin.Test-Llama", "stream": True}, - VALIDATION.MAX_PULL_SSE_WIRE_BYTES, - ), - ), - mock.patch.object(VALIDATION.sys, "stdout", stdout), - ): - VALIDATION._run_stream_worker() - - self.assertFalse(session.trust_env) - session.request.assert_called_once() - session.close.assert_called_once() - self.assertTrue(response.closed) - - def test_pull_rejects_non_loopback_host_before_spawning_worker(self) -> None: - with ( - mock.patch.object(VALIDATION, "_execute_http_worker") as execute, - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "host must be loopback", - ), - ): - VALIDATION._pull_model_streaming( - "builtin.Test-Llama", - 13305, - host="example.com", - ) - - execute.assert_not_called() - - def test_pull_uses_legacy_sse_inside_absolute_deadline_worker(self) -> None: - deadline = time.monotonic() + 60 - response = VALIDATION._BufferedResponse( - 200, - {"Content-Type": "text/event-stream"}, - "utf-8", - b'{"terminal":"complete"}', - ) - with ( - mock.patch.object( - VALIDATION, - "_execute_http_worker", - return_value=response, - ) as execute, - mock.patch.object( - VALIDATION, - "_auth_headers", - return_value={"Authorization": "Bearer secret"}, - ), - ): - result = VALIDATION._pull_model_streaming( - "builtin.Test-Llama", - 13305, - host="127.0.0.1", - wall_clock_deadline=deadline, - ) - - self.assertEqual(result, (200, "")) - execute.assert_called_once_with( - "POST", - "http://127.0.0.1:13305/api/v1/pull", - VALIDATION.TIMEOUT_INFERENCE, - {"Authorization": "Bearer secret"}, - { - "model_name": "builtin.Test-Llama", - "stream": True, - "subscribe": True, - }, - "pull_sse", - deadline, - "model pull", - VALIDATION.MAX_PULL_SSE_WIRE_BYTES, - ) - - def test_pull_maps_only_bounded_worker_terminal_summaries(self) -> None: - cases = ( - ({"terminal": "complete"}, (200, "")), - ( - {"terminal": "error", "status_code": 500, "error": "disk full"}, - (500, "disk full"), - ), - ( - { - "terminal": "error", - "status_code": 400, - "error": "unknown_model", - }, - (400, "unknown_model"), - ), - ) - for summary, expected in cases: - with self.subTest(summary=summary): - response = VALIDATION._BufferedResponse( - 200, - {}, - "utf-8", - json.dumps(summary).encode("utf-8"), - ) - with mock.patch.object( - VALIDATION, - "_execute_http_worker", - return_value=response, - ): - result = VALIDATION._pull_model_streaming( - "builtin.Test-Llama", - 13305, - wall_clock_deadline=time.monotonic() + 60, - ) - self.assertEqual(result, expected) - - malformed = VALIDATION._BufferedResponse(200, {}, "utf-8", b"{}") - with ( - mock.patch.object( - VALIDATION, - "_execute_http_worker", - return_value=malformed, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "terminal summary", - ), - ): - VALIDATION._pull_model_streaming( - "builtin.Test-Llama", - 13305, - wall_clock_deadline=time.monotonic() + 60, - ) - - def test_pull_sse_reducer_handles_crlf_and_chunk_boundaries(self) -> None: - body = ( - b'event: progress\r\ndata: {"percent":50}\r\n\r\n' - b'event: complete\r\ndata: {"status":"ok"}\r\n\r\n' - ) - response = ChunkedResponse( - (body[:1], body[1:17], body[17:41], body[41:]), - content_type="text/event-stream; charset=utf-8", - ) - - summary = VALIDATION._reduce_pull_sse_response(response, len(body)) - - self.assertEqual(json.loads(summary), {"terminal": "complete"}) - self.assertLessEqual( - len(summary), - VALIDATION.MAX_PULL_TERMINAL_SUMMARY_BYTES, - ) - - def test_pull_sse_reducer_discards_large_progress_history(self) -> None: - progress = b'event: progress\ndata: {"percent":50}\n\n' - complete = b'event: complete\ndata: {"status":"ok"}\n\n' - event_total = 20_000 - - def chunks(): - for _index in range(event_total): - yield progress - yield complete - - response = ChunkedResponse(chunks()) - summary = VALIDATION._reduce_pull_sse_response( - response, - len(progress) * event_total + len(complete), - ) - - self.assertEqual(summary, b'{"terminal":"complete"}') - self.assertLess(len(summary), 64) - - def test_pull_sse_reducer_accepts_exact_wire_bound(self) -> None: - body = b'event: complete\ndata: {"status":"ok"}\n\n' - - summary = VALIDATION._reduce_pull_sse_response( - ChunkedResponse((body,)), - len(body), - ) - - self.assertEqual(summary, b'{"terminal":"complete"}') - - def test_pull_sse_reducer_enforces_wire_line_data_and_event_bounds(self) -> None: - complete = b'event: complete\ndata: {"status":"ok"}\n\n' - cases = ( - ( - ChunkedResponse((complete,)), - len(complete) - 1, - contextlib.nullcontext(), - "wire byte count", - ), - ( - ChunkedResponse((b"x" * 9 + b"\n",)), - 100, - mock.patch.object(VALIDATION, "MAX_PULL_SSE_LINE_BYTES", 8), - "line exceeds bounds", - ), - ( - ChunkedResponse((b'event: progress\ndata: {"x":1}\n\n',)), - 100, - mock.patch.object(VALIDATION, "MAX_PULL_SSE_EVENT_DATA_BYTES", 2), - "event data exceeds bounds", - ), - ( - ChunkedResponse( - (b'event: progress\ndata: {"percent":1}\n\n' + complete,) - ), - 1000, - mock.patch.object(VALIDATION, "MAX_PULL_SSE_EVENTS", 1), - "event count exceeds bounds", - ), - ) - for response, wire_bound, patched_bound, expected_error in cases: - with ( - self.subTest(expected_error=expected_error), - patched_bound, - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - expected_error, - ), - ): - VALIDATION._reduce_pull_sse_response(response, wire_bound) - - def test_pull_sse_reducer_requires_one_terminal_and_clean_eof(self) -> None: - progress = b'event: progress\ndata: {"percent":1}\n\n' - complete = b'event: complete\ndata: {"status":"ok"}\n\n' - cases = ( - (progress, "without a terminal"), - (complete + complete, "after its terminal"), - (complete + progress, "after its terminal"), - (complete[:-1], "incomplete event"), - (complete.rstrip(b"\n"), "incomplete line"), - (complete + b"data: {}\n\n", "after its terminal"), - ) - for body, expected_error in cases: - with ( - self.subTest(expected_error=expected_error), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - expected_error, - ), - ): - VALIDATION._reduce_pull_sse_response( - ChunkedResponse((body,)), - len(body), - ) - - def test_pull_sse_reducer_rejects_invalid_content_and_json(self) -> None: - cases = ( - ( - ChunkedResponse( - (b'event: complete\ndata: {"status":"ok"}\n\n',), - content_type="application/json", - ), - "content type", - ), - ( - ChunkedResponse((b'event: progress\ndata: {"x":1,"x":2}\n\n',)), - "event data is invalid", - ), - ( - ChunkedResponse((b'event: progress\ndata: {"x":NaN}\n\n',)), - "event data is invalid", - ), - ( - ChunkedResponse((b"event: complete\ndata: []\n\n",)), - "event data is invalid", - ), - ) - for response, expected_error in cases: - body_size = sum(len(chunk) for chunk in response.chunks) - with ( - self.subTest(expected_error=expected_error), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - expected_error, - ), - ): - VALIDATION._reduce_pull_sse_response(response, body_size) - - def test_pull_worker_always_closes_response_on_parser_failure(self) -> None: - response = ChunkedResponse((b'event: progress\ndata: {"percent":1}\n\n',)) - with ( - mock.patch.object( - VALIDATION.requests, - "request", - return_value=response, - create=True, - ), - mock.patch.object( - VALIDATION, - "_load_stream_worker_request", - return_value=( - "pull_sse", - "POST", - "http://127.0.0.1:13305/api/v1/pull", - 60, - {}, - {"model_name": "builtin.Test-Llama", "stream": True}, - VALIDATION.MAX_PULL_SSE_WIRE_BYTES, - ), - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "without a terminal event", - ), - ): - VALIDATION._run_stream_worker() - - self.assertTrue(response.closed) - - def test_pull_worker_bounds_and_fully_consumes_non_200_body(self) -> None: - response = ChunkedResponse( - (b"temporarily ", b"unavailable"), - status_code=503, - content_type="application/json", - ) - stdout = types.SimpleNamespace(buffer=io.BytesIO()) - with ( - mock.patch.object( - VALIDATION.requests, - "request", - return_value=response, - create=True, - ), - mock.patch.object( - VALIDATION, - "_load_stream_worker_request", - return_value=( - "pull_sse", - "POST", - "http://127.0.0.1:13305/api/v1/pull", - 60, - {}, - {"model_name": "builtin.Test-Llama", "stream": True}, - VALIDATION.MAX_PULL_SSE_WIRE_BYTES, - ), - ), - mock.patch.object(VALIDATION.sys, "stdout", stdout), - ): - VALIDATION._run_stream_worker() - - returned = VALIDATION._decode_http_worker_output(stdout.buffer.getvalue()) - self.assertEqual(returned.status_code, 503) - self.assertEqual(returned.text, "temporarily unavailable") - self.assertTrue(response.closed) - - oversized = ChunkedResponse( - (b"x" * VALIDATION.MAX_PULL_RESPONSE_BYTES, b"x"), - status_code=503, - content_type="application/json", - ) - with ( - mock.patch.object( - VALIDATION.requests, - "request", - return_value=oversized, - create=True, - ), - mock.patch.object( - VALIDATION, - "_load_stream_worker_request", - return_value=( - "pull_sse", - "POST", - "http://127.0.0.1:13305/api/v1/pull", - 60, - {}, - {"model_name": "builtin.Test-Llama", "stream": True}, - VALIDATION.MAX_PULL_SSE_WIRE_BYTES, - ), - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "HTTP response byte count exceeds bounds", - ), - ): - VALIDATION._run_stream_worker() - self.assertTrue(oversized.closed) - - def test_pull_deadline_kills_and_reaps_trickling_worker(self) -> None: - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with trickling_http_response() as (url, body_started, trickle_elapsed): - host, port = url.split("//", maxsplit=1)[1].split(":") - port = int(port.split("/", maxsplit=1)[0]) - with ( - mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ), - self.assertRaisesRegex( - capabilities.CapabilityValidationError, - "model pull exceeded its wall-clock deadline", - ), - ): - VALIDATION._pull_model_streaming( - "builtin.Test-Llama", - port, - host=host, - wall_clock_deadline=time.monotonic() + 5, - ) - - self.assertTrue(body_started.is_set()) - self.assertFalse(trickle_elapsed.is_set()) - - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_pull_success_worker_is_reaped_after_clean_terminal_eof(self) -> None: - prefix = ( - b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" - b"Connection: close\r\n\r\n" - ) - body = b'event: complete\ndata: {"status":"ok"}\n\n' - workers = [] - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - with stalled_http_response(prefix, body, stall_seconds=0) as ( - url, - response_started, - _stall_elapsed, - ): - host, port = url.split("//", maxsplit=1)[1].split(":") - port = int(port.split("/", maxsplit=1)[0]) - with mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ): - result = VALIDATION._pull_model_streaming( - "builtin.Test-Llama", - port, - host=host, - wall_clock_deadline=time.monotonic() + 10, - ) - self.assertTrue(response_started.is_set()) - - self.assertEqual(result, (200, "")) - self.assertEqual(len(workers), 1) - self.assertIsNotNone(workers[0].poll()) - - def test_pull_waits_for_clean_eof_after_terminal_event(self) -> None: - body = b'event: complete\ndata: {"status":"ok"}\n\n' - workers = [] - result = [] - errors = [] - finished = threading.Event() - real_popen = subprocess.Popen - - def record_worker(*args, **kwargs): - worker = real_popen(*args, **kwargs) - workers.append(worker) - return worker - - def pull(host, port): - try: - result.append( - VALIDATION._pull_model_streaming( - "builtin.Test-Llama", - port, - host=host, - wall_clock_deadline=time.monotonic() + 20, - ) - ) - except BaseException as exc: # pylint: disable=broad-exception-caught - errors.append(exc) - finally: - finished.set() - - with held_open_pull_response(body) as (url, terminal_sent, release): - host, port = url.split("//", maxsplit=1)[1].split(":") - port = int(port.split("/", maxsplit=1)[0]) - with mock.patch.object( - VALIDATION.subprocess, - "Popen", - side_effect=record_worker, - ): - caller = threading.Thread(target=pull, args=(host, port)) - caller.start() - self.assertTrue(terminal_sent.wait(10)) - self.assertFalse(finished.wait(0.25)) - self.assertEqual(len(workers), 1) - self.assertIsNone(workers[0].poll()) - release.set() - caller.join(timeout=10) - - self.assertFalse(caller.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(result, [(200, "")]) - self.assertIsNotNone(workers[0].poll()) - - def test_pull_ambiguous_failures_and_control_exceptions_are_not_retried( - self, - ) -> None: - failures = ( - capabilities.CapabilityValidationError("malformed SSE"), - KeyboardInterrupt(), - ) - for failure in failures: - with self.subTest(failure=failure.__class__.__name__): - with ( - mock.patch.object( - VALIDATION, - "_pull_model_streaming", - side_effect=failure, - ) as pull, - mock.patch.object(VALIDATION.time, "sleep") as sleep, - self.assertRaises(failure.__class__) as raised, - ): - VALIDATION.pull_model_with_retry( - "builtin.Test-Llama", - port=13305, - host="127.0.0.1", - ) - - self.assertIs(raised.exception, failure) - pull.assert_called_once() - sleep.assert_not_called() - - def test_pull_retries_only_returned_transient_terminal_status(self) -> None: - with ( - mock.patch.object( - VALIDATION, - "_pull_model_streaming", - side_effect=[(500, "download failed"), (200, "")], - ) as pull, - mock.patch.object(VALIDATION.time, "sleep") as sleep, - contextlib.redirect_stdout(io.StringIO()), - ): - VALIDATION.pull_model_with_retry( - "builtin.Test-Llama", - port=13305, - host="127.0.0.1", - ) - - self.assertEqual(pull.call_count, 2) - self.assertEqual( - pull.call_args_list[0].kwargs["wall_clock_deadline"], - pull.call_args_list[1].kwargs["wall_clock_deadline"], - ) - sleep.assert_called_once_with(2) - - def test_model_rejects_raw_ifm_markers_in_response_fields(self) -> None: - cases = { - "content": ( - {"content": "answer hidden"}, - "content", - ), - "nested_content": ( - { - "content": [ - {"type": "text", "text": "answer "}, - ] - }, - "content", - ), - "reasoning_content": ( - { - "content": "answer", - "reasoning_content": "hidden", - }, - "reasoning_content", - ), - "reasoning": ( - { - "content": "answer", - "reasoning": {"text": "hidden "}, - }, - "reasoning", - ), - "tool_calls": ( - { - "content": "answer", - "tool_calls": [ - { - "function": { - "name": "lookup", - "arguments": '{"query":""}', - } - } - ], - }, - "tool_calls", - ), - "unexpected_field": ( - { - "content": "answer", - "metadata": {"debug": " None: - for token in ("<|ifm|im_start|>", "<|ifm|im_end|>"): - with self.subTest(token=token): - message = {"content": f"answer {token} leaked"} - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat(message), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, error, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - self.assertFalse(success) - self.assertRegex(error, "(?i)raw IFM control marker.*content") - - def test_model_accepts_clean_nested_tool_call_fields(self) -> None: - message = { - "content": "I will use the lookup tool.", - "reasoning_content": "The tool can answer this request.", - "tool_calls": [ - { - "function": { - "name": "lookup", - "arguments": '{"query":"weather"}', - } - } - ], - } - - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat(message), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, response_text, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - self.assertTrue(success) - self.assertEqual(response_text, message["content"]) - - def test_plain_chat_requires_visible_final_content(self) -> None: - message = { - "content": "", - "reasoning_content": "The answer should be four.", - } - - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat(message), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, error, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - self.assertFalse(success) - self.assertIn("visible final content", error) - - def test_plain_smoke_disables_reasoning_to_get_visible_final_content(self) -> None: - def reasoning_model_response(payload): - self.assertEqual(payload["max_completion_tokens"], 50) - if payload.get("chat_template_kwargs") == {"enable_thinking": False}: - return {"content": "The answer is 4."} - return { - "content": "", - "reasoning_content": "The answer should be four.", - } - - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat(reasoning_model_response), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, response, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - self.assertTrue(success, response) - self.assertEqual(response, "The answer is 4.") - - def test_model_runs_requested_capability_profile_and_records_evidence(self) -> None: - matrix = { - "profile": capabilities.K2_HORIZON_PROFILE, - "model": "builtin.Test-Llama", - "pass": True, - "cases": [], - } - evidence = {} - - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat({"content": "The answer is 4."}), - ) as request, - mock.patch.object( - VALIDATION, - "validate_capabilities", - return_value=( - matrix, - True, - f"{len(capabilities.K2_HORIZON_CASE_IDS)}/" - f"{len(capabilities.K2_HORIZON_CASE_IDS)} capability cases passed", - ), - ) as validate, - contextlib.redirect_stdout(io.StringIO()), - ): - success, _response, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - capability_profile=capabilities.K2_HORIZON_PROFILE, - capability_evidence=evidence, - ) - - self.assertTrue(success) - self.assertEqual(evidence, matrix) - validate.assert_called_once_with( - capabilities.K2_HORIZON_PROFILE, - base_url="http://localhost:13305/api/v1", - model="builtin.Test-Llama", - request_json=request, - request_stream=VALIDATION.request_stream, - timeout=VALIDATION.TIMEOUT_INFERENCE, - ) - - def test_capability_failure_fails_model_and_preserves_matrix(self) -> None: - matrix = { - "profile": capabilities.K2_HORIZON_PROFILE, - "model": "builtin.Test-Llama", - "pass": False, - "cases": [{"id": "openai_reasoning_low", "pass": False}], - } - evidence = {} - - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat({"content": "The answer is 4."}), - ), - mock.patch.object( - VALIDATION, - "validate_capabilities", - return_value=(matrix, False, "reasoning low failed"), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, error, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - capability_profile=capabilities.K2_HORIZON_PROFILE, - capability_evidence=evidence, - ) - - self.assertFalse(success) - self.assertIn("reasoning low failed", error) - self.assertEqual(evidence, matrix) - - def test_model_requires_health_to_confirm_the_requested_backend(self) -> None: - cases = ( - (503, {}, "health inventory returned HTTP 503"), - (200, {}, "missing all_models_loaded"), - (200, {"all_models_loaded": "invalid"}, "invalid model inventory"), - (200, {"all_models_loaded": []}, "is absent from the loaded inventory"), - ( - 200, - {"all_models_loaded": [{"model_name": "builtin.Test-Llama"}]}, - "missing recipe_options", - ), - ( - 200, - { - "all_models_loaded": [ - { - "model_name": "builtin.Test-Llama", - "recipe_options": "invalid", - } - ] - }, - "invalid recipe_options", - ), - ( - 200, - { - "all_models_loaded": [ - { - "model_name": "builtin.Test-Llama", - "recipe_options": {}, - } - ] - }, - "missing llamacpp_backend", - ), - ( - 200, - { - "all_models_loaded": [ - { - "model_name": "builtin.Test-Llama", - "recipe_options": {"llamacpp_backend": "cpu"}, - } - ] - }, - "backend 'cpu' instead of 'vulkan'", - ), - ( - 200, - { - "all_models_loaded": [ - { - "model_name": "builtin.Test-Llama", - "recipe_options": {"llamacpp_backend": "vulkan"}, - }, - { - "model_name": "builtin.Stale-Llama", - "recipe_options": {"llamacpp_backend": "vulkan"}, - }, - ] - }, - "exactly the requested model", - ), - ) - - for status_code, health, expected_error in cases: - with self.subTest(health=health, status_code=status_code): - - def request_json(method, url, timeout, **_kwargs): - del method, timeout - operation = url.rsplit("/", maxsplit=1)[-1] - if operation == "load": - return self.response(), {} - if operation == "health": - return self.response(status_code), health - if operation == "unload": - return self.response(), {} - self.fail(f"Unexpected request after invalid health: {operation}") - - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=request_json, - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, error, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - self.assertFalse(success) - self.assertIn(expected_error, error) - - def test_model_binds_top_level_response_model_to_the_request(self) -> None: - for response_model in (None, "builtin.Other-Llama"): - with self.subTest(response_model=response_model): - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat( - {"content": "The answer is 4."}, - response_model=response_model, - ), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, error, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - self.assertFalse(success) - self.assertIn("response model", error) - - def test_model_accepts_canonically_equivalent_bare_response_model(self) -> None: - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat( - {"content": "The answer is 4."}, - response_model="Test-Llama", - ), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - success, response, _stats = VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - self.assertTrue(success, response) - - def test_model_treats_post_inference_unload_failure_as_fatal(self) -> None: - with ( - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat( - {"content": "The answer is 4."}, - unload_status_code=500, - ), - ), - contextlib.redirect_stdout(io.StringIO()), - self.assertRaisesRegex(RuntimeError, "unload.*HTTP 500"), - ): - VALIDATION.test_model( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - ) - - def run_validation(self, explicit): - operations = [] - model = { - "id": "Test-Llama", - "recipe": "llamacpp", - "labels": ["chat", "hot"], - "checkpoint": "example/Test-Llama:model.gguf", - } - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--skip-install", - ] - if explicit: - argv.extend(["--model", "Test-Llama"]) - - with tempfile.TemporaryDirectory() as temp_dir: - argv.extend(["--output", str(Path(temp_dir) / "results.json")]) - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object( - VALIDATION, "get_model_catalog", return_value=[model] - ), - mock.patch.object(VALIDATION, "get_builtin_model", return_value=model), - mock.patch.object( - VALIDATION, - "unload_all_models", - return_value=self.response(), - ), - mock.patch.object( - VALIDATION, - "request_json", - side_effect=self.request_json_for_chat( - {"content": "The answer is 4."}, operations - ), - ), - contextlib.redirect_stdout(io.StringIO()), - ): - VALIDATION.main() - - return operations - - def test_explicit_models_reload_after_unload_but_hot_models_do_not(self) -> None: - expected_loads = {"explicit": 2, "hot": 1} - - for selection_mode, expected_load_count in expected_loads.items(): - with self.subTest(selection_mode=selection_mode): - operations = self.run_validation(selection_mode == "explicit") - - self.assertEqual(operations.count("load"), expected_load_count) - self.assertEqual(operations.count("unload"), expected_load_count) - if selection_mode == "explicit": - first_unload = operations.index("unload") - second_load = operations.index("load", first_unload + 1) - self.assertLess(first_unload, second_load) - - def test_reload_fails_when_the_first_model_remains_loaded(self) -> None: - health = { - "all_models_loaded": [{"model_name": "builtin.Test-Llama"}], - } - with ( - mock.patch.object( - VALIDATION, - "test_model", - return_value=(True, "answer", {}), - ) as test_model, - mock.patch.object( - VALIDATION, - "request_json", - return_value=(self.response(), health), - ), - ): - success, error, _stats = VALIDATION.validate_model_lifecycle( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - reload_after_unload=True, - ) - - self.assertFalse(success) - self.assertRegex(error, "still loaded") - test_model.assert_called_once() - - def test_reload_rejects_any_model_remaining_after_unload(self) -> None: - health = { - "all_models_loaded": [{"model_name": "builtin.Stale-Llama"}], - } - with ( - mock.patch.object( - VALIDATION, - "test_model", - return_value=(True, "answer", {}), - ) as test_model, - mock.patch.object( - VALIDATION, - "request_json", - return_value=(self.response(), health), - ), - ): - success, error, _stats = VALIDATION.validate_model_lifecycle( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - reload_after_unload=True, - ) - - self.assertFalse(success) - self.assertIn("still loaded", error) - test_model.assert_called_once() - - def test_reload_requires_a_valid_loaded_model_inventory(self) -> None: - for health in ({}, {"all_models_loaded": "invalid"}): - with self.subTest(health=health): - with ( - mock.patch.object( - VALIDATION, - "test_model", - return_value=(True, "answer", {}), - ) as test_model, - mock.patch.object( - VALIDATION, - "request_json", - return_value=(self.response(), health), - ), - ): - success, error, _stats = VALIDATION.validate_model_lifecycle( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - reload_after_unload=True, - ) - - self.assertFalse(success) - self.assertRegex(error, "Could not verify unload") - test_model.assert_called_once() - - def test_reload_smoke_does_not_repeat_the_capability_profile(self) -> None: - evidence = {} - health = {"all_models_loaded": []} - - with ( - mock.patch.object( - VALIDATION, - "test_model", - side_effect=[(True, "first", {}), (True, "second", {})], - ) as test_model, - mock.patch.object( - VALIDATION, - "request_json", - return_value=(self.response(), health), - ), - ): - result = VALIDATION.validate_model_lifecycle( - "http://localhost:13305/api/v1", - "builtin.Test-Llama", - "vulkan", - reload_after_unload=True, - capability_profile=capabilities.K2_HORIZON_PROFILE, - capability_evidence=evidence, - ) - - self.assertTrue(result[0]) - self.assertEqual(test_model.call_count, 2) - first_call, second_call = test_model.call_args_list - self.assertEqual( - first_call.kwargs, - { - "capability_profile": capabilities.K2_HORIZON_PROFILE, - "capability_evidence": evidence, - }, - ) - self.assertEqual(second_call.kwargs, {}) - - -class LlamaCppValidationSideEffectTests(unittest.TestCase): - @staticmethod - def response(status_code=200): - return types.SimpleNamespace(status_code=status_code) - - def test_prepare_only_installs_and_pulls_without_running_inference(self) -> None: - model = { - "id": "Test-Llama", - "load_id": "builtin.Test-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/Test-Llama:model.gguf", - } - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--prepare-only", - "--model", - "Test-Llama", - ] - - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object(VALIDATION, "get_builtin_model", return_value=model), - mock.patch.object( - VALIDATION, - "unload_all_models", - return_value=self.response(), - ), - mock.patch.object(VALIDATION, "install_backend") as install, - mock.patch.object(VALIDATION, "pull_model_with_retry") as pull, - mock.patch.object(VALIDATION, "validate_model_lifecycle") as validate, - contextlib.redirect_stdout(io.StringIO()), - ): - try: - VALIDATION.main() - except SystemExit as exc: - self.fail(f"prepare-only exited unexpectedly: {exc}") - - install.assert_called_once() - pull.assert_called_once_with("builtin.Test-Llama", port=13305, host="127.0.0.1") - validate.assert_not_called() - - def test_result_file_creation_is_exclusive_and_rejects_non_finite_values( - self, - ) -> None: - with tempfile.TemporaryDirectory() as directory: - output_path = Path(directory) / "result.json" - output_path.write_text("preserved", encoding="utf-8") - with self.assertRaises(FileExistsError): - VALIDATION.write_results_file(output_path, []) - self.assertEqual(output_path.read_text(encoding="utf-8"), "preserved") - - non_finite_path = Path(directory) / "non-finite.json" - with self.assertRaises(ValueError): - VALIDATION.write_results_file( - non_finite_path, - [{"tokens_per_second": float("inf")}], - ) - - def test_prepare_only_stops_when_a_model_pull_fails(self) -> None: - model = { - "id": "Test-Llama", - "load_id": "builtin.Test-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/Test-Llama:model.gguf", - } - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--prepare-only", - "--model", - "Test-Llama", - ] - - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object(VALIDATION, "get_builtin_model", return_value=model), - mock.patch.object( - VALIDATION, - "unload_all_models", - return_value=self.response(), - ), - mock.patch.object(VALIDATION, "install_backend"), - mock.patch.object( - VALIDATION, - "pull_model_with_retry", - side_effect=AssertionError("pull failed"), - ), - mock.patch.object(VALIDATION, "validate_model_lifecycle") as validate, - contextlib.redirect_stdout(io.StringIO()), - ): - try: - with self.assertRaisesRegex(AssertionError, "pull failed"): - VALIDATION.main() - except SystemExit as exc: - self.fail(f"prepare-only exited before the pull: {exc}") - - validate.assert_not_called() - - def test_prepare_only_rejects_invalid_selection_before_server_mutation( - self, - ) -> None: - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--prepare-only", - "--model", - "Missing-Llama", - ] - - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object(VALIDATION, "get_builtin_model", return_value=None), - mock.patch.object(VALIDATION, "install_backend") as install, - mock.patch.object(VALIDATION, "pull_model_with_retry") as pull, - mock.patch.object(VALIDATION, "unload_all_models") as unload_models, - contextlib.redirect_stderr(io.StringIO()), - self.assertRaises(SystemExit) as raised, - ): - VALIDATION.main() - - self.assertEqual(raised.exception.code, 1) - install.assert_not_called() - pull.assert_not_called() - unload_models.assert_not_called() - - def test_invalid_explicit_model_is_rejected_before_server_mutation(self) -> None: - argv = [ - "validate_llamacpp.py", - "--backend", - "rocm", - "--channel", - "nightly", - "--model", - "Missing-Llama", - ] - - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object(VALIDATION, "get_builtin_model", return_value=None), - mock.patch.object(VALIDATION, "set_rocm_channel") as set_channel, - mock.patch.object(VALIDATION, "unload_all_models") as unload_models, - contextlib.redirect_stderr(io.StringIO()), - self.assertRaises(SystemExit) as raised, - ): - VALIDATION.main() - - self.assertEqual(raised.exception.code, 1) - set_channel.assert_not_called() - unload_models.assert_not_called() - - def test_wrong_builtin_resolver_id_is_rejected_before_server_mutation(self) -> None: - wrong_model = { - "id": "Different-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/different:model.gguf", - } - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--model", - "Test-Llama", - ] - - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object( - VALIDATION, - "get_builtin_model", - return_value=wrong_model, - ), - mock.patch.object(VALIDATION, "install_backend") as install, - mock.patch.object(VALIDATION, "unload_all_models") as unload_models, - contextlib.redirect_stderr(io.StringIO()), - self.assertRaises(SystemExit) as raised, - ): - VALIDATION.main() - - self.assertEqual(raised.exception.code, 1) - install.assert_not_called() - unload_models.assert_not_called() - - def test_initial_clean_state_unload_http_failure_is_fatal(self) -> None: - model = { - "id": "Test-Llama", - "load_id": "builtin.Test-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/Test-Llama:model.gguf", - } - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--skip-install", - "--model", - "Test-Llama", - ] - - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object( - VALIDATION, "select_llamacpp_models", return_value=[model] - ), - mock.patch.object( - VALIDATION, - "unload_all_models", - return_value=self.response(500), - ), - mock.patch.object( - VALIDATION, - "validate_model_lifecycle", - return_value=(True, "Four.", {}), - ) as lifecycle, - contextlib.redirect_stdout(io.StringIO()), - self.assertRaisesRegex(RuntimeError, "clean state.*HTTP 500"), - ): - VALIDATION.main() - - lifecycle.assert_not_called() - - def test_final_cleanup_unload_request_failure_is_fatal(self) -> None: - model = { - "id": "Test-Llama", - "load_id": "builtin.Test-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/Test-Llama:model.gguf", - } - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--skip-install", - "--model", - "Test-Llama", - ] - - with tempfile.TemporaryDirectory() as temp_dir: - argv.extend(["--output", str(Path(temp_dir) / "results.json")]) - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object( - VALIDATION, "select_llamacpp_models", return_value=[model] - ), - mock.patch.object( - VALIDATION, - "unload_all_models", - side_effect=[ - self.response(), - VALIDATION.requests.RequestException("cleanup failed"), - ], - ), - mock.patch.object( - VALIDATION, - "validate_model_lifecycle", - return_value=(True, "Four.", {}), - ), - contextlib.redirect_stdout(io.StringIO()), - self.assertRaisesRegex(RuntimeError, "cleanup failed"), - ): - VALIDATION.main() - - def test_capability_execution_uses_the_requested_canonical_load_id(self) -> None: - model = { - "id": "Untrusted-Display-Id", - "load_id": "builtin.Test-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/Test-Llama:model.gguf", - } - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--skip-install", - "--model", - "Test-Llama", - "--capability-profile", - capabilities.K2_HORIZON_PROFILE, - "--capability-model", - "Test-Llama", - ] - - with tempfile.TemporaryDirectory() as temp_dir: - argv.extend(["--output", str(Path(temp_dir) / "results.json")]) - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object( - VALIDATION, "select_llamacpp_models", return_value=[model] - ), - mock.patch.object( - VALIDATION, - "unload_all_models", - return_value=self.response(), - ), - mock.patch.object( - VALIDATION, - "validate_model_lifecycle", - return_value=(True, "Four.", {}), - ) as lifecycle, - contextlib.redirect_stdout(io.StringIO()), - ): - VALIDATION.main() - - self.assertEqual( - lifecycle.call_args.kwargs["capability_profile"], - capabilities.K2_HORIZON_PROFILE, - ) - - def test_capability_options_must_be_paired(self) -> None: - cases = ( - ["--capability-profile", capabilities.K2_HORIZON_PROFILE], - ["--capability-model", "Test-Llama"], - ) - - for extra_args in cases: - with self.subTest(extra_args=extra_args): - argv = [ - "validate_llamacpp.py", - "--backend", - "rocm", - "--channel", - "nightly", - *extra_args, - ] - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object(VALIDATION, "set_rocm_channel") as set_channel, - mock.patch.object(VALIDATION, "unload_all_models") as unload_models, - contextlib.redirect_stderr(io.StringIO()), - self.assertRaises(SystemExit), - ): - VALIDATION.main() - - set_channel.assert_not_called() - unload_models.assert_not_called() - - def test_capability_profile_can_target_a_selected_hot_model(self) -> None: - model = { - "id": "K2-Horizon-0.9B-GGUF", - "recipe": "llamacpp", - "labels": ["chat", "hot"], - "checkpoint": "IFM/example:model.gguf", - } - matrix = { - "profile": capabilities.K2_HORIZON_PROFILE, - "model": "K2-Horizon-0.9B-GGUF", - "pass": True, - "cases": [], - } - - def validate_lifecycle( - _base_url, - _model_name, - _backend, - reload_after_unload=False, - *, - capability_profile="", - capability_evidence=None, - ): - self.assertFalse(reload_after_unload) - self.assertEqual(capability_profile, capabilities.K2_HORIZON_PROFILE) - capability_evidence.update(matrix) - return True, "Four.", {} - - with tempfile.TemporaryDirectory() as temp_dir: - output_path = Path(temp_dir) / "results.json" - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--skip-install", - "--capability-profile", - capabilities.K2_HORIZON_PROFILE, - "--capability-model", - model["id"], - "--output", - str(output_path), - ] - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object( - VALIDATION, "get_model_catalog", return_value=[model] - ), - mock.patch.object( - VALIDATION, - "unload_all_models", - return_value=self.response(), - ), - mock.patch.object( - VALIDATION, - "validate_model_lifecycle", - side_effect=validate_lifecycle, - ) as lifecycle, - contextlib.redirect_stdout(io.StringIO()), - ): - VALIDATION.main() - - records = json.loads(output_path.read_text(encoding="utf-8")) - - lifecycle.assert_called_once() - self.assertEqual(records[0]["capability_matrix"], matrix) - - def test_capability_model_must_be_selected_and_unique(self) -> None: - cases = ( - ["--capability-model", "Other-Llama"], - [ - "--capability-model", - "Test-Llama", - "--capability-model", - "Test-Llama", - ], - ) - model = { - "id": "Test-Llama", - "recipe": "llamacpp", - "labels": ["chat"], - "checkpoint": "example/Test-Llama:model.gguf", - } - - for capability_args in cases: - with self.subTest(capability_args=capability_args): - argv = [ - "validate_llamacpp.py", - "--backend", - "vulkan", - "--model", - "Test-Llama", - "--capability-profile", - capabilities.K2_HORIZON_PROFILE, - *capability_args, - ] - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(VALIDATION, "require_running_server"), - mock.patch.object( - VALIDATION, - "get_builtin_model", - return_value=model, - ), - mock.patch.object(VALIDATION, "install_backend") as install, - mock.patch.object(VALIDATION, "unload_all_models") as unload_models, - contextlib.redirect_stderr(io.StringIO()), - self.assertRaises(SystemExit), - ): - VALIDATION.main() - - install.assert_not_called() - unload_models.assert_not_called() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_validate_llamacpp_trusted_execution.py b/test/test_validate_llamacpp_trusted_execution.py deleted file mode 100644 index 7aadadfc49..0000000000 --- a/test/test_validate_llamacpp_trusted_execution.py +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env python3 -"""Security-contract tests for protected llama.cpp validation execution.""" - -from __future__ import annotations - -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" -TESTING_GUIDE = ROOT / "docs" / "dev" / "testing.md" -PROTECTED_EVENT = ( - "github.event_name == 'pull_request_target' || " - "github.event_name == 'merge_group'" -) -UNPROTECTED_EVENT = ( - "github.event_name != 'pull_request_target' && " - "github.event_name != 'merge_group'" -) -CHECKOUT_ACTION = "actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0" - - -def job(workflow: str, name: str, following_name: str) -> str: - return workflow.split(f" {name}:\n", 1)[1].split(f" {following_name}:\n", 1)[0] - - -def step(job_text: str, name: str, following_name: str) -> str: - return job_text.split(f" - name: {name}\n", 1)[1].split( - f" - name: {following_name}\n", 1 - )[0] - - -class TrustedExecutionTests(unittest.TestCase): - def assert_trusted_validation_entrypoint(self, validation: str) -> None: - run_script = validation.split(" run: >-\n", 1)[1] - normalized = " ".join(run_script.split()) - trusted_entrypoint = ( - 'python "$VALIDATION_ROOT/.github/scripts/' 'run_llamacpp_validation.py"' - ) - - self.assertEqual(normalized.count(trusted_entrypoint), 1) - self.assertEqual(normalized.count("run_llamacpp_validation.py"), 1) - self.assertNotIn("test/validate_llamacpp.py", normalized) - self.assertNotIn("VALIDATION_CANDIDATE_ROOT", normalized) - self.assertNotIn("candidate/", normalized) - self.assertNotIn("cd candidate", normalized) - - def test_protected_validation_ignores_candidate_control_plane_files(self) -> None: - workflow = WORKFLOW.read_text(encoding="utf-8") - build = job(workflow, "build", "validate") - validate = job(workflow, "validate", "reverify-protected-release-manifest") - - for job_text in (build, validate): - with self.subTest(job=job_text.splitlines()[0]): - control_checkout = step( - job_text, - "Check out validation control plane", - "Check out the validation candidate", - ) - candidate_checkout = step( - job_text, - "Check out the validation candidate", - "Verify authorized merge commit", - ) - self.assertIn(CHECKOUT_ACTION, control_checkout) - self.assertIn("github.event.pull_request.base.sha", control_checkout) - self.assertIn("github.event.merge_group.base_sha", control_checkout) - self.assertIn("github.sha", control_checkout) - self.assertIn("persist-credentials: false", control_checkout) - self.assertNotIn("allow-unsafe-pr-checkout", control_checkout) - self.assertIn("path: candidate", candidate_checkout) - self.assertIn("persist-credentials: false", candidate_checkout) - self.assertIn(f"if: {PROTECTED_EVENT}", candidate_checkout) - self.assertIn( - "github.event.pull_request.merge_commit_sha", candidate_checkout - ) - self.assertIn("github.event.merge_group.head_sha", candidate_checkout) - - protected_catalog = step( - build, - "Stage protected validation-only model catalog", - "Stage branch validation-only model catalog", - ) - self.assertIn(f"if: {PROTECTED_EVENT}", protected_catalog) - self.assertIn( - 'python "$GITHUB_WORKSPACE/test/utils/validation_model_catalog.py"', - protected_catalog, - ) - self.assertIn( - 'candidate_catalog="$GITHUB_WORKSPACE/candidate/src/cpp/resources/' - 'server_models.json"', - protected_catalog, - ) - self.assertIn( - '--overlay "$GITHUB_WORKSPACE/test/fixtures/' - 'llamacpp_validation_models.json"', - protected_catalog, - ) - self.assertIn( - "resolved_catalog.relative_to(candidate_root)", - protected_catalog, - ) - self.assertIn('--base "$candidate_catalog"', protected_catalog) - self.assertIn('--output "$candidate_catalog"', protected_catalog) - - branch_catalog = step( - build, - "Stage branch validation-only model catalog", - "Install Linux build dependencies", - ) - self.assertIn(f"if: {UNPROTECTED_EVENT}", branch_catalog) - self.assertIn( - "working-directory: ${{ env.VALIDATION_CANDIDATE_ROOT }}", - branch_catalog, - ) - self.assertIn("python -m test.utils.validation_model_catalog", branch_catalog) - - protected_setup = step( - validate, - "Set up trusted or branch validation environment", - "Run validation with lemond", - ) - self.assertIn("uses: ./.github/actions/setup-venv", protected_setup) - self.assertIn( - "requirements-file: '${{ github.workspace }}/test/requirements.txt'", - protected_setup, - ) - - before_trusted_runner = validate.split( - " - name: Run validation with lemond\n", 1 - )[0] - self.assertNotIn("lemond --version", before_trusted_runner) - self.assertNotIn("& $lemondExe --version", before_trusted_runner) - self.assertNotIn('"$lemond" --version', before_trusted_runner) - - validation = step(validate, "Run validation with lemond", "Upload results") - self.assertIn( - "VALIDATION_ROOT: ${{ github.workspace }}", - validation, - ) - self.assertIn("PYTHONPATH: ${{ github.workspace }}", validation) - self.assertIn("working-directory: ${{ github.workspace }}", validation) - self.assertIn( - 'python "$VALIDATION_ROOT/.github/scripts/' 'run_llamacpp_validation.py"', - validation, - ) - self.assert_trusted_validation_entrypoint(validation) - self.assertIn( - "CANDIDATE_LEMOND: ${{ runner.os == 'Windows' && " - "format('{0}/build/Release/lemond.exe', " - "env.VALIDATION_CANDIDATE_ROOT) || " - "format('{0}/build/lemond', env.VALIDATION_CANDIDATE_ROOT) }}", - validation, - ) - self.assertIn('--lemond "$CANDIDATE_LEMOND"', validation) - self.assertNotIn("uses: ./candidate/", validate) - self.assertNotIn("$GITHUB_WORKSPACE/candidate/.github", validation) - self.assertNotIn("$GITHUB_WORKSPACE/candidate/test", validation) - - controlled_files = ( - ".github/actions/cleanup-processes-linux/action.yml", - ".github/actions/cleanup-processes-windows/action.yml", - ".github/actions/setup-python/action.yml", - ".github/actions/setup-venv/action.yml", - ".github/scripts/run_llamacpp_validation.py", - "test/fixtures/llamacpp_validation_artifacts.json", - "test/fixtures/llamacpp_validation_models.json", - "test/requirements.txt", - "test/validate_llamacpp.py", - "test/utils/capabilities.py", - "test/utils/llamacpp_capability_validation.py", - "test/utils/llamacpp_validation_artifacts.py", - "test/utils/llamacpp_validation_evidence.py", - "test/utils/server_base.py", - "test/utils/test_models.py", - "test/utils/validation_model_catalog.py", - "test/utils/validation_model_selection.py", - ) - with tempfile.TemporaryDirectory() as directory: - workspace = Path(directory) - for relative_path in controlled_files: - trusted_file = workspace / relative_path - candidate_file = workspace / "candidate" / relative_path - trusted_file.parent.mkdir(parents=True, exist_ok=True) - candidate_file.parent.mkdir(parents=True, exist_ok=True) - trusted_file.write_text("TRUSTED\n", encoding="utf-8") - candidate_file.write_text("MALICIOUS\n", encoding="utf-8") - - execution_root = workspace - for relative_path in controlled_files: - with self.subTest(relative_path=relative_path): - selected = execution_root / relative_path - self.assertEqual(selected.read_text(encoding="utf-8"), "TRUSTED\n") - - def test_relative_candidate_validator_commands_fail_the_contract(self) -> None: - trusted = ( - " run: >-\n" - ' python "$VALIDATION_ROOT/.github/scripts/' - 'run_llamacpp_validation.py"\n' - ' --lemond "$CANDIDATE_LEMOND"\n' - ) - substitutions = ( - trusted.replace( - 'python "$VALIDATION_ROOT/.github/scripts/' - 'run_llamacpp_validation.py"', - "python candidate/test/validate_llamacpp.py", - ), - trusted + " && python candidate/test/validate_llamacpp.py\n", - trusted.replace( - "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py", - "$VALIDATION_CANDIDATE_ROOT/.github/scripts/" - "run_llamacpp_validation.py", - ), - trusted.replace("python ", "cd candidate && python ", 1), - ) - - for substitution in substitutions: - with self.subTest(substitution=substitution): - with self.assertRaises(AssertionError): - self.assert_trusted_validation_entrypoint(substitution) - - def test_protected_ephemeral_mac_has_no_token_bearing_post_run_step( - self, - ) -> None: - workflow = WORKFLOW.read_text(encoding="utf-8") - validate = job(workflow, "validate", "reverify-protected-release-manifest") - validate_header = workflow.split(" validate:\n", 1)[1].split( - " steps:\n", 1 - )[0] - runtime_and_later = validate.split( - " - name: Run validation with lemond\n", 1 - )[1] - after_runtime = runtime_and_later.split(" - name: Upload results\n", 1)[1] - protected_mac_exclusion = ( - "!((github.event_name == 'pull_request_target' || " - "github.event_name == 'merge_group') && matrix.target == " - "'macos-metal' && runner.environment == 'github-hosted' && " - "join(matrix.runner, ',') == 'macos-latest')" - ) - - self.assertIn("permissions:\n contents: read\n", validate_header) - self.assertNotIn("GH_TOKEN:", validate) - for upload_name in ( - "Upload results", - "Upload restart results", - "Upload server logs", - ): - upload = validate.split(f" - name: {upload_name}\n", 1)[1] - upload = upload.split(" - name:", 1)[0] - with self.subTest(upload=upload_name): - self.assertIn(protected_mac_exclusion, " ".join(upload.split())) - self.assertNotIn("secrets.", after_runtime) - self.assertNotIn("actions/cache", after_runtime) - - def test_trust_boundary_does_not_claim_hostile_native_isolation(self) -> None: - guide = TESTING_GUIDE.read_text(encoding="utf-8") - - self.assertIn( - "base-branch checkout supplies the setup and cleanup actions", guide - ) - self.assertIn("every imported `test.utils` helper", guide) - self.assertIn("built `lemond` and adjacent build resources", guide) - self.assertIn( - "does **not** make candidate native code an adversarial sandbox", guide - ) - self.assertIn("execute as the runner's OS user", guide) - self.assertIn("single-job ephemeral protected self-hosted workers", guide) - self.assertIn("expose no repository secrets", guide) - self.assertIn("grant only a read-only repository token", guide) - self.assertIn("no artifact upload, cache upload", guide) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_validate_llamacpp_workflow_plan.py b/test/test_validate_llamacpp_workflow_plan.py deleted file mode 100644 index 94ecf67256..0000000000 --- a/test/test_validate_llamacpp_workflow_plan.py +++ /dev/null @@ -1,4724 +0,0 @@ -#!/usr/bin/env python3 -"""Unit tests for the llama.cpp validation workflow plan.""" - -from __future__ import annotations - -import importlib.util -import json -import os -import subprocess -import sys -import tempfile -import textwrap -import unittest -from pathlib import Path - -import yaml - -from test.utils import llamacpp_release_assets as release_assets -from test.utils import llamacpp_validation_plan as planning - -K2_SMALL = "K2-Horizon-0.9B-GGUF" -K2_MEDIUM = "K2-Horizon-3.7B-GGUF" -K2_LARGE = "K2-Horizon-7B-GGUF" -CHECKOUT_SHA = "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09" -DOWNLOAD_ARTIFACT_SHA = "37930b1c2abaa49bbe596cd826c3c89aef350131" -ROOT = Path(__file__).resolve().parents[1] -LEGACY_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp.yml" -VALIDATION_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_core.yml" -PR_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_pr.yml" -MANUAL_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_manual.yml" -SCHEDULE_WORKFLOW = ROOT / ".github" / "workflows" / "validate_llamacpp_schedule.yml" -DOCS_AND_STYLE_WORKFLOW = ROOT / ".github" / "workflows" / "docs_and_style.yml" -VALIDATION_RUNNER = ROOT / ".github" / "scripts" / "run_llamacpp_validation.py" -REQUIRED_CHECK_POLICY = ( - ROOT / ".github" / "scripts" / "llamacpp_required_check_policy.py" -) -TESTING_GUIDE = ROOT / "docs" / "dev" / "testing.md" - - -def load_validation_runner(): - spec = importlib.util.spec_from_file_location( - "llamacpp_validation_runner", VALIDATION_RUNNER - ) - if spec is None or spec.loader is None: - raise RuntimeError("could not load llama.cpp validation runner") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def load_required_check_policy(): - spec = importlib.util.spec_from_file_location( - "llamacpp_required_check_policy", REQUIRED_CHECK_POLICY - ) - if spec is None or spec.loader is None: - raise RuntimeError("could not load llama.cpp required-check policy") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def load_workflow(path: Path) -> dict: - workflow = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) - if not isinstance(workflow, dict): - raise ValueError(f"workflow must be a mapping: {path}") - return workflow - - -def workflow_step(path: Path, job_name: str, step_name: str) -> dict: - workflow = load_workflow(path) - steps = workflow["jobs"][job_name]["steps"] - matches = [step for step in steps if step.get("name") == step_name] - if len(matches) != 1: - raise ValueError(f"workflow step must be unique: {job_name}/{step_name}") - return matches[0] - - -def extract_workflow_step_script(step_name: str) -> str: - workflow = PR_WORKFLOW.read_text(encoding="utf-8") - step = workflow.split(f" - name: {step_name}\n", 1)[1].split( - " - name:", 1 - )[0] - return textwrap.dedent(step.split(" run: |\n", 1)[1]) - - -def run_identity_step( - snapshots: list[dict[str, object]], - *, - event_merge_sha: str, - event_base_ref: str = "main", -) -> tuple[subprocess.CompletedProcess[str], dict[str, str], int]: - with tempfile.TemporaryDirectory() as directory: - temporary = Path(directory) - snapshots_path = temporary / "snapshots.json" - call_count_path = temporary / "call-count.txt" - output_path = temporary / "github-output.txt" - normalized_snapshots = json.loads(json.dumps(snapshots)) - for snapshot in normalized_snapshots: - snapshot["base"].setdefault("ref", "main") - snapshots_path.write_text(json.dumps(normalized_snapshots), encoding="utf-8") - - fake_gh = temporary / "gh" - fake_gh.write_text( - """#!/usr/bin/env python3 -import json -import os -from pathlib import Path - -snapshots = json.loads(Path(os.environ["FAKE_GH_SNAPSHOTS"]).read_text()) -count_path = Path(os.environ["FAKE_GH_CALL_COUNT"]) -count = int(count_path.read_text()) if count_path.exists() else 0 -count_path.write_text(str(count + 1)) -print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - fake_sleep = temporary / "sleep" - fake_sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - fake_sleep.chmod(0o755) - - environment = os.environ.copy() - base = snapshots[0]["base"] - head = snapshots[0]["head"] - if not isinstance(base, dict) or not isinstance(head, dict): - raise ValueError("fake pull request base and head must be objects") - environment.update( - { - "EVENT_BASE_SHA": str(base["sha"]), - "EVENT_BASE_REF": event_base_ref, - "EVENT_HEAD_SHA": str(head["sha"]), - "EVENT_MERGE_SHA": event_merge_sha, - "EVENT_NAME": "pull_request_target", - "FAKE_GH_CALL_COUNT": str(call_count_path), - "FAKE_GH_SNAPSHOTS": str(snapshots_path), - "GH_TOKEN": "test-token", - "GITHUB_OUTPUT": str(output_path), - "GITHUB_REPOSITORY": "LLM360/lemonade", - "MERGE_GROUP_BASE_SHA": "", - "MERGE_GROUP_HEAD_SHA": "", - "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", - "PR_NUMBER": "7", - } - ) - completed = subprocess.run( - [ - "bash", - "-c", - extract_workflow_step_script("Resolve the live candidate identity"), - ], - cwd=ROOT, - env=environment, - check=False, - capture_output=True, - text=True, - ) - outputs = {} - if output_path.exists(): - outputs = dict( - line.split("=", 1) - for line in output_path.read_text(encoding="utf-8").splitlines() - ) - calls = int(call_count_path.read_text()) if call_count_path.exists() else 0 - return completed, outputs, calls - - -def issue_event( - event_id: int, - event: str, - *, - actor: str = "maintainer", - actor_id: int = 100, - label: str | None = None, -) -> dict[str, object]: - record: dict[str, object] = { - "actor": { - "id": actor_id, - "login": actor, - "node_id": f"USER_{actor_id}", - "type": "User", - }, - "created_at": f"2026-01-01T00:00:{event_id:02d}Z", - "event": event, - "id": event_id, - "node_id": f"EVENT_{event_id}", - "performed_via_github_app": None, - } - if label is not None: - record["label"] = {"name": label} - return record - - -def run_output_step( - step_name: str, environment_values: dict[str, str] -) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: - with tempfile.TemporaryDirectory() as directory: - output_path = Path(directory) / "github-output.txt" - environment = os.environ.copy() - environment.update(environment_values) - environment["GITHUB_OUTPUT"] = str(output_path) - completed = subprocess.run( - ["bash", "-c", extract_workflow_step_script(step_name)], - cwd=ROOT, - env=environment, - check=False, - capture_output=True, - text=True, - ) - outputs = {} - if output_path.exists(): - outputs = dict( - line.split("=", 1) - for line in output_path.read_text(encoding="utf-8").splitlines() - ) - return completed, outputs - - -def run_failed_closed_assessment( - pull_request: dict[str, object], -) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: - with tempfile.TemporaryDirectory() as directory: - temporary = Path(directory) - output_path = temporary / "github-output.txt" - fake_gh = temporary / "gh" - fake_gh.write_text( - f"#!/bin/sh\nprintf '%s\\n' '{json.dumps(pull_request)}'\n", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - environment = os.environ.copy() - environment.update( - { - "EVENT_ACTION": "closed", - "EVENT_BASE_REF": "main", - "EVENT_BASE_SHA": "a" * 40, - "EVENT_HEAD_SHA": "b" * 40, - "EVENT_MERGE_SHA": "c" * 40, - "GITHUB_OUTPUT": str(output_path), - "GITHUB_REPOSITORY": "LLM360/lemonade", - "MODE": "", - "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", - "POLICY_CONCLUSION": "", - "POLICY_RESULT": "failure", - "POLICY_SUMMARY": "", - "PR_NUMBER": "7", - "VALIDATION_RESULT": "skipped", - } - ) - completed = subprocess.run( - [ - "bash", - "-c", - extract_workflow_step_script("Assess policy and validation"), - ], - cwd=ROOT, - env=environment, - check=False, - capture_output=True, - text=True, - ) - outputs = {} - if output_path.exists(): - outputs = dict( - line.split("=", 1) - for line in output_path.read_text(encoding="utf-8").splitlines() - ) - return completed, outputs - - -def run_policy_decision_step( - *, - action: str, - event_label: str, - event_actor: str, - event_actor_id: int, - event_updated_at: str, - issue_events: list[list[dict[str, object]]], - check_pages: list[dict[str, object]], - authorization_label_present: bool = True, - dependabot_pr: bool = False, - fail_check_api: bool = False, - permission: str = "write", - permission_actor_id: int = 100, - run_attempt: int = 1, - run_conclusion: str = "failure", - run_created_at: str = "2026-01-01T00:00:00Z", - run_event: str = "pull_request_target", - run_id: int = 500, - run_number: int = 50, - run_path: str = ".github/workflows/validate_llamacpp_pr.yml@main", - run_repository: str = "LLM360/lemonade", - run_response_attempt: int = 1, - run_response_head_branch: str = "feature/k2-horizon", - run_response_head_repository: str = "LLM360/lemonade", - run_response_head_sha: str = "b" * 40, - run_response_id: int | None = None, - run_response_number: int | None = None, - run_response_pr_base_ref: str = "main", - run_response_pr_base_sha: str = "a" * 40, - run_response_pr_head_ref: str = "feature/k2-horizon", - run_response_pr_head_repository: str = "LLM360/lemonade", - run_response_pr_head_sha: str = "b" * 40, - run_response_pr_number: int = 7, - run_response_pull_request_count: int = 1, - run_response_pull_requests_json: str | None = None, - run_started_at: str | None = None, - run_status: str = "completed", - triggering_actor: str = "maintainer", - identity_matches: bool = True, - relevant: bool = True, -) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: - with tempfile.TemporaryDirectory() as directory: - temporary = Path(directory) - events_path = temporary / "events.json" - checks_path = temporary / "checks.json" - output_path = temporary / "github-output.txt" - events_path.write_text(json.dumps(issue_events), encoding="utf-8") - checks_path.write_text(json.dumps(check_pages), encoding="utf-8") - fake_gh = temporary / "gh" - fake_gh.write_text( - """#!/usr/bin/env python3 -import json -import os -import sys -from pathlib import Path - -endpoint = next( - (argument for argument in sys.argv[1:] if argument.startswith("repos/")), "" -) -if "/issues/" in endpoint: - print(Path(os.environ["FAKE_GH_EVENTS"]).read_text(encoding="utf-8")) -elif "/actions/runs/" in endpoint: - pull_request = { - "base": { - "ref": os.environ["FAKE_RUN_PR_BASE_REF"], - "repo": {"url": os.environ["FAKE_RUN_REPOSITORY_URL"]}, - "sha": os.environ["FAKE_RUN_PR_BASE_SHA"], - }, - "head": { - "ref": os.environ["FAKE_RUN_PR_HEAD_REF"], - "repo": {"url": os.environ["FAKE_RUN_PR_HEAD_REPOSITORY_URL"]}, - "sha": os.environ["FAKE_RUN_PR_HEAD_SHA"], - }, - "id": 4453422506, - "number": int(os.environ["FAKE_RUN_PR_NUMBER"]), - "url": os.environ["FAKE_RUN_PR_URL"], - } - pull_requests = [pull_request] * int(os.environ["FAKE_RUN_PR_COUNT"]) - if os.environ["FAKE_RUN_PULL_REQUESTS_JSON"]: - pull_requests = json.loads(os.environ["FAKE_RUN_PULL_REQUESTS_JSON"]) - print(json.dumps({ - "conclusion": os.environ["FAKE_RUN_CONCLUSION"], - "created_at": os.environ["FAKE_RUN_CREATED_AT"], - "event": os.environ["FAKE_RUN_EVENT"], - "head_branch": os.environ["FAKE_RUN_RESPONSE_HEAD_BRANCH"], - "head_repository": {"full_name": os.environ["FAKE_RUN_RESPONSE_HEAD_REPOSITORY"]}, - "head_sha": os.environ["FAKE_RUN_RESPONSE_HEAD_SHA"], - "id": int(os.environ["FAKE_RUN_RESPONSE_ID"]), - "path": os.environ["FAKE_RUN_PATH"], - "pull_requests": pull_requests, - "repository": {"full_name": os.environ["FAKE_RUN_REPOSITORY"], "id": 1356276914}, - "run_attempt": int(os.environ["FAKE_RUN_RESPONSE_ATTEMPT"]), - "run_number": int(os.environ["FAKE_RUN_RESPONSE_NUMBER"]), - "run_started_at": os.environ["FAKE_RUN_STARTED_AT"], - "status": os.environ["FAKE_RUN_STATUS"], - })) -elif "/collaborators/" in endpoint: - print(json.dumps({ - "permission": os.environ["FAKE_GH_PERMISSION"], - "user": {"id": int(os.environ["FAKE_GH_PERMISSION_ACTOR_ID"])}, - })) -elif "/check-runs" in endpoint: - if os.environ["FAKE_FAIL_CHECK_API"] == "true": - raise SystemExit("Checks API is unavailable") - print(Path(os.environ["FAKE_GH_CHECKS"]).read_text(encoding="utf-8")) -else: - raise SystemExit(f"unexpected endpoint: {endpoint}") -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - environment = os.environ.copy() - environment.update( - { - "AUTHORIZATION_LABEL_PRESENT": str(authorization_label_present).lower(), - "BASE_REF_CHANGED": "false", - "CHECK_NAME": "llama.cpp authorized validation v2", - "CURRENT_RUN_ATTEMPT": str(run_attempt), - "CURRENT_RUN_ID": str(run_id), - "CURRENT_RUN_NUMBER": str(run_number), - "DEPENDABOT_PR": str(dependabot_pr).lower(), - "EVENT_ACTION": action, - "EVENT_ACTOR": event_actor, - "EVENT_ACTOR_ID": str(event_actor_id), - "EVENT_BASE_REF": "main", - "EVENT_BASE_SHA": "a" * 40, - "EVENT_LABEL": event_label, - "EVENT_HEAD_SHA": "b" * 40, - "EVENT_NAME": "pull_request_target", - "EVENT_UPDATED_AT": event_updated_at, - "FAKE_GH_CHECKS": str(checks_path), - "FAKE_GH_EVENTS": str(events_path), - "FAKE_GH_PERMISSION": permission, - "FAKE_GH_PERMISSION_ACTOR_ID": str(permission_actor_id), - "FAKE_FAIL_CHECK_API": str(fail_check_api).lower(), - "FAKE_RUN_CONCLUSION": run_conclusion, - "FAKE_RUN_CREATED_AT": run_created_at, - "FAKE_RUN_EVENT": run_event, - "FAKE_RUN_PATH": run_path, - "FAKE_RUN_REPOSITORY": run_repository, - "FAKE_RUN_REPOSITORY_URL": ( - f"https://api.github.com/repos/{run_repository}" - ), - "FAKE_RUN_RESPONSE_ATTEMPT": str(run_response_attempt), - "FAKE_RUN_RESPONSE_HEAD_BRANCH": run_response_head_branch, - "FAKE_RUN_RESPONSE_HEAD_REPOSITORY": run_response_head_repository, - "FAKE_RUN_RESPONSE_HEAD_SHA": run_response_head_sha, - "FAKE_RUN_RESPONSE_ID": str( - run_id if run_response_id is None else run_response_id - ), - "FAKE_RUN_RESPONSE_NUMBER": str( - run_number if run_response_number is None else run_response_number - ), - "FAKE_RUN_PR_BASE_REF": run_response_pr_base_ref, - "FAKE_RUN_PR_BASE_SHA": run_response_pr_base_sha, - "FAKE_RUN_PR_COUNT": str(run_response_pull_request_count), - "FAKE_RUN_PR_HEAD_REF": run_response_pr_head_ref, - "FAKE_RUN_PR_HEAD_REPOSITORY_URL": ( - "https://api.github.com/repos/" f"{run_response_pr_head_repository}" - ), - "FAKE_RUN_PR_HEAD_SHA": run_response_pr_head_sha, - "FAKE_RUN_PR_NUMBER": str(run_response_pr_number), - "FAKE_RUN_PULL_REQUESTS_JSON": (run_response_pull_requests_json or ""), - "FAKE_RUN_PR_URL": ( - f"https://api.github.com/repos/{run_repository}/pulls/" - f"{run_response_pr_number}" - ), - "FAKE_RUN_STARTED_AT": ( - run_created_at if run_started_at is None else run_started_at - ), - "FAKE_RUN_STATUS": run_status, - "GH_TOKEN": "test-token", - "GITHUB_OUTPUT": str(output_path), - "GITHUB_API_URL": "https://api.github.com", - "GITHUB_REPOSITORY": "LLM360/lemonade", - "IDENTITY_MATCHES": str(identity_matches).lower(), - "MERGED": "false", - "MERGE_CANDIDATE_AVAILABLE": "true", - "MERGE_SHA": "c" * 40, - "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", - "PR_NUMBER": "7", - "RELEVANT": str(relevant).lower(), - "RUNNER_TEMP": str(temporary), - "STATE": "open", - "TRIGGERING_ACTOR": triggering_actor, - } - ) - script = extract_workflow_step_script( - "Decide whether protected validation may run" - ).split("\nvalidate:\n", 1)[0] - script = script.replace( - "trusted/.github/scripts/llamacpp_required_check_policy.py", - str(REQUIRED_CHECK_POLICY), - ) - completed = subprocess.run( - ["bash", "-c", script], - cwd=ROOT, - env=environment, - check=False, - capture_output=True, - text=True, - ) - outputs = {} - if output_path.exists(): - outputs = dict( - line.split("=", 1) - for line in output_path.read_text(encoding="utf-8").splitlines() - ) - return completed, outputs - - -def run_observed_merge_invalidation( - check_pages: list[dict[str, object]], - *, - observed_merge_sha: str, - secondary_observed_merge_sha: str = "", - authorization_epoch: int = 1, -) -> tuple[subprocess.CompletedProcess[str], list[list[str]]]: - with tempfile.TemporaryDirectory() as directory: - temporary = Path(directory) - pages_path = temporary / "check-pages.json" - calls_path = temporary / "gh-calls.jsonl" - pages_path.write_text(json.dumps(check_pages), encoding="utf-8") - - fake_gh = temporary / "gh" - fake_gh.write_text( - """#!/usr/bin/env python3 -import json -import os -import sys -from pathlib import Path - -arguments = sys.argv[1:] -with Path(os.environ["FAKE_GH_CALLS"]).open("a", encoding="utf-8") as stream: - stream.write(json.dumps(arguments) + "\\n") -method = arguments[arguments.index("--method") + 1] if "--method" in arguments else "GET" -if method == "GET": - print(Path(os.environ["FAKE_GH_CHECK_PAGES"]).read_text(encoding="utf-8")) -elif method == "POST": - print('{"id": 80}') -else: - print('{}') -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - environment = os.environ.copy() - environment.update( - { - "CHECK_NAME": "llama.cpp authorization diagnostics v2", - "AUTHORIZATION_EPOCH": str(authorization_epoch), - "FAKE_GH_CALLS": str(calls_path), - "FAKE_GH_CHECK_PAGES": str(pages_path), - "GH_TOKEN": "test-token", - "GITHUB_REPOSITORY": "LLM360/lemonade", - "GITHUB_RUN_ID": "500", - "GITHUB_SERVER_URL": "https://github.com", - "OBSERVED_MERGE_SHA": observed_merge_sha, - "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", - "PR_NUMBER": "7", - "RUN_ATTEMPT": "1", - "RUN_ID": "500", - "RUN_NUMBER": "50", - "SECONDARY_OBSERVED_MERGE_SHA": secondary_observed_merge_sha, - } - ) - completed = subprocess.run( - [ - "bash", - "-c", - extract_workflow_step_script( - "Invalidate an unconfirmed observed merge commit" - ), - ], - cwd=ROOT, - env=environment, - check=False, - capture_output=True, - text=True, - ) - calls = [] - if calls_path.exists(): - calls = [ - json.loads(line) - for line in calls_path.read_text(encoding="utf-8").splitlines() - ] - return completed, calls - - -def run_preserve_step( - *, - pull_request: dict[str, object], - issue_events: list[list[dict[str, object]]], - check_pages: list[dict[str, object]], - run_attempt: int = 1, - run_number: int = 50, -) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: - with tempfile.TemporaryDirectory() as directory: - temporary = Path(directory) - output_path = temporary / "github-output.txt" - inputs_path = temporary / "inputs.json" - inputs_path.write_text( - json.dumps( - { - "checks": check_pages, - "events": issue_events, - "pull_request": pull_request, - } - ), - encoding="utf-8", - ) - fake_gh = temporary / "gh" - fake_gh.write_text( - """#!/usr/bin/env python3 -import json -import os -import sys -from pathlib import Path - -inputs = json.loads(Path(os.environ["FAKE_GH_INPUTS"]).read_text()) -endpoint = next( - (argument for argument in sys.argv[1:] if argument.startswith("repos/")), "" -) -if "/pulls/" in endpoint: - print(json.dumps(inputs["pull_request"])) -elif "/issues/" in endpoint: - print(json.dumps(inputs["events"])) -elif "/collaborators/" in endpoint: - print('{"permission":"write","user":{"id":100}}') -elif "/check-runs" in endpoint: - print(json.dumps(inputs["checks"])) -else: - raise SystemExit(f"unexpected endpoint: {endpoint}") -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - environment = os.environ.copy() - environment.update( - { - "AUTHORIZATION_ACTOR": "maintainer", - "AUTHORIZATION_ACTOR_ID": "100", - "AUTHORIZATION_BOUNDARY": "2025-12-31T23:59:59Z", - "AUTHORIZATION_CREATED_AT": "2026-01-01T00:00:01Z", - "AUTHORIZATION_EPOCH": "1", - "BASE_REF": "main", - "BASE_SHA": "a" * 40, - "CHECK_NAME": "llama.cpp authorized validation v2", - "FAKE_GH_INPUTS": str(inputs_path), - "GH_TOKEN": "test-token", - "GITHUB_OUTPUT": str(output_path), - "GITHUB_REPOSITORY": "LLM360/lemonade", - "HEAD_SHA": "b" * 40, - "MERGE_SHA": "c" * 40, - "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", - "POLICY_HELPER": str(REQUIRED_CHECK_POLICY), - "PR_NUMBER": "7", - "RUN_ATTEMPT": str(run_attempt), - "RUN_NUMBER": str(run_number), - "RUNNER_TEMP": str(temporary), - } - ) - completed = subprocess.run( - [ - "bash", - "-c", - extract_workflow_step_script( - "Preserve the exact-commit gate for unrelated metadata" - ), - ], - cwd=ROOT, - env=environment, - check=False, - capture_output=True, - text=True, - ) - outputs = {} - if output_path.exists(): - outputs = dict( - line.split("=", 1) - for line in output_path.read_text(encoding="utf-8").splitlines() - ) - return completed, outputs - - -def run_report_step( - check_page_snapshots: list[list[dict[str, object]]], - *, - pull_request_snapshots: list[dict[str, object]] | None = None, - issue_event_snapshots: list[list[list[dict[str, object]]]] | None = None, - permission: str = "write", - permission_snapshots: list[str] | None = None, - run_attempt: int = 1, - run_id: int = 500, - run_number: int = 50, - mode: str = "validate", - conclusion: str = "success", - preserve_valid: bool = False, - event_action: str = "synchronize", - event_name: str | None = None, - authorization_created_at: str = "2026-01-01T00:00:01Z", - authorization_epoch: int = 1, - step_name: str = "Report result on the exact candidate commit", -) -> tuple[subprocess.CompletedProcess[str], list[list[str]]]: - with tempfile.TemporaryDirectory() as directory: - temporary = Path(directory) - snapshots_path = temporary / "check-snapshots.json" - check_count_path = temporary / "check-count.txt" - event_count_path = temporary / "event-count.txt" - pr_count_path = temporary / "pr-count.txt" - permission_count_path = temporary / "permission-count.txt" - events_path = temporary / "event-snapshots.json" - pull_requests_path = temporary / "pr-snapshots.json" - calls_path = temporary / "gh-calls.jsonl" - snapshots_path.write_text(json.dumps(check_page_snapshots), encoding="utf-8") - events_path.write_text( - json.dumps(issue_event_snapshots or []), encoding="utf-8" - ) - permission_path = temporary / "permission-snapshots.json" - permission_path.write_text( - json.dumps(permission_snapshots or [permission]), encoding="utf-8" - ) - normalized_pull_requests = json.loads(json.dumps(pull_request_snapshots or [])) - for pull_request in normalized_pull_requests: - pull_request["base"].setdefault("ref", "main") - pull_requests_path.write_text( - json.dumps(normalized_pull_requests), encoding="utf-8" - ) - - fake_gh = temporary / "gh" - fake_gh.write_text( - """#!/usr/bin/env python3 -import json -import os -import sys -from pathlib import Path - -arguments = sys.argv[1:] -with Path(os.environ["FAKE_GH_CALLS"]).open("a", encoding="utf-8") as stream: - stream.write(json.dumps(arguments) + "\\n") -method = arguments[arguments.index("--method") + 1] if "--method" in arguments else "GET" -endpoint = next((argument for argument in arguments if argument.startswith("repos/")), "") -if method == "GET" and "/check-runs" in endpoint and "/commits/" in endpoint: - count_path = Path(os.environ["FAKE_GH_CHECK_COUNT"]) - count = int(count_path.read_text()) if count_path.exists() else 0 - count_path.write_text(str(count + 1)) - snapshots = json.loads(Path(os.environ["FAKE_GH_SNAPSHOTS"]).read_text()) - print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) -elif method == "GET" and "/pulls/" in endpoint: - count_path = Path(os.environ["FAKE_GH_PR_COUNT"]) - count = int(count_path.read_text()) if count_path.exists() else 0 - count_path.write_text(str(count + 1)) - snapshots = json.loads(Path(os.environ["FAKE_GH_PULL_REQUESTS"]).read_text()) - print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) -elif method == "GET" and "/issues/" in endpoint and endpoint.endswith("/events?per_page=100"): - count_path = Path(os.environ["FAKE_GH_EVENT_COUNT"]) - count = int(count_path.read_text()) if count_path.exists() else 0 - count_path.write_text(str(count + 1)) - snapshots = json.loads(Path(os.environ["FAKE_GH_EVENTS"]).read_text()) - print(json.dumps(snapshots[min(count, len(snapshots) - 1)])) -elif method == "GET" and "/collaborators/" in endpoint: - count_path = Path(os.environ["FAKE_GH_PERMISSION_COUNT"]) - count = int(count_path.read_text()) if count_path.exists() else 0 - count_path.write_text(str(count + 1)) - permissions = json.loads(Path(os.environ["FAKE_GH_PERMISSIONS"]).read_text()) - permission = permissions[min(count, len(permissions) - 1)] - if "--jq" in arguments: - print(permission) - else: - print(json.dumps({ - "permission": permission, - "user": {"id": 100}, - })) -elif method == "POST": - print('{"id": 80}') -else: - print('{}') -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - environment = os.environ.copy() - environment.update( - { - "BASE_SHA": "a" * 40, - "BASE_REF": "main", - "CHECK_NAME": "llama.cpp authorized validation v2", - "CHECK_RUN_ID": "", - "CONCLUSION": conclusion, - "EVENT_ACTION": event_action, - "EVENT_NAME": ( - event_name - if event_name is not None - else ( - "pull_request_target" - if normalized_pull_requests - else "merge_group" - ) - ), - "FAKE_GH_CALLS": str(calls_path), - "FAKE_GH_CHECK_COUNT": str(check_count_path), - "FAKE_GH_EVENT_COUNT": str(event_count_path), - "FAKE_GH_EVENTS": str(events_path), - "FAKE_GH_PERMISSION_COUNT": str(permission_count_path), - "FAKE_GH_PERMISSIONS": str(permission_path), - "FAKE_GH_PR_COUNT": str(pr_count_path), - "FAKE_GH_PULL_REQUESTS": str(pull_requests_path), - "FAKE_GH_SNAPSHOTS": str(snapshots_path), - "GH_TOKEN": "test-token", - "GITHUB_REPOSITORY": "LLM360/lemonade", - "GITHUB_RUN_ID": str(run_id), - "GITHUB_SERVER_URL": "https://github.com", - "HEAD_SHA": "b" * 40, - "MERGE_SHA": "c" * 40, - "METADATA_CHECK_NAME": "llama.cpp authorization diagnostics v2", - "MODE": mode, - "PATH": f"{temporary}{os.pathsep}{environment['PATH']}", - "POLICY_HELPER": str(REQUIRED_CHECK_POLICY), - "PR_NUMBER": "7", - "PRESERVE_VALID": str(preserve_valid).lower(), - "RUN_ATTEMPT": str(run_attempt), - "RUN_ID": str(run_id), - "RUN_NUMBER": str(run_number), - "SUMMARY": "Authorized validation passed.", - "TARGET_KEY": "pr-7", - "AUTHORIZATION_ACTOR": "maintainer", - "AUTHORIZATION_ACTOR_ID": "100", - "AUTHORIZATION_BOUNDARY": "2025-12-31T23:59:59Z", - "AUTHORIZATION_CREATED_AT": authorization_created_at, - "AUTHORIZATION_EPOCH": str(authorization_epoch), - } - ) - completed = subprocess.run( - [ - "bash", - "-c", - extract_workflow_step_script(step_name), - ], - cwd=ROOT, - env=environment, - check=False, - capture_output=True, - text=True, - ) - calls = [] - if calls_path.exists(): - calls = [ - json.loads(line) - for line in calls_path.read_text(encoding="utf-8").splitlines() - ] - return completed, calls - - -class LlamaCppValidationPlanTests(unittest.TestCase): - def test_authorization_provenance_is_a_fully_validated_epoch(self) -> None: - policy = load_required_check_policy() - valid = policy.classify_authorization_event_pages( - [ - [ - issue_event(1, "reopened"), - issue_event(2, "labeled", label="ci:upgrades"), - ] - ] - ) - self.assertTrue(valid.epoch_valid) - self.assertEqual(valid.authorization_event_id, 2) - self.assertEqual(valid.authorization_actor, "maintainer") - self.assertEqual(valid.authorization_actor_id, 100) - self.assertEqual(valid.authorization_created_at, "2026-01-01T00:00:02Z") - self.assertTrue( - policy.classify_authorization_event_pages( - [[issue_event(2, "labeled", label="ci:upgrades")]], - authorization_after="2026-01-01T00:00:01Z", - ).epoch_valid - ) - for boundary in ( - "2026-01-01T00:00:02Z", - "2026-01-01T00:00:03Z", - ): - with self.subTest(boundary=boundary): - stale_for_run = policy.classify_authorization_event_pages( - [[issue_event(2, "labeled", label="ci:upgrades")]], - authorization_after=boundary, - ) - self.assertFalse(stale_for_run.epoch_valid) - for invalid_boundary in ("not-a-time", "2026-01-01T00:00:00"): - with self.subTest(invalid_boundary=invalid_boundary): - with self.assertRaises(ValueError): - policy.classify_authorization_event_pages( - [[issue_event(2, "labeled", label="ci:upgrades")]], - authorization_after=invalid_boundary, - ) - - for invalidator in ("base_ref_changed", "reopened", "head_ref_force_pushed"): - with self.subTest(invalidator=invalidator): - invalid = policy.classify_authorization_event_pages( - [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, invalidator), - ] - ] - ) - self.assertFalse(invalid.epoch_valid) - - untrusted_readd = policy.classify_authorization_event_pages( - [ - [ - issue_event(1, "labeled", actor="trusted", label="ci:upgrades"), - issue_event(2, "unlabeled", label="ci:upgrades"), - issue_event(3, "labeled", actor="triage", label="ci:upgrades"), - ] - ] - ) - self.assertTrue(untrusted_readd.epoch_valid) - self.assertEqual(untrusted_readd.authorization_actor, "triage") - unrelated_bot_label = issue_event( - 1, - "labeled", - actor="github-actions[bot]", - actor_id=41898282, - label="automation:classified", - ) - unrelated_bot_label["actor"]["type"] = "Bot" - unrelated_bot_label["performed_via_github_app"] = {"id": 15368} - bot_label_before_authorization = policy.classify_authorization_event_pages( - [ - [ - unrelated_bot_label, - issue_event(2, "labeled", label="ci:upgrades"), - ] - ] - ) - self.assertTrue(bot_label_before_authorization.epoch_valid) - self.assertEqual( - bot_label_before_authorization.authorization_actor, - "maintainer", - ) - stale_epoch = policy.decide_pull_request_policy( - action="labeled", - event_label="ci:upgrades", - base_ref_changed=False, - state="open", - merged=False, - merge_candidate_available=True, - relevant=True, - identity_matches=True, - authorization_label_present=True, - authorization_epoch_valid=False, - actor_permission="write", - has_existing_check=True, - has_newer_check=False, - ) - self.assertEqual(stale_epoch.mode, "awaiting_rerun") - self.assertEqual(stale_epoch.conclusion, "failure") - delayed_delivery = policy.decide_pull_request_policy( - action="labeled", - event_label="ci:upgrades", - base_ref_changed=False, - state="open", - merged=False, - merge_candidate_available=True, - relevant=True, - identity_matches=True, - authorization_label_present=True, - authorization_epoch_valid=True, - authorization_rerun_matches=False, - actor_permission="write", - has_existing_check=False, - has_newer_check=False, - ) - self.assertEqual(delayed_delivery.mode, "awaiting_rerun") - with self.assertRaises(ValueError): - policy.classify_authorization_event_pages( - [ - [ - issue_event(1, "reopened"), - issue_event(1, "labeled", label="ci:upgrades"), - ] - ] - ) - malformed_actor = issue_event(1, "labeled", label="ci:upgrades") - malformed_actor["actor"] = { - "id": 100, - "login": "maintainer", - "node_id": "USER_100", - "type": "Bot", - } - duplicate_node = issue_event(2, "reopened") - duplicate_node["node_id"] = "EVENT_1" - for pages in ( - [[], [issue_event(1, "reopened")]], - [[issue_event(2, "reopened"), issue_event(1, "closed")]], - [[issue_event(1, "reopened"), duplicate_node]], - [[malformed_actor]], - ): - with self.subTest(invalid_pages=pages): - with self.assertRaises(ValueError): - policy.classify_authorization_event_pages(pages) - - def test_live_authorization_epoch_binding_is_enforced(self) -> None: - events = [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, "unlabeled", label="ci:upgrades"), - issue_event(3, "labeled", label="ci:upgrades"), - ] - ] - current_epoch_check = { - "app": {"slug": "github-actions"}, - "conclusion": "success", - "external_id": "lemonade-llamacpp-required-pr-7-3-40-1-400", - "id": 40, - "name": "llama.cpp authorized validation v2", - "status": "completed", - } - old_epoch_check = { - **current_epoch_check, - "external_id": "lemonade-llamacpp-required-pr-7-1-40-1-400", - } - - authorized, authorized_outputs = run_policy_decision_step( - action="synchronize", - event_label="", - event_actor="contributor", - event_actor_id=200, - event_updated_at="2026-01-01T00:00:00Z", - issue_events=events, - check_pages=[{"check_runs": []}], - run_attempt=2, - ) - self.assertEqual(authorized.returncode, 0, authorized.stderr) - self.assertEqual(authorized_outputs["mode"], "validate") - self.assertEqual(authorized_outputs["authorization_event_id"], "3") - - for run_path in ( - ".github/workflows/validate_llamacpp_pr.yml", - ".github/workflows/validate_llamacpp_pr.yml@refs/heads/main", - ): - with self.subTest(allowed_workflow_path=run_path): - allowed, allowed_outputs = run_policy_decision_step( - action="synchronize", - event_label="", - event_actor="contributor", - event_actor_id=200, - event_updated_at="2026-01-01T00:00:00Z", - issue_events=events, - check_pages=[{"check_runs": []}], - run_attempt=2, - run_path=run_path, - ) - self.assertEqual(allowed.returncode, 0, allowed.stderr) - self.assertEqual(allowed_outputs["mode"], "validate") - - fork, fork_outputs = run_policy_decision_step( - action="synchronize", - event_label="", - event_actor="contributor", - event_actor_id=200, - event_updated_at="2026-01-01T00:00:00Z", - issue_events=events, - check_pages=[{"check_runs": []}], - run_attempt=2, - run_response_head_branch="fork/k2-horizon", - run_response_head_repository="contributor/lemonade", - run_response_pr_head_ref="fork/k2-horizon", - run_response_pr_head_repository="contributor/lemonade", - run_response_pull_request_count=0, - ) - self.assertEqual(fork.returncode, 0, fork.stderr) - self.assertEqual(fork_outputs["mode"], "validate") - - multiple, multiple_outputs = run_policy_decision_step( - action="synchronize", - event_label="", - event_actor="contributor", - event_actor_id=200, - event_updated_at="2026-01-01T00:00:00Z", - issue_events=events, - check_pages=[{"check_runs": []}], - run_attempt=2, - run_response_pr_base_ref="release", - run_response_pr_base_sha="d" * 40, - run_response_pr_head_sha="e" * 40, - run_response_pr_number=99, - run_response_pull_request_count=3, - ) - self.assertEqual(multiple.returncode, 0, multiple.stderr) - self.assertEqual(multiple_outputs["mode"], "validate") - - for name, overrides, expected_mode in ( - ("first-attempt", {"run_attempt": 1}, "authorization_required"), - ( - "label-not-after-run", - {"run_created_at": "2026-01-01T00:00:03Z"}, - "authorization_required", - ), - ( - "label-after-queue-but-before-start", - { - "run_created_at": "2025-12-31T23:59:59Z", - "run_started_at": "2026-01-01T00:00:04Z", - }, - "authorization_required", - ), - ( - "missing-run-start", - {"run_started_at": ""}, - "authorization_required", - ), - ( - "wrong-permission-identity", - {"permission_actor_id": 999}, - "authorization_required", - ), - ( - "untrusted-rerun-actor", - {"triggering_actor": "triage"}, - "authorization_required", - ), - ( - "first-attempt-succeeded", - {"run_conclusion": "success"}, - "authorization_required", - ), - ( - "wrong-run-event", - {"run_event": "pull_request"}, - "authorization_required", - ), - ( - "wrong-workflow", - {"run_path": ".github/workflows/other.yml"}, - "authorization_required", - ), - ( - "wrong-workflow-ref", - {"run_path": (".github/workflows/validate_llamacpp_pr.yml@release")}, - "authorization_required", - ), - ( - "malformed-workflow-ref", - { - "run_path": ( - ".github/workflows/validate_llamacpp_pr.yml@main@release" - ) - }, - "authorization_required", - ), - ( - "wrong-repository", - {"run_repository": "attacker/lemonade"}, - "authorization_required", - ), - ( - "wrong-top-level-head-sha", - {"run_response_head_sha": "d" * 40}, - "authorization_required", - ), - ( - "malformed-pull-request-associations", - {"run_response_pull_requests_json": '"not-an-array"'}, - "authorization_required", - ), - ( - "live-identity-drift", - {"identity_matches": False}, - "stale_required", - ), - ( - "wrong-run-id", - {"run_response_id": 501}, - "authorization_required", - ), - ( - "wrong-run-number", - {"run_response_number": 51}, - "authorization_required", - ), - ( - "wrong-attempt-record", - {"run_response_attempt": 2}, - "authorization_required", - ), - ( - "incomplete-first-attempt", - {"run_status": "in_progress"}, - "authorization_required", - ), - ( - "label-event-rerun", - {"action": "labeled", "event_label": "ci:upgrades"}, - "awaiting_rerun", - ), - ): - arguments = { - "action": "synchronize", - "event_label": "", - "event_actor": "contributor", - "event_actor_id": 200, - "event_updated_at": "2026-01-01T00:00:00Z", - "issue_events": events, - "check_pages": [{"check_runs": []}], - "run_attempt": 2, - } - arguments.update(overrides) - with self.subTest(name=name): - denied, denied_outputs = run_policy_decision_step(**arguments) - self.assertEqual(denied.returncode, 0, denied.stderr) - self.assertEqual(denied_outputs["mode"], expected_mode) - self.assertEqual(denied_outputs["conclusion"], "failure") - - for name, checks, expected_mode in ( - ("same-epoch", [current_epoch_check], "preserve"), - ("old-epoch", [old_epoch_check], "authorization_required"), - ( - "in-progress", - [{**current_epoch_check, "conclusion": None, "status": "in_progress"}], - "authorization_required", - ), - ( - "failed", - [{**current_epoch_check, "conclusion": "failure"}], - "authorization_required", - ), - ( - "later-failed-decision", - [ - current_epoch_check, - { - **current_epoch_check, - "conclusion": "failure", - "external_id": ("lemonade-llamacpp-required-pr-7-3-41-1-401"), - "id": 41, - }, - ], - "authorization_required", - ), - ("missing", [], "authorization_required"), - ): - with self.subTest(name=name): - metadata, metadata_outputs = run_policy_decision_step( - action="edited", - event_label="", - event_actor="maintainer", - event_actor_id=100, - event_updated_at="2026-01-01T00:00:04Z", - issue_events=events, - check_pages=[{"check_runs": checks}], - ) - self.assertEqual(metadata.returncode, 0, metadata.stderr) - self.assertEqual(metadata_outputs["mode"], expected_mode) - if expected_mode != "preserve": - self.assertEqual(metadata_outputs["conclusion"], "failure") - - def test_dependabot_policy_does_not_require_the_checks_api(self) -> None: - authorization = [[issue_event(3, "labeled", label="ci:upgrades")]] - cases = ( - ( - "neutral", - { - "action": "synchronize", - "authorization_label_present": False, - "issue_events": [[]], - "relevant": False, - }, - "neutral", - "success", - ), - ( - "unauthorized", - { - "action": "synchronize", - "authorization_label_present": False, - "issue_events": [[]], - }, - "authorization_required", - "failure", - ), - ( - "awaiting-rerun", - { - "action": "labeled", - "event_label": "ci:upgrades", - "issue_events": authorization, - }, - "awaiting_rerun", - "failure", - ), - ( - "metadata-preserve", - {"action": "edited", "issue_events": authorization}, - "preserve", - "", - ), - ( - "authorized-rerun", - { - "action": "synchronize", - "issue_events": authorization, - "run_attempt": 2, - }, - "validate", - "", - ), - ) - for name, overrides, expected_mode, expected_conclusion in cases: - arguments = { - "action": "synchronize", - "event_label": "", - "event_actor": "dependabot[bot]", - "event_actor_id": 49699333, - "event_updated_at": "2026-01-01T00:00:00Z", - "issue_events": authorization, - "check_pages": [{"check_runs": []}], - "dependabot_pr": True, - "fail_check_api": True, - } - arguments.update(overrides) - with self.subTest(name=name): - completed, outputs = run_policy_decision_step(**arguments) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["mode"], expected_mode) - self.assertEqual(outputs["conclusion"], expected_conclusion) - - def test_dependabot_pr_uses_only_the_required_workflow_conclusion(self) -> None: - expected_identity = ( - "${{ github.event_name == 'pull_request_target' && " - "github.event.pull_request.user.login == 'dependabot[bot]' && " - "github.event.pull_request.user.id == 49699333 && " - "github.event.pull_request.user.type == 'Bot' }}" - ) - self.assertEqual( - load_workflow(PR_WORKFLOW)["env"]["DEPENDABOT_PR"], expected_identity - ) - self.assertEqual( - load_workflow(VALIDATION_WORKFLOW)["env"]["DEPENDABOT_PR"], - expected_identity, - ) - - decision = workflow_step( - PR_WORKFLOW, "policy", "Decide whether protected validation may run" - ) - self.assertEqual(decision["env"]["DEPENDABOT_PR"], "${{ env.DEPENDABOT_PR }}") - self.assertIn('if [ "$DEPENDABOT_PR" != "true" ]', decision["run"]) - self.assertIn('--required-workflow-only "$DEPENDABOT_PR"', decision["run"]) - - for step_name in ( - "Preserve the exact-commit gate for unrelated metadata", - "Invalidate an unconfirmed observed merge commit", - "Report result on the exact candidate commit", - ): - with self.subTest(step=step_name): - condition = workflow_step(PR_WORKFLOW, "reconcile", step_name)["if"] - self.assertIn("env.DEPENDABOT_PR != 'true'", condition) - - start = workflow_step( - VALIDATION_WORKFLOW, "validate-invocation", "Start authorized merge check" - ) - self.assertIn("env.DEPENDABOT_PR != 'true'", start["if"]) - enforce = workflow_step(PR_WORKFLOW, "reconcile", "Enforce policy result")["if"] - self.assertIn("env.DEPENDABOT_PR != 'true'", enforce) - - guide = TESTING_GUIDE.read_text(encoding="utf-8") - self.assertIn("Dependabot", guide) - self.assertIn("read-only `GITHUB_TOKEN`", guide) - self.assertIn("required-workflow conclusion", guide) - self.assertIn("omit custom diagnostic Checks API runs", guide) - - def test_required_workflow_only_metadata_preserves_without_a_custom_check( - self, - ) -> None: - policy = load_required_check_policy() - common = { - "action": "edited", - "event_label": "", - "base_ref_changed": False, - "state": "open", - "merged": False, - "merge_candidate_available": True, - "relevant": True, - "identity_matches": True, - "authorization_label_present": True, - "actor_permission": "write", - "has_existing_check": False, - "has_newer_check": False, - "authorization_epoch_valid": True, - "authorization_rerun_matches": False, - "required_workflow_only": True, - } - preserved = policy.decide_pull_request_policy(**common) - self.assertEqual(preserved.mode, "preserve") - self.assertFalse(preserved.run_validation) - self.assertEqual(preserved.conclusion, "") - - for name, overrides, expected_mode in ( - ( - "missing-authorization", - {"authorization_label_present": False}, - "authorization_required", - ), - ( - "untrusted-authorization", - {"actor_permission": "triage"}, - "authorization_required", - ), - ( - "stale-identity", - {"identity_matches": False}, - "stale", - ), - ( - "base-ref-change", - {"base_ref_changed": True}, - "authorization_required", - ), - ( - "protected-synchronize", - {"action": "synchronize"}, - "authorization_required", - ), - ): - arguments = dict(common) - arguments.update(overrides) - with self.subTest(name=name): - self.assertEqual( - policy.decide_pull_request_policy(**arguments).mode, - expected_mode, - ) - - def test_dependabot_authorized_result_is_revalidated_without_writes(self) -> None: - pull_request = { - "base": {"ref": "main", "sha": "a" * 40}, - "head": {"sha": "b" * 40}, - "labels": [{"name": "ci:upgrades"}], - "merge_commit_sha": "c" * 40, - "mergeable": True, - "merged": False, - "number": 7, - "state": "open", - } - events = [[issue_event(1, "labeled", label="ci:upgrades")]] - - valid, valid_calls = run_report_step( - [[]], - pull_request_snapshots=[pull_request], - issue_event_snapshots=[events], - step_name="Revalidate an authorized Dependabot result", - ) - self.assertEqual(valid.returncode, 0, valid.stderr) - self.assertFalse( - any( - "--method" in call - and call[call.index("--method") + 1] in {"POST", "PATCH"} - for call in valid_calls - ) - ) - - invalid_cases = ( - ( - "unlabeled-live-pr", - {**pull_request, "labels": []}, - events, - "write", - ), - ( - "invalidated-epoch", - pull_request, - [ - [ - *events[0], - issue_event(2, "unlabeled", label="ci:upgrades"), - ] - ], - "write", - ), - ("permission-loss", pull_request, events, "read"), - ( - "identity-drift", - {**pull_request, "head": {"sha": "d" * 40}}, - events, - "write", - ), - ) - for name, live_pull_request, live_events, permission in invalid_cases: - with self.subTest(name=name): - invalid, calls = run_report_step( - [[]], - pull_request_snapshots=[live_pull_request], - issue_event_snapshots=[live_events], - permission_snapshots=[permission], - step_name="Revalidate an authorized Dependabot result", - ) - self.assertNotEqual(invalid.returncode, 0) - self.assertFalse( - any( - "--method" in call - and call[call.index("--method") + 1] in {"POST", "PATCH"} - for call in calls - ) - ) - - def test_dependabot_unauthorized_result_still_fails_reconciliation(self) -> None: - assessed, outputs = run_output_step( - "Assess policy and validation", - { - "EVENT_ACTION": "synchronize", - "EVENT_BASE_REF": "main", - "EVENT_BASE_SHA": "a" * 40, - "EVENT_HEAD_SHA": "b" * 40, - "EVENT_MERGE_SHA": "c" * 40, - "GH_TOKEN": "test-token", - "MODE": "authorization_required", - "POLICY_CONCLUSION": "failure", - "POLICY_RESULT": "success", - "POLICY_SUMMARY": "Maintainer authorization is required.", - "PR_NUMBER": "7", - "VALIDATION_RESULT": "skipped", - }, - ) - self.assertEqual(assessed.returncode, 0, assessed.stderr) - self.assertEqual(outputs["mode"], "authorization_required") - self.assertEqual(outputs["passed"], "false") - - dependabot_pr = True - preserve_failed = ( - outputs["mode"] == "preserve" - and not dependabot_pr - and outputs.get("preserve_valid") != "true" - ) - self.assertTrue(outputs["passed"] != "true" or preserve_failed) - enforce_script = extract_workflow_step_script("Enforce policy result").replace( - "${{ steps.assess-policy.outputs.summary || " - "'Validation assessment failed.' }}", - "Maintainer authorization is required.", - ) - enforced = subprocess.run( - ["bash", "-c", enforce_script], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - ) - self.assertNotEqual(enforced.returncode, 0) - - def test_required_workflow_rerun_consumes_only_a_fresh_label(self) -> None: - events = [[issue_event(1, "labeled", label="ci:upgrades")]] - - authorized, outputs = run_policy_decision_step( - action="synchronize", - event_label="", - event_actor="contributor", - event_actor_id=200, - event_updated_at="2026-01-01T00:00:00Z", - issue_events=events, - check_pages=[{"check_runs": []}], - run_attempt=2, - run_created_at="2025-12-31T23:59:59Z", - ) - - self.assertEqual(authorized.returncode, 0, authorized.stderr) - self.assertEqual(outputs["mode"], "validate") - self.assertEqual(outputs["run_validation"], "true") - self.assertEqual(outputs["authorization_event_id"], "1") - - later_attempt, later_outputs = run_policy_decision_step( - action="synchronize", - event_label="", - event_actor="contributor", - event_actor_id=200, - event_updated_at="2026-01-01T00:00:00Z", - issue_events=events, - check_pages=[{"check_runs": []}], - run_attempt=10, - run_created_at="2025-12-31T23:59:59Z", - ) - self.assertEqual(later_attempt.returncode, 0, later_attempt.stderr) - self.assertEqual(later_outputs["mode"], "validate") - - def test_diagnostic_check_cannot_make_a_required_workflow_pass(self) -> None: - policy = load_required_check_policy() - - decision = policy.decide_pull_request_policy( - action="synchronize", - event_label="", - base_ref_changed=False, - state="open", - merged=False, - merge_candidate_available=True, - relevant=True, - identity_matches=True, - authorization_label_present=True, - actor_permission="write", - has_existing_check=True, - has_newer_check=True, - authorization_epoch_valid=True, - authorization_rerun_matches=True, - ) - - self.assertEqual(decision.mode, "superseded") - self.assertFalse(decision.run_validation) - self.assertEqual(decision.conclusion, "failure") - - def test_required_workflow_rerun_rejects_identity_and_lifecycle_races( - self, - ) -> None: - newer_check = { - "app": {"slug": "github-actions"}, - "external_id": "lemonade-llamacpp-required-pr-7-2-51-1-501", - "id": 51, - "name": "llama.cpp authorized validation v2", - } - common = { - "action": "synchronize", - "event_label": "", - "event_actor": "contributor", - "event_actor_id": 200, - "event_updated_at": "2026-01-01T00:00:00Z", - "issue_events": [[issue_event(2, "labeled", label="ci:upgrades")]], - "check_pages": [{"check_runs": []}], - "run_attempt": 2, - "run_created_at": "2026-01-01T00:00:01Z", - } - cases = ( - ( - "label-at-run-boundary", - { - "issue_events": [[issue_event(1, "labeled", label="ci:upgrades")]], - }, - "authorization_required", - ), - ( - "base-retarget-after-label", - { - "issue_events": [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, "base_ref_changed"), - ] - ], - "run_created_at": "2026-01-01T00:00:00Z", - }, - "authorization_required", - ), - ( - "reopen-after-label", - { - "issue_events": [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, "reopened"), - ] - ], - "run_created_at": "2026-01-01T00:00:00Z", - }, - "authorization_required", - ), - ( - "remove-after-label", - { - "issue_events": [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, "unlabeled", label="ci:upgrades"), - ] - ], - "run_created_at": "2026-01-01T00:00:00Z", - }, - "authorization_required", - ), - ( - "push-after-original-run", - {"identity_matches": False}, - "stale_required", - ), - ( - "newer-default-event-run", - {"check_pages": [{"check_runs": [newer_check]}]}, - "superseded", - ), - ) - for name, overrides, expected_mode in cases: - arguments = dict(common) - arguments.update(overrides) - with self.subTest(name=name): - completed, outputs = run_policy_decision_step(**arguments) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["mode"], expected_mode) - self.assertEqual(outputs["run_validation"], "false") - self.assertEqual(outputs["conclusion"], "failure") - - for action in ("opened", "reopened", "synchronize"): - with self.subTest(valid_action=action): - completed, outputs = run_policy_decision_step( - **{ - **common, - "action": action, - "issue_events": [ - [ - issue_event(1, "reopened"), - issue_event(2, "labeled", label="ci:upgrades"), - ] - ], - } - ) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["mode"], "validate") - self.assertEqual(outputs["run_validation"], "true") - - def test_reauthorization_metadata_does_not_deadlock_the_required_rerun( - self, - ) -> None: - events = [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, "unlabeled", label="ci:upgrades"), - issue_event(3, "labeled", label="ci:upgrades"), - ] - ] - common = { - "action": "synchronize", - "event_label": "", - "event_actor": "contributor", - "event_actor_id": 200, - "event_updated_at": "2026-01-01T00:00:00Z", - "issue_events": events, - "run_attempt": 3, - "run_created_at": "2026-01-01T00:00:00Z", - } - prior_required = { - "app": {"slug": "github-actions"}, - "external_id": "lemonade-llamacpp-required-pr-7-1-50-2-500", - "id": 50, - "name": "llama.cpp authorized validation v2", - } - metadata_revocation = { - "app": {"slug": "github-actions"}, - "external_id": "lemonade-llamacpp-metadata-pr-7-2-51-1-501", - "id": 51, - "name": "llama.cpp authorization diagnostics v2", - } - legacy_unscoped_revocation = { - **metadata_revocation, - "external_id": "lemonade-llamacpp-pr-7-2-51-1-501", - "name": "llama.cpp authorized validation v2", - } - - initial_recovery, outputs = run_policy_decision_step( - check_pages=[{"check_runs": [metadata_revocation]}], - **{**common, "run_attempt": 2}, - ) - self.assertEqual(initial_recovery.returncode, 0, initial_recovery.stderr) - self.assertEqual(outputs["mode"], "validate") - self.assertEqual(outputs["run_validation"], "true") - - for name, checks in ( - ("scoped-metadata", [prior_required, metadata_revocation]), - ("legacy-unscoped-metadata", [legacy_unscoped_revocation]), - ): - with self.subTest(name=name): - completed, outputs = run_policy_decision_step( - check_pages=[{"check_runs": checks}], - **common, - ) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["mode"], "validate") - self.assertEqual(outputs["run_validation"], "true") - - newer_required = { - **prior_required, - "external_id": "lemonade-llamacpp-required-pr-7-3-51-1-501", - "id": 52, - } - superseded, outputs = run_policy_decision_step( - check_pages=[{"check_runs": [newer_required]}], - **common, - ) - self.assertEqual(superseded.returncode, 0, superseded.stderr) - self.assertEqual(outputs["mode"], "superseded") - self.assertEqual(outputs["conclusion"], "failure") - - def test_recovered_required_result_is_distinct_from_metadata_diagnostics( - self, - ) -> None: - required_name = "llama.cpp authorized validation v2" - metadata_name = "llama.cpp authorization diagnostics v2" - metadata_report, metadata_calls = run_report_step( - [[]], - authorization_epoch=2, - conclusion="failure", - event_action="unlabeled", - event_name="pull_request_target", - mode="authorization_required", - run_id=501, - run_number=51, - ) - self.assertEqual(metadata_report.returncode, 0, metadata_report.stderr) - metadata_writes = [ - call - for call in metadata_calls - if "--method" in call - and call[call.index("--method") + 1] in {"POST", "PATCH"} - ] - self.assertEqual(len(metadata_writes), 1) - self.assertIn(f"name={metadata_name}", metadata_writes[0]) - self.assertNotIn(f"name={required_name}", metadata_writes[0]) - - metadata_check = { - "app": {"slug": "github-actions"}, - "external_id": "lemonade-llamacpp-metadata-pr-7-2-51-1-501", - "id": 90, - "name": metadata_name, - } - recovered_check = { - "app": {"slug": "github-actions"}, - "external_id": "lemonade-llamacpp-required-pr-7-3-50-3-500", - "id": 80, - "name": required_name, - } - live_pull_request = { - "base": {"ref": "main", "sha": "a" * 40}, - "head": {"sha": "b" * 40}, - "labels": [{"name": "ci:upgrades"}], - "merge_commit_sha": "c" * 40, - "mergeable": True, - "merged": False, - "number": 7, - "state": "open", - } - authorization_events = [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, "unlabeled", label="ci:upgrades"), - issue_event(3, "labeled", label="ci:upgrades"), - ] - ] - recovered, recovered_calls = run_report_step( - [ - [{"check_runs": [metadata_check]}], - [{"check_runs": [metadata_check, recovered_check]}], - ], - authorization_created_at="2026-01-01T00:00:03Z", - authorization_epoch=3, - issue_event_snapshots=[authorization_events, authorization_events], - pull_request_snapshots=[live_pull_request, live_pull_request], - run_attempt=3, - ) - self.assertEqual(recovered.returncode, 0, recovered.stderr) - recovered_writes = [ - call - for call in recovered_calls - if "--method" in call - and call[call.index("--method") + 1] in {"POST", "PATCH"} - ] - self.assertTrue( - any(f"name={required_name}" in call for call in recovered_writes) - ) - self.assertFalse( - any(f"name={metadata_name}" in call for call in recovered_writes) - ) - self.assertFalse( - any( - "repos/LLM360/lemonade/check-runs/90" in call - for call in recovered_writes - ) - ) - - def test_policy_failure_without_outputs_still_reports_and_enforces_failure( - self, - ) -> None: - completed, outputs = run_output_step( - "Assess policy and validation", - { - "EVENT_ACTION": "labeled", - "MODE": "", - "POLICY_CONCLUSION": "", - "POLICY_RESULT": "failure", - "POLICY_SUMMARY": "", - "VALIDATION_RESULT": "skipped", - }, - ) - - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["mode"], "policy_error") - self.assertEqual(outputs["conclusion"], "failure") - self.assertEqual(outputs["passed"], "false") - - workflow = PR_WORKFLOW.read_text(encoding="utf-8") - assess = workflow.split(" - name: Assess policy and validation\n", 1)[ - 1 - ].split(" - name:", 1)[0] - report = workflow.split( - " - name: Report result on the exact candidate commit\n", 1 - )[1].split(" - name: Enforce policy result\n", 1)[0] - enforce = workflow.split(" - name: Enforce policy result\n", 1)[1] - self.assertIn(" if: always()\n", assess) - self.assertIn(" always() &&\n", report) - self.assertIn(" always() &&\n", enforce) - - def test_failed_closed_event_uses_live_lifecycle_before_skipping(self) -> None: - reopened = { - "base": {"ref": "main", "sha": "a" * 40}, - "head": {"sha": "b" * 40}, - "labels": [{"name": "ci:upgrades"}], - "merge_commit_sha": "c" * 40, - "mergeable": True, - "merged": False, - "number": 7, - "state": "open", - } - completed, outputs = run_failed_closed_assessment(reopened) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["mode"], "policy_error") - self.assertEqual(outputs["conclusion"], "failure") - self.assertEqual(outputs["passed"], "false") - self.assertEqual(outputs["fallback_merge_sha"], "c" * 40) - - def test_unconfirmed_observed_merge_gate_is_failed_with_run_ordering( - self, - ) -> None: - workflow = PR_WORKFLOW.read_text(encoding="utf-8") - step_name = "Invalidate an unconfirmed observed merge commit" - self.assertIn(f" - name: {step_name}\n", workflow) - observed_merge_sha = "c" * 40 - prefix = "lemonade-llamacpp-metadata-pr-7-" - old_check = { - "id": 11, - "name": "llama.cpp authorization diagnostics v2", - "app": {"slug": "github-actions"}, - "external_id": f"{prefix}1-10-1-9000", - } - foreign_check = { - "id": 12, - "name": "llama.cpp authorization diagnostics v2", - "app": {"slug": "foreign-app"}, - "external_id": f"{prefix}1-9-1-8000", - } - - completed, calls = run_observed_merge_invalidation( - [{"check_runs": [old_check, foreign_check]}], - observed_merge_sha=observed_merge_sha, - ) - - self.assertEqual(completed.returncode, 0, completed.stderr) - mutations = [ - call for call in calls if call[call.index("--method") + 1] != "GET" - ] - posts = [ - call for call in mutations if call[call.index("--method") + 1] == "POST" - ] - self.assertEqual(len(posts), 1) - self.assertIn(f"head_sha={observed_merge_sha}", posts[0]) - self.assertIn("conclusion=failure", posts[0]) - patched_endpoints = { - next(argument for argument in call if argument.startswith("repos/")) - for call in mutations - if call[call.index("--method") + 1] == "PATCH" - } - self.assertIn("repos/LLM360/lemonade/check-runs/11", patched_endpoints) - self.assertNotIn("repos/LLM360/lemonade/check-runs/12", patched_endpoints) - - newer_check = dict(old_check, id=13, external_id=f"{prefix}1-60-1-10") - superseded, superseded_calls = run_observed_merge_invalidation( - [{"check_runs": [newer_check]}], - observed_merge_sha=observed_merge_sha, - ) - self.assertEqual(superseded.returncode, 0, superseded.stderr) - self.assertTrue( - all(call[call.index("--method") + 1] == "GET" for call in superseded_calls) - ) - - second_observed_merge_sha = "d" * 40 - both, both_calls = run_observed_merge_invalidation( - [{"check_runs": []}], - observed_merge_sha=observed_merge_sha, - secondary_observed_merge_sha=second_observed_merge_sha, - authorization_epoch=0, - ) - self.assertEqual(both.returncode, 0, both.stderr) - observed_posts = [ - call - for call in both_calls - if "--method" in call and call[call.index("--method") + 1] == "POST" - ] - self.assertEqual(len(observed_posts), 2) - self.assertEqual( - { - next(value for value in call if value.startswith("head_sha=")) - for call in observed_posts - }, - {f"head_sha={observed_merge_sha}", f"head_sha={second_observed_merge_sha}"}, - ) - self.assertTrue( - all( - "external_id=lemonade-llamacpp-metadata-pr-7-0-50-1-500" in call - for call in observed_posts - ) - ) - - def test_post_write_recheck_closes_concurrent_check_ordering_races( - self, - ) -> None: - prefix = "lemonade-llamacpp-required-pr-7-" - current = { - "id": 80, - "name": "llama.cpp authorized validation v2", - "app": {"slug": "github-actions"}, - "external_id": f"{prefix}1-50-1-500", - } - newer_revocation = { - "id": 90, - "name": "llama.cpp authorized validation v2", - "app": {"slug": "github-actions"}, - "external_id": f"{prefix}0-51-1-10", - } - - completed, calls = run_report_step( - [[], [{"check_runs": [current, newer_revocation]}]] - ) - self.assertNotEqual(completed.returncode, 0) - mutations = [ - call for call in calls if call[call.index("--method") + 1] != "GET" - ] - self.assertIn("conclusion=success", mutations[0]) - self.assertTrue( - any( - "repos/LLM360/lemonade/check-runs/80" in call - and "conclusion=cancelled" in call - for call in mutations - ) - ) - self.assertFalse( - any("repos/LLM360/lemonade/check-runs/90" in call for call in mutations) - ) - - superseded_before_write, pre_write_calls = run_report_step( - [[{"check_runs": [newer_revocation]}]] - ) - self.assertNotEqual(superseded_before_write.returncode, 0) - self.assertTrue( - all(call[call.index("--method") + 1] == "GET" for call in pre_write_calls) - ) - - prior_attempt = dict(current, id=70, external_id=f"{prefix}9-50-1-9000") - retry = dict(current, external_id=f"{prefix}1-50-2-500") - retried, retry_calls = run_report_step( - [ - [{"check_runs": [prior_attempt]}], - [{"check_runs": [prior_attempt, retry]}], - ], - run_attempt=2, - ) - self.assertEqual(retried.returncode, 0, retried.stderr) - self.assertTrue( - any( - "repos/LLM360/lemonade/check-runs/70" in call - and "conclusion=cancelled" in call - for call in retry_calls - ) - ) - - def test_success_is_demoted_when_live_identity_changes_after_write(self) -> None: - valid_pull_request = { - "base": {"ref": "main", "sha": "a" * 40}, - "head": {"sha": "b" * 40}, - "labels": [{"name": "ci:upgrades"}], - "merge_commit_sha": "c" * 40, - "mergeable": True, - "merged": False, - "number": 7, - "state": "open", - } - current_check = { - "app": {"slug": "github-actions"}, - "external_id": "lemonade-llamacpp-required-pr-7-1-50-1-500", - "id": 80, - "name": "llama.cpp authorized validation v2", - } - valid_events = [[issue_event(1, "labeled", label="ci:upgrades")]] - changed_snapshots = { - "authorization": { - **valid_pull_request, - "labels": [], - }, - "base-ref": { - **valid_pull_request, - "base": {"ref": "release", "sha": "a" * 40}, - }, - "head": { - **valid_pull_request, - "head": {"sha": "d" * 40}, - }, - "merge": { - **valid_pull_request, - "merge_commit_sha": "d" * 40, - }, - "base-epoch": valid_pull_request, - "reopened-epoch": valid_pull_request, - "permission": valid_pull_request, - } - for name, changed_pull_request in changed_snapshots.items(): - with self.subTest(name=name): - post_events = ( - [ - [ - *valid_events[0], - issue_event(2, "unlabeled", label="ci:upgrades"), - ] - ] - if name == "authorization" - else ( - [[*valid_events[0], issue_event(2, "base_ref_changed")]] - if name == "base-epoch" - else ( - [ - [ - *valid_events[0], - issue_event(2, "closed"), - issue_event(3, "reopened"), - ] - ] - if name == "reopened-epoch" - else valid_events - ) - ) - ) - completed, calls = run_report_step( - [[], [{"check_runs": [current_check]}]], - pull_request_snapshots=[valid_pull_request, changed_pull_request], - issue_event_snapshots=[valid_events, post_events], - permission_snapshots=( - ["write", "read"] if name == "permission" else None - ), - ) - self.assertNotEqual(completed.returncode, 0) - pull_reads = [ - call - for call in calls - if any("/pulls/7" in argument for argument in call) - ] - self.assertEqual(len(pull_reads), 2) - self.assertTrue( - any( - "repos/LLM360/lemonade/check-runs/80" in call - and "conclusion=failure" in call - for call in calls - ) - ) - - def test_missing_live_authorization_invalidates_a_prior_protected_gate( - self, - ) -> None: - policy = load_required_check_policy() - common = { - "base_ref_changed": False, - "state": "open", - "merged": False, - "merge_candidate_available": True, - "identity_matches": True, - "actor_permission": "write", - "has_newer_check": False, - } - - authorized = policy.decide_pull_request_policy( - action="synchronize", - event_label="", - relevant=True, - authorization_label_present=True, - authorization_rerun_matches=True, - has_existing_check=False, - **common, - ) - self.assertEqual(authorized.mode, "validate") - self.assertTrue(authorized.run_validation) - - for action, event_label in (("edited", ""), ("labeled", "ci:macos")): - with self.subTest(action=action, event_label=event_label): - missing_live_label = policy.decide_pull_request_policy( - action=action, - event_label=event_label, - relevant=True, - authorization_label_present=False, - has_existing_check=True, - **common, - ) - self.assertEqual(missing_live_label.mode, "authorization_required") - self.assertFalse(missing_live_label.run_validation) - self.assertEqual(missing_live_label.conclusion, "failure") - - neutral = policy.decide_pull_request_policy( - action="edited", - event_label="", - relevant=False, - authorization_label_present=False, - has_existing_check=True, - **common, - ) - self.assertEqual(neutral.mode, "neutral") - self.assertEqual(neutral.conclusion, "success") - - def test_authorization_lifecycle_events_cannot_be_replaced_or_bypassed( - self, - ) -> None: - policy = load_required_check_policy() - common = { - "base_ref_changed": False, - "state": "open", - "merged": False, - "merge_candidate_available": True, - "relevant": True, - "identity_matches": True, - "authorization_label_present": True, - "has_existing_check": True, - "has_newer_check": False, - } - - removed_before_fast_readd = policy.decide_pull_request_policy( - action="unlabeled", - event_label="ci:upgrades", - actor_permission="", - **common, - ) - self.assertEqual(removed_before_fast_readd.mode, "authorization_required") - self.assertEqual(removed_before_fast_readd.conclusion, "failure") - - triage_readd = policy.decide_pull_request_policy( - action="labeled", - event_label="ci:upgrades", - actor_permission="triage", - **common, - ) - self.assertEqual(triage_readd.mode, "awaiting_rerun") - self.assertFalse(triage_readd.run_validation) - - metadata_replacement = policy.decide_pull_request_policy( - action="labeled", - event_label="ci:macos", - actor_permission="write", - **common, - ) - self.assertEqual(metadata_replacement.mode, "preserve") - self.assertFalse(metadata_replacement.run_validation) - - metadata_after_untrusted_readd = policy.decide_pull_request_policy( - action="edited", - event_label="", - actor_permission="triage", - authorization_epoch_valid=True, - **common, - ) - self.assertEqual(metadata_after_untrusted_readd.mode, "authorization_required") - self.assertEqual(metadata_after_untrusted_readd.conclusion, "failure") - - metadata_before_readd = policy.decide_pull_request_policy( - action="edited", - event_label="", - actor_permission="", - has_existing_check=False, - **{ - key: value - for key, value in common.items() - if key != "has_existing_check" - }, - ) - self.assertEqual(metadata_before_readd.mode, "authorization_required") - self.assertEqual(metadata_before_readd.conclusion, "failure") - self.assertFalse(metadata_before_readd.run_validation) - - trusted_readd = policy.decide_pull_request_policy( - action="labeled", - event_label="ci:upgrades", - actor_permission="write", - **common, - ) - self.assertEqual(trusted_readd.mode, "awaiting_rerun") - self.assertFalse(trusted_readd.run_validation) - - trusted_rerun = policy.decide_pull_request_policy( - action="synchronize", - event_label="", - actor_permission="write", - authorization_rerun_matches=True, - **common, - ) - self.assertEqual(trusted_rerun.mode, "validate") - self.assertTrue(trusted_rerun.run_validation) - - delayed_removal = policy.decide_pull_request_policy( - action="unlabeled", - event_label="ci:upgrades", - actor_permission="", - has_newer_check=True, - **{key: value for key, value in common.items() if key != "has_newer_check"}, - ) - self.assertEqual(delayed_removal.mode, "preserve") - - workflow = PR_WORKFLOW.read_text(encoding="utf-8") - decision_environment = workflow.split( - " - name: Decide whether protected validation may run\n", 1 - )[1].split(" run: |\n", 1)[0] - concurrency = workflow.split("concurrency:\n", 1)[1].split("\njobs:\n", 1)[0] - self.assertIn("github.event.action == 'labeled'", concurrency) - self.assertIn("github.event.action == 'unlabeled'", concurrency) - self.assertIn("github.run_number", concurrency) - self.assertIn("github.run_attempt", concurrency) - self.assertNotIn("github.run_id", concurrency) - for action in ( - "opened", - "reopened", - "synchronize", - "edited", - "labeled", - "unlabeled", - "closed", - ): - self.assertIn(action, concurrency) - self.assertNotIn("github.event.pull_request.merge_commit_sha", concurrency) - self.assertIn("issues: read", workflow) - self.assertIn("issues/${PR_NUMBER}/events?per_page=100", workflow) - self.assertIn("--paginate --slurp", workflow) - self.assertIn("authorization_event_id", workflow) - self.assertIn("authorization_created_at", workflow) - self.assertIn("actions: read", workflow) - self.assertIn("CURRENT_RUN_ID: ${{ github.run_id }}", workflow) - self.assertIn( - "EVENT_BASE_REF: ${{ github.event.pull_request.base.ref || '' }}", - decision_environment, - ) - self.assertIn( - "EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || '' }}", - decision_environment, - ) - self.assertIn( - "EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }}", - decision_environment, - ) - self.assertIn( - "TRIGGERING_ACTOR: ${{ github.triggering_actor || '' }}", workflow - ) - self.assertIn("actions/runs/${CURRENT_RUN_ID}/attempts/1", workflow) - self.assertIn(".github/workflows/validate_llamacpp_pr.yml", workflow) - self.assertNotIn(".head_branch == $base_ref", workflow) - self.assertNotIn(".head_repository.full_name == $repository", workflow) - self.assertNotIn(".head_sha == $base_sha", workflow) - self.assertIn('$path + "@" + $base_ref', workflow) - self.assertIn('$path + "@refs/heads/" + $base_ref', workflow) - self.assertIn('((.pull_requests | type) == "array")', workflow) - self.assertNotIn(".pull_requests | length", workflow) - self.assertNotIn(".pull_requests[0]", workflow) - self.assertIn(".head_sha == $head_sha", workflow) - self.assertIn('.conclusion == "failure"', workflow) - self.assertIn(".run_started_at", workflow) - self.assertIn("authorization_boundary", workflow) - self.assertIn("authorization_rerun_matches", workflow) - self.assertIn( - '--authorization-rerun-matches "$authorization_rerun_matches"', - workflow, - ) - self.assertIn('[[ "$CURRENT_RUN_ATTEMPT" =~ ^([2-9]|[1-9][0-9]+)$ ]]', workflow) - self.assertIn(".user.id == $actor_id", workflow) - report = workflow.split( - " - name: Report result on the exact candidate commit\n", 1 - )[1].split(" - name: Enforce policy result\n", 1)[0] - self.assertLess( - report.index('newest_number" -gt "$RUN_NUMBER'), - report.index("--method PATCH"), - ) - self.assertIn(".number < $current_number", report) - self.assertIn("RUN_NUMBER: ${{ github.run_number }}", report) - - def test_preserve_rechecks_the_live_epoch_or_reports_terminal_failure( - self, - ) -> None: - pull_request = { - "base": {"ref": "main", "sha": "a" * 40}, - "head": {"sha": "b" * 40}, - "labels": [{"name": "ci:upgrades"}], - "merge_commit_sha": "c" * 40, - "mergeable": True, - "merged": False, - "number": 7, - "state": "open", - } - current_epoch_check = { - "app": {"slug": "github-actions"}, - "conclusion": "success", - "external_id": "lemonade-llamacpp-required-pr-7-1-40-1-400", - "id": 40, - "name": "llama.cpp authorized validation v2", - "status": "completed", - } - valid, valid_outputs = run_preserve_step( - pull_request=pull_request, - issue_events=[[issue_event(1, "labeled", label="ci:upgrades")]], - check_pages=[{"check_runs": [current_epoch_check]}], - ) - self.assertEqual(valid.returncode, 0, valid.stderr) - self.assertEqual(valid_outputs["valid"], "true") - - for name, events, checks in ( - ( - "unlabeled-after-policy", - [ - [ - issue_event(1, "labeled", label="ci:upgrades"), - issue_event(2, "unlabeled", label="ci:upgrades"), - ] - ], - [current_epoch_check], - ), - ( - "old-epoch-only", - [[issue_event(1, "labeled", label="ci:upgrades")]], - [ - { - **current_epoch_check, - "external_id": "lemonade-llamacpp-required-pr-7-9-40-1-400", - } - ], - ), - ( - "malformed-final-provenance", - [ - [ - { - "actor": {"id": 100, "login": "maintainer"}, - "created_at": "2026-01-01T00:00:01Z", - "event": "labeled", - "id": 1, - "label": {"name": "ci:upgrades"}, - } - ] - ], - [current_epoch_check], - ), - ( - "in-progress-prior-validation", - [[issue_event(1, "labeled", label="ci:upgrades")]], - [{**current_epoch_check, "conclusion": None, "status": "in_progress"}], - ), - ( - "failed-prior-validation", - [[issue_event(1, "labeled", label="ci:upgrades")]], - [{**current_epoch_check, "conclusion": "failure"}], - ), - ( - "newer-required-decision", - [[issue_event(1, "labeled", label="ci:upgrades")]], - [ - current_epoch_check, - { - **current_epoch_check, - "conclusion": None, - "external_id": ("lemonade-llamacpp-required-pr-7-1-51-1-600"), - "id": 51, - "status": "in_progress", - }, - ], - ), - ( - "later-failed-prior-decision", - [[issue_event(1, "labeled", label="ci:upgrades")]], - [ - current_epoch_check, - { - **current_epoch_check, - "conclusion": "failure", - "external_id": ("lemonade-llamacpp-required-pr-7-1-49-1-499"), - "id": 49, - }, - ], - ), - ): - with self.subTest(name=name): - invalid, invalid_outputs = run_preserve_step( - pull_request=pull_request, - issue_events=events, - check_pages=[{"check_runs": checks}], - ) - self.assertEqual(invalid.returncode, 0, invalid.stderr) - self.assertEqual(invalid_outputs["valid"], "false") - - reported_check = { - **current_epoch_check, - "external_id": "lemonade-llamacpp-required-pr-7-1-50-1-500", - "id": 80, - } - reported, calls = run_report_step( - [[], [{"check_runs": [reported_check]}]], - mode="preserve", - conclusion="failure", - preserve_valid=invalid_outputs["valid"] == "true", - ) - self.assertEqual(reported.returncode, 0, reported.stderr) - self.assertTrue( - any( - "--method" in call - and call[call.index("--method") + 1] in {"POST", "PATCH"} - and "conclusion=failure" in call - for call in calls - ) - ) - - def test_live_identity_rejects_stale_merge_sha_until_mergeability_is_known( - self, - ) -> None: - base_sha = "a" * 40 - head_sha = "b" * 40 - stale_merge_sha = "c" * 40 - - def snapshot(mergeable: object) -> dict[str, object]: - return { - "number": 7, - "state": "open", - "merged": False, - "mergeable": mergeable, - "merge_commit_sha": stale_merge_sha, - "base": {"sha": base_sha}, - "head": {"sha": head_sha}, - "changed_files": 1, - "labels": [{"name": "ci:upgrades"}], - } - - cases = ( - ("unknown-through-timeout", [snapshot(None)] * 3, 3), - ("known-unmergeable", [snapshot(False)], 1), - ) - policy = load_required_check_policy() - for name, snapshots, expected_calls in cases: - with self.subTest(name=name): - completed, outputs, calls = run_identity_step( - snapshots, - event_merge_sha=stale_merge_sha, - ) - - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(calls, expected_calls) - self.assertEqual(outputs["merge_candidate_available"], "false") - self.assertEqual(outputs["merge_sha"], head_sha) - self.assertEqual(outputs["observed_merge_sha"], stale_merge_sha) - self.assertEqual(outputs["identity_matches"], "true") - - decision = policy.decide_pull_request_policy( - action="synchronize", - event_label="", - base_ref_changed=False, - state=outputs["state"], - merged=outputs["merged"] == "true", - merge_candidate_available=( - outputs["merge_candidate_available"] == "true" - ), - relevant=True, - identity_matches=outputs["identity_matches"] == "true", - authorization_label_present=True, - authorization_rerun_matches=True, - actor_permission="write", - has_existing_check=False, - has_newer_check=False, - ) - self.assertEqual(decision.mode, "authorization_required") - self.assertFalse(decision.run_validation) - self.assertEqual(decision.conclusion, "failure") - - def test_identity_tracks_event_and_live_unconfirmed_merge_commits(self) -> None: - base_sha = "a" * 40 - head_sha = "b" * 40 - event_merge_sha = "c" * 40 - live_merge_sha = "d" * 40 - - def snapshot(merge_commit_sha: str | None) -> dict[str, object]: - return { - "base": {"ref": "main", "sha": base_sha}, - "changed_files": 1, - "head": {"sha": head_sha}, - "labels": [{"name": "ci:upgrades"}], - "merge_commit_sha": merge_commit_sha, - "mergeable": None, - "merged": False, - "number": 7, - "state": "open", - } - - for name, live_merge, expected_primary, expected_secondary in ( - ("event-only", None, event_merge_sha, ""), - ("live-and-event", live_merge_sha, live_merge_sha, event_merge_sha), - ): - with self.subTest(name=name): - completed, outputs, _ = run_identity_step( - [snapshot(live_merge)] * 3, - event_merge_sha=event_merge_sha, - ) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["merge_sha"], head_sha) - self.assertEqual(outputs["observed_merge_sha"], expected_primary) - self.assertEqual( - outputs["secondary_observed_merge_sha"], expected_secondary - ) - - retargeted = snapshot(live_merge_sha) - retargeted["base"] = {"ref": "release", "sha": base_sha} - completed, outputs, _ = run_identity_step( - [retargeted] * 3, - event_base_ref="main", - event_merge_sha=event_merge_sha, - ) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(outputs["base_ref"], "release") - self.assertEqual(outputs["identity_matches"], "false") - - def test_required_check_classifier_covers_protected_inputs(self) -> None: - policy = load_required_check_policy() - - protected_paths = ( - ".github/llamacpp_release_manifest.json", - ".github/workflows/validate_llamacpp_pr.yml", - ".github/workflows/validate_llamacpp_core.yml", - ".github/scripts/run_llamacpp_validation.py", - ".github/scripts/llamacpp_required_check_policy.py", - ".github/actions/setup-venv/action.yml", - ".github/actions/setup-python/action.yml", - ".github/actions/cleanup-processes-linux/action.yml", - ".github/actions/cleanup-processes-windows/action.yml", - "CMakeLists.txt", - "CMakePresets.json", - "cmake/DetectSystemHttplib.cmake", - "setup.sh", - "src/cpp/resources/backend_versions.json", - "src/cpp/resources/server_models.json", - "src/app/src/renderer/utils/toolDefinitions.json", - "src/cpp/server/backends/llamacpp/llamacpp_server.cpp", - "src/cpp/server/backends/vllm/vllm_server.cpp", - "src/cpp/server/backends/fastflowlm/fastflowlm_models.cpp", - "src/cpp/include/lemon/backend_manager.h", - "src/cpp/server/backend_manager.cpp", - "src/cpp/include/lemon/runtime_config.h", - "src/cpp/server/runtime_config.cpp", - "src/cpp/include/lemon/backends/llamacpp/llamacpp_server.h", - "src/cpp/server/backends/backend_utils.cpp", - "src/cpp/include/lemon/backends/backend_utils.h", - "src/cpp/server/streaming_proxy.cpp", - "src/cpp/include/lemon/streaming_proxy.h", - "src/cpp/server/wrapped_server.cpp", - "src/cpp/include/lemon/wrapped_server.h", - "src/cpp/server/system_info.cpp", - "src/cpp/include/lemon/system_info.h", - "src/cpp/server/utils/platform/process_linux.cpp", - "src/cpp/include/lemon/utils/process_manager.h", - "test/requirements.txt", - "test/validate_llamacpp.py", - "test/utils/llamacpp_capability_validation.py", - "test/utils/capabilities.py", - "test/utils/server_base.py", - "test/utils/test_models.py", - "test/utils/validation_model_catalog.py", - "test/utils/validation_model_selection.py", - "test/fixtures/llamacpp_validation_models.json", - "new-root-build-input.txt", - ".github/actions/future-validation-action/action.yml", - ".github/scripts/future-validation-helper.py", - ) - for path in protected_paths: - with self.subTest(path=path): - self.assertTrue(policy.is_protected_path(path)) - - unrelated_paths = ( - "docs/dev/testing.md", - "src/app/src/renderer/ChatWindow.tsx", - "src/web-app/src/index.ts", - ) - for path in unrelated_paths: - with self.subTest(path=path): - self.assertFalse(policy.is_protected_path(path)) - - def test_required_check_classifier_fails_closed_on_incomplete_file_lists( - self, - ) -> None: - policy = load_required_check_policy() - base_sha = "a" * 40 - stale_target_sha = "b" * 40 - candidate_sha = "c" * 40 - neutral_comparison = { - "ahead_by": 1, - "base_commit": {"sha": base_sha}, - "behind_by": 0, - "commits": [{"sha": candidate_sha}], - "files": [{"filename": "docs/dev/testing.md", "status": "modified"}], - "merge_base_commit": {"sha": base_sha}, - "status": "ahead", - "total_commits": 1, - } - - self.assertFalse( - policy.classify_compare_response( - neutral_comparison, - expected_base_sha=base_sha, - expected_target_sha=candidate_sha, - expected_count=1, - ) - ) - stale_comparison = { - **neutral_comparison, - "commits": [{"sha": stale_target_sha}], - } - with self.assertRaises(ValueError): - policy.classify_compare_response( - stale_comparison, - expected_base_sha=base_sha, - expected_target_sha=candidate_sha, - expected_count=1, - ) - self.assertTrue( - policy.classify_compare_response( - {}, - expected_base_sha=base_sha, - expected_target_sha=candidate_sha, - expected_count=policy.MAX_COMPARE_FILES + 1, - ) - ) - - self.assertTrue( - policy.classify_changed_file_pages( - [ - [ - { - "filename": "docs/renamed.md", - "previous_filename": "CMakeLists.txt", - "status": "renamed", - } - ] - ], - expected_count=1, - ) - ) - self.assertTrue( - policy.classify_changed_file_pages( - [ - [ - { - "filename": "src/cpp/resources/backend_versions.json", - "status": "modified", - } - ] - ], - expected_count=1, - ) - ) - self.assertFalse( - policy.classify_changed_file_pages( - [[{"filename": "docs/dev/testing.md", "status": "modified"}]], - expected_count=1, - ) - ) - - for status in ( - "added", - "removed", - "modified", - "copied", - "changed", - "unchanged", - ): - with self.subTest(valid_status=status): - self.assertFalse( - policy.classify_changed_file_pages( - [[{"filename": "docs/dev/testing.md", "status": status}]], - expected_count=1, - ) - ) - - with self.assertRaises(ValueError): - policy.classify_changed_file_pages( - [ - [ - { - "filename": "docs/dev/testing.md", - "previous_filename": "docs/dev/old.md", - "status": status, - } - ] - ], - expected_count=1, - ) - - for previous_filename in (None, "", 3, "/docs/old.md", "docs/../old.md"): - with self.subTest(invalid_previous_filename=previous_filename): - with self.assertRaises(ValueError): - policy.classify_changed_file_pages( - [ - [ - { - "filename": "docs/new.md", - "previous_filename": previous_filename, - "status": "renamed", - } - ] - ], - expected_count=1, - ) - - invalid_cases = ( - ([], 1), - ([[{"filename": "docs/one.md", "status": "modified"}]], 2), - ( - [ - [ - {"filename": "docs/one.md", "status": "modified"}, - {"filename": "docs/one.md", "status": "modified"}, - ] - ], - 2, - ), - ([[{"filename": "/absolute/path", "status": "modified"}]], 1), - ([[{"filename": "docs/one.md"}]], 1), - ([[{"filename": "docs/one.md", "status": "moved"}]], 1), - ([[{"filename": "docs/one.md", "status": 3}]], 1), - ([[{"filename": "docs/renamed.md", "status": "renamed"}]], 1), - ( - [ - [ - { - "filename": "docs/one.md", - "previous_filename": "docs/old.md", - "status": "modified", - } - ] - ], - 1, - ), - ( - [ - [ - { - "filename": "docs/one.md", - "previous_filename": None, - "status": "modified", - } - ] - ], - 1, - ), - ( - [[{"filename": "docs/one.md", "status": "modified"}]], - policy.MAX_CHANGED_FILES + 1, - ), - ) - for pages, expected_count in invalid_cases: - with self.subTest(pages=pages, expected_count=expected_count): - with self.assertRaises(ValueError): - policy.classify_changed_file_pages( - pages, - expected_count=expected_count, - ) - - def test_required_check_policy_requires_fresh_maintainer_authorization( - self, - ) -> None: - policy = load_required_check_policy() - - def decide(**overrides): - arguments = { - "action": "synchronize", - "event_label": "", - "base_ref_changed": False, - "state": "open", - "merged": False, - "merge_candidate_available": True, - "relevant": True, - "identity_matches": True, - "authorization_label_present": True, - "actor_permission": "write", - "authorization_rerun_matches": False, - "has_existing_check": True, - "has_newer_check": False, - } - arguments.update(overrides) - return policy.decide_pull_request_policy(**arguments) - - for action in ("opened", "reopened", "synchronize"): - with self.subTest(action=action): - decision = decide(action=action) - self.assertEqual(decision.mode, "authorization_required") - self.assertFalse(decision.run_validation) - self.assertIn("remove and reapply", decision.summary) - - base_edit = decide(action="edited", base_ref_changed=True) - self.assertEqual(base_edit.mode, "authorization_required") - - removed = decide( - action="unlabeled", - event_label="ci:upgrades", - authorization_label_present=False, - ) - self.assertEqual(removed.mode, "authorization_required") - - authorized = decide( - action="synchronize", - authorization_rerun_matches=True, - ) - self.assertEqual(authorized.mode, "validate") - self.assertTrue(authorized.run_validation) - - authorized_override = decide( - action="synchronize", - authorization_rerun_matches=True, - relevant=False, - has_existing_check=False, - ) - self.assertEqual(authorized_override.mode, "validate") - self.assertTrue(authorized_override.run_validation) - - unauthorized_override = decide( - action="synchronize", - authorization_rerun_matches=True, - relevant=False, - actor_permission="read", - has_existing_check=False, - ) - self.assertEqual(unauthorized_override.mode, "neutral") - self.assertFalse(unauthorized_override.run_validation) - - for permission in ("read", "none", ""): - with self.subTest(permission=permission): - denied = decide( - action="synchronize", - authorization_rerun_matches=True, - actor_permission=permission, - ) - self.assertEqual(denied.mode, "authorization_required") - self.assertFalse(denied.run_validation) - - stale = decide( - action="synchronize", - authorization_rerun_matches=True, - identity_matches=False, - ) - self.assertEqual(stale.mode, "stale_required") - - stale_without_gate = decide( - action="synchronize", - authorization_rerun_matches=True, - identity_matches=False, - has_existing_check=False, - ) - self.assertEqual(stale_without_gate.mode, "stale_required") - - neutral_without_merge = decide( - action="opened", - merge_candidate_available=False, - relevant=False, - has_existing_check=False, - ) - self.assertEqual(neutral_without_merge.mode, "neutral") - - protected_without_merge = decide( - action="synchronize", - authorization_rerun_matches=True, - merge_candidate_available=False, - has_existing_check=False, - ) - self.assertEqual(protected_without_merge.mode, "authorization_required") - self.assertFalse(protected_without_merge.run_validation) - - preserved = decide(action="labeled", event_label="ci:macos") - self.assertEqual(preserved.mode, "preserve") - - missing_prior = decide( - action="labeled", - event_label="ci:macos", - has_existing_check=False, - ) - self.assertEqual(missing_prior.mode, "authorization_required") - self.assertEqual(missing_prior.conclusion, "failure") - - neutral = decide(relevant=False, has_existing_check=False) - self.assertEqual(neutral.mode, "neutral") - self.assertEqual(neutral.conclusion, "success") - - closed = decide(action="closed", state="closed") - self.assertEqual(closed.mode, "closed") - self.assertEqual(closed.conclusion, "cancelled") - - merged = decide(action="closed", state="closed", merged=True) - self.assertEqual(merged.mode, "merged") - self.assertEqual(merged.conclusion, "") - - stale_after_merge = decide(action="labeled", state="closed", merged=True) - self.assertEqual(stale_after_merge.mode, "merged") - self.assertEqual(stale_after_merge.conclusion, "") - - stale_close_after_reopen = decide( - action="closed", - state="open", - merged=False, - ) - self.assertEqual(stale_close_after_reopen.mode, "stale") - - removal_before_reapply_is_processed = decide( - action="unlabeled", - event_label="ci:upgrades", - authorization_label_present=True, - ) - self.assertEqual( - removal_before_reapply_is_processed.mode, "authorization_required" - ) - self.assertEqual(removal_before_reapply_is_processed.conclusion, "failure") - - delayed_synchronize = decide( - action="synchronize", - has_newer_check=True, - ) - self.assertEqual(delayed_synchronize.mode, "superseded") - self.assertEqual(delayed_synchronize.conclusion, "failure") - - def test_protected_validation_bootstrap_requires_the_trusted_workflow( - self, - ) -> None: - guide = TESTING_GUIDE.read_text(encoding="utf-8") - pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - - self.assertIn( - "| llama.cpp, vLLM, stable-diffusion.cpp validation | " - "Required workflow `Validate llama.cpp protected change`;", - guide, - ) - self.assertNotIn( - "| llama.cpp, vLLM, stable-diffusion.cpp validation | " - "`llama.cpp authorized validation v2`,", - guide, - ) - self.assertIn("Do not require `llama.cpp validation`", guide) - self.assertIn("`ci:upgrades` label", guide) - self.assertIn("protected self-hosted runners", guide) - self.assertIn("CODEOWNERS", guide) - self.assertIn("separate bootstrap merge", guide) - self.assertIn( - "Enforcement therefore starts with later protected pull requests", guide - ) - self.assertIn("current `.github/CODEOWNERS` covers only", guide) - self.assertIn("explicit entries for every trusted validation", guide) - self.assertIn("enforce code-owner review", guide) - self.assertIn("Missing exact CODEOWNERS coverage", guide) - self.assertIn("external activation **NO-GO**", guide) - self.assertIn("merge queue", guide) - self.assertIn( - "Allow GitHub Actions to create and approve pull requests", - guide, - ) - self.assertIn("`LLAMACPP_UPDATE_TOKEN`", guide) - self.assertIn("`LLAMACPP_UPDATE_ACTOR`", guide) - self.assertIn("repository-scoped fine-grained PAT", guide) - self.assertNotIn("short-lived dedicated GitHub App installation token", guide) - self.assertIn("installation tokens expire after one hour", guide) - self.assertIn("mint a fresh token inside each publication job", guide) - self.assertIn("Contents: read and write", guide) - self.assertIn("Pull requests: read and write", guide) - self.assertIn("Never substitute the repository's `GITHUB_TOKEN`", guide) - self.assertIn("does not trigger the required workflow", guide) - self.assertIn("deliberately creates a draft first", guide) - self.assertIn("missing secret or variable configuration", guide) - self.assertIn("neutral success", guide) - self.assertIn("synthetic test-merge commit", guide) - self.assertIn("falls back to the immutable current head", guide) - self.assertIn( - "protected validation never executes without a test-merge commit", guide - ) - self.assertIn("GitHub ignores configured activity filters", guide) - self.assertIn("`opened`, `synchronize`, and `reopened`", guide) - self.assertIn("**Re-run all jobs**", guide) - self.assertIn("The same maintainer must start the rerun", guide) - self.assertIn("a label at or before the original run start time", guide) - self.assertIn("not its queued creation time", guide) - self.assertIn("while the run was queued", guide) - self.assertIn("base-branch retarget", guide) - self.assertIn("supported default-event run", guide) - self.assertIn("Metadata diagnostic runs do not supersede", guide) - self.assertIn("`llama.cpp authorization diagnostics v2`", guide) - self.assertIn("uses a distinct visible name", guide) - self.assertIn("does not conflict with a recovered required result", guide) - self.assertIn("newer `opened`, `synchronize`, or `reopened` run", guide) - self.assertIn("rerun the same selected default-event run", guide) - self.assertIn("Removal after a completed green rerun is diagnostic only", guide) - self.assertIn("one exact default-event run and candidate", guide) - self.assertNotIn("The label authorizes one exact run attempt", guide) - self.assertIn("Activation is **NO-GO** unless", guide) - self.assertIn("the default branch advances after a green rerun", guide) - self.assertIn( - "a merge queue, strict up-to-date enforcement, or a live-proven equivalent", - guide, - ) - self.assertIn("the exact validated base-plus-head candidate", guide) - self.assertIn("exclusive GitHub App", guide) - self.assertIn( - "Do **not** require `llama.cpp authorized validation v2` during bootstrap", - guide, - ) - self.assertIn("immutable K2-capable backend releases", guide) - self.assertIn("all three pinned release repositories", guide) - self.assertIn("controlled immutable mirror", guide) - self.assertIn("all three `sm_121` artifacts", guide) - self.assertIn("source sidecars", guide) - self.assertIn("Require workflows to pass before merging", guide) - self.assertIn("organization-level", guide) - self.assertIn("`LLM360/lemonade`", guide) - self.assertIn("`main`", guide) - self.assertIn("`.github/workflows/validate_llamacpp_pr.yml`", guide) - self.assertIn("organization owner", guide) - self.assertIn("After the bootstrap merge is on the default branch", guide) - self.assertIn("separate, unmerged live fork PRs", guide) - self.assertIn("Do not combine validation-infrastructure bootstrap", guide) - self.assertNotIn("Keep the integration PR in draft until", guide) - self.assertIn("shared `github-actions` App", guide) - self.assertIn("does not identify the workflow", guide) - self.assertIn("diagnostic only", guide) - self.assertIn( - "Never configure `llama.cpp authorized validation v2` as a required " - "status check", - guide, - ) - self.assertNotIn( - "select **GitHub Actions** as the expected check source", guide - ) - self.assertIn(" pull_request_target:\n", pull_request) - self.assertIn(" merge_group:\n", pull_request) - - reconcile = pull_request.split(" reconcile:\n", 1)[1] - enforce = reconcile.split(" - name: Enforce policy result\n", 1)[1] - self.assertIn(" if: always()", reconcile) - self.assertIn("if: >-\n always()", enforce) - self.assertIn("exit 1", enforce) - - def test_pr_required_check_wrapper_covers_every_policy_lifecycle(self) -> None: - pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - classify_step = extract_workflow_step_script( - "Classify the complete pull request file set" - ) - triggers = pull_request.split("permissions:", 1)[0] - report = pull_request.split( - " - name: Report result on the exact candidate commit\n", 1 - )[1].split(" - name: Enforce policy result\n", 1)[0] - - self.assertIn( - "types: [opened, reopened, synchronize, edited, labeled, unlabeled, closed]", - triggers, - ) - self.assertIn(" merge_group:\n", triggers) - self.assertIn( - "compare/${BASE_SHA}...${MERGE_SHA}", - classify_step, - ) - self.assertNotIn("pulls/${PR_NUMBER}/files?per_page=100", classify_step) - self.assertNotIn("--paginate --slurp", classify_step) - self.assertIn("classify-compare", classify_step) - self.assertIn( - "MERGE_SHA: ${{ steps.identity.outputs.merge_sha }}", pull_request - ) - self.assertIn("current_changed_files", pull_request) - self.assertIn("for attempt in 1 2 3; do", pull_request) - self.assertIn("snapshot_mergeable", pull_request) - self.assertIn("sleep 2", pull_request) - self.assertIn("llamacpp_required_check_policy.py", pull_request) - self.assertIn( - "previous_filename", REQUIRED_CHECK_POLICY.read_text(encoding="utf-8") - ) - self.assertIn( - "trusted/.github/scripts/llamacpp_required_check_policy.py", pull_request - ) - self.assertIn("ref: ${{ steps.identity.outputs.base_sha }}", pull_request) - self.assertIn("persist-credentials: false", pull_request) - self.assertIn("collaborators/${authorization_actor}/permission", pull_request) - self.assertIn("EVENT_ACTION: ${{ github.event.action || '' }}", pull_request) - self.assertIn("github.event.changes.base.ref.from", pull_request) - self.assertIn("llama.cpp authorized validation v2", pull_request) - self.assertIn("needs.policy.outputs.run_validation == 'true'", pull_request) - self.assertIn("needs.policy.outputs.mode == 'preserve'", pull_request) - self.assertIn("needs.policy.outputs.conclusion", pull_request) - self.assertIn("github.event.pull_request.merge_commit_sha", pull_request) - self.assertIn("github.event.merge_group.head_sha", pull_request) - self.assertIn("current_merge", pull_request) - self.assertIn("current_mergeable", report) - self.assertIn( - '[ "$current_mergeable" = "true" ] && [[ "$current_merge" =~ $oid_pattern ]]', - report, - ) - self.assertIn("current_base", pull_request) - self.assertIn("current_head", pull_request) - self.assertIn( - "current_merge=$(jq -r '.merge_commit_sha // empty'", pull_request - ) - self.assertIn('candidate_sha="$current_head"', pull_request) - self.assertIn( - '--merge-candidate-available "$MERGE_CANDIDATE_AVAILABLE"', - pull_request, - ) - self.assertIn("steps.assess-policy.outputs.mode != 'stale'", pull_request) - self.assertIn('.app.slug == "github-actions"', pull_request) - self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", pull_request) - - def test_protected_candidate_checkouts_opt_in_without_weakening_trusted_checkouts( - self, - ) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - unsafe_opt_in = ( - "allow-unsafe-pr-checkout: " - "${{ github.event_name == 'pull_request_target' }}" - ) - checkout_v5_1 = ( - "actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 " "# v5.1.0" - ) - - self.assertEqual(validation.count(unsafe_opt_in), 4) - self.assertEqual( - validation.count(checkout_v5_1), - validation.count("uses: actions/checkout@"), - ) - for trusted_checkout in validation.split( - " - name: Check out trusted validation tools\n" - )[1:]: - trusted_block = trusted_checkout.split(" - name:", 1)[0] - self.assertNotIn("allow-unsafe-pr-checkout", trusted_block) - - wrapper = PR_WORKFLOW.read_text(encoding="utf-8") - policy_checkout = wrapper.split( - " - name: Check out trusted policy code\n", 1 - )[1].split(" - name:", 1)[0] - self.assertNotIn("allow-unsafe-pr-checkout", policy_checkout) - - def test_required_check_lookups_are_complete_source_bound_and_race_safe( - self, - ) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - invocation = validation.split(" validate-invocation:\n", 1)[1].split( - " plan:\n", 1 - )[0] - report = pull_request.split( - " - name: Report result on the exact candidate commit\n", 1 - )[1].split(" - name: Enforce policy result\n", 1)[0] - - self.assertIn( - 'EXTERNAL_PREFIX="lemonade-llamacpp-required-${TARGET_KEY}-"', - invocation, - ) - self.assertIn("decision_scope=metadata", report) - self.assertIn("decision_scope=required", report) - self.assertIn( - 'external_prefix="lemonade-llamacpp-${decision_scope}-${TARGET_KEY}-"', - report, - ) - - for lookup in (invocation, report): - with self.subTest(workflow="core" if lookup is invocation else "wrapper"): - self.assertIn("--paginate --slurp", lookup) - self.assertIn('.app.slug == "github-actions"', lookup) - self.assertIn('newest_number" -gt "$RUN_NUMBER', lookup) - self.assertIn(".number < $current_number", lookup) - self.assertLess( - lookup.index('newest_number" -gt "$RUN_NUMBER'), - lookup.index("--method PATCH"), - ) - self.assertGreaterEqual(lookup.count("--paginate --slurp"), 2) - self.assertIn("RUN_NUMBER: ${{ github.run_number }}", lookup) - - def test_required_check_filters_ignore_foreign_and_newer_runs(self) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - lookups = ( - validation.split(" validate-invocation:\n", 1)[1].split(" plan:\n", 1)[0], - pull_request.split( - " - name: Report result on the exact candidate commit\n", 1 - )[1].split(" - name: Enforce policy result\n", 1)[0], - ) - check_name = "llama.cpp authorized validation v2" - external_prefix = "lemonade-llamacpp-required-pr-7-" - check_pages = [ - { - "check_runs": [ - { - "id": 11, - "name": check_name, - "app": {"slug": "github-actions"}, - "conclusion": "success", - "external_id": f"{external_prefix}1-10-1-9000", - "status": "completed", - }, - { - "id": 12, - "name": check_name, - "app": {"slug": "github-actions"}, - "conclusion": "success", - "external_id": f"{external_prefix}1-20-1-8000", - "status": "completed", - }, - { - "id": 13, - "name": check_name, - "app": {"slug": "github-actions"}, - "conclusion": "success", - "external_id": f"{external_prefix}1-30-2-7000", - "status": "completed", - }, - { - "id": 14, - "name": check_name, - "app": {"slug": "foreign-app"}, - "conclusion": "success", - "external_id": f"{external_prefix}1-99-1-6000", - "status": "completed", - }, - { - "id": 15, - "name": "foreign check", - "app": {"slug": "github-actions"}, - "conclusion": "success", - "external_id": f"{external_prefix}1-100-1-5000", - "status": "completed", - }, - ] - } - ] - - def jq_program(block: str, anchor: str) -> str: - tail = block.split(anchor, 1)[1] - starts = [ - position - for token in ("'[.[]", "'.[]") - if (position := tail.find(token)) >= 0 - ] - self.assertTrue(starts) - start = min(starts) - end = tail.index('\' <<<"$check_runs"', start) - return tail[start + 1 : end] - - policy_job = pull_request.split(" policy:\n", 1)[1].split(" validate:\n", 1)[ - 0 - ] - preserve_step = pull_request.split( - " - name: Preserve the exact-commit gate for unrelated metadata\n", - 1, - )[1].split(" - name:", 1)[0] - for lookup in (policy_job, preserve_step): - latest_program = jq_program( - lookup, "read -r newest_check_epoch newest_check_number" - ) - base_command = [ - "jq", - "-r", - "--arg", - "check_name", - check_name, - "--arg", - "external_prefix", - external_prefix, - latest_program, - ] - trusted = subprocess.run( - base_command, - input=json.dumps(check_pages), - text=True, - capture_output=True, - check=True, - ) - self.assertEqual( - trusted.stdout.rstrip("\n").split("\t"), - ["1", "30", "2", "completed", "success"], - ) - foreign_only = [{"check_runs": [check_pages[0]["check_runs"][3]]}] - foreign = subprocess.run( - base_command, - input=json.dumps(foreign_only), - text=True, - capture_output=True, - check=True, - ) - self.assertEqual( - foreign.stdout.rstrip("\n").split("\t"), - ["0", "0", "0", "", ""], - ) - old_epoch_only = [ - { - "check_runs": [ - { - **check_pages[0]["check_runs"][0], - "external_id": f"{external_prefix}9-10-1-9000", - } - ] - } - ] - old_epoch = subprocess.run( - base_command, - input=json.dumps(old_epoch_only), - text=True, - capture_output=True, - check=True, - ) - self.assertEqual( - old_epoch.stdout.rstrip("\n").split("\t"), - ["9", "10", "1", "completed", "success"], - ) - for status, conclusion in ( - ("in_progress", None), - ("completed", "failure"), - ): - unfinished = [ - { - "check_runs": [ - { - **check_pages[0]["check_runs"][0], - "conclusion": conclusion, - "status": status, - } - ] - } - ] - rejected = subprocess.run( - base_command, - input=json.dumps(unfinished), - text=True, - capture_output=True, - check=True, - ) - self.assertEqual( - rejected.stdout.rstrip("\n").split("\t"), - ["1", "10", "1", status, conclusion or ""], - ) - - for lookup in lookups: - with self.subTest(workflow="core" if lookup is lookups[0] else "wrapper"): - newest = subprocess.run( - [ - "jq", - "-r", - "--arg", - "check_name", - check_name, - "--arg", - "external_prefix", - external_prefix, - jq_program(lookup, "read -r newest_number newest_attempt"), - ], - input=json.dumps(check_pages), - text=True, - capture_output=True, - check=True, - ) - self.assertEqual(newest.stdout.strip(), "30\t2") - - older = subprocess.run( - [ - "jq", - "-r", - "--arg", - "check_name", - check_name, - "--arg", - "external_prefix", - external_prefix, - "--argjson", - "current_number", - "20", - "--argjson", - "current_attempt", - "1", - jq_program(lookup, "prior_check_ids=$(jq -r"), - ], - input=json.dumps(check_pages), - text=True, - capture_output=True, - check=True, - ) - self.assertEqual(older.stdout.splitlines(), ["11"]) - - def test_portable_runner_builds_platform_neutral_validation_arguments(self) -> None: - runner = load_validation_runner() - - command = runner.build_validation_command( - python_executable=Path("python"), - backend="rocm", - channel="nightly", - target="windows-rocm-nightly", - models_csv=f"{K2_SMALL},{K2_MEDIUM}", - lite=False, - capability_profile="k2-horizon-v1", - capability_models_csv=K2_SMALL, - port=13305, - ) - - self.assertEqual( - command[0:2], - ["python", str(ROOT / "test" / "validate_llamacpp.py")], - ) - self.assertIn("--channel", command) - self.assertIn("nightly", command) - self.assertEqual(command.count("--model"), 2) - self.assertIn("--capability-profile", command) - self.assertIn("llamacpp_validation_windows-rocm-nightly.json", command) - - with self.assertRaisesRegex(ValueError, "mutually exclusive"): - runner.build_validation_command( - python_executable=Path("python"), - backend="cpu", - channel="", - target="linux-cpu", - models_csv=K2_SMALL, - lite=True, - capability_profile="k2-horizon-v1", - capability_models_csv=K2_SMALL, - port=13305, - ) - - inherited = {"HF_HUB_CACHE": "shared-cache", "PRESERVED": "yes"} - environment = runner.build_server_environment( - Path("target-isolated-cache"), - inherited=inherited, - private_temp_directory=Path("target-private-temp"), - candidate_defaults_path=Path("candidate-defaults.json"), - ) - self.assertEqual( - environment["HF_HUB_CACHE"], - str(Path("target-isolated-cache").resolve()), - ) - self.assertNotIn("PRESERVED", environment) - self.assertEqual(inherited["HF_HUB_CACHE"], "shared-cache") - - builtin_small_command = runner.build_validation_command( - python_executable=Path("python"), - backend="metal", - channel="", - target="macos-metal", - models_csv=f"builtin.{K2_SMALL}", - lite=False, - capability_profile="k2-horizon-v1", - capability_models_csv=K2_SMALL, - port=13305, - ) - self.assertEqual(builtin_small_command.count("--model"), 1) - - def test_validation_plan_supports_direct_script_invocation(self) -> None: - completed = subprocess.run( - [ - sys.executable, - str(ROOT / "test/utils/llamacpp_validation_plan.py"), - "--help", - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - ) - - self.assertEqual(completed.returncode, 0, completed.stderr) - - def test_validation_runner_supports_direct_script_invocation(self) -> None: - completed = subprocess.run( - [sys.executable, str(VALIDATION_RUNNER), "--help"], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - ) - - self.assertEqual(completed.returncode, 0, completed.stderr) - - def assert_active_lanes(self, rows): - self.assertEqual( - [ - ( - row["target"], - row["backend"], - row["channel"], - row["managed_pin"], - row["build_platform"], - ) - for row in rows - ], - [ - ("windows-vulkan", "vulkan", "", "vulkan", "windows"), - ("linux-vulkan", "vulkan", "", "vulkan", "linux"), - ("linux-cpu", "cpu", "", "cpu", "linux"), - ("macos-metal", "metal", "", "metal", "macos"), - ("windows-cuda", "cuda", "", "cuda", "windows"), - ( - "windows-rocm-stable", - "rocm", - "stable", - "rocm-stable", - "windows", - ), - ( - "windows-rocm-nightly", - "rocm", - "nightly", - "rocm-nightly", - "windows", - ), - ], - ) - self.assertEqual( - {row["managed_pin"] for row in rows}, - {"cpu", "cuda", "metal", "rocm-nightly", "rocm-stable", "vulkan"}, - ) - vulkan_platforms = { - row["build_platform"] for row in rows if row["backend"] == "vulkan" - } - self.assertEqual(vulkan_platforms, {"linux", "windows"}) - self.assertTrue( - all(row["capability_profile"] == "k2-horizon-v1" for row in rows) - ) - self.assertTrue(all(row["capability_models"] == [K2_SMALL] for row in rows)) - self.assertTrue(all(K2_SMALL in row["models"] for row in rows if row["models"])) - - for row in rows: - if K2_MEDIUM in row["models"] or K2_LARGE in row["models"]: - self.assertEqual(row["target"], "windows-vulkan") - self.assertIn("128gb", row["runner"]) - - def test_pull_request_and_merge_group_require_k2(self) -> None: - for event_name in ("pull_request", "merge_group"): - with self.subTest(event_name=event_name): - rows = planning.create_validation_plan(event_name)["include"] - - self.assert_active_lanes(rows) - self.assertEqual(rows[0]["models"], [K2_SMALL, K2_MEDIUM, K2_LARGE]) - self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) - self.assertTrue(all(not row["lite"] for row in rows)) - self.assertIn("128gb", rows[0]["runner"]) - - def test_schedule_captures_exact_validation_candidate_contract(self) -> None: - rows = planning.create_validation_plan("schedule")["include"] - - self.assert_active_lanes(rows) - expected = planning.load_validation_candidate_model_ids() - self.assertIn(K2_SMALL, expected) - self.assertIn(K2_MEDIUM, expected) - self.assertIn(K2_LARGE, expected) - self.assertEqual(rows[0]["models"], expected) - self.assertEqual(rows[0]["expected_models"], expected) - self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) - self.assertTrue(all(row["expected_models"] == [K2_SMALL] for row in rows[1:])) - self.assertTrue(all(not row["lite"] for row in rows)) - self.assertIn("128gb", rows[0]["runner"]) - - def test_validation_candidate_set_reads_overlay_outside_product_catalog( - self, - ) -> None: - with tempfile.TemporaryDirectory() as directory: - registry = Path(directory) / "server_models.json" - overlay = Path(directory) / "validation_models.json" - registry.write_text( - json.dumps( - { - "Existing-Hot-Model": { - "recipe": "llamacpp", - "labels": ["chat", "hot"], - } - } - ), - encoding="utf-8", - ) - overlay.write_text( - json.dumps( - { - "Candidate-Not-Hot": { - "recipe": "llamacpp", - "labels": ["chat"], - } - } - ), - encoding="utf-8", - ) - - models = planning.load_validation_candidate_model_ids(registry, overlay) - - self.assertEqual( - models, - ["Candidate-Not-Hot", "Existing-Hot-Model"], - ) - - def test_default_dispatch_selects_hot_models_and_its_capability_model( - self, - ) -> None: - rows = planning.create_validation_plan("workflow_dispatch")["include"] - - self.assert_active_lanes(rows) - self.assertEqual( - set(rows[0]["models"]), - {K2_SMALL, *planning.load_hot_llamacpp_model_ids()}, - ) - self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) - self.assertTrue( - all( - set(row["capability_models"]).issubset( - {model.removeprefix("builtin.") for model in row["models"]} - ) - for row in rows - ) - ) - self.assertTrue(all(not row["lite"] for row in rows)) - self.assertIn("128gb", rows[0]["runner"]) - - def test_manual_lite_selection_runs_only_k2_small(self) -> None: - rows = planning.create_validation_plan("workflow_dispatch", lite=True)[ - "include" - ] - - self.assert_active_lanes(rows) - self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows)) - self.assertTrue(all(not row["lite"] for row in rows)) - - def test_manual_models_are_trimmed_and_keep_order_and_duplicates(self) -> None: - rows = planning.create_validation_plan( - "workflow_dispatch", - models_csv=" Zulu, ,Alpha,Zulu ", - )["include"] - - self.assert_active_lanes(rows) - self.assertEqual( - rows[0]["models"], - [K2_SMALL, "Zulu", "Alpha", "Zulu"], - ) - self.assertTrue( - all( - row["models"] == [K2_SMALL, "Zulu", "Alpha", "Zulu"] for row in rows[1:] - ) - ) - self.assertTrue(all(not row["lite"] for row in rows)) - self.assertIn("128gb", rows[0]["runner"]) - - def test_manual_large_k2_models_are_confined_to_large_vulkan_lane(self) -> None: - rows = planning.create_validation_plan( - "workflow_dispatch", - models_csv=f"{K2_MEDIUM},{K2_LARGE}", - )["include"] - - self.assert_active_lanes(rows) - self.assertEqual(rows[0]["models"], [K2_SMALL, K2_MEDIUM, K2_LARGE]) - self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) - - def test_manual_builtin_large_k2_aliases_are_confined_to_large_lane(self) -> None: - builtin_medium = f"builtin.{K2_MEDIUM}" - builtin_large = f"builtin.{K2_LARGE}" - rows = planning.create_validation_plan( - "workflow_dispatch", - models_csv=f"{builtin_medium},{builtin_large}", - )["include"] - - self.assert_active_lanes(rows) - self.assertEqual(rows[0]["models"], [K2_SMALL, builtin_medium, builtin_large]) - self.assertTrue(all(row["models"] == [K2_SMALL] for row in rows[1:])) - - def test_manual_models_and_lite_are_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "mutually exclusive"): - planning.create_validation_plan( - "workflow_dispatch", - models_csv=K2_SMALL, - lite=True, - ) - - def test_non_manual_events_reject_manual_inputs(self) -> None: - with self.assertRaisesRegex(ValueError, "workflow_dispatch"): - planning.create_validation_plan("schedule", models_csv=K2_SMALL) - - def test_unknown_event_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "Unsupported"): - planning.create_validation_plan("push") - - def test_only_scheduled_runs_are_promotion_eligible(self) -> None: - cases = ( - ("schedule", "", False, True), - ("workflow_dispatch", "", False, False), - ("workflow_dispatch", K2_SMALL, False, False), - ("workflow_dispatch", "", True, False), - ("pull_request", "", False, False), - ("merge_group", "", False, False), - ) - - for event_name, models_csv, lite, expected in cases: - with self.subTest( - event_name=event_name, - models_csv=models_csv, - lite=lite, - ): - self.assertEqual( - planning.is_promotion_eligible(event_name, models_csv, lite), - expected, - ) - - def test_promotion_coverage_rejects_missing_or_mapped_wrong_lanes(self) -> None: - complete = planning.create_validation_plan("schedule") - planning.require_complete_promotion_coverage(complete) - - missing = json.loads(json.dumps(complete)) - missing["include"].pop() - with self.assertRaisesRegex(ValueError, "exactly"): - planning.require_complete_promotion_coverage(missing) - - wrong_mapping = json.loads(json.dumps(complete)) - wrong_mapping["include"][2]["managed_pin"] = "cuda" - with self.assertRaisesRegex(ValueError, "mapping"): - planning.require_complete_promotion_coverage(wrong_mapping) - - wrong_vulkan_platform = json.loads(json.dumps(complete)) - wrong_vulkan_platform["include"][1]["build_platform"] = "windows" - with self.assertRaisesRegex(ValueError, "mapping"): - planning.require_complete_promotion_coverage(wrong_vulkan_platform) - - unexecuted_models = json.loads(json.dumps(complete)) - unexecuted_models["include"][0]["models"] = [] - with self.assertRaisesRegex(ValueError, "selected models"): - planning.require_complete_promotion_coverage(unexecuted_models) - - lite_lane = json.loads(json.dumps(complete)) - lite_lane["include"][0]["lite"] = True - with self.assertRaisesRegex(ValueError, "lite"): - planning.require_complete_promotion_coverage(lite_lane) - - missing_candidate = json.loads(json.dumps(complete)) - for field in ("models", "expected_models"): - missing_candidate["include"][0][field].remove(K2_MEDIUM) - with self.assertRaisesRegex(ValueError, "exact candidate model set"): - planning.require_complete_promotion_coverage(missing_candidate) - - unexpected_small_lane_model = json.loads(json.dumps(complete)) - for field in ("models", "expected_models"): - unexpected_small_lane_model["include"][1][field].append("Other-Model") - with self.assertRaisesRegex(ValueError, "exact K2 small model set"): - planning.require_complete_promotion_coverage(unexpected_small_lane_model) - - def test_github_output_contains_compact_matrix_json(self) -> None: - plan = planning.create_validation_plan("merge_group") - with tempfile.TemporaryDirectory() as directory: - output_path = Path(directory) / "github-output.txt" - - planning.write_github_output( - output_path, - plan, - promotion_eligible=False, - ) - - outputs = dict( - line.split("=", 1) - for line in output_path.read_text(encoding="utf-8").splitlines() - ) - self.assertEqual(json.loads(outputs["matrix"]), plan) - self.assertNotIn("\n", outputs["matrix"]) - self.assertEqual(outputs["promotion_eligible"], "false") - - def test_validation_core_has_constrained_permissions_and_no_legacy_entrypoint( - self, - ) -> None: - self.assertFalse(LEGACY_WORKFLOW.exists()) - - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - validation_triggers = validation.split("permissions:", 1)[0] - workflow_scope = validation.split("jobs:", 1)[0] - invocation_job = validation.split(" validate-invocation:\n", 1)[1].split( - " plan:\n", 1 - )[0] - plan_job = validation.split(" plan:\n", 1)[1].split( - " get-latest-releases:\n", 1 - )[0] - build_job = validation.split(" build:\n", 1)[1].split(" validate:\n", 1)[0] - validate_job = validation.split(" validate:\n", 1)[1].split( - " validation-gate:\n", 1 - )[0] - validation_step = validate_job.split( - " - name: Run validation with lemond\n", 1 - )[1].split(" - name: Upload results\n", 1)[0] - result_upload = validate_job.split(" - name: Upload results\n", 1)[ - 1 - ].split(" - name: Upload restart results\n", 1)[0] - restart_result_upload = validate_job.split( - " - name: Upload restart results\n", 1 - )[1].split(" - name: Upload server logs\n", 1)[0] - server_log_upload = validate_job.split(" - name: Upload server logs\n", 1)[ - 1 - ].split(" - name: Cleanup\n", 1)[0] - - self.assertIn(" workflow_call:\n", validation_triggers) - self.assertNotIn(" pull_request_target:\n", validation_triggers) - self.assertNotIn(" pull_request:\n", validation_triggers) - self.assertNotIn(" merge_group:\n", validation_triggers) - self.assertNotIn(" workflow_dispatch:\n", validation_triggers) - self.assertNotIn(" schedule:\n", validation_triggers) - self.assertIn("permissions:\n contents: read\n", validation) - self.assertNotIn("contents: write", validation) - self.assertNotIn("pull-requests: write", validation) - self.assertEqual(validation.count("checks: write"), 1) - self.assertIn("checks: write", invocation_job) - self.assertNotIn("concurrency:", workflow_scope) - self.assertNotIn(" create-pr:\n", validation) - self.assertIn(" HUGGINGFACE_ACCESS_TOKEN:\n", validation_triggers) - self.assertNotIn("HF_TOKEN:", workflow_scope) - self.assertNotIn("secrets.HUGGINGFACE_ACCESS_TOKEN", workflow_scope) - self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", plan_job) - self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", build_job) - self.assertEqual(validate_job.count("HF_TOKEN:"), 1) - self.assertIn( - " HF_TOKEN: ${{ github.event_name == 'schedule' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }}\n", - validation_step, - ) - self.assertNotIn( - " HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}\n", - validation_step, - ) - self.assertIn(".github/scripts/run_llamacpp_validation.py", validation_step) - self.assertIn(" id: run-validation\n", validation_step) - self.assertIn("steps.validation-venv.outputs.python-path", validation_step) - self.assertIn("--lemond", validation_step) - self.assertIn( - "${{ github.event_name == 'schedule' && '--allow-huggingface-download-credentials' || '' }}", - validation_step, - ) - self.assertEqual(validation_step.count("LEMONADE_PROTECTED_RUNNER_CONTEXT:"), 1) - self.assertEqual( - validation_step.count("--allow-darwin-github-hosted-ephemeral-runner"), - 1, - ) - self.assertIn("matrix.target == 'macos-metal'", validation_step) - self.assertIn("runner.environment == 'github-hosted'", validation_step) - self.assertIn("join(matrix.runner, ',') == 'macos-latest'", validation_step) - self.assertIn("github-hosted:macos-latest:macos-metal", validation_step) - self.assertNotIn("shell: PowerShell", validation_step) - protected_hosted_mac_exclusion = ( - "!((github.event_name == 'pull_request_target' || " - "github.event_name == 'merge_group') &&" - ) - for upload in (result_upload, restart_result_upload, server_log_upload): - self.assertIn(" if: >-\n", upload) - self.assertIn( - " steps.run-validation.outcome == 'success' &&\n", upload - ) - self.assertIn(protected_hosted_mac_exclusion, upload) - self.assertIn("matrix.target == 'macos-metal'", upload) - self.assertIn("runner.environment == 'github-hosted'", upload) - self.assertIn("join(matrix.runner, ',') == 'macos-latest'", upload) - self.assertIn( - "path: ${{ steps.run-validation.outputs.validation_result }}", - result_upload, - ) - self.assertIn(" if-no-files-found: error\n", result_upload) - self.assertNotIn("llamacpp_restart_validation_", restart_result_upload) - self.assertIn(" if-no-files-found: error\n", restart_result_upload) - self.assertIn( - "path: ${{ steps.run-validation.outputs.restart_result }}", - restart_result_upload, - ) - self.assertNotIn("server-restart-logs", server_log_upload) - for output_name in ( - "prepare_stdout_log", - "prepare_stderr_log", - "validation_stdout_log", - "validation_stderr_log", - "restart_stdout_log", - "restart_stderr_log", - ): - self.assertIn( - f"steps.run-validation.outputs.{output_name}", - server_log_upload, - ) - self.assertIn(" if-no-files-found: error\n", server_log_upload) - self.assertNotIn("if: always()", result_upload) - self.assertNotIn("if: always()", restart_result_upload) - self.assertNotIn("if: always()", server_log_upload) - - runner = VALIDATION_RUNNER.read_text(encoding="utf-8") - self.assertIn("verify_validation_artifacts", runner) - self.assertIn('cache_directory / "hf-hub"', runner) - self.assertIn('environment["HF_HUB_CACHE"]', runner) - self.assertIn("VALIDATION_ARTIFACT_LOCKS", runner) - - def test_build_and_validation_orchestration_are_native_per_platform(self) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - build_job = validation.split(" build:\n", 1)[1].split(" validate:\n", 1)[0] - validate_job = validation.split(" validate:\n", 1)[1].split( - " reverify-protected-release-manifest:\n", 1 - )[0] - - for platform, runner in ( - ("windows", "windows-latest"), - ("linux", "ubuntu-latest"), - ("macos", "macos-latest"), - ): - with self.subTest(platform=platform): - self.assertIn(f"platform: {platform}", build_job) - self.assertIn(f"runner: {runner}", build_job) - self.assertIn("runs-on: ${{ matrix.runner }}", build_job) - self.assertIn("llamacpp-build-${{ matrix.platform }}", build_job) - self.assertIn("if: runner.os == 'Windows'", build_job) - self.assertIn("if: runner.os != 'Windows'", build_job) - self.assertIn("llamacpp-build-${{ matrix.build_platform }}", validate_job) - self.assertIn("if: runner.os == 'Windows'", validate_job) - self.assertIn("if: runner.os == 'Linux'", validate_job) - self.assertIn("if: runner.os != 'Windows'", validate_job) - - protected_overlay_step = build_job.split( - " - name: Stage protected validation-only model catalog\n", 1 - )[1].split(" - name: Stage branch validation-only model catalog\n", 1)[0] - branch_overlay_step = build_job.split( - " - name: Stage branch validation-only model catalog\n", 1 - )[1].split(" - name: Install Linux build dependencies\n", 1)[0] - self.assertIn( - "$GITHUB_WORKSPACE/test/utils/validation_model_catalog.py", - protected_overlay_step, - ) - self.assertIn("$GITHUB_WORKSPACE/candidate", protected_overlay_step) - self.assertIn( - "$GITHUB_WORKSPACE/test/fixtures/llamacpp_validation_models.json", - protected_overlay_step, - ) - self.assertIn( - "python -m test.utils.validation_model_catalog", branch_overlay_step - ) - self.assertIn( - "--base src/cpp/resources/server_models.json", branch_overlay_step - ) - self.assertIn( - "--overlay test/fixtures/llamacpp_validation_models.json", - branch_overlay_step, - ) - self.assertIn( - "--output src/cpp/resources/server_models.json", branch_overlay_step - ) - self.assertLess( - build_job.index( - " - name: Stage protected validation-only model catalog\n" - ), - build_job.index(" cmake --preset vs18"), - ) - self.assertLess( - build_job.index( - " - name: Stage branch validation-only model catalog\n" - ), - build_job.index(" cmake --preset default"), - ) - - def test_focused_ci_runs_the_capability_contract_tests(self) -> None: - workflow = DOCS_AND_STYLE_WORKFLOW.read_text(encoding="utf-8") - - self.assertIn("test.test_llamacpp_capabilities", workflow) - self.assertIn("test.test_llamacpp_validation_artifacts", workflow) - self.assertIn("test.test_llamacpp_validation_evidence", workflow) - self.assertIn("test.test_llamacpp_pr_file_classification", workflow) - self.assertIn("test.test_run_llamacpp_validation", workflow) - self.assertIn("test.test_validation_model_catalog", workflow) - self.assertIn("test.test_validate_llamacpp_action_pins", workflow) - self.assertIn("test.test_validate_llamacpp_selection", workflow) - self.assertIn("test.test_validate_llamacpp_trusted_execution", workflow) - self.assertIn("test.test_verify_llamacpp_pin_manifest", workflow) - - def test_focused_ci_installs_process_tracking_dependency(self) -> None: - workflow = DOCS_AND_STYLE_WORKFLOW.read_text(encoding="utf-8") - focused_tests = workflow.split( - " - name: Run focused Python unit tests\n", 1 - )[0] - - self.assertIn("pip install pre-commit psutil", focused_tests) - - def test_release_asset_manifest_is_captured_and_exported(self) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - release_job = validation.split(" get-latest-releases:\n", 1)[1].split( - " verify-release-assets:\n", 1 - )[0] - asset_job = validation.split(" verify-release-assets:\n", 1)[1].split( - " build:\n", 1 - )[0] - - self.assertIn(" release_asset_manifest:\n", validation) - self.assertIn( - "jobs.verify-release-assets.outputs.release_asset_manifest", - validation, - ) - self.assertIn("release_asset_manifest:", asset_job) - self.assertIn("llamacpp_release_manifest", asset_job) - self.assertIn('--github-output "$GITHUB_OUTPUT"', asset_job) - self.assertIn("jq -r '.assets[].name'", asset_job) - self.assertIn("attach_release_claims", asset_job) - self.assertIn("--source-manifest-asset-id", asset_job) - self.assertIn("--validate-source-manifest", asset_job) - self.assertIn("--build-target-attestation-output", asset_job) - self.assertIn(".build_target_attestations", asset_job) - self.assertIn("Accept: application/octet-stream", asset_job) - self.assertIn("source_manifest_max_bytes=65536", asset_job) - self.assertIn("source_manifest_directory=$(mktemp -d)", asset_job) - self.assertIn( - 'source_manifest_path="${source_manifest_directory}/source-' - '${source_manifest_asset_id}.json"', - asset_job, - ) - self.assertIn('head -c "$((source_manifest_max_bytes + 1))"', asset_job) - self.assertIn('publisher_claim_type="immutable-source-manifest"', asset_job) - self.assertNotIn("--extract-source-commit", asset_job) - self.assertIn("--require-upstream-ancestry", asset_job) - self.assertIn("/compare/", asset_job) - self.assertRegex( - asset_job, - r"attach_release_claims lemonade-sdk/llamacpp-rocm " - r'\s*\\?\s*"\$ROCM_RELEASE" rocm_release\.json', - ) - self.assertRegex( - asset_job, - r"attach_release_claims lemonade-sdk/llama\.cpp " - r'\s*\\?\s*"\$LEMONADE_RELEASE" lemonade_release\.json', - ) - self.assertIn("source_repository=ggml-org/llama.cpp", asset_job) - self.assertIn("upstream_reference_head", asset_job) - self.assertIn("publisher_claimed_source_commit", asset_job) - self.assertIn("publisher_claim_type", asset_job) - self.assertIn("release_tag_commit", asset_job) - self.assertIn("source_repository", asset_job) - self.assertNotIn("attach_source_commit", asset_job) - self.assertNotIn( - 'source_commit=$(gh api "repos/${repository}/commits/${tag}"', - asset_job, - ) - self.assertNotIn("release_asset_manifest", release_job) - - def test_protected_pin_changes_require_live_committed_manifest(self) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - - self.assertIn(" verify-protected-release-manifest:\n", validation) - manifest_job = validation.split(" verify-protected-release-manifest:\n", 1)[ - 1 - ].split(" build:\n", 1)[0] - build_job = validation.split(" build:\n", 1)[1].split(" validate:\n", 1)[0] - verifier = ( - ROOT / ".github" / "scripts" / "verify_llamacpp_pin_manifest.sh" - ).read_text(encoding="utf-8") - - self.assertIn("pull_request_target", manifest_job) - self.assertIn("merge_group", manifest_job) - self.assertIn("base.sha", manifest_job) - self.assertIn("merge_group.base_sha", manifest_job) - self.assertIn("verify_llamacpp_pin_manifest.sh", manifest_job) - self.assertNotIn("id: managed-pin-changes", manifest_job) - self.assertNotIn("steps.managed-pin-changes.outputs.changed", manifest_job) - self.assertIn(".github/llamacpp_release_manifest.json", verifier) - self.assertIn("capture_llamacpp_release_manifest.sh", verifier) - self.assertIn("changed managed llama.cpp pins", verifier) - self.assertIn("verify-protected-release-manifest", build_job) - self.assertIn(" reverify-protected-release-manifest:\n", validation) - final_manifest_job = validation.split( - " reverify-protected-release-manifest:\n", 1 - )[1].split(" validation-gate:\n", 1)[0] - validation_gate = validation.split(" validation-gate:\n", 1)[1] - self.assertIn("needs: [validate", final_manifest_job) - self.assertIn("verify_llamacpp_pin_manifest.sh", final_manifest_job) - self.assertIn("reverify-protected-release-manifest", validation_gate) - - def test_protected_sidecar_only_changes_fail_closed(self) -> None: - verifier = ( - ROOT / ".github" / "scripts" / "verify_llamacpp_pin_manifest.sh" - ).read_text(encoding="utf-8") - - self.assertIn("manifest_path=.github/llamacpp_release_manifest.json", verifier) - self.assertIn( - 'base_manifest_entry=$(git -C "$trusted_repository" ls-tree', verifier - ) - self.assertIn( - 'candidate_manifest_entry=$(git -C "$candidate_repository" ls-tree', - verifier, - ) - self.assertIn("if [[ \"$changes\" == '{}' ]]", verifier) - self.assertIn( - '[[ "$base_manifest_entry" != "$candidate_manifest_entry" ]]', verifier - ) - self.assertIn( - "Release manifest changes require changed managed llama.cpp pins.", - verifier, - ) - - def test_manifest_recheck_anchors_to_live_trusted_upstream(self) -> None: - capture = ( - ROOT / ".github" / "scripts" / "capture_llamacpp_release_manifest.sh" - ).read_text(encoding="utf-8") - - self.assertIn("trusted_source_repository=ggml-org/llama.cpp", capture) - self.assertIn(".default_branch", capture) - self.assertIn("live_upstream_head", capture) - self.assertIn("upstream-anchor-comparison", capture) - self.assertIn("--source-manifest-asset-id", capture) - self.assertIn("--validate-source-manifest", capture) - self.assertIn("Accept: application/octet-stream", capture) - self.assertIn("source_manifest_max_bytes=65536", capture) - self.assertIn("source_manifest_directory=$(mktemp -d)", capture) - self.assertIn( - 'source_manifest_path="${source_manifest_directory}/source-' - '${source_manifest_asset_id}.json"', - capture, - ) - self.assertIn('head -c "$((source_manifest_max_bytes + 1))"', capture) - self.assertIn('publisher_claim_type="immutable-source-manifest"', capture) - self.assertIn( - "${upstream_reference_head}...${live_upstream_head}", - capture, - ) - self.assertIn("release_pattern='^b[0-9]+$'", capture) - - def test_release_commit_resolution_uses_explicit_ref_namespaces(self) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - asset_job = validation.split(" verify-release-assets:\n", 1)[1].split( - " build:\n", 1 - )[0] - capture = ( - ROOT / ".github" / "scripts" / "capture_llamacpp_release_manifest.sh" - ).read_text(encoding="utf-8") - - for source in (asset_job, capture): - with self.subTest(source="workflow" if source is asset_job else "capture"): - self.assertIn( - "commits/heads/${source_default_branch}", - source, - ) - self.assertNotIn( - "commits/${source_default_branch}", - source, - ) - self.assertIn( - '"repos/${repository}/commits/tags/${tag}"', - source, - ) - self.assertNotIn( - '"repos/${repository}/commits/${tag}"', - source, - ) - - def test_release_asset_eligibility_uses_size_and_rocm_target_metadata(self) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - asset_job = validation.split(" verify-release-assets:\n", 1)[1].split( - " build:\n", 1 - )[0] - - self.assertIn("def read_asset_metadata(path):", asset_job) - self.assertIn('asset_sizes[asset["name"]] = asset["size"]', asset_job) - self.assertIn('read_asset_metadata("ggml_release.json")', asset_job) - self.assertIn('read_asset_metadata("rocm_release.json")', asset_job) - self.assertIn('read_asset_metadata("lemonade_release.json")', asset_job) - self.assertIn("build_rocm_asset_target_requirements", asset_job) - self.assertIn("required_asset_targets=rocm_target_requirements", asset_job) - self.assertIn("attested_asset_targets=rocm_attested_targets", asset_job) - - versions = { - "therock": {"version": "7.14.0"}, - "rocm_asset_families": { - "gfx1033": "gfx103X", - "gfx1035": "gfx103X", - "gfx1036": "gfx103X", - }, - } - requirements = release_assets.build_asset_requirements( - ggml_release="b100", - rocm_release="b300", - lemonade_release="b200", - backend_versions=versions, - ) - required_targets = release_assets.build_rocm_asset_target_requirements( - rocm_release="b300", - backend_versions=versions, - ) - available = { - asset_name: 1 for asset_name in requirements["rocm"]["rocm-nightly"] - } - for platform in ("windows", "ubuntu"): - target_asset = f"llama-b300-{platform}-rocm-gfx103X-x64.zip" - for missing_target in ("gfx1033", "gfx1035", "gfx1036"): - with self.subTest( - platform=platform, - missing_target=missing_target, - ): - attested_targets = dict(required_targets) - attested_targets[target_asset] = tuple( - target - for target in required_targets[target_asset] - if target != missing_target - ) - eligible, missing = release_assets.evaluate_asset_group( - available, - requirements["rocm"], - required_asset_targets=required_targets, - attested_asset_targets=attested_targets, - ) - self.assertEqual(eligible, []) - self.assertEqual(missing["rocm-nightly"], [target_asset]) - - def test_pr_validation_uses_a_trusted_merge_check_gate(self) -> None: - pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - pull_request_triggers = pull_request.split("permissions:", 1)[0] - concurrency = pull_request.split("concurrency:\n", 1)[1].split("\njobs:\n", 1)[ - 0 - ] - policy_job = pull_request.split(" policy:\n", 1)[1].split(" validate:\n", 1)[ - 0 - ] - invocation_job = validation.split(" validate-invocation:\n", 1)[1].split( - " plan:\n", 1 - )[0] - validation_job = pull_request.split(" validate:\n", 1)[1].split( - " reconcile:\n", 1 - )[0] - report_job = pull_request.split(" reconcile:\n", 1)[1] - report_step = report_job.split( - " - name: Report result on the exact candidate commit\n", 1 - )[1].split(" - name: Enforce policy result\n", 1)[0] - - self.assertIn(" pull_request_target:\n", pull_request_triggers) - self.assertIn(" merge_group:\n", pull_request_triggers) - self.assertNotIn(" pull_request:\n", pull_request_triggers) - self.assertNotIn(" workflow_call:\n", pull_request_triggers) - self.assertIn( - "types: [opened, reopened, synchronize, edited, labeled, unlabeled, closed]", - pull_request_triggers, - ) - self.assertIn("github.event.pull_request.number", concurrency) - self.assertIn("github.event.merge_group.head_sha", concurrency) - self.assertIn("github.run_number", concurrency) - self.assertIn("github.run_attempt", concurrency) - self.assertNotIn("github.run_id", concurrency) - self.assertNotIn("cancel-in-progress: true", concurrency) - - self.assertIn('if [ "$EVENT_NAME" = "merge_group" ]', policy_job) - self.assertIn("collaborators/${authorization_actor}/permission", policy_job) - self.assertIn("merge_candidate_available", policy_job) - self.assertIn('candidate_sha="$current_head"', policy_job) - self.assertIn("--paginate --slurp", policy_job) - self.assertIn('.app.slug == "github-actions"', policy_job) - - self.assertNotIn(" start-check:\n", pull_request) - self.assertIn("checks: write", invocation_job) - self.assertIn("github.event.pull_request.merge_commit_sha", invocation_job) - self.assertIn("github.event.merge_group.head_sha", invocation_job) - self.assertNotIn("github.event.pull_request.head.sha", invocation_job) - self.assertIn('head_sha="$MERGE_SHA"', invocation_job) - self.assertIn("CHECK_NAME: llama.cpp authorized validation v2", invocation_job) - self.assertIn('name="$CHECK_NAME"', invocation_job) - self.assertIn('external_id="$EXTERNAL_ID"', invocation_job) - self.assertIn("github.run_id", invocation_job) - self.assertIn("github.run_number", invocation_job) - self.assertIn("github.run_attempt", invocation_job) - self.assertIn("status=in_progress", invocation_job) - self.assertIn("check_run_id", invocation_job) - self.assertIn("--paginate --slurp", invocation_job) - self.assertIn('.app.slug == "github-actions"', invocation_job) - self.assertIn("newest_number", invocation_job) - self.assertIn(".number < $current_number", invocation_job) - self.assertIn( - "value: ${{ jobs.validate-invocation.outputs.check_run_id }}", validation - ) - - self.assertIn("needs: policy", validation_job) - self.assertNotIn("start-check", validation_job) - self.assertIn( - "needs.policy.outputs.run_validation == 'true'", - validation_job, - ) - self.assertIn( - "uses: ./.github/workflows/validate_llamacpp_core.yml", validation_job - ) - self.assertIn( - "permissions:\n checks: write\n contents: read", validation_job - ) - self.assertNotIn("secrets:", validation_job) - self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", pull_request) - - self.assertIn("needs: [policy, validate]", report_job) - self.assertIn("if: always()", report_job) - self.assertIn("checks: write", report_job) - self.assertIn("repos/${GITHUB_REPOSITORY}/check-runs", report_job) - self.assertIn("github.event.pull_request.head.sha", report_job) - self.assertIn("needs.validate.outputs.check_run_id", report_job) - self.assertIn("commits/${MERGE_SHA}/check-runs", report_step) - self.assertIn(".external_id", report_step) - self.assertIn("--arg external_id", report_step) - self.assertIn('head_sha="$MERGE_SHA"', report_step) - self.assertIn("CHECK_NAME: llama.cpp authorized validation v2", report_job) - self.assertIn('name="$CHECK_NAME"', report_step) - self.assertIn('external_id="$external_id"', report_step) - self.assertNotIn('head_sha="$HEAD_SHA"', report_step) - self.assertNotIn('-f name="llama.cpp validation"', pull_request) - self.assertNotIn(" name: llama.cpp validation\n", pull_request) - self.assertIn("--method PATCH", report_step) - self.assertIn("check-runs/${CHECK_RUN_ID}", report_step) - self.assertIn("pull-requests: read", report_job) - self.assertIn("repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", report_job) - self.assertIn("github.event.pull_request.base.sha", report_job) - self.assertIn("github.event.pull_request.merge_commit_sha", report_job) - self.assertIn("github.event.merge_group.head_sha", report_job) - self.assertIn(".base.sha", report_job) - self.assertIn(".head.sha", report_job) - self.assertIn(".merge_commit_sha", report_job) - self.assertIn('current_candidate="$current_head"', report_step) - self.assertIn('.app.slug == "github-actions"', report_step) - self.assertIn("newest_number", report_step) - self.assertIn(".number < $current_number", report_step) - self.assertLess( - report_step.index("repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}"), - report_step.index("--method PATCH"), - ) - self.assertIn("needs.validate.result", report_job) - - self.assertIn(" validate-invocation:\n", validation) - plan_job = validation.split(" plan:\n", 1)[1].split( - " get-latest-releases:\n", 1 - )[0] - build_job_header = validation.split(" build:\n", 1)[1].split( - " runs-on:", 1 - )[0] - validation_gate = validation.split(" validation-gate:\n", 1)[1] - - self.assertIn( - "merge_group|pull_request_target|schedule|workflow_dispatch", - invocation_job, - ) - self.assertIn("Unsupported invocation event", invocation_job) - self.assertNotIn("actions/checkout", invocation_job) - self.assertIn("needs: validate-invocation", plan_job) - self.assertIn("validate-invocation", build_job_header) - self.assertIn( - "needs: [validate-invocation, build, validate, " - "reverify-protected-release-manifest, plan]", - validation_gate, - ) - self.assertNotIn("checks: write", validation_gate) - self.assertNotIn("check-runs", validation_gate) - - merge_repository = "github.repository" - merge_sha = "github.event.pull_request.merge_commit_sha" - self.assertGreaterEqual(validation.count(merge_repository), 4) - self.assertGreaterEqual(validation.count(merge_sha), 2) - self.assertGreaterEqual( - validation.count("github.event.merge_group.head_sha"), 4 - ) - self.assertIn("github.event.pull_request.base.sha", validation) - self.assertIn("github.event.pull_request.head.sha", validation) - self.assertEqual(validation.count("persist-credentials: false"), 10) - - def test_authorization_removal_reconciles_from_fresh_live_state(self) -> None: - pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - policy_job = pull_request.split(" policy:\n", 1)[1].split(" validate:\n", 1)[ - 0 - ] - helper = REQUIRED_CHECK_POLICY.read_text(encoding="utf-8") - - self.assertIn("current_authorized", policy_job) - self.assertIn("authorization_label_present=$current_authorized", policy_job) - self.assertIn("EVENT_ACTION", policy_job) - self.assertIn("EVENT_LABEL", policy_job) - self.assertIn("AUTHORIZATION_LABEL_PRESENT", policy_job) - self.assertIn("not authorization_label_present", helper) - self.assertIn("conclusion=failure", pull_request) - self.assertNotIn(" revoke-check:\n", pull_request) - - def test_closed_pull_requests_use_live_merged_state_before_mutating(self) -> None: - pull_request = PR_WORKFLOW.read_text(encoding="utf-8") - helper = REQUIRED_CHECK_POLICY.read_text(encoding="utf-8") - - self.assertIn("current_state", pull_request) - self.assertIn("current_merged", pull_request) - self.assertIn("if merged:", helper) - self.assertIn('if state == "closed":', helper) - self.assertIn("mode != 'merged'", pull_request) - self.assertIn("mode != 'skip_closed'", pull_request) - self.assertNotIn(" authorize-invocation:\n", pull_request) - self.assertNotIn(" revoke-check:\n", pull_request) - - def test_protected_change_runner_matrix_is_defined_by_trusted_yaml(self) -> None: - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - plan_job = validation.split(" plan:\n", 1)[1].split( - " get-latest-releases:\n", 1 - )[0] - - self.assertIn(" - name: Build trusted protected-change matrix\n", plan_job) - trusted_step = plan_job.split( - " - name: Build trusted protected-change matrix\n", 1 - )[1].split(" - name: Build branch validation matrix\n", 1)[0] - branch_step = plan_job.split( - " - name: Build branch validation matrix\n", 1 - )[1] - - self.assertIn("github.event_name == 'pull_request_target'", trusted_step) - self.assertIn("github.event_name == 'merge_group'", trusted_step) - self.assertIn('"lemon-prod"', trusted_step) - self.assertIn('"K2-Horizon-0.9B-GGUF"', trusted_step) - self.assertIn('"K2-Horizon-3.7B-GGUF"', trusted_step) - self.assertIn('"K2-Horizon-7B-GGUF"', trusted_step) - self.assertIn('target: "windows-vulkan"', trusted_step) - for target in ( - "windows-vulkan", - "linux-vulkan", - "linux-cpu", - "macos-metal", - "windows-cuda", - "windows-rocm-stable", - "windows-rocm-nightly", - ): - with self.subTest(target=target): - self.assertIn(f'target: "{target}"', trusted_step) - self.assertEqual(trusted_step.count('capability_profile: "k2-horizon-v1"'), 7) - self.assertEqual( - trusted_step.count('capability_models: ["K2-Horizon-0.9B-GGUF"]'), - 7, - ) - self.assertEqual(trusted_step.count('managed_pin: "vulkan"'), 2) - for managed_pin in ( - "cpu", - "cuda", - "metal", - "rocm-stable", - "rocm-nightly", - ): - self.assertEqual(trusted_step.count(f'managed_pin: "{managed_pin}"'), 1) - self.assertEqual(trusted_step.count('build_platform: "windows"'), 4) - self.assertEqual(trusted_step.count('build_platform: "linux"'), 2) - self.assertEqual(trusted_step.count('build_platform: "macos"'), 1) - self.assertEqual(trusted_step.count('runner: ["macos-latest"]'), 1) - self.assertNotIn("python -m test.utils", trusted_step) - self.assertIn("github.event_name != 'pull_request_target'", branch_step) - self.assertIn("github.event_name != 'merge_group'", branch_step) - self.assertIn("python -m test.utils.llamacpp_validation_plan", branch_step) - self.assertIn("steps.protected-plan.outputs.matrix", plan_job) - self.assertIn("steps.branch-plan.outputs.matrix", plan_job) - - validate_job = validation.split(" validate:\n", 1)[1].split( - " validation-gate:\n", 1 - )[0] - self.assertIn(".github/scripts/run_llamacpp_validation.py", validate_job) - self.assertIn(' TARGET: "${{ matrix.target }}"\n', validate_job) - self.assertIn( - " CAPABILITY_PROFILE: ${{ matrix.capability_profile || '' }}\n", - validate_job, - ) - self.assertIn(" CAPABILITY_MODELS:", validate_job) - self.assertIn(" LLAMACPP_BACKEND: ${{ matrix.backend }}", validate_job) - self.assertIn(" LLAMACPP_CHANNEL: ${{ matrix.channel }}", validate_job) - self.assertIn("validation-results-${{ matrix.target }}", validate_job) - self.assertIn("steps.run-validation.outputs.validation_result", validate_job) - - def test_manual_validation_calls_the_read_only_core(self) -> None: - manual = MANUAL_WORKFLOW.read_text(encoding="utf-8") - manual_triggers = manual.split("permissions:", 1)[0] - - self.assertIn(" workflow_dispatch:\n", manual_triggers) - self.assertNotIn(" schedule:\n", manual_triggers) - self.assertIn("permissions:\n contents: read\n", manual) - self.assertNotIn("contents: write", manual) - self.assertIn("permissions:\n checks: write\n contents: read", manual) - self.assertNotIn("pull-requests: write", manual) - self.assertNotIn("secrets: inherit", manual) - self.assertNotIn("HUGGINGFACE_ACCESS_TOKEN", manual) - self.assertIn( - "uses: ./.github/workflows/validate_llamacpp_core.yml", - manual, - ) - - def test_validation_credential_aliases_are_scrubbed_by_trusted_control_plane( - self, - ) -> None: - step = workflow_step( - VALIDATION_WORKFLOW, - "validate", - "Run validation with lemond", - ) - environment = step["env"] - run = step["run"] - credential_aliases = { - "HF_TOKEN", - "HUGGINGFACE_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - } - - self.assertEqual( - {key for key in environment if key.upper() in credential_aliases}, - credential_aliases, - ) - self.assertEqual( - environment["HF_TOKEN"], - "${{ github.event_name == 'schedule' && secrets.HUGGINGFACE_ACCESS_TOKEN || '' }}", - ) - self.assertEqual(environment["HUGGINGFACE_TOKEN"], "") - self.assertEqual(environment["HUGGING_FACE_HUB_TOKEN"], "") - self.assertEqual(run.count("--allow-huggingface-download-credentials"), 1) - self.assertIn( - "${{ github.event_name == 'schedule' && '--allow-huggingface-download-credentials' || '' }}", - run, - ) - self.assertIn( - 'python "$VALIDATION_ROOT/.github/scripts/run_llamacpp_validation.py"', - run, - ) - self.assertNotIn("VALIDATION_CANDIDATE_ROOT/.github/scripts", run) - - manual_job = load_workflow(MANUAL_WORKFLOW)["jobs"]["validate"] - schedule_job = load_workflow(SCHEDULE_WORKFLOW)["jobs"]["validate"] - self.assertNotIn("secrets", manual_job) - self.assertEqual( - schedule_job["secrets"], - {"HUGGINGFACE_ACCESS_TOKEN": ("${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}")}, - ) - - def test_scheduled_publication_is_isolated_and_serialized(self) -> None: - schedule = SCHEDULE_WORKFLOW.read_text(encoding="utf-8") - schedule_triggers = schedule.split("permissions:", 1)[0] - validation_job = schedule.split(" validate:\n", 1)[1].split(" publish:\n", 1)[ - 0 - ] - publication = schedule.split(" publish:\n", 1)[1] - checkout_marker = f" - uses: actions/checkout@{CHECKOUT_SHA} # v5.1.0\n" - publication_identity_step = publication.split( - " - name: Require dedicated publication identity\n", 1 - )[1].split(checkout_marker, 1)[0] - checkout_step = publication.split(checkout_marker, 1)[1].split( - "\n - name:", 1 - )[0] - - self.assertIn(" schedule:\n", schedule_triggers) - self.assertNotIn(" workflow_dispatch:\n", schedule_triggers) - self.assertNotIn(" workflow_call:\n", schedule_triggers) - self.assertNotIn(" pull_request:\n", schedule_triggers) - self.assertNotIn(" merge_group:\n", schedule_triggers) - self.assertIn("group: llamacpp-auto-update\n", schedule) - self.assertIn( - "uses: ./.github/workflows/validate_llamacpp_core.yml", - validation_job, - ) - self.assertIn( - "permissions:\n checks: write\n contents: read", validation_job - ) - self.assertNotIn("contents: write", validation_job) - self.assertIn( - "permissions:\n checks: write\n contents: read", validation_job - ) - self.assertNotIn("secrets: inherit", schedule) - self.assertIn( - "HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}", - validation_job, - ) - self.assertIn("group: llamacpp-auto-update-publication", publication) - self.assertIn("cancel-in-progress: false", publication) - self.assertIn( - "needs.validate.outputs.promotion_eligible == 'true'", publication - ) - self.assertIn("permissions:\n contents: read", publication) - self.assertNotIn("contents: write", publication) - self.assertNotIn("pull-requests: write", publication) - self.assertNotIn("secrets.GITHUB_TOKEN", publication) - self.assertIn("publish_llamacpp_update.sh", publication) - self.assertIn("fetch-depth: 0", checkout_step) - self.assertIn("persist-credentials: false", checkout_step) - self.assertIn("token: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", checkout_step) - self.assertEqual( - publication.count( - f"uses: actions/download-artifact@{DOWNLOAD_ARTIFACT_SHA} # v7.0.0" - ), - 2, - ) - self.assertNotIn("actions/checkout@v5", publication) - self.assertNotIn("actions/download-artifact@v7", publication) - self.assertIn("gh api graphql", publication_identity_step) - self.assertIn(".data.viewer.login", publication_identity_step) - self.assertIn( - "GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", - publication_identity_step, - ) - self.assertIn( - "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", - publication_identity_step, - ) - self.assertLess( - publication.index(" - name: Require dedicated publication identity"), - publication.index(f" - uses: actions/checkout@{CHECKOUT_SHA}"), - ) - self.assertIn("VALIDATED_BASE_SHA: ${{ github.sha }}", publication) - self.assertIn( - "BASE_BRANCH: ${{ github.event.repository.default_branch }}", - publication, - ) - self.assertIn("pattern: validation-results-*", publication) - self.assertIn("pattern: restart-validation-results-*", publication) - evidence_step = publication.split( - " - name: Validate complete validation evidence\n", 1 - )[1].split( - " - name: Update backend_versions.json from release manifest\n", 1 - )[ - 0 - ] - update_step = publication.split( - " - name: Update backend_versions.json from release manifest\n", 1 - )[1].split(" - name: Generate PR body\n", 1)[0] - manifest_step = publication.split( - " - name: Verify release asset manifest\n", 1 - )[1].split(" - name: Publish update pull request\n", 1)[0] - self.assertIn("llamacpp_validation_evidence", evidence_step) - self.assertIn("--expected-matrix-json", evidence_step) - self.assertIn( - "needs.validate.outputs.validation_matrix", - evidence_step, - ) - self.assertEqual(update_step.count("set -euo pipefail"), 1) - self.assertIn("llamacpp_release_manifest", manifest_step) - self.assertIn( - "needs.validate.outputs.release_asset_manifest", - manifest_step, - ) - self.assertIn("capture_llamacpp_release_manifest.sh", manifest_step) - self.assertIn("--materialize-manifest", manifest_step) - self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", manifest_step) - self.assertIn("EXPECTED_RELEASE_ASSET_MANIFEST", publication) - self.assertNotIn("\n - name:", manifest_step) - self.assertLess( - publication.index(" - name: Generate PR body\n"), - publication.index(" - name: Verify release asset manifest\n"), - ) - self.assertLess( - publication.index(" - name: Verify release asset manifest\n"), - publication.index(" - name: Publish update pull request\n"), - ) - publisher_step = publication.split( - " - name: Publish update pull request\n", 1 - )[1] - self.assertIn("GH_TOKEN: ${{ secrets.LLAMACPP_UPDATE_TOKEN }}", publisher_step) - self.assertIn( - "EXPECTED_PUBLICATION_ACTOR: ${{ vars.LLAMACPP_UPDATE_ACTOR }}", - publisher_step, - ) - - validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8") - output_names = ( - "promotion_eligible", - "validation_matrix", - "llamacpp_release", - "llamacpp_rocm_release", - "llamacpp_lemonade_release", - "ggml_update_backends", - "lemonade_update_backends", - "rocm_update_backends", - "ggml_missing_count", - "rocm_nightly_missing_count", - "rocm_stable_missing_count", - "cuda_missing_count", - "vulkan_available", - "cpu_available", - "metal_available", - "rocm_nightly_available", - "rocm_stable_available", - "cuda_available", - "release_asset_manifest", - ) - for output_name in output_names: - with self.subTest(output_name=output_name): - self.assertIn(f" {output_name}:\n", validation) - self.assertIn(f"needs.validate.outputs.{output_name}", schedule) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_validation_model_catalog.py b/test/test_validation_model_catalog.py deleted file mode 100644 index 40c084e123..0000000000 --- a/test/test_validation_model_catalog.py +++ /dev/null @@ -1,178 +0,0 @@ -import json -import tempfile -import unittest -from pathlib import Path - -from test.utils.validation_model_catalog import ( - ModelCatalogOverlayError, - merge_model_catalog_files, - merge_model_catalogs, -) - - -class ValidationModelCatalogTests(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.TemporaryDirectory() - self.addCleanup(self.temp_dir.cleanup) - self.root = Path(self.temp_dir.name) - - def _write(self, name, payload): - path = self.root / name - path.write_text(payload, encoding="utf-8") - return path - - def _merge(self, base, overlay): - output = self.root / "merged.json" - merge_model_catalog_files(base, overlay, output) - return output.read_text(encoding="utf-8") - - def test_merges_disjoint_models_deterministically(self): - base = self._write( - "base.json", - json.dumps({"Existing": {"recipe": "llamacpp", "size": 1.0}}), - ) - overlay = self._write( - "overlay.json", - json.dumps({"Candidate": {"recipe": "llamacpp", "size": 2.0}}), - ) - - first = self._merge(base, overlay) - second = self._merge(base, overlay) - - self.assertEqual(first, second) - self.assertEqual( - json.loads(first), - { - "Candidate": {"recipe": "llamacpp", "size": 2.0}, - "Existing": {"recipe": "llamacpp", "size": 1.0}, - }, - ) - self.assertTrue(first.endswith("\n")) - - def test_accepts_an_identical_promoted_model(self): - model = {"recipe": "llamacpp", "size": 2.0} - base = self._write("base.json", json.dumps({"Candidate": model})) - overlay = self._write("overlay.json", json.dumps({"Candidate": model})) - - merged = json.loads(self._merge(base, overlay)) - - self.assertEqual(merged, {"Candidate": model}) - - def test_returns_the_merged_catalog_without_writing_a_file(self): - base = self._write( - "base.json", - json.dumps({"Existing": {"recipe": "llamacpp"}}), - ) - overlay = self._write( - "overlay.json", - json.dumps({"Candidate": {"recipe": "llamacpp"}}), - ) - - self.assertEqual( - merge_model_catalogs(base, overlay), - { - "Existing": {"recipe": "llamacpp"}, - "Candidate": {"recipe": "llamacpp"}, - }, - ) - self.assertFalse((self.root / "merged.json").exists()) - - def test_rejects_a_conflicting_promoted_model(self): - base = self._write( - "base.json", json.dumps({"Candidate": {"recipe": "llamacpp"}}) - ) - overlay = self._write( - "overlay.json", json.dumps({"Candidate": {"recipe": "vllm"}}) - ) - - with self.assertRaisesRegex( - ModelCatalogOverlayError, "conflicting model definition: Candidate" - ): - self._merge(base, overlay) - - def test_rejects_duplicate_json_keys(self): - base = self._write( - "base.json", - '{"Duplicate":{"recipe":"llamacpp"},' '"Duplicate":{"recipe":"llamacpp"}}', - ) - overlay = self._write("overlay.json", "{}") - - with self.assertRaisesRegex( - ModelCatalogOverlayError, "duplicate JSON member: Duplicate" - ): - self._merge(base, overlay) - - def test_rejects_non_finite_json_numbers(self): - valid = self._write("valid.json", "{}") - for literal in ("NaN", "Infinity", "-Infinity"): - with self.subTest(literal=literal): - invalid = self._write( - "invalid.json", - '{"Candidate":{"recipe":"llamacpp","size":' + literal + "}}", - ) - - with self.assertRaisesRegex( - ModelCatalogOverlayError, "non-finite JSON number" - ): - self._merge(valid, invalid) - - def test_rejects_floating_point_overflow_in_both_catalogs(self): - for invalid_side in ("base", "overlay"): - with self.subTest(invalid_side=invalid_side): - invalid = self._write( - f"{invalid_side}-overflow.json", - '{"Candidate":{"recipe":"llamacpp","size":1e400}}', - ) - valid = self._write(f"{invalid_side}-valid.json", "{}") - base, overlay = ( - (invalid, valid) if invalid_side == "base" else (valid, invalid) - ) - - with self.assertRaisesRegex( - ModelCatalogOverlayError, "non-finite JSON number" - ): - self._merge(base, overlay) - - def test_promoted_model_comparison_is_type_sensitive(self): - base = self._write( - "base.json", - '{"Candidate":{"recipe":"llamacpp","value":1}}', - ) - overlay = self._write( - "overlay.json", - '{"Candidate":{"recipe":"llamacpp","value":true}}', - ) - - with self.assertRaisesRegex( - ModelCatalogOverlayError, "conflicting model definition: Candidate" - ): - self._merge(base, overlay) - - def test_rejects_non_object_catalogs_and_models(self): - valid = self._write("valid.json", "{}") - catalog_list = self._write("list.json", "[]") - invalid_model = self._write("model.json", '{"Candidate":[]}') - - with self.assertRaisesRegex( - ModelCatalogOverlayError, "catalog must be a JSON object" - ): - self._merge(catalog_list, valid) - with self.assertRaisesRegex( - ModelCatalogOverlayError, "Candidate must be a JSON object" - ): - self._merge(valid, invalid_model) - - def test_does_not_replace_output_after_a_validation_failure(self): - output = self.root / "merged.json" - output.write_text('{"preserved":true}\n', encoding="utf-8") - base = self._write("base.json", "{}") - overlay = self._write("overlay.json", '{"Candidate":[]}') - - with self.assertRaises(ModelCatalogOverlayError): - merge_model_catalog_files(base, overlay, output) - - self.assertEqual(output.read_text(encoding="utf-8"), '{"preserved":true}\n') - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_verify_llamacpp_pin_manifest.py b/test/test_verify_llamacpp_pin_manifest.py deleted file mode 100644 index ecf0a1dbe7..0000000000 --- a/test/test_verify_llamacpp_pin_manifest.py +++ /dev/null @@ -1,436 +0,0 @@ -#!/usr/bin/env python3 -"""Behavior tests for trusted llama.cpp pin-manifest verification.""" - -from __future__ import annotations - -import copy -import json -import os -import shutil -import subprocess -import tempfile -import unittest -from pathlib import Path - -from test.utils import llamacpp_release_assets as release_assets -from test.utils import llamacpp_release_manifest as manifest - -ROOT = Path(__file__).resolve().parents[1] -VERIFIER = ROOT / ".github" / "scripts" / "verify_llamacpp_pin_manifest.sh" -MANIFEST_TOOL = ROOT / "test" / "utils" / "llamacpp_release_manifest.py" -ASSET_TOOL = ROOT / "test" / "utils" / "llamacpp_release_assets.py" -VERSIONS_PATH = Path("src/cpp/resources/backend_versions.json") -RELEASE_MANIFEST_PATH = Path(".github/llamacpp_release_manifest.json") -MISSING = object() - - -def release_payload( - repository: str, - tag: str, - release_id: int, - asset_names: list[str] | None = None, -) -> dict: - release_tag_commit = f"{release_id:040x}" - claim_type = ( - "source-release-tag" - if repository == manifest.TRUSTED_SOURCE_REPOSITORY - else "immutable-source-manifest" - ) - names = asset_names or [f"llama-{release_id}.zip"] - assets = [ - { - "digest": "sha256:" + f"{release_id * 1000 + index:064x}"[-64:], - "id": release_id * 1000 + index, - "name": name, - "size": release_id, - "state": "uploaded", - "updated_at": "2026-09-05T12:30:00Z", - } - for index, name in enumerate(names, start=1) - ] - if claim_type == "immutable-source-manifest": - assets.append( - { - "digest": "sha256:" + "a" * 64, - "id": release_id * 1000 + len(assets) + 1, - "name": manifest.SOURCE_MANIFEST_ASSET_NAME, - "size": 512, - "state": "uploaded", - "updated_at": "2026-09-05T12:31:00Z", - } - ) - return { - "assets": assets, - "draft": False, - "id": release_id, - "immutable": True, - "publisher_claim_type": claim_type, - "publisher_claimed_source_commit": release_tag_commit, - "release_tag_commit": release_tag_commit, - "source_repository": manifest.TRUSTED_SOURCE_REPOSITORY, - "tag_name": tag, - "upstream_reference": "refs/heads/master", - "upstream_reference_head": "f" * 40, - } - - -class VerifyLlamaCppPinManifestTests(unittest.TestCase): - def setUp(self) -> None: - self.temporary_directory = tempfile.TemporaryDirectory() - self.directory = Path(self.temporary_directory.name) - self.tool_root = self.directory / "tools" - scripts = self.tool_root / ".github" / "scripts" - utilities = self.tool_root / "test" / "utils" - scripts.mkdir(parents=True) - utilities.mkdir(parents=True) - shutil.copy2(VERIFIER, scripts / VERIFIER.name) - shutil.copy2(MANIFEST_TOOL, utilities / MANIFEST_TOOL.name) - shutil.copy2(ASSET_TOOL, utilities / ASSET_TOOL.name) - capture = scripts / "capture_llamacpp_release_manifest.sh" - capture.write_text( - "#!/usr/bin/env bash\n" - 'printf \'%s\\n\' "$*" >> "$CAPTURE_LOG"\n' - 'exit "$CAPTURE_EXIT"\n', - encoding="utf-8", - ) - capture.chmod(0o755) - self.verifier = scripts / VERIFIER.name - self.case_number = 0 - self.capture_log = self.directory / "capture-0.log" - self.base_versions = { - "llamacpp": { - "cpu": "b1", - "cuda": "b2", - "metal": "b1", - "rocm-nightly": "b3", - "rocm-stable": "b2", - "vulkan": "b1", - }, - "rocm_asset_families": {}, - "therock": {"version": "7.14.0"}, - } - - def tearDown(self) -> None: - self.temporary_directory.cleanup() - - @staticmethod - def initialize_repository( - path: Path, - versions: dict, - release_manifest: object = MISSING, - ) -> str: - path.mkdir() - subprocess.run(["git", "init", "-q", str(path)], check=True) - subprocess.run( - ["git", "-C", str(path), "config", "user.name", "Test"], check=True - ) - subprocess.run( - ["git", "-C", str(path), "config", "user.email", "test@example.com"], - check=True, - ) - subprocess.run( - ["git", "-C", str(path), "config", "commit.gpgsign", "false"], - check=True, - ) - versions_path = path / VERSIONS_PATH - versions_path.parent.mkdir(parents=True) - versions_path.write_text(json.dumps(versions) + "\n", encoding="utf-8") - if release_manifest is not MISSING: - manifest_path = path / RELEASE_MANIFEST_PATH - manifest_path.parent.mkdir(parents=True) - manifest_path.write_text(str(release_manifest), encoding="utf-8") - subprocess.run(["git", "-C", str(path), "add", "."], check=True) - subprocess.run( - ["git", "-C", str(path), "commit", "-q", "-m", "fixture"], - check=True, - ) - return subprocess.run( - ["git", "-C", str(path), "rev-parse", "HEAD"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - - def matching_manifest( - self, - ggml_tag: str = "b9", - rocm_tag: str = "b3", - lemonade_tag: str = "b2", - *, - missing_asset: str | None = None, - missing_attested_target: tuple[str, str] | None = None, - zero_size_asset: str | None = None, - versions: dict | None = None, - ) -> str: - selected_versions = versions or self.base_versions - requirements = release_assets.build_asset_requirements( - ggml_release=ggml_tag, - rocm_release=rocm_tag, - lemonade_release=lemonade_tag, - backend_versions=selected_versions, - ) - rocm_target_requirements = release_assets.build_rocm_asset_target_requirements( - rocm_release=rocm_tag, - backend_versions=selected_versions, - ) - release_inputs = ( - ("ggml-org/llama.cpp", ggml_tag, 9, "ggml"), - ("lemonade-sdk/llama.cpp", lemonade_tag, 2, "lemonade"), - ("lemonade-sdk/llamacpp-rocm", rocm_tag, 3, "rocm"), - ) - releases = [] - for repository, tag, release_id, requirement_group in release_inputs: - names = [ - name - for family in requirements[requirement_group].values() - for name in family - ] - payload = release_payload(repository, tag, release_id, names) - if missing_asset is not None: - payload["assets"] = [ - asset - for asset in payload["assets"] - if asset["name"] != missing_asset - ] - if zero_size_asset is not None: - for asset in payload["assets"]: - if asset["name"] == zero_size_asset: - asset["size"] = 0 - if repository == "lemonade-sdk/llamacpp-rocm": - payload["build_target_attestations"] = [] - for asset in payload["assets"]: - required_targets = rocm_target_requirements.get(asset["name"]) - if required_targets is None: - continue - build_targets = list(required_targets) - if ( - missing_attested_target is not None - and asset["name"] == missing_attested_target[0] - ): - build_targets.remove(missing_attested_target[1]) - payload["build_target_attestations"].append( - { - "build_targets": build_targets, - "digest": asset["digest"], - "name": asset["name"], - "size": asset["size"], - } - ) - releases.append((repository, tag, payload)) - return manifest.build_release_asset_manifest(releases) - - def run_verifier( - self, - candidate_versions: dict, - *, - base_manifest: object = MISSING, - candidate_manifest: object = MISSING, - capture_exit: int = 37, - ) -> subprocess.CompletedProcess[str]: - self.case_number += 1 - trusted = self.directory / f"trusted-{self.case_number}" - candidate = self.directory / f"candidate-{self.case_number}" - self.capture_log = self.directory / f"capture-{self.case_number}.log" - base_sha = self.initialize_repository( - trusted, self.base_versions, base_manifest - ) - candidate_sha = self.initialize_repository( - candidate, candidate_versions, candidate_manifest - ) - env = os.environ.copy() - env.update( - { - "CAPTURE_EXIT": str(capture_exit), - "CAPTURE_LOG": str(self.capture_log), - } - ) - return subprocess.run( - [ - "bash", - str(self.verifier), - str(trusted), - str(candidate), - base_sha, - candidate_sha, - ], - capture_output=True, - check=False, - env=env, - text=True, - ) - - def test_unchanged_pins_skip_capture(self) -> None: - result = self.run_verifier(copy.deepcopy(self.base_versions)) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("No changed managed llama.cpp pins", result.stdout) - self.assertFalse(self.capture_log.exists()) - - def test_changed_pin_rejects_missing_and_mismatched_manifests(self) -> None: - candidate = copy.deepcopy(self.base_versions) - candidate["llamacpp"]["vulkan"] = "b9" - - missing = self.run_verifier(candidate) - self.assertNotEqual(missing.returncode, 0) - self.assertIn("require a committed release manifest", missing.stderr) - self.assertFalse(self.capture_log.exists()) - - mismatched = self.run_verifier( - candidate, - candidate_manifest=self.matching_manifest("b8"), - ) - self.assertNotEqual(mismatched.returncode, 0) - self.assertIn("does not match changed managed llama.cpp pin", mismatched.stderr) - self.assertFalse(self.capture_log.exists()) - - def test_sidecar_only_change_is_rejected(self) -> None: - result = self.run_verifier( - copy.deepcopy(self.base_versions), - candidate_manifest="{}\n", - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn( - "Release manifest changes require changed managed llama.cpp pins", - result.stderr, - ) - self.assertFalse(self.capture_log.exists()) - - def test_capture_failure_is_propagated(self) -> None: - candidate = copy.deepcopy(self.base_versions) - candidate["llamacpp"]["vulkan"] = "b9" - - result = self.run_verifier( - candidate, - candidate_manifest=self.matching_manifest(), - capture_exit=37, - ) - - self.assertEqual(result.returncode, 37, result.stderr) - self.assertTrue(self.capture_log.exists()) - - def test_changed_pins_require_complete_positive_cross_platform_assets( - self, - ) -> None: - cases = ( - ( - "cpu", - {"ggml_tag": "b9"}, - "llama-b9-bin-win-cpu-x64.zip", - ), - ( - "vulkan", - {"ggml_tag": "b9"}, - "llama-b9-bin-ubuntu-vulkan-arm64.tar.gz", - ), - ( - "metal", - {"ggml_tag": "b9"}, - "llama-b9-bin-macos-arm64.tar.gz", - ), - ( - "cuda", - {"ggml_tag": "b1", "lemonade_tag": "b9"}, - "llama-b9-ubuntu-cuda-sm_121-arm64.tar.xz", - ), - ( - "rocm-stable", - {"ggml_tag": "b1", "lemonade_tag": "b9"}, - "llama-b9-bin-win-rocm-7.14-x64.zip", - ), - ( - "rocm-nightly", - {"ggml_tag": "b1", "rocm_tag": "b9"}, - "llama-b9-ubuntu-rocm-gfx942-x64.zip", - ), - ) - for backend, tags, required_asset in cases: - candidate = copy.deepcopy(self.base_versions) - candidate["llamacpp"][backend] = "b9" - for defect in ("missing", "zero-size"): - with self.subTest(backend=backend, defect=defect): - manifest_json = self.matching_manifest( - **tags, - missing_asset=required_asset if defect == "missing" else None, - zero_size_asset=( - required_asset if defect == "zero-size" else None - ), - ) - result = self.run_verifier( - candidate, - candidate_manifest=manifest_json, - capture_exit=0, - ) - - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertIn(f"llamacpp.{backend}", result.stderr) - self.assertFalse(self.capture_log.exists()) - - def test_rocm_nightly_pin_rejects_incomplete_concrete_target_attestation( - self, - ) -> None: - candidate = copy.deepcopy(self.base_versions) - candidate["llamacpp"]["rocm-nightly"] = "b9" - candidate["rocm_asset_families"] = { - "gfx1033": "gfx103X", - "gfx1035": "gfx103X", - "gfx1036": "gfx103X", - } - target_asset = "llama-b9-windows-rocm-gfx103X-x64.zip" - - result = self.run_verifier( - candidate, - candidate_manifest=self.matching_manifest( - ggml_tag="b1", - rocm_tag="b9", - versions=candidate, - missing_attested_target=(target_asset, "gfx1035"), - ), - capture_exit=0, - ) - - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertIn("incomplete build-target attestation", result.stderr) - self.assertIn(target_asset, result.stderr) - self.assertFalse(self.capture_log.exists()) - - def test_selector_only_changes_require_live_manifests(self) -> None: - mapping_candidate = copy.deepcopy(self.base_versions) - mapping_candidate["rocm_asset_families"] = {"gfx1033": "gfx103X"} - missing_mapping_manifest = self.run_verifier(mapping_candidate) - self.assertNotEqual(missing_mapping_manifest.returncode, 0) - self.assertIn( - "require a committed release manifest", missing_mapping_manifest.stderr - ) - self.assertFalse(self.capture_log.exists()) - - therock_candidates = [] - for field, value in ( - ("version", "7.14.1"), - ("architectures", ["gfx103X", "gfx110X"]), - ("url_mapping", {"windows": "runtime.zip"}), - ): - candidate = copy.deepcopy(self.base_versions) - candidate["therock"][field] = value - therock_candidates.append(candidate) - with self.subTest(therock_selector=field): - missing_therock_manifest = self.run_verifier(candidate) - self.assertNotEqual(missing_therock_manifest.returncode, 0) - self.assertIn( - "require a committed release manifest", - missing_therock_manifest.stderr, - ) - self.assertFalse(self.capture_log.exists()) - - therock_candidate = therock_candidates[-1] - accepted = self.run_verifier( - therock_candidate, - candidate_manifest=self.matching_manifest(versions=therock_candidate), - capture_exit=0, - ) - self.assertEqual(accepted.returncode, 0, accepted.stderr) - self.assertTrue(self.capture_log.exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/utils/llamacpp_capability_validation.py b/test/utils/llamacpp_capability_validation.py deleted file mode 100644 index 15619f9315..0000000000 --- a/test/utils/llamacpp_capability_validation.py +++ /dev/null @@ -1,1238 +0,0 @@ -"""Fail-closed API capability validation for llama.cpp models.""" - -from __future__ import annotations - -import json -import time -from collections.abc import Callable, Iterable -from typing import Any - -K2_HORIZON_PROFILE = "k2-horizon-v1" -K2_HORIZON_CASE_IDS = ( - "openai_plain_off_nonstream", - "openai_plain_off_stream", - "openai_reasoning_high", - "openai_reasoning_high_stream", - "openai_reasoning_medium", - "openai_reasoning_low", - "openai_tool_json", - "openai_tool_xml", - "openai_tool_xml_typed", - "openai_tool_default_xml", - "openai_tool_stream_xml", - "openai_tool_result_followup", - "ollama_thinking", - "anthropic_translation", -) -CASE_CONTRACTS = { - "openai_plain_off_nonstream": ("openai", False, "stop", True, False, 0, None), - "openai_plain_off_stream": ("openai", True, "stop", True, False, 0, None), - "openai_reasoning_high": ("openai", False, "stop", True, True, 0, None), - "openai_reasoning_high_stream": ( - "openai", - True, - "stop", - True, - True, - 0, - None, - ), - "openai_reasoning_medium": ("openai", False, "stop", True, True, 0, None), - "openai_reasoning_low": ("openai", False, "stop", True, False, 0, None), - "openai_tool_json": ("openai", False, "tool_calls", False, False, 1, None), - "openai_tool_xml": ("openai", False, "tool_calls", False, False, 1, None), - "openai_tool_xml_typed": ( - "openai", - False, - "tool_calls", - False, - False, - 1, - None, - ), - "openai_tool_default_xml": ( - "openai", - False, - "tool_calls", - False, - False, - 1, - "openai_tool_xml", - ), - "openai_tool_stream_xml": ( - "openai", - True, - "tool_calls", - False, - False, - 1, - "openai_tool_xml", - ), - "openai_tool_result_followup": ( - "openai", - False, - "stop", - True, - False, - 0, - None, - ), - "ollama_thinking": ("ollama", False, "stop", True, True, 0, None), - "anthropic_translation": ( - "anthropic", - False, - "end_turn", - True, - False, - 0, - None, - ), -} -TOOL_NAME = "lookup_weather" -EXPECTED_TOOL_ARGUMENTS = { - "city": "Paris", - "unit": "celsius", - "days": 2, - "include_hourly": False, -} -IFM_CONTROL_PREFIXES = ("", start) - if end < 0: - end = min(start + 79, len(candidate) - 1) - return candidate[start : end + 1], "".join(path_components) - elif isinstance(candidate, (dict, list, tuple)): - pending.append( - ( - iter(_marker_children(candidate)), - len(path_components), - ) - ) - return None - - -def _require(condition: bool, message: str) -> None: - if not condition: - raise CapabilityValidationError(message) - - -def _marker_free(value: object, field: str = "response") -> None: - marker = find_raw_ifm_control_marker(value, field) - if marker: - marker_text, marker_path = marker - raise CapabilityValidationError( - f"raw IFM control marker {marker_text!r} in {marker_path}" - ) - - -def _status_code(response: object) -> int: - status_code = getattr(response, "status_code", None) - _require( - isinstance(status_code, int) and not isinstance(status_code, bool), - "response has no integer status code", - ) - return status_code - - -def _canonical_model_id(model: str) -> str: - return model.removeprefix("builtin.") - - -def _require_response_model(body: dict, expected_model: str, field: str) -> None: - response_model = body.get("model") - _require( - isinstance(response_model, str) and bool(response_model), - f"{field} model is missing", - ) - _require( - _canonical_model_id(response_model) == _canonical_model_id(expected_model), - f"{field} model {response_model!r} does not match {expected_model!r}", - ) - - -def reject_duplicate_json_object(pairs: list[tuple[str, object]]) -> dict: - result = {} - for key, value in pairs: - if key in result: - raise CapabilityValidationError(f"JSON object repeats key {key!r}") - result[key] = value - return result - - -def _stream_event_size(event: object) -> int: - if isinstance(event, bytes): - return len(event) + 1 - if isinstance(event, str): - return len(event.encode("utf-8")) + 1 - try: - encoded = json.dumps( - event, - allow_nan=False, - ensure_ascii=False, - separators=(",", ":"), - ).encode("utf-8") - except (TypeError, ValueError) as exc: - raise CapabilityValidationError("stream event cannot be encoded") from exc - return len(encoded) + 1 - - -def _validate_tool_arguments(arguments: object) -> dict: - _require(isinstance(arguments, dict), "tool arguments must be an object") - _require( - set(arguments) == set(EXPECTED_TOOL_ARGUMENTS), - "tool argument keys do not match request", - ) - for field in ("city", "unit"): - _require( - type(arguments[field]) is str - and arguments[field] == EXPECTED_TOOL_ARGUMENTS[field], - f"tool argument {field} has the wrong value or type", - ) - _require( - type(arguments["days"]) is int - and arguments["days"] == EXPECTED_TOOL_ARGUMENTS["days"], - "tool argument days has the wrong value or type", - ) - _require( - type(arguments["include_hourly"]) is bool - and arguments["include_hourly"] is EXPECTED_TOOL_ARGUMENTS["include_hourly"], - "tool argument include_hourly has the wrong value or type", - ) - return arguments - - -def _decode_tool_arguments(arguments_text: str) -> dict: - try: - arguments = json.loads( - arguments_text, - object_pairs_hook=reject_duplicate_json_object, - ) - except json.JSONDecodeError as exc: - raise CapabilityValidationError( - f"tool arguments are invalid JSON: {exc}" - ) from exc - return _validate_tool_arguments(arguments) - - -def _post_json( - request_json: Callable[..., tuple[object, object]], - url: str, - timeout: int, - payload: dict, -) -> tuple[dict, int]: - response, body = request_json("POST", url, timeout=timeout, json=payload) - status_code = _status_code(response) - _require(status_code == 200, f"HTTP {status_code}: {body}") - _require(isinstance(body, dict), "response body must be an object") - _marker_free(body) - expected_model = payload.get("model") - _require( - isinstance(expected_model, str) and expected_model, "request model is missing" - ) - _require_response_model(body, expected_model, "response") - return body, status_code - - -def _openai_message(body: dict, finish_reason: str) -> dict: - choices = body.get("choices") - _require(isinstance(choices, list) and len(choices) == 1, "expected one choice") - choice = choices[0] - _require(isinstance(choice, dict), "choice must be an object") - _require( - choice.get("finish_reason") == finish_reason, - f"expected finish_reason {finish_reason!r}", - ) - message = choice.get("message") - _require(isinstance(message, dict), "choice.message must be an object") - _require( - message.get("role") in (None, "assistant"), "message role is not assistant" - ) - _marker_free(message, "message") - return message - - -def _visible_content(message: dict, token: str | None = None) -> str: - content = message.get("content") - _require(isinstance(content, str) and bool(content.strip()), "content is blank") - if token is not None: - _require(token in content, f"content does not contain {token!r}") - return content - - -def _reasoning_content(message: dict) -> str: - reasoning = message.get("reasoning_content") - _require( - isinstance(reasoning, str) and bool(reasoning.strip()), - "reasoning_content is blank", - ) - return reasoning - - -def _require_no_reasoning(message: dict) -> None: - reasoning = message.get("reasoning_content") - _require( - reasoning is None or (isinstance(reasoning, str) and not reasoning.strip()), - "thinking-disabled response contains reasoning_content", - ) - - -def _require_no_tool_calls(message: dict) -> None: - tool_calls = message.get("tool_calls") - _require( - tool_calls is None or (isinstance(tool_calls, list) and not tool_calls), - "ordinary response contains tool calls", - ) - - -def _parse_tool_call(message: dict) -> tuple[dict, dict]: - content = message.get("content") - _require( - content is None or (isinstance(content, str) and not content.strip()), - "tool response contains unstructured visible content", - ) - calls = message.get("tool_calls") - _require(isinstance(calls, list) and len(calls) == 1, "expected one tool call") - call = calls[0] - _require(isinstance(call, dict), "tool call must be an object") - _require(isinstance(call.get("id"), str) and call["id"], "tool call id is missing") - _require(call.get("type") == "function", "tool call type must be function") - function = call.get("function") - _require(isinstance(function, dict), "tool call function must be an object") - _require(function.get("name") == TOOL_NAME, "unexpected tool function name") - arguments_text = function.get("arguments") - _require(isinstance(arguments_text, str), "tool arguments must be a JSON string") - arguments = _decode_tool_arguments(arguments_text) - return call, arguments - - -def _base_openai_payload(model: str, *, stream: bool, max_tokens: int) -> dict: - return { - "model": model, - "messages": [{"role": "user", "content": ARITHMETIC_PROMPT}], - "max_completion_tokens": max_tokens, - "temperature": 0, - "top_p": 1, - "seed": 42, - "stream": stream, - } - - -def _tool_payload(model: str, tool_format: str | None, *, stream: bool) -> dict: - payload = { - "model": model, - "messages": [{"role": "user", "content": TOOL_PROMPT}], - "tools": [TOOL_DEFINITION], - "tool_choice": "required", - "max_completion_tokens": 256, - "temperature": 0, - "top_p": 1, - "seed": 42, - "stream": stream, - "chat_template_kwargs": {"enable_thinking": False}, - } - if tool_format is not None: - payload["chat_template_kwargs"]["tool_call_format"] = tool_format - return payload - - -def _decode_openai_stream( - events: Iterable[object], *, deadline: float | None = None -) -> tuple[dict, str, bool, str]: - if deadline is None: - deadline = time.monotonic() + STREAM_WALL_CLOCK_TIMEOUT_SECONDS - content_parts: list[str] = [] - reasoning_parts: list[str] = [] - tools: dict[int, dict[str, Any]] = {} - finish_reason = None - stream_model = None - saw_choice = False - saw_done = False - total_sse_bytes = 0 - semantic_event_count = 0 - total_text_bytes = 0 - total_tool_argument_bytes = 0 - - for event_index, event in enumerate(events): - _require( - time.monotonic() < deadline, - "stream exceeded its wall-clock deadline", - ) - total_sse_bytes += _stream_event_size(event) - _require( - total_sse_bytes <= MAX_STREAM_SSE_BYTES, - "stream SSE byte count exceeds bounds", - ) - if isinstance(event, bytes): - try: - event = event.decode("utf-8") - except UnicodeDecodeError as exc: - raise CapabilityValidationError( - f"stream event {event_index} is not UTF-8" - ) from exc - if isinstance(event, str): - line = event.strip() - if not line or line.startswith(":"): - continue - semantic_event_count += 1 - _require( - semantic_event_count <= MAX_STREAM_EVENTS, - "stream event count exceeds bounds", - ) - _require( - not saw_done, "stream contains data after the terminal [DONE] frame" - ) - _marker_free(line, f"stream[{event_index}]") - _require(line.startswith("data:"), "stream line is not an SSE data frame") - data = line[len("data:") :].strip() - if data == "[DONE]": - _require(not saw_done, "stream contains duplicate [DONE] frames") - _require( - isinstance(finish_reason, str), - "stream ended before a terminal finish_reason", - ) - saw_done = True - continue - try: - chunk = json.loads( - data, - object_pairs_hook=reject_duplicate_json_object, - ) - except json.JSONDecodeError as exc: - raise CapabilityValidationError( - f"stream event {event_index} contains invalid JSON: {exc}" - ) from exc - else: - semantic_event_count += 1 - _require( - semantic_event_count <= MAX_STREAM_EVENTS, - "stream event count exceeds bounds", - ) - _require( - not saw_done, "stream contains data after the terminal [DONE] frame" - ) - chunk = event - _require(isinstance(chunk, dict), "stream chunk must be an object") - _marker_free(chunk, f"stream[{event_index}]") - _require( - "error" not in chunk, f"stream returned an error: {chunk.get('error')}" - ) - choices = chunk.get("choices") - _require(isinstance(choices, list), "stream choices must be an array") - if not choices: - continue - _require( - finish_reason is None, - "stream contains a choice chunk after a terminal finish_reason", - ) - _require( - len(choices) == 1 and isinstance(choices[0], dict), "invalid stream choice" - ) - response_model = chunk.get("model") - _require( - isinstance(response_model, str) and bool(response_model.strip()), - f"stream[{event_index}] model must be a nonempty string", - ) - _require( - stream_model in (None, response_model), - f"stream model changed from {stream_model!r} to {response_model!r}", - ) - stream_model = response_model - saw_choice = True - choice = choices[0] - delta = choice.get("delta", {}) - _require(isinstance(delta, dict), "stream delta must be an object") - for key, destination in ( - ("content", content_parts), - ("reasoning_content", reasoning_parts), - ): - value = delta.get(key) - if value is not None: - _require(isinstance(value, str), f"stream {key} must be a string") - total_text_bytes += len(value.encode("utf-8")) - _require( - total_text_bytes <= MAX_STREAM_TEXT_BYTES, - "stream content and reasoning exceed bounds", - ) - destination.append(value) - - tool_deltas = delta.get("tool_calls", []) - _require(isinstance(tool_deltas, list), "stream tool_calls must be an array") - for tool_delta in tool_deltas: - _require( - isinstance(tool_delta, dict), "stream tool delta must be an object" - ) - index = tool_delta.get("index") - _require( - isinstance(index, int) and not isinstance(index, bool) and index >= 0, - "stream tool delta has an invalid index", - ) - target = tools.setdefault( - index, - { - "id": None, - "type": None, - "name_parts": [], - "argument_parts": [], - }, - ) - for key in ("id", "type"): - value = tool_delta.get(key) - if value is not None: - _require( - isinstance(value, str) and value, - f"stream tool {key} is invalid", - ) - _require( - target[key] in (None, value), - f"stream tool {key} changed between chunks", - ) - target[key] = value - function = tool_delta.get("function") - if function is not None: - _require(isinstance(function, dict), "stream tool function is invalid") - name = function.get("name") - if name is not None: - _require(isinstance(name, str), "stream tool name must be a string") - target["name_parts"].append(name) - arguments = function.get("arguments") - if arguments is not None: - _require( - isinstance(arguments, str), - "stream tool arguments must be a string", - ) - total_tool_argument_bytes += len(arguments.encode("utf-8")) - _require( - total_tool_argument_bytes <= MAX_STREAM_TOOL_ARGUMENT_BYTES, - "stream tool arguments exceed bounds", - ) - target["argument_parts"].append(arguments) - - current_finish = choice.get("finish_reason") - if current_finish is not None: - _require(isinstance(current_finish, str), "finish_reason must be a string") - _require( - finish_reason in (None, current_finish), - "finish_reason changed between chunks", - ) - finish_reason = current_finish - - _require(saw_choice, "stream contains no choice chunks") - _require(saw_done, "stream is missing the terminal [DONE] frame") - _require(isinstance(finish_reason, str), "stream has no terminal finish_reason") - _require(isinstance(stream_model, str), "stream model is missing") - tool_calls = [] - for index in sorted(tools): - tool = tools[index] - tool_calls.append( - { - "id": tool["id"], - "type": tool["type"], - "function": { - "name": "".join(tool["name_parts"]), - "arguments": "".join(tool["argument_parts"]), - }, - } - ) - message = { - "role": "assistant", - "content": "".join(content_parts), - "reasoning_content": "".join(reasoning_parts), - "tool_calls": tool_calls, - } - _marker_free(message, "assembled_message") - return message, finish_reason, saw_done, stream_model - - -def _post_openai_stream( - request_stream: Callable[..., tuple[object, Iterable[object]]], - url: str, - timeout: int, - payload: dict, -) -> tuple[dict, str, int, bool]: - deadline = time.monotonic() + min(timeout, STREAM_WALL_CLOCK_TIMEOUT_SECONDS) - response, events = request_stream( - "POST", - url, - timeout=timeout, - wall_clock_deadline=deadline, - json=payload, - ) - status_code = _status_code(response) - _require(status_code == 200, f"stream returned HTTP {status_code}") - expected_model = payload.get("model") - _require( - isinstance(expected_model, str) and expected_model, "request model is missing" - ) - message, finish_reason, terminal, response_model = _decode_openai_stream( - events, - deadline=deadline, - ) - _require_response_model( - {"model": response_model}, - expected_model, - "stream response", - ) - return message, finish_reason, status_code, terminal - - -def _new_case(case_id: str, protocol: str, stream: bool) -> dict: - return { - "id": case_id, - "protocol": protocol, - "stream": stream, - "pass": False, - "status_code": 0, - "finish_reason": None, - "content_chars": 0, - "reasoning_chars": 0, - "tool_call_count": 0, - "terminal_frame": not stream, - "tool_name": None, - "tool_arguments": None, - "parity_with": None, - "raw_ifm_marker": None, - "error": None, - } - - -def _run_case( - cases: list[dict], - case_id: str, - protocol: str, - stream: bool, - action: Callable[[], tuple[dict, object]], -): - case = _new_case(case_id, protocol, stream) - artifact = None - try: - values, artifact = action() - _require(isinstance(values, dict), "case metrics must be an object") - unknown = set(values) - CASE_FIELDS - _require(not unknown, f"case returned unknown metrics: {sorted(unknown)}") - case.update(values) - case["pass"] = True - except Exception as exc: # pylint: disable=broad-exception-caught - marker = find_raw_ifm_control_marker(str(exc), "error") - if marker: - case["raw_ifm_marker"] = marker[0] - case["error"] = str(exc)[:512] or exc.__class__.__name__ - cases.append(case) - return artifact - - -def _openai_plain_case(request_json, url, timeout, payload): - body, status = _post_json(request_json, url, timeout, payload) - message = _openai_message(body, "stop") - content = _visible_content(message, "42") - _require_no_reasoning(message) - _require_no_tool_calls(message) - return { - "status_code": status, - "finish_reason": "stop", - "content_chars": len(content), - }, message - - -def _openai_plain_stream_case(request_stream, url, timeout, payload): - message, finish, status, terminal = _post_openai_stream( - request_stream, url, timeout, payload - ) - _require(finish == "stop", "plain stream did not finish with stop") - content = _visible_content(message, "42") - _require_no_reasoning(message) - _require_no_tool_calls(message) - return { - "status_code": status, - "finish_reason": finish, - "content_chars": len(content), - "terminal_frame": terminal, - }, message - - -def _openai_reasoning_case( - request_json, - url, - timeout, - payload, - *, - require_reasoning: bool, -): - body, status = _post_json(request_json, url, timeout, payload) - message = _openai_message(body, "stop") - content = _visible_content(message, "42") - if require_reasoning: - reasoning = _reasoning_content(message) - else: - _require_no_reasoning(message) - reasoning = "" - _require_no_tool_calls(message) - return { - "status_code": status, - "finish_reason": "stop", - "content_chars": len(content), - "reasoning_chars": len(reasoning), - }, message - - -def _openai_reasoning_stream_case(request_stream, url, timeout, payload): - message, finish, status, terminal = _post_openai_stream( - request_stream, url, timeout, payload - ) - _require(finish == "stop", "reasoning stream did not finish with stop") - content = _visible_content(message, "42") - reasoning = _reasoning_content(message) - _require_no_tool_calls(message) - return { - "status_code": status, - "finish_reason": finish, - "content_chars": len(content), - "reasoning_chars": len(reasoning), - "terminal_frame": terminal, - }, message - - -def _openai_tool_case(request_json, url, timeout, payload): - body, status = _post_json(request_json, url, timeout, payload) - message = _openai_message(body, "tool_calls") - _require_no_reasoning(message) - call, arguments = _parse_tool_call(message) - return { - "status_code": status, - "finish_reason": "tool_calls", - "tool_call_count": 1, - "tool_name": TOOL_NAME, - "tool_arguments": arguments, - }, call - - -def _openai_tool_parity_case( - request_json, - url, - timeout, - payload, - expected_call, - parity_with, -): - _require(isinstance(expected_call, dict), f"{parity_with} tool case did not pass") - values, call = _openai_tool_case(request_json, url, timeout, payload) - expected_function = expected_call.get("function", {}) - _require( - call["function"]["name"] == expected_function.get("name") - and _decode_tool_arguments(call["function"]["arguments"]) - == _decode_tool_arguments(expected_function.get("arguments", "null")), - f"tool call differs from {parity_with}", - ) - values["parity_with"] = parity_with - return values, call - - -def _openai_tool_stream_case(request_stream, url, timeout, payload, expected_call): - _require( - isinstance(expected_call, dict), "non-streaming XML tool case did not pass" - ) - message, finish, status, terminal = _post_openai_stream( - request_stream, url, timeout, payload - ) - _require(finish == "tool_calls", "tool stream did not finish with tool_calls") - _require_no_reasoning(message) - call, arguments = _parse_tool_call(message) - expected_function = expected_call.get("function", {}) - _require( - call["function"]["name"] == expected_function.get("name") - and arguments - == _decode_tool_arguments(expected_function.get("arguments", "null")), - "streaming tool call differs from non-streaming XML result", - ) - return { - "status_code": status, - "finish_reason": finish, - "tool_call_count": 1, - "terminal_frame": terminal, - "tool_name": TOOL_NAME, - "tool_arguments": arguments, - "parity_with": "openai_tool_xml", - }, call - - -def _openai_followup_case(request_json, url, timeout, model, call): - _require(isinstance(call, dict), "non-streaming XML tool case did not pass") - payload = { - "model": model, - "messages": [ - {"role": "user", "content": TOOL_PROMPT}, - {"role": "assistant", "content": None, "tool_calls": [call]}, - { - "role": "tool", - "tool_call_id": call["id"], - "content": json.dumps( - { - "validation_token": "K2-SUNNY-731", - "forecast": "sunny", - }, - separators=(",", ":"), - ), - }, - { - "role": "user", - "content": ( - "Return only the validation_token from the tool result. " - "Do not call a tool." - ), - }, - ], - "tools": [TOOL_DEFINITION], - "tool_choice": "none", - "max_completion_tokens": 128, - "temperature": 0, - "top_p": 1, - "seed": 42, - "stream": False, - "chat_template_kwargs": {"enable_thinking": False}, - } - body, status = _post_json(request_json, url, timeout, payload) - message = _openai_message(body, "stop") - content = _visible_content(message, "K2-SUNNY-731") - _require_no_tool_calls(message) - _require_no_reasoning(message) - return { - "status_code": status, - "finish_reason": "stop", - "content_chars": len(content), - }, message - - -def _ollama_case(request_json, root_url, timeout, model): - payload = { - "model": model, - "messages": [{"role": "user", "content": ARITHMETIC_PROMPT}], - "think": True, - "stream": False, - "options": { - "num_predict": 3072, - "temperature": 0, - "top_p": 1, - "seed": 42, - }, - } - body, status = _post_json(request_json, f"{root_url}/api/chat", timeout, payload) - _require(body.get("done") is True, "Ollama response is not complete") - _require(body.get("done_reason") == "stop", "Ollama response did not stop") - message = body.get("message") - _require(isinstance(message, dict), "Ollama message must be an object") - _require(message.get("role") == "assistant", "Ollama role is not assistant") - content = _visible_content(message, "42") - thinking = message.get("thinking") - _require(isinstance(thinking, str) and thinking.strip(), "Ollama thinking is blank") - _require_no_tool_calls(message) - return { - "status_code": status, - "finish_reason": "stop", - "content_chars": len(content), - "reasoning_chars": len(thinking), - }, message - - -def _anthropic_case(request_json, root_url, timeout, model): - payload = { - "model": model, - "system": [{"type": "text", "text": "Be concise."}], - "messages": [ - { - "role": "user", - "content": [{"type": "text", "text": "Reply with ANTHROPIC-OK."}], - } - ], - "max_tokens": 128, - "temperature": 0, - "stream": False, - "thinking": {"type": "disabled"}, - } - body, status = _post_json( - request_json, f"{root_url}/v1/messages?beta=true", timeout, payload - ) - _require(body.get("type") == "message", "Anthropic type is not message") - _require(body.get("role") == "assistant", "Anthropic role is not assistant") - _require(isinstance(body.get("id"), str) and body["id"], "Anthropic id is missing") - _require( - isinstance(body.get("model"), str) and body["model"], - "Anthropic model is missing", - ) - _require(body.get("stop_reason") == "end_turn", "Anthropic response did not end") - blocks = body.get("content") - _require(isinstance(blocks, list) and blocks, "Anthropic content is empty") - _require( - not any( - isinstance(block, dict) and block.get("type") == "tool_use" - for block in blocks - ), - "ordinary Anthropic response contains tool use", - ) - text = "".join( - block.get("text", "") - for block in blocks - if isinstance(block, dict) and block.get("type") == "text" - ) - _require("ANTHROPIC-OK" in text, "Anthropic text does not contain sentinel") - usage = body.get("usage") - _require(isinstance(usage, dict), "Anthropic usage is missing") - for field in ("input_tokens", "output_tokens"): - value = usage.get(field) - _require( - isinstance(value, int) and not isinstance(value, bool) and value >= 0, - f"Anthropic {field} is invalid", - ) - return { - "status_code": status, - "finish_reason": "end_turn", - "content_chars": len(text), - }, body - - -def validate_capabilities( - profile: str, - *, - base_url: str, - model: str, - request_json: Callable[..., tuple[object, object]], - request_stream: Callable[..., tuple[object, Iterable[object]]], - timeout: int, -) -> tuple[dict, bool, str]: - """Run a deterministic capability profile and return compact evidence.""" - if profile != K2_HORIZON_PROFILE: - raise CapabilityValidationError(f"Unsupported capability profile: {profile}") - if not base_url.endswith("/api/v1"): - raise CapabilityValidationError("base_url must end with /api/v1") - if not isinstance(model, str) or not model: - raise CapabilityValidationError("model must be a nonempty string") - - root_url = base_url[: -len("/api/v1")] - openai_url = f"{base_url}/chat/completions" - cases: list[dict] = [] - - off_payload = _base_openai_payload(model, stream=False, max_tokens=256) - off_payload["chat_template_kwargs"] = {"enable_thinking": False} - _run_case( - cases, - "openai_plain_off_nonstream", - "openai", - False, - lambda: _openai_plain_case(request_json, openai_url, timeout, off_payload), - ) - - off_stream_payload = _base_openai_payload(model, stream=True, max_tokens=256) - off_stream_payload["stream_options"] = {"include_usage": True} - off_stream_payload["chat_template_kwargs"] = {"enable_thinking": False} - _run_case( - cases, - "openai_plain_off_stream", - "openai", - True, - lambda: _openai_plain_stream_case( - request_stream, openai_url, timeout, off_stream_payload - ), - ) - - for effort in ("high", "medium", "low"): - payload = _base_openai_payload(model, stream=False, max_tokens=3072) - payload["chat_template_kwargs"] = {"reasoning_effort": effort} - _run_case( - cases, - f"openai_reasoning_{effort}", - "openai", - False, - lambda payload=payload, effort=effort: _openai_reasoning_case( - request_json, - openai_url, - timeout, - payload, - require_reasoning=effort != "low", - ), - ) - if effort == "high": - stream_payload = _base_openai_payload( - model, - stream=True, - max_tokens=3072, - ) - stream_payload["stream_options"] = {"include_usage": True} - stream_payload["chat_template_kwargs"] = {"reasoning_effort": effort} - _run_case( - cases, - "openai_reasoning_high_stream", - "openai", - True, - lambda: _openai_reasoning_stream_case( - request_stream, - openai_url, - timeout, - stream_payload, - ), - ) - - xml_call = None - for tool_format in ("json", "xml", "xml_typed"): - call = _run_case( - cases, - f"openai_tool_{tool_format}", - "openai", - False, - lambda tool_format=tool_format: _openai_tool_case( - request_json, - openai_url, - timeout, - _tool_payload(model, tool_format, stream=False), - ), - ) - if tool_format == "xml": - xml_call = call - - _run_case( - cases, - "openai_tool_default_xml", - "openai", - False, - lambda: _openai_tool_parity_case( - request_json, - openai_url, - timeout, - _tool_payload(model, None, stream=False), - xml_call, - "openai_tool_xml", - ), - ) - - _run_case( - cases, - "openai_tool_stream_xml", - "openai", - True, - lambda: _openai_tool_stream_case( - request_stream, - openai_url, - timeout, - _tool_payload(model, "xml", stream=True), - xml_call, - ), - ) - _run_case( - cases, - "openai_tool_result_followup", - "openai", - False, - lambda: _openai_followup_case( - request_json, openai_url, timeout, model, xml_call - ), - ) - _run_case( - cases, - "ollama_thinking", - "ollama", - False, - lambda: _ollama_case(request_json, root_url, timeout, model), - ) - _run_case( - cases, - "anthropic_translation", - "anthropic", - False, - lambda: _anthropic_case(request_json, root_url, timeout, model), - ) - - passed = all(case["pass"] for case in cases) - matrix = { - "profile": profile, - "model": model, - "pass": passed, - "cases": cases, - } - if passed: - summary = f"{len(cases)}/{len(cases)} capability cases passed" - else: - failed = [case["id"] for case in cases if not case["pass"]] - details = "; ".join( - f"{case['id']}: {case['error']}" for case in cases if not case["pass"] - ) - summary = f"Failed capability cases {', '.join(failed)}: {details}" - return matrix, passed, summary[:1024] - - -def validate_capability_matrix_evidence( - value: object, - expected_profile: str, - expected_model: str, -) -> None: - """Validate one compact capability evidence envelope.""" - _require( - expected_profile == K2_HORIZON_PROFILE, - f"Unsupported capability profile: {expected_profile}", - ) - _require(isinstance(value, dict), "capability matrix must be an object") - _require( - set(value) == {"profile", "model", "pass", "cases"}, - "capability matrix has unexpected fields", - ) - _require(value["profile"] == expected_profile, "capability profile mismatch") - _require( - isinstance(expected_model, str) and expected_model, "expected model is missing" - ) - _require_response_model(value, expected_model, "capability matrix") - _require(value["pass"] is True, "capability matrix did not pass") - cases = value["cases"] - _require(isinstance(cases, list), "capability cases must be an array") - _require( - [case.get("id") if isinstance(case, dict) else None for case in cases] - == list(K2_HORIZON_CASE_IDS), - "capability cases do not match the profile contract", - ) - - for case in cases: - _require(set(case) == CASE_FIELDS, f"{case['id']}: unexpected case fields") - _require(case["pass"] is True, f"{case['id']}: case did not pass") - _require(case["error"] is None, f"{case['id']}: case has an error") - _require( - case["raw_ifm_marker"] is None, - f"{case['id']}: raw IFM marker was recorded", - ) - _require( - type(case["status_code"]) is int and case["status_code"] == 200, - f"{case['id']}: HTTP status is not 200", - ) - _require( - case["terminal_frame"] is True, - f"{case['id']}: response is not terminal", - ) - for field in ("content_chars", "reasoning_chars", "tool_call_count"): - value_field = case[field] - _require( - isinstance(value_field, int) - and not isinstance(value_field, bool) - and value_field >= 0, - f"{case['id']}: {field} is invalid", - ) - ( - protocol, - stream, - finish_reason, - has_content, - has_reasoning, - tool_call_count, - parity_with, - ) = CASE_CONTRACTS[case["id"]] - _require(case["protocol"] == protocol, f"{case['id']}: protocol mismatch") - _require(case["stream"] is stream, f"{case['id']}: stream flag mismatch") - _require( - case["finish_reason"] == finish_reason, - f"{case['id']}: finish reason mismatch", - ) - _require( - (case["content_chars"] > 0) is has_content, - f"{case['id']}: content contract mismatch", - ) - _require( - (case["reasoning_chars"] > 0) is has_reasoning, - f"{case['id']}: reasoning contract mismatch", - ) - _require( - case["tool_call_count"] == tool_call_count, - f"{case['id']}: tool-call count mismatch", - ) - _require( - case["parity_with"] == parity_with, - f"{case['id']}: parity binding mismatch", - ) - if tool_call_count: - _require( - case["tool_name"] == TOOL_NAME, f"{case['id']}: tool name mismatch" - ) - _validate_tool_arguments(case["tool_arguments"]) - else: - _require(case["tool_name"] is None, f"{case['id']}: unexpected tool name") - _require( - case["tool_arguments"] is None, - f"{case['id']}: unexpected tool arguments", - ) diff --git a/test/utils/llamacpp_release_assets.py b/test/utils/llamacpp_release_assets.py deleted file mode 100644 index 29e2feea4d..0000000000 --- a/test/utils/llamacpp_release_assets.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Managed llama.cpp release-asset requirements used by CI.""" - -from __future__ import annotations - -import re -from collections.abc import Collection, Iterable, Mapping - -_RELEASE_RE = re.compile(r"^b[0-9]+$") -_ROCM_CONCRETE_ISA_RE = re.compile(r"^gfx[0-9a-f]{3,4}$") -_CUDA_COMPUTE_CAPABILITIES = ( - "sm_75", - "sm_80", - "sm_86", - "sm_89", - "sm_90", - "sm_100", - "sm_120", - "sm_121", -) -_ROCM_RDNA_NIGHTLY_ISAS = ( - "gfx1030", - "gfx1031", - "gfx1032", - "gfx1033", - "gfx1034", - "gfx1035", - "gfx1036", - "gfx1100", - "gfx1101", - "gfx1102", - "gfx1103", - "gfx1150", - "gfx1151", - "gfx1152", - "gfx1200", - "gfx1201", -) -_ROCM_NIGHTLY_ISAS_BY_PLATFORM = { - "windows": _ROCM_RDNA_NIGHTLY_ISAS, - "ubuntu": (*_ROCM_RDNA_NIGHTLY_ISAS, "gfx908", "gfx90a", "gfx942"), -} - - -def _validated_release(value: str, name: str) -> str: - release = value.strip() - if not _RELEASE_RE.fullmatch(release): - raise ValueError(f"Invalid {name}: {release!r}") - return release - - -def _therock_major_minor(backend_versions: Mapping[str, object]) -> str: - therock = backend_versions.get("therock", {}) - if not isinstance(therock, Mapping): - raise ValueError("backend_versions.json therock must be an object") - version = str(therock.get("version", "")).strip().removeprefix("v") - major_minor = ".".join(version.split(".")[:2]) - if not major_minor: - raise ValueError("backend_versions.json is missing therock.version") - return major_minor - - -def _validate_rocm_asset_families(asset_families: Mapping[str, object]) -> None: - for isa, family in asset_families.items(): - if isa == "comment": - continue - if not isinstance(isa, str) or _ROCM_CONCRETE_ISA_RE.fullmatch(isa) is None: - raise ValueError( - f"ROCm asset family key must be a concrete ROCm ISA: {isa!r}" - ) - if not isinstance(family, str) or not family: - raise ValueError(f"Invalid ROCm asset family for {isa}: {family!r}") - - -def _rocm_asset_families( - backend_versions: Mapping[str, object], -) -> Mapping[str, object]: - asset_families = backend_versions.get("rocm_asset_families", {}) - if not isinstance(asset_families, Mapping): - raise ValueError("backend_versions.json rocm_asset_families must be an object") - _validate_rocm_asset_families(asset_families) - return asset_families - - -def build_rocm_asset_target_requirements( - *, - rocm_release: str, - backend_versions: Mapping[str, object], -) -> dict[str, tuple[str, ...]]: - """Bind every required ROCm nightly asset to its concrete build targets.""" - release = _validated_release(rocm_release, "ROCM_RELEASE") - asset_families = _rocm_asset_families(backend_versions) - grouped_targets: dict[str, list[str]] = {} - for platform, concrete_isas in _ROCM_NIGHTLY_ISAS_BY_PLATFORM.items(): - for concrete_isa in concrete_isas: - family = asset_families.get(concrete_isa, concrete_isa) - if not isinstance(family, str) or not family: - raise ValueError( - f"Invalid ROCm asset family for {concrete_isa}: {family!r}" - ) - asset_name = f"llama-{release}-{platform}-rocm-{family}-x64.zip" - grouped_targets.setdefault(asset_name, []).append(concrete_isa) - return { - asset_name: tuple(concrete_isas) - for asset_name, concrete_isas in grouped_targets.items() - } - - -def build_asset_requirements( - *, - ggml_release: str, - rocm_release: str, - lemonade_release: str, - backend_versions: Mapping[str, object], -) -> dict[str, dict[str, list[str]]]: - """Return required asset names grouped by publisher and Lemonade backend.""" - ggml_release = _validated_release(ggml_release, "GGML_RELEASE") - rocm_release = _validated_release(rocm_release, "ROCM_RELEASE") - lemonade_release = _validated_release(lemonade_release, "LEMONADE_RELEASE") - therock = _therock_major_minor(backend_versions) - - rocm_nightly = list( - build_rocm_asset_target_requirements( - rocm_release=rocm_release, - backend_versions=backend_versions, - ) - ) - - cuda = [] - for compute_capability in _CUDA_COMPUTE_CAPABILITIES: - cuda.extend( - ( - f"llama-{lemonade_release}-windows-cuda-{compute_capability}-x64.7z", - f"llama-{lemonade_release}-ubuntu-cuda-{compute_capability}-x64.tar.xz", - f"llama-{lemonade_release}-ubuntu-cuda-{compute_capability}-arm64.tar.xz", - ) - ) - - return { - "ggml": { - "vulkan": [ - f"llama-{ggml_release}-bin-win-vulkan-x64.zip", - f"llama-{ggml_release}-bin-ubuntu-vulkan-x64.tar.gz", - f"llama-{ggml_release}-bin-ubuntu-vulkan-arm64.tar.gz", - ], - "cpu": [ - f"llama-{ggml_release}-bin-win-cpu-x64.zip", - f"llama-{ggml_release}-bin-ubuntu-x64.tar.gz", - f"llama-{ggml_release}-bin-ubuntu-arm64.tar.gz", - ], - "metal": [ - f"llama-{ggml_release}-bin-macos-arm64.tar.gz", - ], - }, - "rocm": {"rocm-nightly": rocm_nightly}, - "lemonade": { - "rocm-stable": [ - f"llama-{lemonade_release}-bin-win-rocm-{therock}-x64.zip", - f"llama-{lemonade_release}-bin-ubuntu-rocm-{therock}-x64.tar.gz", - ], - "cuda": cuda, - }, - } - - -def evaluate_asset_group( - available_assets: Collection[str] | Mapping[str, object], - backend_requirements: Mapping[str, Iterable[str]], - *, - required_asset_targets: Mapping[str, Iterable[str]] | None = None, - attested_asset_targets: Mapping[str, Iterable[str]] | None = None, -) -> tuple[list[str], dict[str, list[str]]]: - """Return eligible backends and their missing required assets.""" - - def has_required_targets(asset: str) -> bool: - if required_asset_targets is None: - return True - required = required_asset_targets.get(asset) - if required is None or isinstance(required, str): - return False - if attested_asset_targets is None: - return False - attested = attested_asset_targets.get(asset) - if attested is None or isinstance(attested, str): - return False - return set(required).issubset(attested) - - def is_available(asset: str) -> bool: - if not isinstance(available_assets, Mapping): - has_asset = asset in available_assets - else: - size = available_assets.get(asset) - has_asset = ( - isinstance(size, int) and not isinstance(size, bool) and size > 0 - ) - return has_asset and has_required_targets(asset) - - eligible = [] - missing_by_backend = {} - for backend, required_assets in backend_requirements.items(): - missing = [asset for asset in required_assets if not is_available(asset)] - missing_by_backend[backend] = missing - if not missing: - eligible.append(backend) - return eligible, missing_by_backend diff --git a/test/utils/llamacpp_release_manifest.py b/test/utils/llamacpp_release_manifest.py deleted file mode 100644 index ab26fb7244..0000000000 --- a/test/utils/llamacpp_release_manifest.py +++ /dev/null @@ -1,1205 +0,0 @@ -#!/usr/bin/env python3 -"""Capture and compare canonical GitHub release-asset manifests.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import re -from pathlib import Path -from typing import Iterable -from urllib.parse import urlparse - -if __package__: - from .llamacpp_release_assets import ( - build_asset_requirements, - build_rocm_asset_target_requirements, - evaluate_asset_group, - ) -else: - from llamacpp_release_assets import ( # type: ignore[import-not-found] - build_asset_requirements, - build_rocm_asset_target_requirements, - evaluate_asset_group, - ) - -SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-fA-F]{64}$") -LOWER_SHA256_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") -GIT_OID_RE = re.compile(r"^[0-9a-fA-F]{40}$") -RELEASE_TAG_RE = re.compile(r"^b[0-9]+$") -ROCM_CONCRETE_ISA_RE = re.compile(r"^gfx[0-9a-f]{3,4}$") -SOURCE_MANIFEST_ASSET_NAME = ".llamacpp-source.json" -SOURCE_MANIFEST_MAX_BYTES = 65_536 -TRUSTED_SOURCE_REPOSITORY = "ggml-org/llama.cpp" -ROCM_RELEASE_REPOSITORY = "lemonade-sdk/llamacpp-rocm" -PUBLISHER_CLAIM_TYPES = frozenset({"immutable-source-manifest", "source-release-tag"}) -FORK_RELEASE_REPOSITORIES = frozenset( - {"lemonade-sdk/llama.cpp", "lemonade-sdk/llamacpp-rocm"} -) -MANAGED_PIN_REPOSITORIES = { - "cpu": "ggml-org/llama.cpp", - "cuda": "lemonade-sdk/llama.cpp", - "metal": "ggml-org/llama.cpp", - "rocm-nightly": "lemonade-sdk/llamacpp-rocm", - "rocm-stable": "lemonade-sdk/llama.cpp", - "vulkan": "ggml-org/llama.cpp", -} -ASSET_REQUIREMENT_GROUPS = { - "ggml-org/llama.cpp": "ggml", - "lemonade-sdk/llama.cpp": "lemonade", - "lemonade-sdk/llamacpp-rocm": "rocm", -} - - -class ReleaseManifestError(ValueError): - """Raised when release metadata is incomplete, malformed, or changed.""" - - -def _is_git_oid(value: object) -> bool: - return isinstance(value, str) and GIT_OID_RE.fullmatch(value) is not None - - -def _reject_duplicate_json_members(pairs: list[tuple[str, object]]) -> dict: - result = {} - for key, value in pairs: - if key in result: - raise ReleaseManifestError(f"duplicate JSON member: {key}") - result[key] = value - return result - - -def _strict_json_loads(value: str, label: str) -> object: - try: - return json.loads(value, object_pairs_hook=_reject_duplicate_json_members) - except (TypeError, json.JSONDecodeError) as exc: - raise ReleaseManifestError(f"{label} is not valid JSON: {exc}") from exc - - -def require_upstream_ancestry( - comparison: object, - source_repository: str, - publisher_claimed_source_commit: str, - upstream_reference_head: str, -) -> None: - expected_path = ( - f"/repos/{source_repository}/compare/" - f"{publisher_claimed_source_commit}...{upstream_reference_head}" - ) - if not isinstance(comparison, dict): - raise ReleaseManifestError("publisher-claimed source commit is not reachable") - base_commit = comparison.get("base_commit") - merge_base_commit = comparison.get("merge_base_commit") - comparison_url = comparison.get("url") - if ( - not isinstance(source_repository, str) - or source_repository.count("/") != 1 - or not _is_git_oid(publisher_claimed_source_commit) - or not _is_git_oid(upstream_reference_head) - or not isinstance(base_commit, dict) - or not isinstance(merge_base_commit, dict) - or not isinstance(comparison_url, str) - or urlparse(comparison_url).path != expected_path - or base_commit.get("sha", "").lower() != publisher_claimed_source_commit.lower() - or merge_base_commit.get("sha", "").lower() - != publisher_claimed_source_commit.lower() - or comparison.get("status") not in {"ahead", "identical"} - ): - raise ReleaseManifestError("publisher-claimed source commit is not reachable") - - -def _positive_integer(value: object, label: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise ReleaseManifestError(f"{label} must be a positive integer") - return value - - -def locate_immutable_source_manifest_asset( - release_payload: object, - release_repository: str, -) -> dict: - """Return the single bounded source-manifest asset for a fork release.""" - if release_repository not in FORK_RELEASE_REPOSITORIES: - raise ReleaseManifestError( - f"{release_repository}: immutable source manifests are only valid for " - "managed fork releases" - ) - if not isinstance(release_payload, dict): - raise ReleaseManifestError( - f"{release_repository}: release payload must be an object" - ) - if release_payload.get("immutable") is not True: - raise ReleaseManifestError( - f"{release_repository}: release immutable must be true" - ) - if release_payload.get("draft") is not False: - raise ReleaseManifestError(f"{release_repository}: release draft must be false") - raw_assets = release_payload.get("assets") - if not isinstance(raw_assets, list): - raise ReleaseManifestError( - f"{release_repository}: release assets must be an array" - ) - matches = [ - asset - for asset in raw_assets - if isinstance(asset, dict) and asset.get("name") == SOURCE_MANIFEST_ASSET_NAME - ] - if len(matches) != 1: - raise ReleaseManifestError( - f"{release_repository}: release must contain exactly one " - f"{SOURCE_MANIFEST_ASSET_NAME} asset" - ) - asset = matches[0] - _positive_integer(asset.get("id"), f"{release_repository}: source manifest id") - size = _positive_integer( - asset.get("size"), f"{release_repository}: source manifest size" - ) - if size > SOURCE_MANIFEST_MAX_BYTES: - raise ReleaseManifestError( - f"{release_repository}: source manifest size exceeds " - f"{SOURCE_MANIFEST_MAX_BYTES} bytes" - ) - if asset.get("state") != "uploaded": - raise ReleaseManifestError( - f"{release_repository}: source manifest must be uploaded" - ) - digest = asset.get("digest") - if not isinstance(digest, str) or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None: - raise ReleaseManifestError( - f"{release_repository}: source manifest must have a lowercase sha256 digest" - ) - _source_manifest_asset_bindings(release_payload, release_repository) - return asset - - -def _source_manifest_asset_bindings( - release_payload: object, - release_repository: str, -) -> list[dict]: - if not isinstance(release_payload, dict) or not isinstance( - release_payload.get("assets"), list - ): - raise ReleaseManifestError( - f"{release_repository}: release assets must be an array" - ) - bindings = [] - names: set[str] = set() - for index, asset in enumerate(release_payload["assets"]): - prefix = f"{release_repository}: release asset {index}" - if not isinstance(asset, dict): - raise ReleaseManifestError(f"{prefix} must be an object") - name = asset.get("name") - if not isinstance(name, str) or not name: - raise ReleaseManifestError(f"{prefix} name must be nonempty") - if name in names: - raise ReleaseManifestError( - f"{release_repository}: duplicate release asset name {name!r}" - ) - names.add(name) - if asset.get("state") != "uploaded": - raise ReleaseManifestError(f"{prefix} must be uploaded") - size = asset.get("size") - if isinstance(size, bool) or not isinstance(size, int) or size < 0: - raise ReleaseManifestError(f"{prefix} size must be nonnegative") - digest = asset.get("digest") - if ( - not isinstance(digest, str) - or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None - ): - raise ReleaseManifestError(f"{prefix} must have a lowercase sha256 digest") - if name != SOURCE_MANIFEST_ASSET_NAME: - bindings.append({"digest": digest, "name": name, "size": size}) - if not bindings: - raise ReleaseManifestError( - f"{release_repository}: source manifest must bind at least one release asset" - ) - return sorted(bindings, key=lambda asset: asset["name"]) - - -def _normalize_build_targets(value: object, prefix: str) -> list[str]: - if not isinstance(value, list): - raise ReleaseManifestError(f"{prefix} build_targets must be an array") - targets = [] - seen: set[str] = set() - for target in value: - if ( - not isinstance(target, str) - or ROCM_CONCRETE_ISA_RE.fullmatch(target) is None - ): - raise ReleaseManifestError( - f"{prefix} build target must be a concrete ROCm ISA" - ) - if target in seen: - raise ReleaseManifestError( - f"{prefix} has duplicate build target {target!r}" - ) - seen.add(target) - targets.append(target) - return sorted(targets) - - -def validate_immutable_source_manifest_with_attestation( - evidence: bytes, - release_payload: object, - release_repository: str, - release_tag: str, - release_tag_commit: str, -) -> tuple[str, list[dict]]: - """Validate one immutable publisher claim and normalize its build targets.""" - source_asset = locate_immutable_source_manifest_asset( - release_payload, - release_repository, - ) - if not isinstance(evidence, bytes): - raise ReleaseManifestError( - f"{release_repository}: source manifest must be raw bytes" - ) - if len(evidence) != source_asset["size"]: - raise ReleaseManifestError( - f"{release_repository}: source manifest size does not match release metadata" - ) - measured_digest = "sha256:" + hashlib.sha256(evidence).hexdigest() - if measured_digest != source_asset["digest"]: - raise ReleaseManifestError( - f"{release_repository}: source manifest sha256 digest does not match " - "release metadata" - ) - try: - evidence_text = evidence.decode("utf-8") - except UnicodeDecodeError as exc: - raise ReleaseManifestError( - f"{release_repository}: source manifest must be UTF-8 JSON" - ) from exc - document = _strict_json_loads( - evidence_text, - f"{release_repository}: source manifest", - ) - if not isinstance(document, dict): - raise ReleaseManifestError( - f"{release_repository}: source manifest must be an object" - ) - expected_keys = { - "assets", - "release_repository", - "release_tag", - "release_tag_commit", - "schema_version", - "source_commit", - "source_repository", - } - if set(document) != expected_keys: - raise ReleaseManifestError( - f"{release_repository}: source manifest has unexpected fields" - ) - schema_version = document["schema_version"] - if isinstance(schema_version, bool) or not isinstance(schema_version, int): - raise ReleaseManifestError( - f"{release_repository}: source manifest schema_version must be 1 or 2" - ) - if schema_version == 2 and release_repository != ROCM_RELEASE_REPOSITORY: - raise ReleaseManifestError( - f"{release_repository}: source manifest schema_version 2 is only valid " - f"for {ROCM_RELEASE_REPOSITORY}" - ) - if schema_version not in {1, 2}: - raise ReleaseManifestError( - f"{release_repository}: source manifest schema_version must be 1 or 2" - ) - if document["release_repository"] != release_repository: - raise ReleaseManifestError( - f"{release_repository}: source manifest release repository mismatch" - ) - if ( - not isinstance(release_tag, str) - or RELEASE_TAG_RE.fullmatch(release_tag) is None - or document["release_tag"] != release_tag - ): - raise ReleaseManifestError( - f"{release_repository}: source manifest release tag mismatch" - ) - if not _is_git_oid(release_tag_commit) or ( - not isinstance(document["release_tag_commit"], str) - or document["release_tag_commit"].lower() != release_tag_commit.lower() - ): - raise ReleaseManifestError( - f"{release_repository}: source manifest release tag commit mismatch" - ) - if document["source_repository"] != TRUSTED_SOURCE_REPOSITORY: - raise ReleaseManifestError( - f"{release_repository}: source manifest source repository must be " - f"{TRUSTED_SOURCE_REPOSITORY}" - ) - source_commit = document["source_commit"] - if not _is_git_oid(source_commit): - raise ReleaseManifestError( - f"{release_repository}: source manifest source commit must be a full " - "40-character Git OID" - ) - raw_bindings = document["assets"] - if not isinstance(raw_bindings, list): - raise ReleaseManifestError( - f"{release_repository}: source manifest assets must be an array" - ) - bindings = [] - names: set[str] = set() - for index, binding in enumerate(raw_bindings): - prefix = f"{release_repository}: source manifest asset {index}" - expected_binding_keys = {"digest", "name", "size"} - if schema_version == 2: - expected_binding_keys.add("build_targets") - if not isinstance(binding, dict) or set(binding) != expected_binding_keys: - raise ReleaseManifestError(f"{prefix} has unexpected fields") - name = binding["name"] - if not isinstance(name, str) or not name: - raise ReleaseManifestError(f"{prefix} name must be nonempty") - if name == SOURCE_MANIFEST_ASSET_NAME: - raise ReleaseManifestError( - f"{release_repository}: source manifest must exclude itself" - ) - if name in names: - raise ReleaseManifestError( - f"{release_repository}: source manifest has duplicate asset {name!r}" - ) - names.add(name) - size = binding["size"] - if isinstance(size, bool) or not isinstance(size, int) or size < 0: - raise ReleaseManifestError(f"{prefix} size must be nonnegative") - digest = binding["digest"] - if ( - not isinstance(digest, str) - or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None - ): - raise ReleaseManifestError(f"{prefix} must have a lowercase sha256 digest") - normalized_binding = {"digest": digest, "name": name, "size": size} - if schema_version == 2: - normalized_binding["build_targets"] = _normalize_build_targets( - binding["build_targets"], - prefix, - ) - bindings.append(normalized_binding) - expected_bindings = _source_manifest_asset_bindings( - release_payload, - release_repository, - ) - bound_asset_metadata = [ - {"digest": binding["digest"], "name": binding["name"], "size": binding["size"]} - for binding in bindings - ] - if ( - sorted(bound_asset_metadata, key=lambda asset: asset["name"]) - != expected_bindings - ): - raise ReleaseManifestError( - f"{release_repository}: source manifest asset metadata does not match " - "the immutable release" - ) - attestations = ( - sorted(bindings, key=lambda asset: asset["name"]) if schema_version == 2 else [] - ) - return source_commit.lower(), attestations - - -def validate_immutable_source_manifest( - evidence: bytes, - release_payload: object, - release_repository: str, - release_tag: str, - release_tag_commit: str, -) -> str: - """Validate one immutable publisher claim and return its upstream source OID.""" - source_commit, _ = validate_immutable_source_manifest_with_attestation( - evidence, - release_payload, - release_repository, - release_tag, - release_tag_commit, - ) - return source_commit - - -def _canonical_build_target_attestations( - repository: str, - value: object, - release_assets: list[dict], -) -> list[dict]: - if not isinstance(value, list): - raise ReleaseManifestError( - f"{repository}: build_target_attestations must be an array" - ) - if repository != ROCM_RELEASE_REPOSITORY and value: - raise ReleaseManifestError( - f"{repository}: build target attestations are only valid for " - f"{ROCM_RELEASE_REPOSITORY}" - ) - release_bindings = { - (asset["name"], asset["size"], asset["digest"]) - for asset in release_assets - if asset["name"] != SOURCE_MANIFEST_ASSET_NAME - } - attestations = [] - names: set[str] = set() - for index, attestation in enumerate(value): - prefix = f"{repository}: build target attestation {index}" - if not isinstance(attestation, dict) or set(attestation) != { - "build_targets", - "digest", - "name", - "size", - }: - raise ReleaseManifestError(f"{prefix} has unexpected fields") - name = attestation["name"] - if not isinstance(name, str) or not name: - raise ReleaseManifestError(f"{prefix} name must be nonempty") - if name in names: - raise ReleaseManifestError( - f"{repository}: duplicate build target attestation for {name!r}" - ) - names.add(name) - size = attestation["size"] - if isinstance(size, bool) or not isinstance(size, int) or size < 0: - raise ReleaseManifestError(f"{prefix} size must be nonnegative") - digest = attestation["digest"] - if ( - not isinstance(digest, str) - or LOWER_SHA256_DIGEST_RE.fullmatch(digest) is None - ): - raise ReleaseManifestError(f"{prefix} must have a lowercase sha256 digest") - if (name, size, digest) not in release_bindings: - raise ReleaseManifestError( - f"{prefix} does not match release asset name, size, and digest" - ) - attestations.append( - { - "build_targets": _normalize_build_targets( - attestation["build_targets"], prefix - ), - "digest": digest, - "name": name, - "size": size, - } - ) - return sorted(attestations, key=lambda item: item["name"]) - - -def _canonical_release( - repository: str, - expected_tag: str, - payload: object, -) -> dict: - if not isinstance(repository, str) or repository.count("/") != 1: - raise ReleaseManifestError(f"invalid repository name: {repository!r}") - if not isinstance(payload, dict): - raise ReleaseManifestError(f"{repository}: release payload must be an object") - - actual_tag = payload.get("tag_name") - if actual_tag != expected_tag: - raise ReleaseManifestError( - f"{repository}: release tag mismatch: expected {expected_tag!r}, " - f"got {actual_tag!r}" - ) - if ( - not isinstance(expected_tag, str) - or RELEASE_TAG_RE.fullmatch(expected_tag) is None - ): - raise ReleaseManifestError(f"{repository}: release tag must match bNNNN") - release_id = _positive_integer(payload.get("id"), f"{repository}: release id") - release_immutable = payload.get("immutable") - if release_immutable is not True: - raise ReleaseManifestError(f"{repository}: release immutable must be true") - release_draft = payload.get("draft") - if release_draft is not False: - raise ReleaseManifestError(f"{repository}: release draft must be false") - release_tag_commit = payload.get("release_tag_commit") - if not _is_git_oid(release_tag_commit): - raise ReleaseManifestError( - f"{repository}: release tag commit must be a full 40-character Git OID" - ) - source_repository = payload.get("source_repository") - if source_repository != TRUSTED_SOURCE_REPOSITORY: - raise ReleaseManifestError( - f"{repository}: source repository must be {TRUSTED_SOURCE_REPOSITORY}" - ) - publisher_claim_type = payload.get("publisher_claim_type") - if publisher_claim_type not in PUBLISHER_CLAIM_TYPES: - raise ReleaseManifestError( - f"{repository}: publisher claim type must identify its source" - ) - publisher_claimed_source_commit = payload.get("publisher_claimed_source_commit") - if not _is_git_oid(publisher_claimed_source_commit): - raise ReleaseManifestError( - f"{repository}: publisher-claimed source commit must be a full " - "40-character Git OID" - ) - upstream_reference = payload.get("upstream_reference") - if not isinstance(upstream_reference, str) or not upstream_reference.startswith( - "refs/heads/" - ): - raise ReleaseManifestError( - f"{repository}: upstream reference must be a branch reference" - ) - upstream_reference_head = payload.get("upstream_reference_head") - if not _is_git_oid(upstream_reference_head): - raise ReleaseManifestError( - f"{repository}: upstream reference head must be a full 40-character Git OID" - ) - if ( - repository == source_repository - and release_tag_commit.lower() != publisher_claimed_source_commit.lower() - ): - raise ReleaseManifestError( - f"{repository}: release tag commit must equal publisher-claimed source commit " - "for a source release" - ) - expected_claim_type = ( - "source-release-tag" - if repository == source_repository - else "immutable-source-manifest" - ) - if publisher_claim_type != expected_claim_type: - raise ReleaseManifestError( - f"{repository}: publisher claim type must be {expected_claim_type}" - ) - raw_assets = payload.get("assets") - if not isinstance(raw_assets, list) or not raw_assets: - raise ReleaseManifestError(f"{repository}: release assets must be nonempty") - - asset_ids: set[int] = set() - asset_names: set[str] = set() - assets = [] - for index, raw_asset in enumerate(raw_assets): - prefix = f"{repository}: asset {index}" - if not isinstance(raw_asset, dict): - raise ReleaseManifestError(f"{prefix} must be an object") - - name = raw_asset.get("name") - if not isinstance(name, str) or not name: - raise ReleaseManifestError(f"{prefix}: asset name must be nonempty") - asset_id = _positive_integer(raw_asset.get("id"), f"{prefix}: asset id") - size = raw_asset.get("size") - if isinstance(size, bool) or not isinstance(size, int) or size < 0: - raise ReleaseManifestError( - f"{prefix}: asset size must be a nonnegative integer" - ) - digest = raw_asset.get("digest") - if not isinstance(digest, str) or not SHA256_DIGEST_RE.fullmatch(digest): - raise ReleaseManifestError( - f"{prefix}: asset must have a complete sha256 digest" - ) - updated_at = raw_asset.get("updated_at") - if not isinstance(updated_at, str) or not updated_at: - raise ReleaseManifestError(f"{prefix}: asset updated_at must be nonempty") - state = raw_asset.get("state") - if state != "uploaded": - raise ReleaseManifestError(f"{prefix}: asset must be uploaded") - - if asset_id in asset_ids: - raise ReleaseManifestError(f"{repository}: duplicate asset id {asset_id}") - if name in asset_names: - raise ReleaseManifestError(f"{repository}: duplicate asset name {name!r}") - asset_ids.add(asset_id) - asset_names.add(name) - assets.append( - { - "digest": digest.lower(), - "id": asset_id, - "name": name, - "size": size, - "state": state, - "updated_at": updated_at, - } - ) - - assets.sort(key=lambda asset: (asset["name"], asset["id"])) - build_target_attestations = _canonical_build_target_attestations( - repository, - payload.get("build_target_attestations", []), - assets, - ) - if publisher_claim_type == "immutable-source-manifest": - source_assets = [ - asset for asset in assets if asset["name"] == SOURCE_MANIFEST_ASSET_NAME - ] - if len(source_assets) != 1: - raise ReleaseManifestError( - f"{repository}: immutable source claim requires exactly one " - f"{SOURCE_MANIFEST_ASSET_NAME} asset" - ) - if ( - source_assets[0]["size"] <= 0 - or source_assets[0]["size"] > SOURCE_MANIFEST_MAX_BYTES - or LOWER_SHA256_DIGEST_RE.fullmatch(source_assets[0]["digest"]) is None - ): - raise ReleaseManifestError( - f"{repository}: immutable source manifest metadata is invalid" - ) - return { - "assets": assets, - "build_target_attestations": build_target_attestations, - "publisher_claim_type": publisher_claim_type, - "publisher_claimed_source_commit": publisher_claimed_source_commit.lower(), - "release_draft": release_draft, - "release_id": release_id, - "release_immutable": release_immutable, - "repository": repository, - "release_tag_commit": release_tag_commit.lower(), - "source_repository": source_repository, - "tag_name": expected_tag, - "upstream_reference": upstream_reference, - "upstream_reference_head": upstream_reference_head.lower(), - } - - -def build_release_asset_manifest( - releases: Iterable[tuple[str, str, object]], -) -> str: - canonical_releases = [ - _canonical_release(repository, expected_tag, payload) - for repository, expected_tag, payload in releases - ] - if not canonical_releases: - raise ReleaseManifestError("release manifest must contain at least one release") - repositories = [release["repository"] for release in canonical_releases] - if len(repositories) != len(set(repositories)): - raise ReleaseManifestError("release manifest contains duplicate repositories") - upstream_bindings = { - ( - release["source_repository"], - release["upstream_reference"], - release["upstream_reference_head"], - ) - for release in canonical_releases - } - if len(upstream_bindings) != 1: - raise ReleaseManifestError( - "release manifest must use one immutable upstream reference head" - ) - canonical_releases.sort(key=lambda release: release["repository"]) - return json.dumps( - {"releases": canonical_releases, "schema_version": 5}, - separators=(",", ":"), - sort_keys=True, - ) - - -def canonicalize_release_asset_manifest(manifest_json: str) -> str: - document = _strict_json_loads(manifest_json, "release manifest") - if ( - not isinstance(document, dict) - or isinstance(document.get("schema_version"), bool) - or not isinstance(document.get("schema_version"), int) - or document.get("schema_version") != 5 - ): - raise ReleaseManifestError("release manifest schema_version must be 5") - releases = document.get("releases") - if not isinstance(releases, list): - raise ReleaseManifestError("release manifest releases must be an array") - - inputs = [] - for release in releases: - if not isinstance(release, dict): - raise ReleaseManifestError("release manifest entry must be an object") - expected_keys = { - "assets", - "build_target_attestations", - "publisher_claim_type", - "publisher_claimed_source_commit", - "release_draft", - "release_id", - "release_immutable", - "repository", - "release_tag_commit", - "source_repository", - "tag_name", - "upstream_reference", - "upstream_reference_head", - } - if set(release) != expected_keys: - raise ReleaseManifestError("release manifest entry has unexpected fields") - inputs.append( - ( - release["repository"], - release["tag_name"], - { - "assets": release["assets"], - "build_target_attestations": release["build_target_attestations"], - "draft": release["release_draft"], - "id": release["release_id"], - "immutable": release["release_immutable"], - "publisher_claim_type": release["publisher_claim_type"], - "publisher_claimed_source_commit": release[ - "publisher_claimed_source_commit" - ], - "release_tag_commit": release["release_tag_commit"], - "source_repository": release["source_repository"], - "tag_name": release["tag_name"], - "upstream_reference": release["upstream_reference"], - "upstream_reference_head": release["upstream_reference_head"], - }, - ) - ) - - canonical = build_release_asset_manifest(inputs) - if json.loads(canonical) != document: - raise ReleaseManifestError("release manifest has unexpected fields") - return canonical - - -def require_exact_manifest_match(expected_json: str, actual_json: str) -> None: - expected = canonicalize_release_asset_manifest(expected_json) - actual = canonicalize_release_asset_manifest(actual_json) - if expected != actual: - raise ReleaseManifestError( - "release manifest changed since validation; refusing publication" - ) - - -def managed_llamacpp_pin_changes( - base_versions: object, - candidate_versions: object, -) -> dict[str, tuple[str, str]]: - sections = [] - for label, document in ( - ("base", base_versions), - ("candidate", candidate_versions), - ): - if not isinstance(document, dict) or not isinstance( - document.get("llamacpp"), dict - ): - raise ReleaseManifestError( - f"{label} backend versions must contain a llamacpp object" - ) - section = document["llamacpp"] - for backend in MANAGED_PIN_REPOSITORIES: - value = section.get(backend) - if not isinstance(value, str) or RELEASE_TAG_RE.fullmatch(value) is None: - raise ReleaseManifestError( - f"{label} llamacpp.{backend} must be a bNNNN release tag" - ) - sections.append(section) - - base_section, candidate_section = sections - changes = { - backend: (base_section[backend], candidate_section[backend]) - for backend in MANAGED_PIN_REPOSITORIES - if base_section[backend] != candidate_section[backend] - } - - def therock_selector(document: object) -> object: - if not isinstance(document, dict): - return None - therock = document.get("therock") - if not isinstance(therock, dict): - return therock - return {key: value for key, value in therock.items() if key != "comment"} - - def rocm_asset_families(document: object) -> object: - if not isinstance(document, dict): - return None - families = document.get("rocm_asset_families") - if not isinstance(families, dict): - return families - return {key: value for key, value in families.items() if key != "comment"} - - if therock_selector(base_versions) != therock_selector(candidate_versions): - changes.setdefault( - "rocm-stable", - (base_section["rocm-stable"], candidate_section["rocm-stable"]), - ) - if rocm_asset_families(base_versions) != rocm_asset_families(candidate_versions): - changes.setdefault( - "rocm-nightly", - (base_section["rocm-nightly"], candidate_section["rocm-nightly"]), - ) - return changes - - -def require_release_manifest_for_managed_pin_changes( - base_versions: object, - candidate_versions: object, - manifest_json: str | None, -) -> bool: - changes = managed_llamacpp_pin_changes(base_versions, candidate_versions) - if not changes: - return False - if manifest_json is None: - raise ReleaseManifestError( - "changed managed llama.cpp pins require a committed release manifest" - ) - - canonical = canonicalize_release_asset_manifest(manifest_json) - releases = { - release["repository"]: release for release in json.loads(canonical)["releases"] - } - expected_repositories = set(MANAGED_PIN_REPOSITORIES.values()) - if set(releases) != expected_repositories: - raise ReleaseManifestError( - "changed managed llama.cpp pins require all managed release repositories" - ) - for backend, (_, candidate_tag) in changes.items(): - repository = MANAGED_PIN_REPOSITORIES[backend] - if releases[repository]["tag_name"] != candidate_tag: - raise ReleaseManifestError( - f"release manifest does not match changed managed llama.cpp pin " - f"llamacpp.{backend}={candidate_tag}" - ) - - try: - requirements = build_asset_requirements( - ggml_release=releases["ggml-org/llama.cpp"]["tag_name"], - rocm_release=releases["lemonade-sdk/llamacpp-rocm"]["tag_name"], - lemonade_release=releases["lemonade-sdk/llama.cpp"]["tag_name"], - backend_versions=candidate_versions, - ) - rocm_target_requirements = build_rocm_asset_target_requirements( - rocm_release=releases[ROCM_RELEASE_REPOSITORY]["tag_name"], - backend_versions=candidate_versions, - ) - except ValueError as exc: - raise ReleaseManifestError( - f"candidate backend versions cannot define release assets: {exc}" - ) from exc - - for backend in changes: - repository = MANAGED_PIN_REPOSITORIES[backend] - group = ASSET_REQUIREMENT_GROUPS[repository] - required_assets = requirements[group][backend] - asset_sizes = { - asset["name"]: asset["size"] for asset in releases[repository]["assets"] - } - evaluation_options = {} - if backend == "rocm-nightly": - evaluation_options = { - "required_asset_targets": rocm_target_requirements, - "attested_asset_targets": { - attestation["name"]: attestation["build_targets"] - for attestation in releases[repository]["build_target_attestations"] - }, - } - _, missing_by_backend = evaluate_asset_group( - asset_sizes, - {backend: required_assets}, - **evaluation_options, - ) - for asset_name in missing_by_backend[backend]: - if backend == "rocm-nightly" and asset_name in asset_sizes: - raise ReleaseManifestError( - "release manifest has incomplete build-target attestation for " - f"llamacpp.{backend}: {asset_name}" - ) - if asset_name in asset_sizes: - raise ReleaseManifestError( - f"release manifest required asset for llamacpp.{backend} must " - f"have positive size: {asset_name}" - ) - raise ReleaseManifestError( - f"release manifest is missing required asset for " - f"llamacpp.{backend}: {asset_name}" - ) - return True - - -def materialize_release_asset_manifest( - manifest_json: str, - output_path: Path, -) -> tuple[str, str]: - canonical = canonicalize_release_asset_manifest(manifest_json) - digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - document = json.loads(canonical) - summary_lines = [ - "## Release manifest evidence", - "", - ( - "Only immutable GitHub releases are accepted. Fork source commits are " - "bound to the release tag and binary assets by an immutable publisher " - "manifest; this is not cryptographic build provenance." - ), - "", - f"Canonical manifest SHA-256: `{digest}`", - "", - "| Publisher release | Immutable | Release tag commit | Publisher source claim | Upstream boundary |", - "|---|---|---|---|---|", - ] - for release in document["releases"]: - summary_lines.append( - f"| `{release['repository']}@{release['tag_name']}` " - f"| `{str(release['release_immutable']).lower()}` " - f"| `{release['release_tag_commit']}` " - f"| `{release['publisher_claimed_source_commit']}` " - f"(`{release['publisher_claim_type']}`) " - f"| `{release['source_repository']}@{release['upstream_reference_head']}` " - f"(`{release['upstream_reference']}`) |" - ) - try: - output_path.write_text(canonical + "\n", encoding="utf-8") - except OSError as exc: - raise ReleaseManifestError(f"could not write {output_path}: {exc}") from exc - return digest, "\n".join(summary_lines) - - -def _load_json_file(path: Path) -> object: - try: - return _strict_json_loads(path.read_text(encoding="utf-8"), str(path)) - except OSError as exc: - raise ReleaseManifestError(f"could not read {path}: {exc}") from exc - - -def _load_release_specs(specs: list[list[str]]) -> list[tuple[str, str, object]]: - releases = [] - for repository, expected_tag, path_text in specs: - path = Path(path_text) - releases.append((repository, expected_tag, _load_json_file(path))) - return releases - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - "--release", - action="append", - nargs=3, - metavar=("REPOSITORY", "TAG", "JSON_PATH"), - ) - parser.add_argument( - "--source-manifest-asset-id", - nargs=2, - metavar=("RELEASE_JSON_PATH", "RELEASE_REPOSITORY"), - ) - parser.add_argument( - "--validate-source-manifest", - nargs=5, - metavar=( - "SOURCE_MANIFEST_PATH", - "RELEASE_JSON_PATH", - "RELEASE_REPOSITORY", - "RELEASE_TAG", - "RELEASE_TAG_COMMIT", - ), - ) - parser.add_argument("--build-target-attestation-output", type=Path) - parser.add_argument( - "--require-upstream-ancestry", - nargs=4, - metavar=("JSON_PATH", "REPOSITORY", "CLAIMED_COMMIT", "UPSTREAM_HEAD"), - ) - parser.add_argument("--expected-json") - parser.add_argument("--github-output", type=Path) - parser.add_argument("--manifest-json") - parser.add_argument("--materialize-manifest", type=Path) - parser.add_argument("--append-summary-to", type=Path) - parser.add_argument( - "--managed-pin-changes", - nargs=2, - type=Path, - metavar=("BASE_VERSIONS", "CANDIDATE_VERSIONS"), - ) - parser.add_argument( - "--require-managed-pin-manifest", - nargs=3, - type=Path, - metavar=("BASE_VERSIONS", "CANDIDATE_VERSIONS", "MANIFEST"), - ) - args = parser.parse_args() - - source_modes = sum( - mode is not None - for mode in (args.source_manifest_asset_id, args.validate_source_manifest) - ) - if ( - args.build_target_attestation_output is not None - and args.validate_source_manifest is None - ): - parser.error( - "--build-target-attestation-output requires --validate-source-manifest" - ) - if source_modes: - if ( - source_modes != 1 - or ( - args.source_manifest_asset_id is not None - and args.build_target_attestation_output is not None - ) - or any( - ( - args.release, - args.require_upstream_ancestry, - args.expected_json is not None, - args.github_output, - args.manifest_json is not None, - args.materialize_manifest, - args.append_summary_to, - args.managed_pin_changes, - args.require_managed_pin_manifest, - ) - ) - ): - parser.error("source-manifest modes cannot be combined with other modes") - try: - if args.source_manifest_asset_id is not None: - release_path, release_repository = args.source_manifest_asset_id - asset = locate_immutable_source_manifest_asset( - _load_json_file(Path(release_path)), - release_repository, - ) - print(asset["id"]) - else: - ( - source_path, - release_path, - release_repository, - release_tag, - release_tag_commit, - ) = args.validate_source_manifest - source_commit, build_target_attestations = ( - validate_immutable_source_manifest_with_attestation( - Path(source_path).read_bytes(), - _load_json_file(Path(release_path)), - release_repository, - release_tag, - release_tag_commit, - ) - ) - if args.build_target_attestation_output is not None: - args.build_target_attestation_output.write_text( - json.dumps( - build_target_attestations, - separators=(",", ":"), - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - print(source_commit) - except (OSError, ReleaseManifestError) as exc: - parser.exit(1, f"ERROR: {exc}\n") - return - - if args.managed_pin_changes: - if any( - ( - args.release, - args.require_upstream_ancestry, - args.expected_json is not None, - args.github_output, - args.manifest_json is not None, - args.materialize_manifest, - args.append_summary_to, - args.require_managed_pin_manifest, - ) - ): - parser.error("--managed-pin-changes cannot be combined with other modes") - try: - base_path, candidate_path = args.managed_pin_changes - changes = managed_llamacpp_pin_changes( - _load_json_file(base_path), - _load_json_file(candidate_path), - ) - except ReleaseManifestError as exc: - parser.exit(1, f"ERROR: {exc}\n") - print( - json.dumps( - { - backend: {"from": old, "to": new} - for backend, (old, new) in changes.items() - }, - separators=(",", ":"), - sort_keys=True, - ) - ) - return - if args.require_managed_pin_manifest: - if any( - ( - args.release, - args.require_upstream_ancestry, - args.expected_json is not None, - args.github_output, - args.manifest_json is not None, - args.materialize_manifest, - args.append_summary_to, - ) - ): - parser.error( - "--require-managed-pin-manifest cannot be combined with other modes" - ) - base_path, candidate_path, manifest_path = args.require_managed_pin_manifest - try: - require_release_manifest_for_managed_pin_changes( - _load_json_file(base_path), - _load_json_file(candidate_path), - manifest_path.read_text(encoding="utf-8"), - ) - except (OSError, ReleaseManifestError) as exc: - parser.exit(1, f"ERROR: {exc}\n") - print("changed") - return - - if args.require_upstream_ancestry: - if ( - args.release - or args.expected_json is not None - or args.github_output - or args.manifest_json is not None - or args.materialize_manifest - or args.append_summary_to - or args.managed_pin_changes - or args.require_managed_pin_manifest - ): - parser.error( - "--require-upstream-ancestry cannot be combined with manifest options" - ) - path_text, repository, claimed_commit, upstream_head = ( - args.require_upstream_ancestry - ) - try: - require_upstream_ancestry( - _load_json_file(Path(path_text)), - repository, - claimed_commit, - upstream_head, - ) - except ReleaseManifestError as exc: - parser.exit(1, f"ERROR: {exc}\n") - return - if args.materialize_manifest is not None: - if ( - args.release - or args.expected_json is not None - or args.github_output - or args.manifest_json is None - ): - parser.error( - "--materialize-manifest requires only --manifest-json and optional " - "--append-summary-to" - ) - try: - digest, summary = materialize_release_asset_manifest( - args.manifest_json, - args.materialize_manifest, - ) - if args.append_summary_to is not None: - with args.append_summary_to.open("a", encoding="utf-8") as output_file: - output_file.write(f"\n\n{summary}\n") - except (OSError, ReleaseManifestError) as exc: - parser.exit(1, f"ERROR: {exc}\n") - print(digest) - return - if args.manifest_json is not None or args.append_summary_to is not None: - parser.error("--manifest-json requires --materialize-manifest") - if not args.release: - parser.error("at least one --release is required") - - try: - actual = build_release_asset_manifest(_load_release_specs(args.release)) - if args.expected_json is not None: - require_exact_manifest_match(args.expected_json, actual) - if args.github_output: - with args.github_output.open("a", encoding="utf-8") as output_file: - output_file.write(f"release_asset_manifest={actual}\n") - except ReleaseManifestError as exc: - parser.exit(1, f"ERROR: {exc}\n") - - print(actual) - - -if __name__ == "__main__": - main() diff --git a/test/utils/llamacpp_validation_artifacts.py b/test/utils/llamacpp_validation_artifacts.py deleted file mode 100644 index 66b8efcff2..0000000000 --- a/test/utils/llamacpp_validation_artifacts.py +++ /dev/null @@ -1,448 +0,0 @@ -#!/usr/bin/env python3 -"""Verify validation-only K2-Horizon artifacts in an isolated HF cache.""" - -from __future__ import annotations - -import argparse -import hashlib -import os -import re -import stat -import struct -import sys -from dataclasses import dataclass -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from test.utils.validation_model_catalog import load_model_catalog - - -class ValidationArtifactError(ValueError): - """Raised when a validation artifact does not match its immutable lock.""" - - -@dataclass(frozen=True) -class VerifiedValidationArtifact: - model_id: str - resolved_path: Path - size_bytes: int - sha256: str - - -LOCK_FIELDS = { - "filename", - "gguf_architecture", - "max_context_window", - "revision", - "sha256", - "size_bytes", -} -LOWER_HEX_40 = re.compile(r"^[0-9a-f]{40}$") -LOWER_HEX_64 = re.compile(r"^[0-9a-f]{64}$") -REPOSITORY_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") -ARCHITECTURE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") -MAX_METADATA_ITEMS = 1_000_000 -MAX_STRING_BYTES = 64 * 1024 * 1024 -SCALAR_FORMATS = { - 0: " bytes: - if count < 0 or count > self.size - self.stream.tell(): - raise ValidationArtifactError( - f"{self.path}: GGUF metadata exceeds file bounds" - ) - value = self.stream.read(count) - if len(value) != count: - raise ValidationArtifactError( - f"{self.path}: GGUF metadata exceeds file bounds" - ) - return value - - def unpack(self, format_string: str): - size = struct.calcsize(format_string) - return struct.unpack(format_string, self.read(size))[0] - - def string(self, *, decode: bool) -> str | None: - length = self.unpack(" MAX_STRING_BYTES: - raise ValidationArtifactError( - f"{self.path}: GGUF string length exceeds bounds" - ) - value = self.read(length) - if not decode: - return None - try: - return value.decode("utf-8") - except UnicodeDecodeError as exc: - raise ValidationArtifactError( - f"{self.path}: GGUF metadata is not UTF-8" - ) from exc - - -def _read_scalar(reader: _Reader, value_type: int): - format_string = SCALAR_FORMATS.get(value_type) - if format_string is None: - raise ValidationArtifactError( - f"{reader.path}: unsupported GGUF metadata type {value_type}" - ) - return reader.unpack(format_string) - - -def _skip_value(reader: _Reader, value_type: int) -> None: - if value_type == 8: - reader.string(decode=False) - return - if value_type != 9: - _read_scalar(reader, value_type) - return - - element_type = reader.unpack(" MAX_METADATA_ITEMS or element_type == 9: - raise ValidationArtifactError( - f"{reader.path}: GGUF array count or type exceeds bounds" - ) - if element_type == 8: - for _ in range(count): - reader.string(decode=False) - return - format_string = SCALAR_FORMATS.get(element_type) - if format_string is None: - raise ValidationArtifactError( - f"{reader.path}: unsupported GGUF array type {element_type}" - ) - reader.read(struct.calcsize(format_string) * count) - - -def _read_gguf_identity(open_artifact, path: Path) -> tuple[str, int]: - open_artifact.seek(0) - reader = _Reader(open_artifact, path) - if reader.read(4) != b"GGUF": - raise ValidationArtifactError(f"{path}: invalid GGUF magic") - version = reader.unpack(" MAX_METADATA_ITEMS: - raise ValidationArtifactError(f"{path}: GGUF metadata count exceeds bounds") - - metadata = {} - for _ in range(metadata_count): - key = reader.string(decode=True) - if key in metadata: - raise ValidationArtifactError(f"{path}: duplicate GGUF key {key}") - value_type = reader.unpack(" tuple[str, int]: - with path.open("rb") as artifact: - return _read_gguf_identity(artifact, path) - - -def _parse_checkpoint(model_id: str, checkpoint: object) -> tuple[str, str]: - if not isinstance(checkpoint, str) or checkpoint.count(":") != 1: - raise ValidationArtifactError( - f"{model_id}: validation checkpoint must be repository:filename" - ) - repository, filename = checkpoint.split(":", 1) - components = repository.split("/") - if ( - len(components) != 2 - or any(not REPOSITORY_COMPONENT.fullmatch(part) for part in components) - or any(part in {".", ".."} or "--" in part for part in components) - ): - raise ValidationArtifactError(f"{model_id}: unsafe HF repository name") - if ( - not filename - or Path(filename).name != filename - or "/" in filename - or "\\" in filename - or ":" in filename - or "\0" in filename - or filename in {".", ".."} - ): - raise ValidationArtifactError(f"{model_id}: filename must be a safe basename") - return repository, filename - - -def _validate_lock(model_id: str, lock: dict) -> None: - if set(lock) != LOCK_FIELDS: - raise ValidationArtifactError(f"{model_id}: artifact lock fields are invalid") - if not isinstance(lock["revision"], str) or not LOWER_HEX_40.fullmatch( - lock["revision"] - ): - raise ValidationArtifactError(f"{model_id}: revision must be lowercase hex") - if not isinstance(lock["sha256"], str) or not LOWER_HEX_64.fullmatch( - lock["sha256"] - ): - raise ValidationArtifactError(f"{model_id}: sha256 must be lowercase hex") - for field in ("size_bytes", "max_context_window"): - if type(lock[field]) is not int or not 0 < lock[field] <= 2**63 - 1: - raise ValidationArtifactError(f"{model_id}: {field} is out of bounds") - if not isinstance(lock["gguf_architecture"], str) or not ARCHITECTURE.fullmatch( - lock["gguf_architecture"] - ): - raise ValidationArtifactError(f"{model_id}: invalid GGUF architecture") - if not isinstance(lock["filename"], str): - raise ValidationArtifactError(f"{model_id}: filename must be a safe basename") - _parse_checkpoint(model_id, f"x/y:{lock['filename']}") - - -def _sha256(open_artifact) -> str: - digest = hashlib.sha256() - open_artifact.seek(0) - for chunk in iter(lambda: open_artifact.read(8 * 1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _artifact_state(file_status: os.stat_result) -> tuple[int, ...]: - return ( - file_status.st_dev, - file_status.st_ino, - file_status.st_mode, - file_status.st_size, - file_status.st_mtime_ns, - file_status.st_ctime_ns, - ) - - -def _verify_locked_artifact( - cache_root: Path, - repository_path: Path, - model_id: str, - lock: dict, -) -> VerifiedValidationArtifact: - artifact_path = repository_path / "snapshots" / lock["revision"] / lock["filename"] - try: - resolved_artifact = artifact_path.resolve(strict=True) - resolved_artifact.relative_to(cache_root) - except (OSError, ValueError) as exc: - raise ValidationArtifactError( - f"{model_id}: artifact path escapes or is absent from the HF cache" - ) from exc - try: - with resolved_artifact.open("rb") as artifact: - status_before = os.fstat(artifact.fileno()) - if not stat.S_ISREG(status_before.st_mode): - raise ValidationArtifactError(f"{model_id}: artifact is not a file") - if status_before.st_size != lock["size_bytes"]: - raise ValidationArtifactError( - f"{model_id}: artifact size {status_before.st_size} != " - f"{lock['size_bytes']}" - ) - artifact_sha256 = _sha256(artifact) - if artifact_sha256 != lock["sha256"]: - raise ValidationArtifactError(f"{model_id}: artifact SHA-256 mismatch") - architecture, context = _read_gguf_identity(artifact, resolved_artifact) - status_after = os.fstat(artifact.fileno()) - - path_after = artifact_path.resolve(strict=True) - path_after.relative_to(cache_root) - path_status = path_after.stat() - except ValidationArtifactError: - raise - except (OSError, ValueError) as exc: - raise ValidationArtifactError( - f"{model_id}: artifact changed during verification" - ) from exc - if ( - path_after != resolved_artifact - or _artifact_state(status_before) != _artifact_state(status_after) - or _artifact_state(status_after) != _artifact_state(path_status) - ): - raise ValidationArtifactError( - f"{model_id}: artifact changed during verification" - ) - if architecture != lock["gguf_architecture"]: - raise ValidationArtifactError( - f"{model_id}: GGUF architecture {architecture} != " - f"{lock['gguf_architecture']}" - ) - if context != lock["max_context_window"]: - raise ValidationArtifactError( - f"{model_id}: GGUF context {context} != {lock['max_context_window']}" - ) - return VerifiedValidationArtifact( - model_id=model_id, - resolved_path=resolved_artifact, - size_bytes=status_after.st_size, - sha256=artifact_sha256, - ) - - -def _verify_reference_and_artifact( - cache_root: Path, - repository_path: Path, - model_id: str, - lock: dict, -) -> VerifiedValidationArtifact: - reference_path = repository_path / "refs" / "main" - try: - resolved_reference = reference_path.resolve(strict=True) - resolved_reference.relative_to(cache_root) - except OSError as exc: - raise ValidationArtifactError( - f"{model_id}: could not read HF refs/main" - ) from exc - except ValueError as exc: - raise ValidationArtifactError( - f"{model_id}: HF refs/main escapes the cache" - ) from exc - - expected_reference = lock["revision"].encode("ascii") - canonical_references = {expected_reference, expected_reference + b"\n"} - try: - with resolved_reference.open("rb") as reference_file: - status_before = os.fstat(reference_file.fileno()) - if not stat.S_ISREG(status_before.st_mode): - raise ValidationArtifactError(f"{model_id}: HF refs/main is not a file") - reference_bytes = reference_file.read(42) - if reference_bytes not in canonical_references: - raise ValidationArtifactError( - f"{model_id}: HF refs/main does not match locked revision" - ) - - verified_artifact = _verify_locked_artifact( - cache_root, - repository_path, - model_id, - lock, - ) - - reference_file.seek(0) - reference_bytes_after = reference_file.read(42) - status_after = os.fstat(reference_file.fileno()) - - path_after = reference_path.resolve(strict=True) - path_after.relative_to(cache_root) - path_status = path_after.stat() - except ValidationArtifactError: - raise - except (OSError, ValueError) as exc: - raise ValidationArtifactError( - f"{model_id}: HF refs/main changed during verification" - ) from exc - if ( - reference_bytes_after not in canonical_references - or reference_bytes_after != reference_bytes - or path_after != resolved_reference - or _artifact_state(status_before) != _artifact_state(status_after) - or _artifact_state(status_after) != _artifact_state(path_status) - ): - raise ValidationArtifactError( - f"{model_id}: HF refs/main changed during verification" - ) - return verified_artifact - - -def verify_validation_artifacts( - cache_path: Path | str, - catalog_path: Path | str, - lock_path: Path | str, - model_ids: list[str], -) -> dict[str, VerifiedValidationArtifact]: - cache_path = Path(cache_path) - catalog = load_model_catalog(catalog_path) - locks = load_model_catalog(lock_path) - if set(catalog) != set(locks): - raise ValidationArtifactError( - "validation catalog and artifact lock IDs must match exactly" - ) - - requested = list( - dict.fromkeys(value.removeprefix("builtin.") for value in model_ids) - ) - locked_requested = [model_id for model_id in requested if model_id in locks] - if not locked_requested: - return {} - - try: - cache_root = cache_path.resolve(strict=True) - except OSError as exc: - raise ValidationArtifactError("HF cache does not exist") from exc - verified_artifacts = {} - for model_id in locked_requested: - model = catalog[model_id] - lock = locks[model_id] - _validate_lock(model_id, lock) - repository, checkpoint_filename = _parse_checkpoint( - model_id, model.get("checkpoint") - ) - if checkpoint_filename != lock["filename"]: - raise ValidationArtifactError( - f"{model_id}: checkpoint and artifact filenames differ" - ) - - repository_path = cache_path / ("models--" + repository.replace("/", "--")) - verified_artifacts[model_id] = _verify_reference_and_artifact( - cache_root, - repository_path, - model_id, - lock, - ) - - return verified_artifacts - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--cache", required=True, type=Path) - parser.add_argument("--catalog", required=True, type=Path) - parser.add_argument("--locks", required=True, type=Path) - parser.add_argument("--model", action="append", required=True) - args = parser.parse_args() - verify_validation_artifacts( - args.cache, - args.catalog, - args.locks, - args.model, - ) - - -if __name__ == "__main__": - main() diff --git a/test/utils/llamacpp_validation_evidence.py b/test/utils/llamacpp_validation_evidence.py deleted file mode 100644 index 9cd21a54aa..0000000000 --- a/test/utils/llamacpp_validation_evidence.py +++ /dev/null @@ -1,498 +0,0 @@ -#!/usr/bin/env python3 -"""Validate complete scheduled llama.cpp result evidence.""" - -from __future__ import annotations - -import argparse -import json -import math -import os -import re -import stat -from pathlib import Path - -if __package__: - from .llamacpp_capability_validation import ( - K2_HORIZON_PROFILE, - CapabilityValidationError, - validate_capability_matrix_evidence, - ) - from .llamacpp_validation_plan import ( - K2_LARGE, - K2_MEDIUM, - K2_SMALL, - VALIDATION_LANES, - require_complete_promotion_coverage, - ) -else: - from llamacpp_capability_validation import ( # type: ignore[import-not-found] - K2_HORIZON_PROFILE, - CapabilityValidationError, - validate_capability_matrix_evidence, - ) - from llamacpp_validation_plan import ( # type: ignore[import-not-found] - K2_LARGE, - K2_MEDIUM, - K2_SMALL, - VALIDATION_LANES, - require_complete_promotion_coverage, - ) - -EXPECTED_RESULT_FILES = tuple( - f"llamacpp_validation_{lane['target']}.json" for lane in VALIDATION_LANES -) -EXPECTED_RESTART_RESULT_FILES = tuple( - f"llamacpp_restart_validation_{lane['target']}.json" for lane in VALIDATION_LANES -) -REQUIRED_RESULT_FIELDS = { - "model", - "pass", - "response", - "input_tokens", - "output_tokens", - "time_to_first_token", - "tokens_per_second", -} -MAX_VALIDATION_EVIDENCE_BYTES = 4 * 1024 * 1024 -_EVIDENCE_READ_CHUNK_BYTES = 64 * 1024 - - -class ValidationEvidenceError(ValueError): - """Raised when scheduled validation evidence is incomplete or failed.""" - - -def _unique_json_object(pairs: list[tuple[str, object]]) -> dict: - result = {} - for key, value in pairs: - if key in result: - raise ValidationEvidenceError(f"duplicate JSON key: {key}") - result[key] = value - return result - - -def _reject_json_constant(value: str) -> None: - raise ValidationEvidenceError(f"nonstandard JSON numeric constant: {value}") - - -def _parse_finite_float(value: str) -> float: - parsed = float(value) - if not math.isfinite(parsed): - raise ValidationEvidenceError(f"non-finite JSON number: {value}") - return parsed - - -def _validate_record(filename: str, index: int, record: object) -> dict: - if not isinstance(record, dict): - raise ValidationEvidenceError(f"{filename}: result {index} must be an object") - unexpected_fields = set(record) - REQUIRED_RESULT_FIELDS - {"capability_matrix"} - if unexpected_fields: - raise ValidationEvidenceError( - f"{filename}: result {index} has unexpected fields: " - + ", ".join(sorted(unexpected_fields)) - ) - missing_fields = REQUIRED_RESULT_FIELDS - set(record) - if missing_fields: - raise ValidationEvidenceError( - f"{filename}: result {index} missing fields: " - + ", ".join(sorted(missing_fields)) - ) - model = record["model"] - if not isinstance(model, str) or not model: - raise ValidationEvidenceError( - f"{filename}: result {index} model must be nonempty" - ) - passed = record["pass"] - if not isinstance(passed, bool): - raise ValidationEvidenceError( - f"{filename}: result {index} pass must be boolean" - ) - if not passed: - raise ValidationEvidenceError(f"{filename}: {model} did not pass validation") - response = record["response"] - if not isinstance(response, str) or not response.strip(): - raise ValidationEvidenceError( - f"{filename}: result {index} response must be nonempty" - ) - for field in ("input_tokens", "output_tokens"): - value = record[field] - if not ( - value == "N/A" - or (isinstance(value, int) and not isinstance(value, bool) and value >= 0) - ): - raise ValidationEvidenceError( - f"{filename}: result {index} {field} must be nonnegative or N/A" - ) - for field in ("time_to_first_token", "tokens_per_second"): - value = record[field] - if not ( - value == "N/A" - or ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and value >= 0 - and (isinstance(value, int) or math.isfinite(value)) - ) - ): - raise ValidationEvidenceError( - f"{filename}: result {index} {field} must be nonnegative or N/A" - ) - return record - - -def _canonical_model_id(model_id: str) -> str: - return model_id.removeprefix("builtin.") - - -def _stat_snapshot(status: os.stat_result) -> tuple[int, int, int, int, int, int]: - return ( - status.st_dev, - status.st_ino, - status.st_mode, - status.st_size, - status.st_mtime_ns, - status.st_ctime_ns, - ) - - -def _read_bounded_result_file(path: Path) -> str: - filename = path.name - try: - path_status = os.stat(path, follow_symlinks=False) - except FileNotFoundError as exc: - raise ValidationEvidenceError(f"missing validation result: {filename}") from exc - except OSError as exc: - raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc - if not stat.S_ISREG(path_status.st_mode): - raise ValidationEvidenceError( - f"validation result must be a regular file: {filename}" - ) - - flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_CLOEXEC", 0) - flags |= getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(path, flags) - except OSError as exc: - raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc - - try: - opened_status = os.fstat(descriptor) - if not stat.S_ISREG(opened_status.st_mode): - raise ValidationEvidenceError( - f"validation result must be a regular file: {filename}" - ) - if (opened_status.st_dev, opened_status.st_ino) != ( - path_status.st_dev, - path_status.st_ino, - ): - raise ValidationEvidenceError( - f"validation result changed before reading: {filename}" - ) - if opened_status.st_size > MAX_VALIDATION_EVIDENCE_BYTES: - raise ValidationEvidenceError( - f"validation result exceeds byte limit: {filename}" - ) - - chunks = [] - total_bytes = 0 - while total_bytes <= MAX_VALIDATION_EVIDENCE_BYTES: - chunk = os.read( - descriptor, - min( - _EVIDENCE_READ_CHUNK_BYTES, - MAX_VALIDATION_EVIDENCE_BYTES + 1 - total_bytes, - ), - ) - if not chunk: - break - chunks.append(chunk) - total_bytes += len(chunk) - final_status = os.fstat(descriptor) - except ValidationEvidenceError: - raise - except OSError as exc: - raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc - finally: - os.close(descriptor) - - if total_bytes > MAX_VALIDATION_EVIDENCE_BYTES: - raise ValidationEvidenceError( - f"validation result exceeds byte limit: {filename}" - ) - if _stat_snapshot(opened_status) != _stat_snapshot(final_status): - raise ValidationEvidenceError( - f"validation result changed while reading: {filename}" - ) - if total_bytes != final_status.st_size: - raise ValidationEvidenceError( - f"validation result size changed while reading: {filename}" - ) - try: - final_path_status = os.stat(path, follow_symlinks=False) - except OSError as exc: - raise ValidationEvidenceError( - f"validation result changed while reading: {filename}" - ) from exc - if not stat.S_ISREG(final_path_status.st_mode) or ( - final_path_status.st_dev, - final_path_status.st_ino, - ) != (final_status.st_dev, final_status.st_ino): - raise ValidationEvidenceError( - f"validation result changed while reading: {filename}" - ) - try: - return b"".join(chunks).decode("utf-8") - except UnicodeError as exc: - raise ValidationEvidenceError(f"could not read {filename}: {exc}") from exc - - -def load_and_validate_result_file( - path: Path, - expected_models: list[str], - *, - capability_profile: str = "", - capability_models: list[str] | None = None, -) -> list[dict]: - filename = path.name - if ( - not expected_models - or not all(isinstance(model, str) and model for model in expected_models) - or len({_canonical_model_id(model) for model in expected_models}) - != len(expected_models) - ): - raise ValidationEvidenceError( - f"{filename}: expected models must be unique nonempty model identifiers" - ) - expected_capability_models = capability_models or [] - if bool(capability_profile) != bool(expected_capability_models): - raise ValidationEvidenceError( - f"{filename}: capability profile and models must be paired" - ) - canonical_expected_models = [ - _canonical_model_id(model) for model in expected_models - ] - canonical_capability_models = { - _canonical_model_id(model) for model in expected_capability_models - } - if len(canonical_capability_models) != len(expected_capability_models): - raise ValidationEvidenceError(f"{filename}: capability models must be unique") - if not canonical_capability_models.issubset(canonical_expected_models): - raise ValidationEvidenceError(f"{filename}: capability models must be selected") - raw = _read_bounded_result_file(path) - if not raw.strip(): - raise ValidationEvidenceError(f"validation result is empty: {filename}") - try: - records = json.loads( - raw, - object_pairs_hook=_unique_json_object, - parse_constant=_reject_json_constant, - parse_float=_parse_finite_float, - ) - except json.JSONDecodeError as exc: - raise ValidationEvidenceError( - f"validation result is invalid JSON: {filename}: {exc}" - ) from exc - if not isinstance(records, list) or not records: - raise ValidationEvidenceError( - f"validation result must be a nonempty array: {filename}" - ) - checked = [ - _validate_record(filename, index, record) - for index, record in enumerate(records) - ] - models = [record["model"] for record in checked] - canonical_models = [_canonical_model_id(model) for model in models] - if len(canonical_models) != len(set(canonical_models)): - raise ValidationEvidenceError( - f"validation result contains duplicate models: {filename}" - ) - if canonical_models != canonical_expected_models: - raise ValidationEvidenceError( - f"{filename}: result models do not match planned models" - ) - for record, model_id in zip(checked, canonical_models): - if model_id not in canonical_capability_models: - if "capability_matrix" in record: - raise ValidationEvidenceError( - f"{filename}: {record['model']} has an unexpected capability matrix" - ) - continue - if "capability_matrix" not in record: - raise ValidationEvidenceError( - f"{filename}: {record['model']} is missing capability evidence" - ) - try: - validate_capability_matrix_evidence( - record["capability_matrix"], - capability_profile, - record["model"], - ) - except CapabilityValidationError as exc: - raise ValidationEvidenceError( - f"{filename}: {record['model']} capability evidence is invalid: {exc}" - ) from exc - return checked - - -def _expected_models_by_file(expected_matrix: object) -> dict[str, dict]: - if not isinstance(expected_matrix, dict): - raise ValidationEvidenceError("validation matrix must be an object") - try: - require_complete_promotion_coverage(expected_matrix) - except ValueError as exc: - raise ValidationEvidenceError( - f"invalid scheduled validation lanes: {exc}" - ) from exc - rows = expected_matrix.get("include") - if not isinstance(rows, list): - raise ValidationEvidenceError("validation matrix include must be an array") - - expected = {} - for index, row in enumerate(rows): - if not isinstance(row, dict): - raise ValidationEvidenceError( - f"validation matrix row {index} must be an object" - ) - backend = row.get("backend") - channel = row.get("channel") - target = row.get("target") - if not isinstance(backend, str) or not isinstance(channel, str): - raise ValidationEvidenceError( - f"validation matrix row {index} backend and channel must be strings" - ) - if not isinstance(target, str) or not re.fullmatch(r"[a-z0-9-]+", target): - raise ValidationEvidenceError( - f"validation matrix row {index} target must be a safe nonempty label" - ) - filename = f"llamacpp_validation_{target}.json" - if filename in expected: - raise ValidationEvidenceError(f"duplicate validation lane: {target}") - selected_models = row.get("models") - if ( - not isinstance(selected_models, list) - or not all(isinstance(model, str) and model for model in selected_models) - or len(selected_models) != len(set(selected_models)) - ): - raise ValidationEvidenceError( - f"validation matrix lane {target} needs unique selected models" - ) - models = row.get("expected_models") - if ( - not isinstance(models, list) - or not models - or not all(isinstance(model, str) and model for model in models) - or len(models) != len(set(models)) - ): - raise ValidationEvidenceError( - f"validation matrix lane {target} needs unique expected models" - ) - if selected_models != models: - raise ValidationEvidenceError( - f"validation matrix lane {target} selected and expected models differ" - ) - if not any(_canonical_model_id(model) == K2_SMALL for model in models): - raise ValidationEvidenceError( - f"validation matrix lane {target} must include {K2_SMALL}" - ) - has_capability_profile = "capability_profile" in row - has_capability_models = "capability_models" in row - if has_capability_profile != has_capability_models: - raise ValidationEvidenceError( - f"validation matrix lane {target} must pair capability profile and models" - ) - if not has_capability_profile: - raise ValidationEvidenceError( - f"scheduled validation lane {target} needs capability evidence" - ) - capability_profile = row["capability_profile"] - capability_models = row["capability_models"] - if capability_profile != K2_HORIZON_PROFILE: - raise ValidationEvidenceError( - f"validation matrix lane {target} has unsupported capability profile" - ) - if ( - not isinstance(capability_models, list) - or not capability_models - or not all(isinstance(model, str) and model for model in capability_models) - or len(capability_models) != len(set(capability_models)) - ): - raise ValidationEvidenceError( - f"validation matrix lane {target} needs unique capability models" - ) - canonical_models = {_canonical_model_id(model) for model in models} - if any( - _canonical_model_id(model) not in canonical_models - for model in capability_models - ): - raise ValidationEvidenceError( - f"validation matrix lane {target} has an unselected capability model" - ) - if K2_SMALL not in capability_models: - raise ValidationEvidenceError( - f"validation matrix lane {target} must validate {K2_SMALL} capabilities" - ) - if any( - _canonical_model_id(model) in {K2_MEDIUM, K2_LARGE} for model in models - ) and (target != "windows-vulkan" or "128gb" not in row["runner"]): - raise ValidationEvidenceError( - f"validation matrix lane {target} needs the 128gb Vulkan runner " - "for K2-Horizon-3.7B/7B" - ) - expected[filename] = { - "models": models, - "capability_profile": capability_profile, - "capability_models": capability_models, - "restart_model": capability_models[0], - } - - if set(expected) != set(EXPECTED_RESULT_FILES): - raise ValidationEvidenceError( - "validation matrix must contain exactly the scheduled validation lanes" - ) - return expected - - -def load_and_validate_results( - root: Path, - expected_matrix: object, -) -> dict[str, list[dict]]: - expected_lanes = _expected_models_by_file(expected_matrix) - validated = {} - for filename in EXPECTED_RESULT_FILES: - expected_lane = expected_lanes[filename] - validated[filename] = load_and_validate_result_file( - root / filename, - expected_lane["models"], - capability_profile=expected_lane["capability_profile"], - capability_models=expected_lane["capability_models"], - ) - restart_filename = filename.replace( - "llamacpp_validation_", "llamacpp_restart_validation_", 1 - ) - validated[restart_filename] = load_and_validate_result_file( - root / restart_filename, - [expected_lane["restart_model"]], - ) - return validated - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--directory", type=Path, default=Path.cwd()) - parser.add_argument("--expected-matrix-json", required=True) - args = parser.parse_args() - try: - expected_matrix = json.loads( - args.expected_matrix_json, - object_pairs_hook=_unique_json_object, - parse_constant=_reject_json_constant, - parse_float=_parse_finite_float, - ) - results = load_and_validate_results(args.directory, expected_matrix) - except (json.JSONDecodeError, ValidationEvidenceError) as exc: - parser.exit(1, f"ERROR: {exc}\n") - for filename, records in results.items(): - print(f"{filename}: {len(records)} passing result(s)") - - -if __name__ == "__main__": - main() diff --git a/test/utils/llamacpp_validation_plan.py b/test/utils/llamacpp_validation_plan.py deleted file mode 100644 index 2f922a9dbf..0000000000 --- a/test/utils/llamacpp_validation_plan.py +++ /dev/null @@ -1,432 +0,0 @@ -"""Build the llama.cpp validation matrix for GitHub Actions.""" - -from __future__ import annotations - -import argparse -import json -import os -from pathlib import Path - -if __package__: - from .validation_model_catalog import ( - load_model_catalog, - merge_model_catalogs, - ) -else: - from validation_model_catalog import ( # type: ignore[import-not-found] - load_model_catalog, - merge_model_catalogs, - ) - -K2_SMALL = "K2-Horizon-0.9B-GGUF" -K2_MEDIUM = "K2-Horizon-3.7B-GGUF" -K2_LARGE = "K2-Horizon-7B-GGUF" -SUPPORTED_EVENTS = {"merge_group", "pull_request", "schedule", "workflow_dispatch"} -K2_CAPABILITY_PROFILE = "k2-horizon-v1" -MANAGED_PINS = frozenset( - {"cpu", "cuda", "metal", "rocm-nightly", "rocm-stable", "vulkan"} -) -MODEL_REGISTRY = ( - Path(__file__).resolve().parents[2] - / "src" - / "cpp" - / "resources" - / "server_models.json" -) -VALIDATION_MODEL_OVERLAY = ( - Path(__file__).resolve().parents[1] / "fixtures" / "llamacpp_validation_models.json" -) - -LARGE_VULKAN_RUNNER = [ - "self-hosted", - "Windows", - "128gb", - "vulkan", - "lemon-prod", -] -LINUX_VULKAN_RUNNER = [ - "self-hosted", - "Linux", - "vulkan", - "lemon-prod", -] -LINUX_CPU_RUNNER = ["self-hosted", "Linux", "lemon-prod"] -MACOS_METAL_RUNNER = ["macos-latest"] -WINDOWS_CUDA_RUNNER = [ - "self-hosted", - "Windows", - "cuda", - "lemon-prod", -] -WINDOWS_ROCM_RUNNER = [ - "self-hosted", - "Windows", - "stx-halo", - "rocm", - "lemon-prod", -] - -VALIDATION_LANES = ( - { - "target": "windows-vulkan", - "backend": "vulkan", - "channel": "", - "managed_pin": "vulkan", - "build_platform": "windows", - "runner": LARGE_VULKAN_RUNNER, - }, - { - "target": "linux-vulkan", - "backend": "vulkan", - "channel": "", - "managed_pin": "vulkan", - "build_platform": "linux", - "runner": LINUX_VULKAN_RUNNER, - }, - { - "target": "linux-cpu", - "backend": "cpu", - "channel": "", - "managed_pin": "cpu", - "build_platform": "linux", - "runner": LINUX_CPU_RUNNER, - }, - { - "target": "macos-metal", - "backend": "metal", - "channel": "", - "managed_pin": "metal", - "build_platform": "macos", - "runner": MACOS_METAL_RUNNER, - }, - { - "target": "windows-cuda", - "backend": "cuda", - "channel": "", - "managed_pin": "cuda", - "build_platform": "windows", - "runner": WINDOWS_CUDA_RUNNER, - }, - { - "target": "windows-rocm-stable", - "backend": "rocm", - "channel": "stable", - "managed_pin": "rocm-stable", - "build_platform": "windows", - "runner": WINDOWS_ROCM_RUNNER, - }, - { - "target": "windows-rocm-nightly", - "backend": "rocm", - "channel": "nightly", - "managed_pin": "rocm-nightly", - "build_platform": "windows", - "runner": WINDOWS_ROCM_RUNNER, - }, -) - - -def _parse_models(models_csv: str) -> list[str]: - return [model.strip() for model in models_csv.split(",") if model.strip()] - - -def _canonical_model_id(model_id: str) -> str: - return model_id.removeprefix("builtin.") - - -def _hot_llamacpp_model_ids(registry: dict, source: Path) -> list[str]: - models = sorted( - model_id - for model_id, model in registry.items() - if isinstance(model, dict) - and model.get("recipe") == "llamacpp" - and isinstance(model.get("labels"), list) - and "hot" in model["labels"] - ) - if not models: - raise ValueError(f"Model registry {source} has no hot llama.cpp models") - return models - - -def load_hot_llamacpp_model_ids(path: Path = MODEL_REGISTRY) -> list[str]: - return _hot_llamacpp_model_ids(load_model_catalog(path), path) - - -def load_validation_candidate_model_ids( - path: Path = MODEL_REGISTRY, - overlay_path: Path = VALIDATION_MODEL_OVERLAY, -) -> list[str]: - product_hot_models = _hot_llamacpp_model_ids(load_model_catalog(path), path) - overlay = load_model_catalog(overlay_path) - merged = merge_model_catalogs(path, overlay_path) - - for model_id in overlay: - model = merged[model_id] - if model.get("recipe") != "llamacpp": - raise ValueError( - f"Validation model {model_id} must use the llamacpp recipe" - ) - labels = model.get("labels") - if not isinstance(labels, list) or "chat" not in labels: - raise ValueError(f"Validation model {model_id} must support chat") - - return sorted(set(product_hot_models) | set(overlay)) - - -def _row( - lane: dict[str, object], - models: list[str], - expected_models: list[str] | None = None, -) -> dict[str, object]: - row = { - "target": lane["target"], - "backend": lane["backend"], - "channel": lane["channel"], - "managed_pin": lane["managed_pin"], - "build_platform": lane["build_platform"], - "runner": list(lane["runner"]), - "models": list(models), - "lite": False, - "capability_profile": K2_CAPABILITY_PROFILE, - "capability_models": [K2_SMALL], - } - if expected_models is not None: - row["expected_models"] = list(expected_models) - return row - - -def _with_required_k2_small(models: list[str]) -> list[str]: - if any(_canonical_model_id(model) == K2_SMALL for model in models): - return list(models) - return [K2_SMALL, *models] - - -def _small_lane_models(models: list[str]) -> list[str]: - return [ - model - for model in _with_required_k2_small(models) - if _canonical_model_id(model) not in {K2_MEDIUM, K2_LARGE} - ] - - -def require_complete_promotion_coverage( - plan: dict[str, list[dict[str, object]]], -) -> None: - rows = plan.get("include") if isinstance(plan, dict) else None - if not isinstance(rows, list) or len(rows) != len(VALIDATION_LANES): - raise ValueError("Promotion requires exactly the seven validation lanes") - - candidate_models = load_validation_candidate_model_ids() - actual_targets = [] - for index, (row, lane) in enumerate(zip(rows, VALIDATION_LANES)): - if not isinstance(row, dict): - raise ValueError(f"Validation lane {index} must be an object") - actual_targets.append(row.get("target")) - for field in ( - "target", - "backend", - "channel", - "managed_pin", - "build_platform", - ): - if row.get(field) != lane[field]: - raise ValueError( - f"Validation lane {index} has an invalid {field} mapping" - ) - if row.get("runner") != lane["runner"]: - raise ValueError(f"Validation lane {index} has an invalid runner mapping") - if row.get("capability_profile") != K2_CAPABILITY_PROFILE: - raise ValueError(f"Validation lane {index} lacks the K2 capability profile") - if row.get("capability_models") != [K2_SMALL]: - raise ValueError( - f"Validation lane {index} lacks the K2 small capability model" - ) - - if row.get("lite") is not False: - raise ValueError(f"Validation lane {index} cannot use lite mode") - - selected_models = row.get("models") - expected_models = row.get("expected_models") - for field_name, model_ids in ( - ("selected", selected_models), - ("expected", expected_models), - ): - if ( - not isinstance(model_ids, list) - or not model_ids - or not all( - isinstance(model_id, str) and model_id for model_id in model_ids - ) - or len(model_ids) != len(set(model_ids)) - ): - raise ValueError( - f"Validation lane {index} {field_name} models must be " - "unique nonempty strings" - ) - if selected_models != expected_models: - raise ValueError( - f"Validation lane {index} selected models do not match expected models" - ) - - planned_models = expected_models - if not any( - _canonical_model_id(model_id) == K2_SMALL for model_id in planned_models - ): - raise ValueError(f"Validation lane {index} does not include {K2_SMALL}") - if any( - _canonical_model_id(model_id) in {K2_MEDIUM, K2_LARGE} - for model_id in planned_models - ): - if row["target"] != "windows-vulkan" or "128gb" not in row["runner"]: - raise ValueError( - "K2-Horizon-3.7B/7B validation requires the 128gb Vulkan lane" - ) - - if row["target"] == "windows-vulkan": - if planned_models != candidate_models: - raise ValueError( - "The 128gb Vulkan lane must use the exact candidate model set" - ) - elif planned_models != [K2_SMALL]: - raise ValueError( - f"Validation lane {index} must use the exact K2 small model set" - ) - - if actual_targets != [lane["target"] for lane in VALIDATION_LANES]: - raise ValueError("Promotion validation lanes must use the exact target order") - if {row["managed_pin"] for row in rows} != MANAGED_PINS: - raise ValueError("Promotion validation does not cover every managed pin") - - -def create_validation_plan( - event_name: str, - models_csv: str = "", - lite: bool = False, -) -> dict[str, list[dict[str, object]]]: - if event_name not in SUPPORTED_EVENTS: - raise ValueError(f"Unsupported workflow event: {event_name}") - - models = _parse_models(models_csv) - if event_name != "workflow_dispatch" and (models or lite): - raise ValueError("Manual model inputs are valid only for workflow_dispatch") - if models and lite: - raise ValueError("Explicit models and lite mode are mutually exclusive") - - if event_name in {"pull_request", "merge_group"}: - return { - "include": [ - _row( - lane, - ( - [K2_SMALL, K2_MEDIUM, K2_LARGE] - if lane["target"] == "windows-vulkan" - else [K2_SMALL] - ), - ) - for lane in VALIDATION_LANES - ] - } - - if event_name == "schedule": - candidate_models = load_validation_candidate_model_ids() - plan = { - "include": [ - _row( - lane, - ( - candidate_models - if lane["target"] == "windows-vulkan" - else [K2_SMALL] - ), - ( - candidate_models - if lane["target"] == "windows-vulkan" - else [K2_SMALL] - ), - ) - for lane in VALIDATION_LANES - ] - } - require_complete_promotion_coverage(plan) - return plan - - if lite: - large_lane_models = [K2_SMALL] - small_lane_models = [K2_SMALL] - elif models: - large_lane_models = _with_required_k2_small(models) - small_lane_models = _small_lane_models(models) - else: - large_lane_models = _with_required_k2_small(load_hot_llamacpp_model_ids()) - small_lane_models = [K2_SMALL] - return { - "include": [ - _row( - lane, - ( - large_lane_models - if lane["target"] == "windows-vulkan" - else small_lane_models - ), - ) - for lane in VALIDATION_LANES - ] - } - - -def is_promotion_eligible( - event_name: str, - models_csv: str = "", - lite: bool = False, -) -> bool: - plan = create_validation_plan(event_name, models_csv, lite) - if event_name != "schedule": - return False - require_complete_promotion_coverage(plan) - return True - - -def write_github_output( - output_path: Path, - plan: dict[str, list[dict[str, object]]], - *, - promotion_eligible: bool, -) -> None: - matrix = json.dumps(plan, separators=(",", ":")) - with output_path.open("a", encoding="utf-8") as output: - output.write(f"matrix={matrix}\n") - output.write(f"promotion_eligible={str(promotion_eligible).lower()}\n") - - -def _bool_argument(value: str) -> bool: - normalized = value.strip().lower() - if normalized == "true": - return True - if normalized in {"", "false"}: - return False - raise argparse.ArgumentTypeError("expected true or false") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--event", required=True) - parser.add_argument("--models", default="") - parser.add_argument("--lite", default=False, type=_bool_argument) - args = parser.parse_args() - - output_value = os.environ.get("GITHUB_OUTPUT") - if not output_value: - parser.error("GITHUB_OUTPUT is required") - - plan = create_validation_plan(args.event, args.models, args.lite) - promotion_eligible = is_promotion_eligible(args.event, args.models, args.lite) - write_github_output( - Path(output_value), - plan, - promotion_eligible=promotion_eligible, - ) - print(json.dumps(plan, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/test/utils/server_base.py b/test/utils/server_base.py index bfa095bea0..9e22b60743 100644 --- a/test/utils/server_base.py +++ b/test/utils/server_base.py @@ -131,7 +131,7 @@ def get_cli_binary(): return _config["cli_binary"] -def wait_for_server(port=PORT, timeout=60, host="localhost"): +def wait_for_server(port=PORT, timeout=60): """ Wait for the server to start by checking if the port is available. @@ -147,7 +147,7 @@ def wait_for_server(port=PORT, timeout=60, host="localhost"): if time.time() - start_time > timeout: raise TimeoutError(f"Server failed to start within {timeout} seconds") try: - conn = socket.create_connection((host, port)) + conn = socket.create_connection(("localhost", port)) conn.close() return True except socket.error: @@ -181,13 +181,13 @@ def set_server_config(config: dict, port=PORT): return response.json() -def unload_all_models(port=PORT, attempts=3, host="localhost"): +def unload_all_models(port=PORT, attempts=3): """POST /api/v1/unload to unload all models for clean state.""" response = None for attempt in range(1, attempts + 1): try: response = requests.post( - f"http://{host}:{port}/api/v1/unload", + f"http://localhost:{port}/api/v1/unload", json={}, headers=_auth_headers(), timeout=30, @@ -280,7 +280,7 @@ def _is_transient_pull_status(status_code): return status_code in {408, 409, 429, 500, 502, 503, 504} -def _pull_model_streaming(model_name, port, host="localhost"): +def _pull_model_streaming(model_name, port): """Pull via the SSE streaming mode and block until the download completes. Large models (10+ GB) exceed any fixed read timeout on the synchronous @@ -288,7 +288,7 @@ def _pull_model_streaming(model_name, port, host="localhost"): applies between events, not to the whole download. """ with requests.post( - f"http://{host}:{port}/api/v1/pull", + f"http://localhost:{port}/api/v1/pull", json={"model_name": model_name, "stream": True}, stream=True, timeout=TIMEOUT_MODEL_OPERATION, @@ -312,7 +312,7 @@ def _pull_model_streaming(model_name, port, host="localhost"): return 500, "SSE stream ended without a 'complete' event" -def pull_model_with_retry(model_name, attempts=3, port=PORT, host="localhost"): +def pull_model_with_retry(model_name, attempts=3, port=PORT): """Pull a model with bounded retry for transient setup failures. Test setup should tolerate one-off transient pull failures, but persistent @@ -327,7 +327,7 @@ def pull_model_with_retry(model_name, attempts=3, port=PORT, host="localhost"): time.sleep(min(30, 2 ** (attempt - 1))) try: - status, body = _pull_model_streaming(model_name, port, host=host) + status, body = _pull_model_streaming(model_name, port) except requests.RequestException as exc: last_error = exc if attempt < attempts: diff --git a/test/utils/validation_model_catalog.py b/test/utils/validation_model_catalog.py deleted file mode 100644 index 34745a5f49..0000000000 --- a/test/utils/validation_model_catalog.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -"""Merge validation-only model records into a disposable Lemonade catalog.""" - -from __future__ import annotations - -import argparse -import json -import math -import os -import tempfile -from pathlib import Path - - -class ModelCatalogOverlayError(ValueError): - """Raised when a validation catalog overlay cannot be merged safely.""" - - -def _reject_duplicate_members(pairs: list[tuple[str, object]]) -> dict: - result = {} - for key, value in pairs: - if key in result: - raise ModelCatalogOverlayError(f"duplicate JSON member: {key}") - result[key] = value - return result - - -def _reject_non_finite_number(value: str): - raise ModelCatalogOverlayError(f"non-finite JSON number: {value}") - - -def _parse_finite_float(value: str) -> float: - parsed = float(value) - if not math.isfinite(parsed): - _reject_non_finite_number(value) - return parsed - - -def _json_values_equal(left: object, right: object) -> bool: - if type(left) is not type(right): - return False - if isinstance(left, dict): - return left.keys() == right.keys() and all( - _json_values_equal(left[key], right[key]) for key in left - ) - if isinstance(left, list): - return len(left) == len(right) and all( - _json_values_equal(left_value, right_value) - for left_value, right_value in zip(left, right) - ) - return left == right - - -def load_model_catalog(path: Path | str) -> dict: - path = Path(path) - try: - catalog = json.loads( - path.read_text(encoding="utf-8"), - object_pairs_hook=_reject_duplicate_members, - parse_constant=_reject_non_finite_number, - parse_float=_parse_finite_float, - ) - except (OSError, UnicodeError, json.JSONDecodeError) as exc: - raise ModelCatalogOverlayError(f"{path}: invalid JSON: {exc}") from exc - if not isinstance(catalog, dict): - raise ModelCatalogOverlayError(f"{path}: catalog must be a JSON object") - for model_id, model in catalog.items(): - if not isinstance(model, dict): - raise ModelCatalogOverlayError( - f"{path}: model {model_id} must be a JSON object" - ) - return catalog - - -def merge_model_catalogs( - base_path: Path | str, - overlay_path: Path | str, -) -> dict: - base_path = Path(base_path) - overlay_path = Path(overlay_path) - merged = load_model_catalog(base_path) - overlay = load_model_catalog(overlay_path) - - for model_id, candidate in overlay.items(): - if model_id in merged and not _json_values_equal(merged[model_id], candidate): - raise ModelCatalogOverlayError(f"conflicting model definition: {model_id}") - merged[model_id] = candidate - return merged - - -def merge_model_catalog_files( - base_path: Path | str, - overlay_path: Path | str, - output_path: Path | str, -) -> None: - output_path = Path(output_path) - merged = merge_model_catalogs(base_path, overlay_path) - - output_path.parent.mkdir(parents=True, exist_ok=True) - temporary_path = None - try: - with tempfile.NamedTemporaryFile( - "w", - dir=output_path.parent, - encoding="utf-8", - delete=False, - ) as temporary: - temporary_path = Path(temporary.name) - json.dump(merged, temporary, indent=4, sort_keys=True, allow_nan=False) - temporary.write("\n") - temporary.flush() - os.fsync(temporary.fileno()) - os.replace(temporary_path, output_path) - finally: - if temporary_path is not None and temporary_path.exists(): - temporary_path.unlink() - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Merge a validation-only model overlay into a catalog" - ) - parser.add_argument("--base", required=True, type=Path) - parser.add_argument("--overlay", required=True, type=Path) - parser.add_argument("--output", required=True, type=Path) - args = parser.parse_args() - merge_model_catalog_files(args.base, args.overlay, args.output) - - -if __name__ == "__main__": - main() diff --git a/test/utils/validation_model_selection.py b/test/utils/validation_model_selection.py deleted file mode 100644 index 276279084a..0000000000 --- a/test/utils/validation_model_selection.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Model selection shared by llama.cpp validation entry points.""" - -from __future__ import annotations - -import argparse -import math -from typing import Any, Callable, Iterable - -BuiltinModelResolver = Callable[[str], dict[str, Any] | None] - - -class ModelSelectionError(ValueError): - """Raised when requested validation models cannot be selected safely.""" - - -def add_model_selection_arguments(parser: argparse.ArgumentParser) -> None: - selection = parser.add_mutually_exclusive_group() - selection.add_argument( - "--model", - action="append", - default=None, - help="Built-in llama.cpp model to validate. May be repeated.", - ) - selection.add_argument( - "--lite", - action="store_true", - help="Lite mode: only test the smallest hot model", - ) - - -def _model_size(model: dict[str, Any]) -> float: - size = model.get("size") - if isinstance(size, (int, float)) and not isinstance(size, bool): - return float(size) - return math.inf - - -def _canonical_builtin_model_id(model_id: str) -> str: - return model_id if model_id.startswith("builtin.") else f"builtin.{model_id}" - - -def _require_pullable_checkpoint(model: dict[str, Any]) -> None: - checkpoint = model.get("checkpoint") - if isinstance(checkpoint, str) and checkpoint.strip(): - return - - checkpoints = model.get("checkpoints") - if isinstance(checkpoints, dict): - main_checkpoint = checkpoints.get("main") - if isinstance(main_checkpoint, str) and main_checkpoint.strip(): - return - - model_id = model.get("id", "") - raise ModelSelectionError( - f"Model '{model_id}' cannot be pulled because its catalog checkpoint " - "is missing or invalid" - ) - - -def select_llamacpp_models( - catalog: Iterable[dict[str, Any]], - requested_model_ids: list[str] | None = None, - lite: bool = False, - builtin_model_resolver: BuiltinModelResolver | None = None, -) -> list[dict[str, Any]]: - models = list(catalog) - - if requested_model_ids: - if lite: - raise ModelSelectionError( - "Explicit models and lite mode are mutually exclusive" - ) - if builtin_model_resolver is None: - raise ModelSelectionError( - "Explicit model selection requires a built-in model resolver" - ) - selected = [] - for requested_model_id in requested_model_ids: - canonical_model_id = _canonical_builtin_model_id(requested_model_id) - model = builtin_model_resolver(canonical_model_id) - if model is None: - raise ModelSelectionError( - f"Model '{requested_model_id}' was not found in the built-in " - "model catalog" - ) - model = dict(model) - resolved_model_id = model.get("id") - if not isinstance(resolved_model_id, str) or ( - _canonical_builtin_model_id(resolved_model_id) != canonical_model_id - ): - raise ModelSelectionError( - f"requested built-in model '{canonical_model_id}' resolved to " - f"unrelated id '{resolved_model_id}'" - ) - model["load_id"] = canonical_model_id - recipe = model.get("recipe") - if recipe != "llamacpp": - raise ModelSelectionError( - f"Model '{requested_model_id}' uses recipe '{recipe}', " - "expected 'llamacpp'" - ) - labels = model.get("labels") - if not isinstance(labels, list) or "chat" not in labels: - raise ModelSelectionError( - f"Model '{requested_model_id}' does not support chat validation" - ) - _require_pullable_checkpoint(model) - selected.append(model) - return selected - - selected = [ - model - for model in models - if model.get("recipe") == "llamacpp" and "hot" in model.get("labels", []) - ] - selected.sort(key=lambda model: model["id"]) - if not selected: - raise ModelSelectionError("No hot llama.cpp models found in the model catalog") - - if lite: - selected = [min(selected, key=_model_size)] - - return selected diff --git a/test/validate_llamacpp.py b/test/validate_llamacpp.py index 8e07b3ef74..a321730f52 100644 --- a/test/validate_llamacpp.py +++ b/test/validate_llamacpp.py @@ -5,17 +5,15 @@ Usage: python test/validate_llamacpp.py --backend vulkan python test/validate_llamacpp.py --backend rocm - python test/validate_llamacpp.py --backend vulkan --model MODEL_ID This script expects `lemond` to already be running on the target port. This script: -1. Queries `/api/v1/models?show_all=true` and selects either the requested - llama.cpp models or all models with recipe `llamacpp` and label `hot` +1. Queries `/api/v1/models?show_all=true` and selects models with recipe + `llamacpp` and label `hot` 2. Installs the requested llamacpp backend via `POST /api/v1/install` -3. For each selected model, loads it with the requested backend, sends a - `chat/completions` request, optionally runs an API capability profile, - queries `/api/v1/stats`, and unloads it +3. For each hot model, loads it with the requested backend, sends a + `chat/completions` request, queries `/api/v1/stats`, and unloads it 4. Outputs a JSON results file for CI consumption """ @@ -24,54 +22,16 @@ import json import os import shutil -import socket -import subprocess import sys import tempfile -import time -from urllib.parse import quote import requests -from utils.llamacpp_capability_validation import ( - CapabilityValidationError, - K2_HORIZON_PROFILE, - MAX_STREAM_EVENTS, - MAX_STREAM_SSE_BYTES, - STREAM_WALL_CLOCK_TIMEOUT_SECONDS, - find_raw_ifm_control_marker, - reject_duplicate_json_object, - validate_capabilities, -) -from utils.server_base import _auth_headers +from utils.server_base import _auth_headers, unload_all_models, wait_for_server from utils.test_models import PORT, TIMEOUT_DEFAULT -from utils.validation_model_selection import ( - ModelSelectionError, - add_model_selection_arguments, - select_llamacpp_models, -) TIMEOUT_HEALTH = 60 TIMEOUT_INFERENCE = 1800 # 30 minutes; large models may need 60+ GB download -PULL_WALL_CLOCK_TIMEOUT_SECONDS = 3 * 60 * 60 -MAX_PULL_RESPONSE_BYTES = 64 * 1024 -MAX_PULL_SSE_WIRE_BYTES = 1024 * 1024 * 1024 -MAX_PULL_SSE_EVENTS = 4 * 1024 * 1024 -MAX_PULL_SSE_LINE_BYTES = 64 * 1024 -MAX_PULL_SSE_EVENT_DATA_BYTES = 64 * 1024 -MAX_PULL_TERMINAL_SUMMARY_BYTES = 4096 -SERVER_CONNECT_ATTEMPT_TIMEOUT_SECONDS = 1 -SERVER_CONNECT_RETRY_DELAY_SECONDS = 1 -VALIDATION_CTX_SIZE = 8192 -VALIDATION_HOST = "127.0.0.1" -INTERNAL_STREAM_WORKER_ARGUMENT = "--internal-stream-worker" -STREAM_WORKER_PROTOCOL_VERSION = 2 -STREAM_WORKER_MAGIC = b"LEMONADE_STREAM_WORKER_V2\n" -MAX_STREAM_WORKER_INPUT_BYTES = 1024 * 1024 -MAX_STREAM_WORKER_HEADER_BYTES = 64 * 1024 -MAX_STREAM_WORKER_ERROR_BYTES = 4096 -STREAM_WORKER_CHUNK_BYTES = 64 * 1024 -MAX_STREAM_SSE_LINE_BYTES = 64 * 1024 CHAT_PROMPT = [ {"role": "user", "content": "What is 2+2? Reply in one sentence."}, ] @@ -100,759 +60,23 @@ def collect_server_logs(output_dir): def request_json(method, url, timeout, **kwargs): - """Perform a bounded HTTP request and parse the JSON response when present.""" - deadline = kwargs.pop("wall_clock_deadline", None) - if deadline is None: - deadline = time.monotonic() + timeout - unsupported_options = set(kwargs) - {"headers", "json"} - if unsupported_options: - raise CapabilityValidationError("JSON request options are invalid") + """Perform an HTTP request and parse the JSON response when present.""" auth_headers = _auth_headers() headers = {**(kwargs.get("headers") or {}), **auth_headers} - response = _execute_http_worker( - method, - url, - timeout, - headers, - kwargs.get("json"), - "json", - deadline, - "HTTP response", - ) + request_kwargs = {**kwargs, "headers": headers} + response = requests.request(method, url, timeout=timeout, **request_kwargs) body = {} if response.content: try: - body = response.json(object_pairs_hook=reject_duplicate_json_object) + body = response.json() except ValueError: body = {"raw_text": response.text} - if time.monotonic() >= deadline: - raise CapabilityValidationError( - "HTTP response exceeded its wall-clock deadline" - ) return response, body -class _BufferedResponse: - def __init__(self, status_code, headers, encoding, content): - self.status_code = status_code - self.headers = headers - self.encoding = encoding - self.content = content - - @property - def text(self): - encoding = self.encoding or "utf-8" - try: - return self.content.decode(encoding, errors="replace") - except LookupError: - return self.content.decode("utf-8", errors="replace") - - def json(self, **kwargs): - return json.loads(self.text, **kwargs) - - def close(self): - return None - - -def _encode_stream_worker_output( - status_code, - body, - headers=None, - encoding=None, -): - header = json.dumps( - { - "version": STREAM_WORKER_PROTOCOL_VERSION, - "status_code": status_code, - "body_bytes": len(body), - "headers": headers or {}, - "encoding": encoding, - }, - allow_nan=False, - separators=(",", ":"), - ).encode("ascii") - if len(header) > MAX_STREAM_WORKER_HEADER_BYTES: - raise CapabilityValidationError("HTTP worker header exceeds bounds") - return STREAM_WORKER_MAGIC + header + b"\n" + body - - -def _decode_http_worker_output(output): - maximum_output_bytes = ( - len(STREAM_WORKER_MAGIC) - + MAX_STREAM_WORKER_HEADER_BYTES - + 1 - + MAX_STREAM_SSE_BYTES - ) - if not isinstance(output, bytes) or len(output) > maximum_output_bytes: - raise CapabilityValidationError("HTTP worker output exceeds bounds") - if not output.startswith(STREAM_WORKER_MAGIC): - raise CapabilityValidationError("HTTP worker output has invalid framing") - raw_header, separator, body = output[len(STREAM_WORKER_MAGIC) :].partition(b"\n") - if not separator or len(raw_header) > MAX_STREAM_WORKER_HEADER_BYTES: - raise CapabilityValidationError("HTTP worker output has invalid framing") - try: - header = json.loads( - raw_header.decode("ascii"), - object_pairs_hook=reject_duplicate_json_object, - ) - except (UnicodeError, json.JSONDecodeError) as exc: - raise CapabilityValidationError( - "HTTP worker output has an invalid header" - ) from exc - if not isinstance(header, dict) or set(header) != { - "version", - "status_code", - "body_bytes", - "headers", - "encoding", - }: - raise CapabilityValidationError("HTTP worker output has an invalid header") - status_code = header["status_code"] - body_bytes = header["body_bytes"] - headers = header["headers"] - encoding = header["encoding"] - if ( - header["version"] != STREAM_WORKER_PROTOCOL_VERSION - or not isinstance(status_code, int) - or isinstance(status_code, bool) - or not 100 <= status_code <= 599 - or not isinstance(body_bytes, int) - or isinstance(body_bytes, bool) - or body_bytes != len(body) - or body_bytes > MAX_STREAM_SSE_BYTES - or not isinstance(headers, dict) - or not all( - isinstance(key, str) and isinstance(value, str) - for key, value in headers.items() - ) - or not (encoding is None or isinstance(encoding, str)) - ): - raise CapabilityValidationError("HTTP worker output has an invalid header") - return _BufferedResponse(status_code, headers, encoding, body) - - -def _decode_stream_worker_output(output): - response = _decode_http_worker_output(output) - return response, _split_stream_lines(response.content) - - -def _split_stream_lines(body): - lines = [] - event_count = 0 - line_start = 0 - while line_start < len(body): - newline = body.find(b"\n", line_start) - if newline < 0: - line = body[line_start:] - line_start = len(body) - else: - line = body[line_start:newline] - line_start = newline + 1 - if len(line) > MAX_STREAM_SSE_LINE_BYTES: - raise CapabilityValidationError("stream SSE line exceeds bounds") - stripped = line.strip() - if not stripped or stripped.startswith(b":"): - continue - if not stripped.startswith(b"data:"): - raise CapabilityValidationError("stream line is not an SSE data frame") - event_count += 1 - if event_count > MAX_STREAM_EVENTS: - raise CapabilityValidationError("stream event count exceeds bounds") - lines.append(line) - return tuple(lines) - - -def _iter_bounded_response_lines(response, max_response_bytes): - total_bytes = 0 - pending = bytearray() - for chunk in response.iter_content( - chunk_size=STREAM_WORKER_CHUNK_BYTES, - decode_unicode=False, - ): - if not isinstance(chunk, bytes): - raise CapabilityValidationError("HTTP response chunk must be bytes") - total_bytes += len(chunk) - if total_bytes > max_response_bytes: - raise CapabilityValidationError( - "model pull SSE wire byte count exceeds bounds" - ) - start = 0 - while start < len(chunk): - newline = chunk.find(b"\n", start) - end = len(chunk) if newline < 0 else newline - segment = chunk[start:end] - if len(pending) + len(segment) > MAX_PULL_SSE_LINE_BYTES: - raise CapabilityValidationError("model pull SSE line exceeds bounds") - pending.extend(segment) - if newline < 0: - break - line = bytes(pending) - pending.clear() - if line.endswith(b"\r"): - line = line[:-1] - yield line - start = newline + 1 - if pending: - raise CapabilityValidationError("model pull SSE ended with an incomplete line") - - -def _reject_non_finite_json(value): - raise ValueError(f"non-finite JSON value: {value}") - - -def _parse_pull_event_data(data): - try: - payload = json.loads( - data.decode("utf-8"), - object_pairs_hook=reject_duplicate_json_object, - parse_constant=_reject_non_finite_json, - ) - except (UnicodeError, ValueError) as exc: - raise CapabilityValidationError("model pull SSE event data is invalid") from exc - if not isinstance(payload, dict): - raise CapabilityValidationError("model pull SSE event data is invalid") - return payload - - -def _pull_content_type(response): - for key, value in response.headers.items(): - if str(key).lower() == "content-type": - return str(value).split(";", maxsplit=1)[0].strip().lower() - return "" - - -def _bounded_pull_error(error): - encoded = str(error).encode("utf-8", errors="replace")[:1000] - return encoded.decode("utf-8", errors="ignore") - - -def _reduce_pull_sse_response(response, max_response_bytes): - if _pull_content_type(response) != "text/event-stream": - raise CapabilityValidationError( - "model pull returned an invalid SSE content type" - ) - - event_name = None - event_data = bytearray() - data_seen = False - event_count = 0 - terminal = None - - for line in _iter_bounded_response_lines(response, max_response_bytes): - if not line: - if event_name is None and not data_seen: - continue - event_count += 1 - if event_count > MAX_PULL_SSE_EVENTS: - raise CapabilityValidationError( - "model pull SSE event count exceeds bounds" - ) - if terminal is not None: - raise CapabilityValidationError( - "model pull SSE contained an event after its terminal event" - ) - if event_name not in {"progress", "complete", "error"} or not data_seen: - raise CapabilityValidationError("model pull SSE event is invalid") - payload = _parse_pull_event_data(bytes(event_data)) - if event_name == "complete": - terminal = {"terminal": "complete"} - elif event_name == "error": - error = payload.get("error") - code = payload.get("code") - if ( - not isinstance(error, str) - or not error - or not (code is None or isinstance(code, str)) - ): - raise CapabilityValidationError( - "model pull SSE error event is invalid" - ) - message = _bounded_pull_error(error) - terminal = { - "terminal": "error", - "status_code": ( - 400 - if code == "unknown_model" or "unknown_model" in message - else 500 - ), - "error": message, - } - event_name = None - event_data.clear() - data_seen = False - continue - - if terminal is not None: - raise CapabilityValidationError( - "model pull SSE contained data after its terminal event" - ) - if line.startswith(b"event:"): - if event_name is not None: - raise CapabilityValidationError("model pull SSE event is invalid") - raw_name = line[len(b"event:") :] - if raw_name.startswith(b" "): - raw_name = raw_name[1:] - try: - event_name = raw_name.decode("ascii") - except UnicodeError as exc: - raise CapabilityValidationError( - "model pull SSE event is invalid" - ) from exc - elif line.startswith(b"data:"): - raw_data = line[len(b"data:") :] - if raw_data.startswith(b" "): - raw_data = raw_data[1:] - additional_bytes = len(raw_data) + (1 if data_seen else 0) - if len(event_data) + additional_bytes > MAX_PULL_SSE_EVENT_DATA_BYTES: - raise CapabilityValidationError( - "model pull SSE event data exceeds bounds" - ) - if data_seen: - event_data.extend(b"\n") - event_data.extend(raw_data) - data_seen = True - else: - raise CapabilityValidationError("model pull SSE line is invalid") - - if event_name is not None or data_seen: - raise CapabilityValidationError("model pull SSE ended with an incomplete event") - if terminal is None: - raise CapabilityValidationError("model pull SSE ended without a terminal event") - summary = json.dumps( - terminal, - allow_nan=False, - separators=(",", ":"), - ).encode("utf-8") - if len(summary) > MAX_PULL_TERMINAL_SUMMARY_BYTES: - raise CapabilityValidationError("model pull terminal summary exceeds bounds") - return summary - - -def _buffer_response_body(response, max_response_bytes, failure_message): - buffered_body = bytearray() - for chunk in response.iter_content( - chunk_size=STREAM_WORKER_CHUNK_BYTES, - decode_unicode=False, - ): - if not isinstance(chunk, bytes): - raise CapabilityValidationError("HTTP response chunk must be bytes") - if len(buffered_body) + len(chunk) > max_response_bytes: - raise CapabilityValidationError(failure_message) - buffered_body.extend(chunk) - return bytes(buffered_body) - - -def _load_stream_worker_request(): - raw_request = sys.stdin.buffer.read(MAX_STREAM_WORKER_INPUT_BYTES + 1) - if len(raw_request) > MAX_STREAM_WORKER_INPUT_BYTES: - raise CapabilityValidationError("HTTP worker input exceeds bounds") - try: - request = json.loads( - raw_request.decode("utf-8"), - object_pairs_hook=reject_duplicate_json_object, - ) - except (UnicodeError, json.JSONDecodeError) as exc: - raise CapabilityValidationError("HTTP worker input is invalid") from exc - if not isinstance(request, dict) or set(request) != { - "version", - "response_kind", - "max_response_bytes", - "method", - "url", - "timeout", - "headers", - "json", - }: - raise CapabilityValidationError("HTTP worker input is invalid") - response_kind = request["response_kind"] - max_response_bytes = request["max_response_bytes"] - method = request["method"] - url = request["url"] - timeout = request["timeout"] - headers = request["headers"] - payload = request["json"] - if ( - request["version"] != STREAM_WORKER_PROTOCOL_VERSION - or response_kind not in ("json", "stream", "pull_sse") - or not isinstance(max_response_bytes, int) - or isinstance(max_response_bytes, bool) - or not 1 - <= max_response_bytes - <= ( - MAX_PULL_SSE_WIRE_BYTES - if response_kind == "pull_sse" - else MAX_STREAM_SSE_BYTES - ) - or not isinstance(method, str) - or not method - or not isinstance(url, str) - or not url - or not isinstance(timeout, (int, float)) - or isinstance(timeout, bool) - or timeout <= 0 - or not isinstance(headers, dict) - or not all( - isinstance(key, str) and isinstance(value, str) - for key, value in headers.items() - ) - or not (payload is None or isinstance(payload, dict)) - ): - raise CapabilityValidationError("HTTP worker input is invalid") - if response_kind in {"stream", "pull_sse"} and not isinstance(payload, dict): - raise CapabilityValidationError("HTTP worker input is invalid") - return ( - response_kind, - method, - url, - timeout, - headers, - payload, - max_response_bytes, - ) - - -def _run_stream_worker(): - response_kind, method, url, timeout, headers, payload, max_response_bytes = ( - _load_stream_worker_request() - ) - request_options = { - "timeout": timeout, - "headers": headers, - "stream": True, - } - if payload is not None: - request_options["json"] = payload - session = requests.Session() - session.trust_env = False - try: - response = session.request(method, url, **request_options) - try: - if response_kind == "pull_sse" and response.status_code == 200: - body = _reduce_pull_sse_response(response, max_response_bytes) - else: - response_bound = ( - min(max_response_bytes, MAX_PULL_RESPONSE_BYTES) - if response_kind == "pull_sse" - else max_response_bytes - ) - failure_message = ( - "stream SSE byte count exceeds bounds" - if response_kind == "stream" - else "HTTP response byte count exceeds bounds" - ) - body = _buffer_response_body( - response, - response_bound, - failure_message, - ) - finally: - response.close() - output = _encode_stream_worker_output( - response.status_code, - body, - dict(response.headers), - response.encoding, - ) - sys.stdout.buffer.write(output) - sys.stdout.buffer.flush() - finally: - session.close() - - -def _loopback_pull_url(host, port): - if host not in {"127.0.0.1", "localhost", "::1"}: - raise CapabilityValidationError("model pull host must be loopback") - if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535: - raise CapabilityValidationError("model pull port is invalid") - authority = f"[{host}]" if host == "::1" else host - return f"http://{authority}:{port}/api/v1/pull" - - -def _stream_worker_main(): - try: - _run_stream_worker() - except BaseException as exc: # pylint: disable=broad-exception-caught - error = f"{exc.__class__.__name__}: {exc}".encode("utf-8", errors="replace") - sys.stderr.buffer.write(error[:MAX_STREAM_WORKER_ERROR_BYTES]) - sys.stderr.buffer.flush() - return 1 - return 0 - - -def _kill_and_reap_stream_worker(process): - try: - if process.poll() is None: - process.kill() - finally: - process.communicate() - - -def _encode_stream_worker_request( - method, - url, - timeout, - headers, - payload, - response_kind="stream", - max_response_bytes=MAX_STREAM_SSE_BYTES, -): - try: - encoded = json.dumps( - { - "version": STREAM_WORKER_PROTOCOL_VERSION, - "response_kind": response_kind, - "max_response_bytes": max_response_bytes, - "method": method, - "url": url, - "timeout": timeout, - "headers": headers, - "json": payload, - }, - allow_nan=False, - separators=(",", ":"), - ).encode("utf-8") - except (TypeError, ValueError) as exc: - raise CapabilityValidationError("HTTP request cannot be encoded") from exc - if len(encoded) > MAX_STREAM_WORKER_INPUT_BYTES: - raise CapabilityValidationError("HTTP worker input exceeds bounds") - return encoded - - -def _execute_http_worker( - method, - url, - timeout, - headers, - payload, - response_kind, - deadline, - deadline_subject, - max_response_bytes=MAX_STREAM_SSE_BYTES, -): - worker_input = _encode_stream_worker_request( - method, - url, - timeout, - headers, - payload, - response_kind, - max_response_bytes, - ) - deadline_message = f"{deadline_subject} exceeded its wall-clock deadline" - remaining = deadline - time.monotonic() - if remaining <= 0: - raise CapabilityValidationError(deadline_message) - process = None - worker_reaped = False - try: - process = subprocess.Popen( - [ - sys.executable, - os.path.realpath(__file__), - INTERNAL_STREAM_WORKER_ARGUMENT, - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - remaining = deadline - time.monotonic() - if remaining <= 0: - raise CapabilityValidationError(deadline_message) - try: - output, error = process.communicate(input=worker_input, timeout=remaining) - except subprocess.TimeoutExpired as exc: - raise CapabilityValidationError(deadline_message) from exc - worker_reaped = True - if time.monotonic() >= deadline: - raise CapabilityValidationError(deadline_message) - if process.returncode != 0: - detail = error[:MAX_STREAM_WORKER_ERROR_BYTES].decode( - "utf-8", errors="replace" - ) - raise CapabilityValidationError( - f"{deadline_subject} transport worker failed: " - f"{detail or 'unknown error'}" - ) - response = _decode_http_worker_output(output) - if time.monotonic() >= deadline: - raise CapabilityValidationError(deadline_message) - return response - except BaseException: # pylint: disable=broad-exception-caught - if process is not None and not worker_reaped: - _kill_and_reap_stream_worker(process) - raise - - -def request_stream(method, url, timeout, **kwargs): - """Perform a bounded streaming HTTP request and return raw response lines.""" - deadline = kwargs.pop("wall_clock_deadline", None) - if deadline is None: - deadline = time.monotonic() + min( - timeout, - STREAM_WALL_CLOCK_TIMEOUT_SECONDS, - ) - unsupported_options = set(kwargs) - {"headers", "json"} - if unsupported_options or not isinstance(kwargs.get("json"), dict): - raise CapabilityValidationError("stream request options are invalid") - auth_headers = _auth_headers() - headers = {**(kwargs.get("headers") or {}), **auth_headers} - response = _execute_http_worker( - method, - url, - timeout, - headers, - kwargs["json"], - "stream", - deadline, - "stream", - ) - response_and_lines = response, _split_stream_lines(response.content) - if time.monotonic() >= deadline: - raise CapabilityValidationError("stream exceeded its wall-clock deadline") - return response_and_lines - - -def wait_for_server(port=PORT, timeout=TIMEOUT_HEALTH, host="localhost"): - """Wait for a server while bounding every loopback connection attempt.""" - deadline = time.monotonic() + timeout - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError(f"Server failed to start within {timeout} seconds") - try: - connection = socket.create_connection( - (host, port), - timeout=min(SERVER_CONNECT_ATTEMPT_TIMEOUT_SECONDS, remaining), - ) - connection.close() - return True - except OSError: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError( - f"Server failed to start within {timeout} seconds" - ) from None - time.sleep(min(SERVER_CONNECT_RETRY_DELAY_SECONDS, remaining)) - - -def unload_all_models(port=PORT, attempts=3, host="localhost"): - """Unload all models through the bounded HTTP worker.""" - response = None - for attempt in range(1, attempts + 1): - try: - response, _body = request_json( - "POST", - f"http://{host}:{port}/api/v1/unload", - timeout=TIMEOUT_DEFAULT, - json={}, - ) - if response.status_code in (200, 404): - return response - except (requests.RequestException, CapabilityValidationError): - if attempt == attempts: - raise - if attempt < attempts: - time.sleep(1) - return response - - -def _is_transient_pull_status(status_code): - return status_code in {408, 409, 429, 500, 502, 503, 504} - - -def _pull_model_streaming( - model_name, - port, - host=VALIDATION_HOST, - *, - wall_clock_deadline=None, -): - if wall_clock_deadline is None: - wall_clock_deadline = time.monotonic() + PULL_WALL_CLOCK_TIMEOUT_SECONDS - pull_url = _loopback_pull_url(host, port) - response = _execute_http_worker( - "POST", - pull_url, - TIMEOUT_INFERENCE, - _auth_headers(), - {"model_name": model_name, "stream": True, "subscribe": True}, - "pull_sse", - wall_clock_deadline, - "model pull", - MAX_PULL_SSE_WIRE_BYTES, - ) - if response.status_code != 200: - return response.status_code, response.text[:1000] - try: - summary = response.json( - object_pairs_hook=reject_duplicate_json_object, - parse_constant=_reject_non_finite_json, - ) - except (UnicodeError, ValueError) as exc: - raise CapabilityValidationError( - "model pull worker returned an invalid terminal summary" - ) from exc - if summary == {"terminal": "complete"}: - return 200, "" - if ( - isinstance(summary, dict) - and set(summary) == {"terminal", "status_code", "error"} - and summary["terminal"] == "error" - and summary["status_code"] in {400, 500} - and not isinstance(summary["status_code"], bool) - and isinstance(summary["error"], str) - and summary["error"] - ): - return summary["status_code"], summary["error"] - raise CapabilityValidationError( - "model pull worker returned an invalid terminal summary" - ) - - -def pull_model_with_retry(model_name, attempts=3, port=PORT, host=VALIDATION_HOST): - """Pull a model with bounded retry for transient setup failures.""" - last_status = None - last_body = "" - wall_clock_deadline = time.monotonic() + PULL_WALL_CLOCK_TIMEOUT_SECONDS - - for attempt in range(1, attempts + 1): - if attempt > 1: - remaining = wall_clock_deadline - time.monotonic() - if remaining <= 0: - raise CapabilityValidationError( - "model pull exceeded its wall-clock deadline" - ) - time.sleep(min(30, 2 ** (attempt - 1), remaining)) - - status, body = _pull_model_streaming( - model_name, - port, - host=host, - wall_clock_deadline=wall_clock_deadline, - ) - - if status == 200: - return - - last_status = status - last_body = body - if _is_transient_pull_status(status) and attempt < attempts: - print( - f"Transient /pull setup failure for {model_name}: " - f"status={status}, attempt={attempt}/{attempts}. Retrying..." - ) - continue - break - - raise AssertionError( - f"Expected 200 from /api/v1/pull for {model_name} after " - f"{attempts} attempt(s), got {last_status}. Body: {last_body}" - ) - - def require_running_server(base_url, port): """Wait for a running server and confirm the health endpoint responds.""" - wait_for_server(port=port, timeout=TIMEOUT_HEALTH, host=VALIDATION_HOST) + wait_for_server(port=port, timeout=TIMEOUT_HEALTH) response, body = request_json( "GET", f"{base_url}/health", @@ -868,8 +92,8 @@ def require_running_server(base_url, port): ) -def get_model_catalog(base_url): - """Return all model catalog entries from the API.""" +def get_hot_llamacpp_models(base_url): + """Return the catalog entries for hot llamacpp models from the API.""" response, body = request_json( "GET", f"{base_url}/models?show_all=true", @@ -880,29 +104,14 @@ def get_model_catalog(base_url): f"Failed to query model catalog: HTTP {response.status_code} - {body}" ) - return body.get("data", []) + hot_models = [] + for model in body.get("data", []): + labels = model.get("labels", []) + if model.get("recipe") == "llamacpp" and "hot" in labels: + hot_models.append(model) - -def get_builtin_model(base_url, canonical_model_id): - """Resolve one canonical built-in model from the API.""" - encoded_model_id = quote(canonical_model_id, safe="") - response, body = request_json( - "GET", - f"{base_url}/models/{encoded_model_id}", - timeout=TIMEOUT_DEFAULT, - ) - if response.status_code == 404: - return None - if response.status_code != 200: - raise RuntimeError( - f"Failed to query built-in model '{canonical_model_id}': " - f"HTTP {response.status_code} - {body}" - ) - if not isinstance(body, dict): - raise RuntimeError( - f"Built-in model '{canonical_model_id}' returned an invalid response" - ) - return body + hot_models.sort(key=lambda model: model["id"]) + return hot_models def set_rocm_channel(base_url, channel): @@ -949,31 +158,13 @@ def unload_model(base_url, model_name=None): json=payload, ) if response.status_code not in (200, 404): - raise RuntimeError(f"Model unload failed: HTTP {response.status_code} - {body}") - - -def require_all_models_unloaded(port, phase): - """Unload every model and reject an unsuccessful cleanup response.""" - response = unload_all_models(port=port, host=VALIDATION_HOST) - if response is None or response.status_code not in (200, 404): - status = "no response" if response is None else f"HTTP {response.status_code}" - raise RuntimeError(f"{phase} model unload failed: {status}") - - -def canonical_model_id(model_name): - """Return the canonical built-in identity used at API boundaries.""" - return model_name if model_name.startswith("builtin.") else f"builtin.{model_name}" + print( + f" Warning: unload returned HTTP {response.status_code}: {body}", + flush=True, + ) -def test_model( - base_url, - model_name, - backend, - max_tokens=50, - *, - capability_profile="", - capability_evidence=None, -): +def test_model(base_url, model_name, backend, max_tokens=50): """Send a chat/completions request and return (success, response_text, stats).""" print(f" Loading model: {model_name} (backend={backend})", flush=True) try: @@ -981,11 +172,7 @@ def test_model( "POST", f"{base_url}/load", timeout=TIMEOUT_INFERENCE, - json={ - "model_name": model_name, - "llamacpp_backend": backend, - "ctx_size": VALIDATION_CTX_SIZE, - }, + json={"model_name": model_name, "llamacpp_backend": backend}, ) if load_resp.status_code != 200: return ( @@ -999,68 +186,20 @@ def test_model( f"{base_url}/health", timeout=TIMEOUT_DEFAULT, ) - if health_resp.status_code != 200: - return ( - False, - f"Loaded-model health inventory returned HTTP " - f"{health_resp.status_code}: {health_body}", - {}, - ) - if not isinstance(health_body, dict): - return False, "Loaded-model health response is not an object", {} - if "all_models_loaded" not in health_body: - return ( - False, - "Loaded-model health response is missing all_models_loaded", - {}, - ) - loaded_models = health_body["all_models_loaded"] - if not isinstance(loaded_models, list) or not all( - isinstance(model, dict) and isinstance(model.get("model_name"), str) - for model in loaded_models - ): - return ( - False, - "Loaded-model health response has an invalid model inventory", - {}, - ) - - requested_model_id = canonical_model_id(model_name) - requested_models = [ - model - for model in loaded_models - if canonical_model_id(model["model_name"]) == requested_model_id - ] - if not requested_models: - return ( - False, - f"Model '{model_name}' is absent from the loaded inventory", - {}, - ) - if len(loaded_models) != 1: - inventory = ", ".join(model["model_name"] for model in loaded_models) - return ( - False, - f"Loaded inventory must contain exactly the requested model " - f"'{model_name}', found: {inventory or ''}", - {}, - ) - loaded_model = requested_models[0] - - if "recipe_options" not in loaded_model: - return False, f"Loaded model '{model_name}' is missing recipe_options", {} - recipe_options = loaded_model["recipe_options"] - if not isinstance(recipe_options, dict): - return False, f"Loaded model '{model_name}' has invalid recipe_options", {} - actual_backend = recipe_options.get("llamacpp_backend") - if not isinstance(actual_backend, str) or not actual_backend: - return False, f"Loaded model '{model_name}' is missing llamacpp_backend", {} - if actual_backend != backend: - return ( - False, - f"Model loaded with backend '{actual_backend}' instead of '{backend}'", - {}, - ) + if health_resp.status_code == 200: + loaded = { + model["model_name"]: model + for model in health_body.get("all_models_loaded", []) + } + loaded_model = loaded.get(model_name, {}) + recipe_options = loaded_model.get("recipe_options", {}) + actual_backend = recipe_options.get("llamacpp_backend") + if actual_backend and actual_backend != backend: + return ( + False, + f"Model loaded with backend '{actual_backend}' instead of '{backend}'", + {}, + ) print(" Sending chat/completions request...", flush=True) chat_resp, chat_body = request_json( @@ -1071,54 +210,17 @@ def test_model( "model": model_name, "messages": CHAT_PROMPT, "max_completion_tokens": max_tokens, - "chat_template_kwargs": {"enable_thinking": False}, }, ) if chat_resp.status_code != 200: return False, f"HTTP {chat_resp.status_code}: {chat_body}", {} - response_model = chat_body.get("model") - if not isinstance(response_model, str) or ( - canonical_model_id(response_model) != requested_model_id - ): - return ( - False, - f"Chat response model '{response_model}' does not match " - f"requested model '{model_name}'", - {}, - ) - message = chat_body["choices"][0]["message"] - marker = find_raw_ifm_control_marker(message, "message") - if marker: - marker_text, marker_path = marker - return ( - False, - f"Raw IFM control marker {marker_text!r} found in " - f"response field {marker_path}", - {}, - ) - - content = message.get("content") - if not isinstance(content, str) or not content.strip(): - return False, "Plain chat response has no visible final content", {} - - if capability_profile: - if not isinstance(capability_evidence, dict): - return False, "Capability evidence output is required", {} - matrix, capabilities_passed, capability_summary = validate_capabilities( - capability_profile, - base_url=base_url, - model=model_name, - request_json=request_json, - request_stream=request_stream, - timeout=TIMEOUT_INFERENCE, - ) - capability_evidence.clear() - capability_evidence.update(matrix) - if not capabilities_passed: - return False, f"Capability validation failed: {capability_summary}", {} - print(f" Capabilities: {capability_summary}", flush=True) + content = message.get("content") or "" + reasoning = message.get("reasoning_content") or "" + combined = content + reasoning + if not combined: + return False, "Empty response (no content or reasoning_content)", {} stats = {} stats_resp, stats_body = request_json( @@ -1135,82 +237,22 @@ def test_model( flush=True, ) - return True, content, stats + response_text = content if content else f"[reasoning] {reasoning}" + return True, response_text, stats except (KeyError, IndexError, TypeError, ValueError) as exc: return False, f"Bad response format: {exc}", {} except requests.RequestException as exc: return False, f"Request failed: {exc}", {} finally: - unload_model(base_url, model_name) - - -def validate_model_lifecycle( - base_url, - model_name, - backend, - reload_after_unload=False, - *, - capability_profile="", - capability_evidence=None, -): - """Validate one model and optionally validate a second load after unload.""" - capability_args = {} - if capability_profile: - capability_args = { - "capability_profile": capability_profile, - "capability_evidence": capability_evidence, - } - result = test_model(base_url, model_name, backend, **capability_args) - if not reload_after_unload or not result[0]: - return result - - try: - health_response, health_body = request_json( - "GET", - f"{base_url}/health", - timeout=TIMEOUT_DEFAULT, - ) - except requests.RequestException as exc: - return False, f"Could not verify unload before reload: {exc}", result[2] - - if health_response.status_code != 200: - return ( - False, - f"Could not verify unload before reload: HTTP " - f"{health_response.status_code} - {health_body}", - result[2], - ) - - if not isinstance(health_body, dict): - return False, "Could not verify unload: invalid health response", result[2] - loaded_models = health_body.get("all_models_loaded") - if not isinstance(loaded_models, list) or not all( - isinstance(model, dict) and isinstance(model.get("model_name"), str) - for model in loaded_models - ): - return False, "Could not verify unload: invalid model inventory", result[2] - - if loaded_models: - remaining_models = ", ".join(model["model_name"] for model in loaded_models) - return ( - False, - f"Models are still loaded after unloading '{model_name}': " - f"{remaining_models}", - result[2], - ) - - print(" Reloading model after unload...", flush=True) - return test_model(base_url, model_name, backend) - - -def write_results_file(output_path, results): - with open(output_path, "x", encoding="utf-8") as output_file: - json.dump(results, output_file, indent=2, allow_nan=False) + try: + unload_model(base_url, model_name) + except requests.RequestException as exc: + print(f" Warning: failed to unload {model_name}: {exc}", flush=True) def main(): parser = argparse.ArgumentParser( - description="Validate a llama.cpp backend against selected Lemonade models" + description="Validate a llama.cpp backend against hot Lemonade models" ) parser.add_argument( "--backend", @@ -1247,144 +289,64 @@ def main(): action="store_true", help="Skip backend installation step", ) - parser.add_argument( - "--prepare-only", - action="store_true", - help="Install the backend and pull selected models without inference", - ) parser.add_argument( "--logs-dir", default=None, help="Directory to collect server log files into (for CI artifact upload)", ) parser.add_argument( - "--capability-profile", - choices=[K2_HORIZON_PROFILE], - default="", - help="Optional API capability contract to validate", - ) - parser.add_argument( - "--capability-model", - action="append", - default=[], - help="Selected validation model that must run the capability profile", + "--lite", + action="store_true", + help="Lite mode: only test the smallest hot model", ) - add_model_selection_arguments(parser) args = parser.parse_args() - has_capability_profile = bool(args.capability_profile) - has_capability_models = bool(args.capability_model) - if has_capability_profile != has_capability_models: - parser.error( - "--capability-profile and --capability-model must be supplied together" - ) - - canonical_capability_models = [ - model_id.removeprefix("builtin.") for model_id in args.capability_model - ] - if len(set(canonical_capability_models)) != len(canonical_capability_models): - parser.error("--capability-model values must be unique") - # Label used for output filenames and artifact names label = f"{args.backend}-{args.channel}" if args.channel else args.backend - base_url = f"http://{VALIDATION_HOST}:{args.port}/api/v1" + base_url = f"http://localhost:{args.port}/api/v1" output_path = args.output or f"llamacpp_validation_{label}.json" require_running_server(base_url, args.port) - catalog = get_model_catalog(base_url) if not args.model else [] - try: - selected_models = select_llamacpp_models( - catalog, - requested_model_ids=args.model, - lite=args.lite, - builtin_model_resolver=lambda model_id: get_builtin_model( - base_url, model_id - ), - ) - except ModelSelectionError as exc: - print(f"ERROR: {exc}", file=sys.stderr, flush=True) - sys.exit(1) - - selected_model_ids = { - canonical_model_id(model.get("load_id", model["id"])).removeprefix("builtin.") - for model in selected_models - } - missing_capability_models = sorted( - set(canonical_capability_models) - selected_model_ids - ) - if missing_capability_models: - print( - "ERROR: Capability models are absent from the selected validation set: " - + ", ".join(missing_capability_models), - file=sys.stderr, - flush=True, - ) - sys.exit(1) - capability_model_ids = set(canonical_capability_models) - if args.channel: set_rocm_channel(base_url, args.channel) print("Unloading all models for clean state...", flush=True) - require_all_models_unloaded(args.port, "Pre-validation clean state") + try: + unload_all_models(port=args.port) + except requests.RequestException as exc: + print(f"Warning: failed to unload pre-existing models: {exc}", flush=True) - selection_mode = "explicitly selected" if args.model else "hot" - print( - f"Found {len(selected_models)} {selection_mode} llamacpp models:", - flush=True, - ) - for model in selected_models: + hot_models = get_hot_llamacpp_models(base_url) + print(f"Found {len(hot_models)} hot llamacpp models:", flush=True) + for model in hot_models: print(f" - {model['id']} ({model.get('size', '?')} GB)", flush=True) - if args.lite: - selected = selected_models[0] + if not hot_models: + print("ERROR: No hot llamacpp models found!", file=sys.stderr, flush=True) + sys.exit(1) + + if args.lite and len(hot_models) > 1: + smallest = min(hot_models, key=lambda m: m.get("size", float("inf"))) print( f"Lite mode: testing only smallest model: " - f"{selected['id']} ({selected.get('size', '?')} GB)", + f"{smallest['id']} ({smallest.get('size', '?')} GB)", flush=True, ) + hot_models = [smallest] if not args.skip_install: install_backend(base_url, args.backend) - if args.prepare_only: - try: - for model in selected_models: - model_name = model.get("load_id", model["id"]) - print(f"Pulling model: {model_name}", flush=True) - pull_model_with_retry( - model_name, - port=args.port, - host=VALIDATION_HOST, - ) - finally: - if args.logs_dir: - collect_server_logs(args.logs_dir) - return - results = [] all_passed = True try: - for model in selected_models: + for model in hot_models: model_name = model["id"] - load_id = model.get("load_id", model_name) print(f"\nTesting: {model_name}", flush=True) - runs_capability_profile = ( - canonical_model_id(load_id).removeprefix("builtin.") - in capability_model_ids - ) - capability_evidence = {} if runs_capability_profile else None - success, response_text, stats = validate_model_lifecycle( - base_url, - load_id, - args.backend, - reload_after_unload=bool(args.model), - capability_profile=( - args.capability_profile if runs_capability_profile else "" - ), - capability_evidence=capability_evidence, + success, response_text, stats = test_model( + base_url, model_name, args.backend ) result = { "model": model_name, @@ -1395,8 +357,6 @@ def main(): "time_to_first_token": stats.get("time_to_first_token", "N/A"), "tokens_per_second": stats.get("tokens_per_second", "N/A"), } - if runs_capability_profile: - result["capability_matrix"] = capability_evidence results.append(result) status = "PASS" if success else "FAIL" print(f" Result: {status}", flush=True) @@ -1405,12 +365,14 @@ def main(): print(f" Error: {response_text}", flush=True) finally: try: - require_all_models_unloaded(args.port, "Final cleanup") - finally: - if args.logs_dir: - collect_server_logs(args.logs_dir) - - write_results_file(output_path, results) + unload_all_models(port=args.port) + except requests.RequestException as exc: + print(f"Warning: failed to unload models during cleanup: {exc}", flush=True) + if args.logs_dir: + collect_server_logs(args.logs_dir) + + with open(output_path, "w", encoding="utf-8") as output_file: + json.dump(results, output_file, indent=2) print(f"\nResults written to {output_path}", flush=True) print(f"\n{'=' * 60}", flush=True) @@ -1426,6 +388,4 @@ def main(): if __name__ == "__main__": - if sys.argv[1:] == [INTERNAL_STREAM_WORKER_ARGUMENT]: - sys.exit(_stream_worker_main()) main() From bfd1b6eabd07f8e0af9b86cd55e4814784a2de66 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:38:20 -0700 Subject: [PATCH 09/14] docs: remove IFM development validation caveat --- docs/embeddable/backends.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index 882d52c61f..6c06c3fe4f 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -225,10 +225,6 @@ curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/jso ``` These are custom models, not a claim of support in Lemonade's managed binaries. -IFM commit `35999d101cf2233fc54f09c3c8d599da7303ce02` was used for development -validation; its medium/low reasoning modes can leak control markers. Prefer high -reasoning and the default XML tool format, and verify any other modes against your -build. K2-Horizon Uno is not covered by this example. To return to Lemonade's managed Metal binary, unload the model and run `lemonade config set llamacpp.metal_bin=builtin` (use the corresponding `*_bin` From 9d372306887ede34bc1a27fcf77a8f945f381fca Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:22:19 -0700 Subject: [PATCH 10/14] docs: add complete macOS K2 setup --- docs/embeddable/backends.md | 58 +++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index 6c06c3fe4f..5cae1cab91 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -180,7 +180,35 @@ K2-Horizon can use the existing `llamacpp` recipe with a build of IFM's [`model/K2Horizon` branch](https://github.com/MBZUAI-IFM/llama.cpp/tree/model/K2Horizon). No Lemonade source changes or managed-backend version updates are required. -Build `llama-server` using the branch's [build instructions](https://github.com/MBZUAI-IFM/llama.cpp/blob/model/K2Horizon/docs/build.md) +The following is a complete macOS source-build example. From the Lemonade +source tree, check the dependencies and build the server and CLI: + +```bash +./setup.sh +cmake --preset default -DBUILD_WEB_APP=OFF -DBUILD_TESTING=OFF +cmake --build --preset default --target lemond lemonade -j +``` + +If `setup.sh` reaches the pre-commit installation step and exits because the +repository has `core.hooksPath` configured, that does not prevent the runtime +build after all dependency checks have passed; continue with the CMake commands. +This minimal build intentionally omits the desktop and web apps. The message +"This build of Lemonade has been built without a desktop app or a web app" at +the server root is expected and does not affect the REST API. + +To include the browser UI instead, configure with `BUILD_WEB_APP=ON`, build the +`web-app` target, and restart `lemond`: + +```bash +cmake --preset default -DBUILD_WEB_APP=ON -DBUILD_TESTING=OFF +cmake --build --preset default --target lemond lemonade web-app -j +``` + +The UI is then available at the server root. A native desktop app has additional +prerequisites; see the [application development guide](../dev/app.md). + +Build `llama-server` from the Lemonade source tree using the branch's +[build instructions](https://github.com/MBZUAI-IFM/llama.cpp/blob/model/K2Horizon/docs/build.md) for your accelerator. For example, on macOS with Metal: ```bash @@ -189,11 +217,21 @@ cmake -S llama.cpp-ifm -B llama.cpp-ifm/build -DGGML_METAL=ON -DCMAKE_BUILD_TYPE cmake --build llama.cpp-ifm/build --config Release --target llama-server -j ``` -With `lemond` running, select the matching backend and the **full executable path** -(not its directory). Keep the build's shared libraries alongside the executable. +Start Lemonade on an explicit port in one terminal: + +```bash +./build/lemond --port 13305 +``` + +In a second terminal, select the matching backend and the **full executable +path** (not its directory). Keep the build's shared libraries alongside the +executable. Supplying `--port` makes the source-built CLI use the same server +without relying on discovery. ```bash -lemonade config set llamacpp.backend=metal "llamacpp.metal_bin=$(pwd)/llama.cpp-ifm/build/bin/llama-server" +./build/lemonade --port 13305 config set \ + llamacpp.backend=metal \ + "llamacpp.metal_bin=$(pwd)/llama.cpp-ifm/build/bin/llama-server" ``` For a Vulkan build on Windows or Linux, use `llamacpp.backend=vulkan` and @@ -204,7 +242,9 @@ Unload any already-loaded model before switching binaries. Register and download a model through the existing [custom-model CLI](../guide/configuration/custom-models.md): ```bash -lemonade pull user.K2-Horizon-0.9B --recipe llamacpp --checkpoint main IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf +./build/lemonade --port 13305 pull user.K2-Horizon-0.9B \ + --recipe llamacpp \ + --checkpoint main IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf ``` For the larger models, substitute the corresponding name and checkpoint: @@ -218,14 +258,14 @@ For the larger models, substitute the corresponding name and checkpoint: Context and compute buffers require additional memory. Start with a bounded context: ```bash -curl http://localhost:8000/v1/load -H "Content-Type: application/json" \ +curl http://localhost:13305/v1/load -H "Content-Type: application/json" \ -d '{"model_name":"user.K2-Horizon-0.9B","ctx_size":4096}' -curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" \ +curl http://localhost:13305/v1/chat/completions -H "Content-Type: application/json" \ -d '{"model":"user.K2-Horizon-0.9B","messages":[{"role":"user","content":"What is 2 + 2?"}],"reasoning_effort":"high","max_tokens":1024}' ``` These are custom models, not a claim of support in Lemonade's managed binaries. To return to Lemonade's managed Metal binary, unload the model and run -`lemonade config set llamacpp.metal_bin=builtin` (use the corresponding `*_bin` -key for another backend). +`./build/lemonade --port 13305 config set llamacpp.metal_bin=builtin` (use the +corresponding `*_bin` key for another backend). From 8b3425e462feba1a7639800311d653a3d330e092 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:25:37 -0700 Subject: [PATCH 11/14] docs: list supported K2 models --- docs/embeddable/backends.md | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index 5cae1cab91..76a167b95d 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -239,21 +239,43 @@ For a Vulkan build on Windows or Linux, use `llamacpp.backend=vulkan` and under `build/bin/Release`. The override applies to all models using that backend. Unload any already-loaded model before switching binaries. -Register and download a model through the existing [custom-model CLI](../guide/configuration/custom-models.md): +Register and download all non-Uno models supported by the IFM llama.cpp branch +through the existing [custom-model CLI](../guide/configuration/custom-models.md): ```bash ./build/lemonade --port 13305 pull user.K2-Horizon-0.9B \ --recipe llamacpp \ --checkpoint main IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf + +./build/lemonade --port 13305 pull user.K2-Horizon-3.7B \ + --recipe llamacpp \ + --checkpoint main IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf + +./build/lemonade --port 13305 pull user.K2-Horizon-7B \ + --recipe llamacpp \ + --checkpoint main IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf + +./build/lemonade --port 13305 pull user.K2-Horizon-32B \ + --recipe llamacpp \ + --checkpoint main IFM/K2-Horizon-32B-GGUF:K2-Horizon-32B-BF16.gguf + +./build/lemonade --port 13305 pull user.K2-Horizon-MoVA-36B-A4B \ + --recipe llamacpp \ + --checkpoint main IFM/K2-Horizon-MoVA-36B-A4B-GGUF:K2-Horizon-36B-BF16.gguf ``` -For the larger models, substitute the corresponding name and checkpoint: +These BF16 weights require approximately 175 GB of disk space in total: + +| Name | BF16 weight size | +|---|---:| +| `user.K2-Horizon-0.9B` | 2.16 GB | +| `user.K2-Horizon-3.7B` | 10.13 GB | +| `user.K2-Horizon-7B` | 18.01 GB | +| `user.K2-Horizon-32B` | 69.57 GB | +| `user.K2-Horizon-MoVA-36B-A4B` | 74.92 GB | -| Name | Checkpoint | BF16 weight size | -|---|---|---:| -| `user.K2-Horizon-0.9B` | `IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf` | 2.16 GB | -| `user.K2-Horizon-3.7B` | `IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf` | 10.13 GB | -| `user.K2-Horizon-7B` | `IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf` | 18.01 GB | +The 375B model is intentionally not implemented in the IFM llama.cpp branch +and cannot be used through this integration. Context and compute buffers require additional memory. Start with a bounded context: From 6674946850e5c6a377593d4d3323ee8c24cecd30 Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:30:53 -0700 Subject: [PATCH 12/14] docs: record K2 context limits --- docs/embeddable/backends.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index 76a167b95d..e4e92c433c 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -264,28 +264,37 @@ through the existing [custom-model CLI](../guide/configuration/custom-models.md) --checkpoint main IFM/K2-Horizon-MoVA-36B-A4B-GGUF:K2-Horizon-36B-BF16.gguf ``` -These BF16 weights require approximately 175 GB of disk space in total: - -| Name | BF16 weight size | -|---|---:| -| `user.K2-Horizon-0.9B` | 2.16 GB | -| `user.K2-Horizon-3.7B` | 10.13 GB | -| `user.K2-Horizon-7B` | 18.01 GB | -| `user.K2-Horizon-32B` | 69.57 GB | -| `user.K2-Horizon-MoVA-36B-A4B` | 74.92 GB | +These BF16 weights require approximately 175 GB of disk space in total. The +maximum context windows come from the corresponding model cards and are also +stored in the GGUF metadata: + +| Name | BF16 weight size | Maximum context | +|---|---:|---:| +| `user.K2-Horizon-0.9B` | 2.16 GB | 131,072 tokens (128K) | +| `user.K2-Horizon-3.7B` | 10.13 GB | 524,288 tokens (512K) | +| `user.K2-Horizon-7B` | 18.01 GB | 524,288 tokens (512K) | +| `user.K2-Horizon-32B` | 69.57 GB | 524,288 tokens (512K) | +| `user.K2-Horizon-MoVA-36B-A4B` | 74.92 GB | 524,288 tokens (512K) | The 375B model is intentionally not implemented in the IFM llama.cpp branch and cannot be used through this integration. -Context and compute buffers require additional memory. Start with a bounded context: +The custom-model registration does not need a separate context-length option. +Use `ctx_size` when loading to select the runtime context window. For example, +load the 0.9B model at its full model-card limit: ```bash curl http://localhost:13305/v1/load -H "Content-Type: application/json" \ - -d '{"model_name":"user.K2-Horizon-0.9B","ctx_size":4096}' + -d '{"model_name":"user.K2-Horizon-0.9B","ctx_size":131072}' curl http://localhost:13305/v1/chat/completions -H "Content-Type: application/json" \ -d '{"model":"user.K2-Horizon-0.9B","messages":[{"role":"user","content":"What is 2 + 2?"}],"reasoning_effort":"high","max_tokens":1024}' ``` +For any other model in the table, use its model name and `"ctx_size":524288`. +These are model-supported maxima, not hardware recommendations. KV cache and +compute buffers require substantial additional memory; choose a smaller +`ctx_size` if the full window does not fit on the target system. + These are custom models, not a claim of support in Lemonade's managed binaries. To return to Lemonade's managed Metal binary, unload the model and run From 6d1c4f1f014a863f5980ebefc53213394f126abe Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:04:21 -0700 Subject: [PATCH 13/14] docs: clarify custom backend executable paths --- docs/guide/configuration/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/configuration/README.md b/docs/guide/configuration/README.md index 4d42ad5e38..ecdba83e6a 100644 --- a/docs/guide/configuration/README.md +++ b/docs/guide/configuration/README.md @@ -345,7 +345,7 @@ Every `*_bin` key (e.g. `llamacpp.vulkan_bin`, `whispercpp.cpu_bin`, `sdcpp.rocm | `""` | Same as `"builtin"`. | | `"latest"` | Resolve to the most-recent upstream GitHub release on first install or first status query for that backend, then install on demand. The resolved tag is recorded in `/bin///version.txt`. | | `"b8664"` / `"v1.8.2"` / etc. | A specific upstream release tag. Lemonade downloads that exact version from GitHub. | -| `"/path/to/bin"` | A directory you populated yourself (e.g. a local build). Lemonade uses the executable inside this directory and never downloads. The path must exist when set. | +| `"/path/to/backend-server"` | The full path to an executable you built or installed yourself. Lemonade runs this executable directly and never downloads a replacement. The path must exist when set. | > Note: the `latest` setting is experimental. @@ -361,7 +361,7 @@ lemonade config set llamacpp.vulkan_bin=latest lemonade config set llamacpp.vulkan_bin=b8664 # Use your own llama.cpp build -lemonade config set llamacpp.vulkan_bin=/home/me/llama.cpp/build/bin +lemonade config set llamacpp.vulkan_bin=/home/me/llama.cpp/build/bin/llama-server # Revert to the version lemonade ships lemonade config set llamacpp.vulkan_bin=builtin From 7d4bf9b8fdf7bbb202aa5f9ef905d8f14aaf4ccb Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:04:26 -0700 Subject: [PATCH 14/14] docs: pin IFM model registry source --- docs/embeddable/backends.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/embeddable/backends.md b/docs/embeddable/backends.md index e4e92c433c..ff42d41304 100644 --- a/docs/embeddable/backends.md +++ b/docs/embeddable/backends.md @@ -245,22 +245,27 @@ through the existing [custom-model CLI](../guide/configuration/custom-models.md) ```bash ./build/lemonade --port 13305 pull user.K2-Horizon-0.9B \ --recipe llamacpp \ + --source huggingface \ --checkpoint main IFM/K2-Horizon-0.9B-GGUF:K2-Horizon-1B-BF16.gguf ./build/lemonade --port 13305 pull user.K2-Horizon-3.7B \ --recipe llamacpp \ + --source huggingface \ --checkpoint main IFM/K2-Horizon-3.7B-GGUF:K2-Horizon-4B-BF16.gguf ./build/lemonade --port 13305 pull user.K2-Horizon-7B \ --recipe llamacpp \ + --source huggingface \ --checkpoint main IFM/K2-Horizon-7B-GGUF:K2-Horizon-7B-BF16.gguf ./build/lemonade --port 13305 pull user.K2-Horizon-32B \ --recipe llamacpp \ + --source huggingface \ --checkpoint main IFM/K2-Horizon-32B-GGUF:K2-Horizon-32B-BF16.gguf ./build/lemonade --port 13305 pull user.K2-Horizon-MoVA-36B-A4B \ --recipe llamacpp \ + --source huggingface \ --checkpoint main IFM/K2-Horizon-MoVA-36B-A4B-GGUF:K2-Horizon-36B-BF16.gguf ```