diff --git a/skills/lemonade-router-builder/SKILL.md b/skills/lemonade-router-builder/SKILL.md index 081903ce..89e784ad 100644 --- a/skills/lemonade-router-builder/SKILL.md +++ b/skills/lemonade-router-builder/SKILL.md @@ -26,6 +26,8 @@ try and stays editable in the desktop app's Hybrid Router editor. - **Lemonade Server v11.5.0+** running locally (`lemonade server start`). Required only to register and test the generated policy - the skill itself (JSON generation + offline validation) works without a live server. + `min_total_chars` / `max_total_chars` need **v11.8.0+**; the + `POST /api/v1/routing/validate` check in Step 8b needs **v11.6.0+**. - **No GPU or ROCm dependency** for authoring. The router policy is a JSON document; no hardware is needed to generate or validate it. - **Python** (any 3.x) in PATH - used by the bundled offline validator @@ -202,7 +204,8 @@ condition per leaf object; nesting is allowed: |------|---------|-------| | `keywords_any` / `keywords_all` | `{ "keywords_any": ["SSN", "Email"] }` | case-insensitive substring - `"hi"` matches inside `"this"`, `"shipping"`, `"high"`, etc. Use `regex` with `\b...\b` when word-boundary precision is needed | | `regex` | `{ "regex": "\\b\\d{3}-?\\d{2}-?\\d{4}\\b" }` | ECMAScript flavor | -| `min_chars` / `max_chars` | `{ "min_chars": 4000 }` | input length, UTF-8 bytes, non-negative integer | +| `min_chars` / `max_chars` | `{ "min_chars": 4000 }` | length of the **latest user turn**, UTF-8 bytes, non-negative integer | +| `min_total_chars` / `max_total_chars` | `{ "min_total_chars": 60000 }` | length of **all** text in the request (every message, every role), UTF-8 bytes; use for "long conversation" rules - `min_chars` sees only the last turn. ~4 bytes per token. v11.8.0+; not editable in the desktop UI | | `has_tools` / `has_images` | `{ "has_images": true }` | booleans | | `classifier` | `{ "classifier": "clf-1", "label": "PII", "min_score": 0.5 }` | band test; `min_score`/`max_score` in [0,1]; default `min_score` 0.5; omit `label` only if the classifier has `default_label` | | `metadata` | `{ "metadata": { "key": "consent", "equals": "denied" } }` | exactly one of `equals` / `any` / `exists`; note: not editable in the desktop UI yet - use only when the user asks for metadata routing | @@ -228,7 +231,17 @@ It exits 0 with `"ready": true` when there are no errors. If it reports errors, fix the JSON and re-run. Do not present a policy that fails this check. -**8b. Immediately after validation passes, print these three curl commands** +The validator takes its vocabulary (keys, conditions, classifier types and +their required fields) from `scripts/route_policy.schema.json`, a copy of the +server's own schema; the output's `schema` field says which Lemonade ref it +came from. If the validator rejects a field the user says their server or the +Lemonade docs support, the copy is stale - run +`python scripts/sync_schema.py --ref ` +(fetches from GitHub) and re-validate, rather than editing the policy around +it. A `schema_drift` warning means the schema has a constraint the validator +can't check; rely on the server-side check in 8b. + +**8b. Immediately after validation passes, print these curl commands** as plain text for the user to copy and run. This is not optional. Fill in `` and `` from the policy, and a short `` that should hit the first rule. Do not execute these with Bash or any tool — @@ -237,9 +250,16 @@ print them as text only. `ready: true` from the validator only means the JSON is schema-valid - it says nothing about whether these models exist on the target host. Run #1 for every candidate/classifier model before #2, or `/pull` will 400 on a policy -that just passed validation. +that just passed validation. #0 runs the target server's own parser on the +policy without registering it - the authoritative answer if the server's +version differs from the bundled schema. ```bash +# 0. Parse and test the policy on the server itself, without registering it +curl -X POST http://localhost:13305/api/v1/routing/validate \ + -H "Content-Type: application/json" \ + -d '{"policy": , "prompt": ""}' + # 1. Check a model exists before registering curl http://localhost:13305/api/v1/models/ diff --git a/skills/lemonade-router-builder/evals/evals.json b/skills/lemonade-router-builder/evals/evals.json index aaf8028b..107f5e95 100644 --- a/skills/lemonade-router-builder/evals/evals.json +++ b/skills/lemonade-router-builder/evals/evals.json @@ -52,6 +52,24 @@ "router.json" ] }, + { + "id": "long-conversation-router", + "skill_should_trigger": true, + "prompt": "Once a conversation's total history gets past about 15,000 tokens, send it to fireworks.deepseek-v3; otherwise keep it on Qwen3.5-9B-GGUF.", + "expected_behavior": [ + "Use a min_total_chars condition (whole-conversation length), not min_chars", + "Size the bound at roughly 4 bytes per token (about 60000)", + "Set Qwen3.5-9B-GGUF as default_model", + "Run the offline validator scripts/validate.py and report it passed" + ], + "unexpected_behavior": [ + "Use min_chars as a substitute for min_total_chars", + "Claim the validator or the server does not support min_total_chars" + ], + "files_exist": [ + "router.json" + ] + }, { "id": "nginx-load-balance", "skill_should_trigger": false, diff --git a/skills/lemonade-router-builder/reference.md b/skills/lemonade-router-builder/reference.md index aab1ffba..d78bbd57 100644 --- a/skills/lemonade-router-builder/reference.md +++ b/skills/lemonade-router-builder/reference.md @@ -5,6 +5,15 @@ generation steps in `SKILL.md` leave a question open. The server-side parser (`routing_policy_parser.cpp`) is strict: it rejects unknown keys at every level, so never emit fields not listed here. +The machine-readable source of truth is `scripts/route_policy.schema.json`, a +verbatim copy of the server's +[`route_policy.schema.json`](https://github.com/lemonade-sdk/lemonade/blob/main/src/cpp/resources/schemas/route_policy.schema.json); +`scripts/route_policy.schema.source.json` records the ref it was copied from. +`scripts/validate.py` derives its key sets from it, so the validator follows +the schema automatically after `python scripts/sync_schema.py`. The tables +below are hand-written. When a sync reports new keys or types, update them too. +`python scripts/sync_schema.py --check` exits 1 when the copy is stale. + ## Contents - [Root document](#root-document) @@ -14,6 +23,7 @@ level, so never emit fields not listed here. - [Model capability requirements](#model-capability-requirements) - [Validation checklist](#validation-checklist) - [Registering, invoking, tracing](#registering-invoking-tracing) +- [Router scope and limits](#router-scope-and-limits) - [Desktop-editor compatibility](#desktop-editor-compatibility) --- @@ -114,15 +124,17 @@ Match-expression grammar: | `keywords_any` | non-empty array of non-empty strings | case-insensitive substring, any - `"hi"` also matches inside `"this"`, `"shipping"`, `"high"`, etc. Use `regex` `\b...\b` for word-boundary matching | | `keywords_all` | same | all must appear; same substring semantics as `keywords_any` | | `regex` | non-empty string | ECMAScript regex over the input text | -| `min_chars` / `max_chars` | integer ≥ 0 | input length in **UTF-8 bytes** | +| `min_chars` / `max_chars` | integer ≥ 0 | length of the routing input (latest user turn) in **UTF-8 bytes** | +| `min_total_chars` / `max_total_chars` | integer ≥ 0 | length of **all** text in the request in UTF-8 bytes: every chat message of every role (system/user/assistant/tool), the `/completions` `prompt`, or every Responses `input` item. Images/audio count as 0. Fires on a long history even when the last turn is `"go on"`, which `min_chars` sees as 5 bytes. Estimate ~4 bytes per token when sizing against a context window. Server v11.8.0+ | | `has_tools` / `has_images` | boolean | request carries `tools[]` / image parts. `false` is legal and means "must NOT have"; `{"not": {"has_tools": true}}` is the equivalent preferred spelling | | `classifier` | classifier id (string) | band test on that classifier's score | | `label` | string | only with `classifier`; must exist on that classifier; may be omitted only if the classifier has `default_label` | | `min_score` / `max_score` | number in [0,1], min ≤ max | only with `classifier`; omitting both defaults to `min_score: 0.5` | | `metadata` | `{ "key": , ... }` | plus exactly one of `equals` (string), `any` (non-empty string array), `exists` (boolean); evaluated over the request's OpenAI `metadata` object. Comparisons are case-sensitive. `any` comma-decodes the metadata value before matching: a caller sending `{"dept": "legal,compliance"}` (scalar) matches `any: ["legal", "compliance"]` - same as if two separate values were in the array. | -The routed input text is the last user message (chat), the `prompt` -(completions), or the `input` (responses). +The routed input text is the last user message (`/chat/completions`), the +`prompt` (`/completions`), or the `input` (`/responses`). All three endpoints +route a `collection.router` model on every supported server (v11.5.0+). ## Model capability requirements @@ -178,6 +190,11 @@ authoring judgment and aren't (fully) checkable by a script. ## Registering, invoking, tracing ```bash +curl -X POST http://localhost:13305/api/v1/routing/validate \ + -H "Content-Type: application/json" \ + -d '{"policy": {...}, "prompt": "My SSN is 123-45-6789", + "has_tools": false, "has_images": false, "metadata": {"consent": "denied"}}' + # register (idempotent - re-POST to update) curl -X POST http://localhost:13305/api/v1/pull \ -H "Content-Type: application/json" --data-binary @router.json @@ -221,8 +238,10 @@ there: - One condition per leaf object. Compound leaves (`{"min_chars": 5, "has_tools": true}`) are valid server-side but the editor cannot display them and will warn it must drop them on save. -- `metadata` conditions are JSON-only for now - the editor shows a lossy-edit - warning for rules containing them. Generate them only when the user - explicitly wants metadata routing, and say so in your reply. +- `metadata` and `min_total_chars` / `max_total_chars` conditions are + JSON-only in released builds - the editor shows a lossy-edit warning for + rules containing them. (The editor on Lemonade `main` edits `metadata` but + refuses to import `*_total_chars`.) Generate them only when the user's + intent needs them, and say so in your reply. - Everything else in this reference (nested `all`/`any`/`not`, all three classifier types, negated booleans) round-trips cleanly. diff --git a/skills/lemonade-router-builder/scripts/route_policy.schema.json b/skills/lemonade-router-builder/scripts/route_policy.schema.json new file mode 100644 index 00000000..218f719d --- /dev/null +++ b/skills/lemonade-router-builder/scripts/route_policy.schema.json @@ -0,0 +1,245 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://lemonade-sdk.github.io/schemas/route_policy.schema.json", + "title": "Lemonade route policy (collection.router)", + "description": "The routing block embedded in a collection.router collection JSON. Distributed and invoked exactly like collection.omni: point the OpenAI `model` field at the collection name and recipe:collection.router flips the server into engine mode. Generic engine; trust is a consumer. Vocabulary mirrors vLLM Semantic Router for migration.", + "type": "object", + "required": ["version", "recipe", "routing"], + "properties": { + "version": { + "description": "Schema major version this document targets. This file defines version \"1\"; a server validates a policy against the schema matching its major version and rejects unknown majors with a clear message rather than a per-key error. Required so parsers never have to guess.", + "const": "1" + }, + "model_name": { "type": "string" }, + "recipe": { "const": "collection.router" }, + "components": { + "description": "Lean local form: names of already-registered models (candidates AND classifier models). The full models[] manifest is only needed for Hugging Face redistribution.", + "type": "array", + "items": { "type": "string" } + }, + "models": { + "description": "Embedded component definitions for HF redistribution only; omitted when authoring locally.", + "type": "array" + }, + "routing": { "$ref": "#/$defs/routing" } + }, + "$defs": { + "routing": { + "type": "object", + "required": ["candidates", "default_model"], + "properties": { + "candidates": { + "description": "Explicit routing-target set (analog of vLLM SR providers.models). default_model and every rule.route_to must be a member.", + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "default_model": { + "description": "Fail-open target; must be a member of candidates.", + "type": "string" + }, + "router": { "$ref": "#/$defs/router_sugar" }, + "classifiers": { + "type": "array", + "items": { "$ref": "#/$defs/classifier" } + }, + "rules": { + "type": "array", + "items": { "$ref": "#/$defs/rule" } + } + }, + "allOf": [ + { "anyOf": [{ "required": ["router"] }, { "required": ["rules"] }] }, + { "if": { "required": ["router"] }, + "then": { "not": { "anyOf": [{ "required": ["rules"] }, { "required": ["classifiers"] }] } } } + ], + "additionalProperties": false + }, + "router_sugar": { + "description": "Zero-config L0(a) LLM-as-router. Desugars at load into one `llm` classifier (labels = candidates) + identity rules (label X -> route_to X). This desugaring is pure, deterministic engine logic and is frozen for v1: its expansion must stay behavior-equivalent so existing L0(a) policies keep routing identically across server versions.", + "type": "object", + "required": ["type", "model", "prompt"], + "properties": { + "type": { "const": "llm" }, + "model": { "type": "string", "minLength": 1 }, + "prompt": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + }, + "classifier": { + "type": "object", + "required": ["id", "type"], + "properties": { + "id": { "type": "string" }, + "type": { + "description": "v1: semantic_similarity, classifier, llm. Reserved vLLM-SR presets ship post-v1 as thin presets over `classifier`.", + "enum": [ + "semantic_similarity", + "classifier", + "llm", + "pii_detection", + "prompt_safety", + "language_detection", + "domain_classification", + "complexity", + "sentiment" + ] + }, + "model": { "type": "string" }, + "prompt": { "type": "string" }, + "labels": { + "description": "classifier output labels; dangling condition `label` refs are rejected.", + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "default_label": { + "description": "Used when a condition omits `label`.", + "type": "string" + }, + "reference_phrases": { + "description": "semantic_similarity only: a map of concept label -> exemplar phrases. Each concept's phrases are embedded once and compared to the input by cosine; the classifier reports the max cosine per concept as that concept's label score. The concept labels are this classifier's output labels (so a condition references a concept via `label`).", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + } + }, + "on_error": { + "description": "Behavior when the classifier fails to evaluate. match_true = fail-closed authoring. Frozen v1 default when omitted: match_false (fail-open).", + "enum": ["match_true", "match_false"], + "default": "match_false" + } + }, + "additionalProperties": false, + "allOf": [ + { "if": { "required": ["type"], "properties": { "type": { "const": "semantic_similarity" } } }, + "then": { "required": ["model", "reference_phrases"], "not": { "required": ["labels"] } } }, + { "if": { "required": ["type"], "properties": { "type": { "const": "classifier" } } }, + "then": { "required": ["model"] } }, + { "if": { "required": ["type"], "properties": { "type": { "const": "llm" } } }, + "then": { "required": ["model", "prompt", "labels"], + "properties": { "labels": { "minItems": 1 } } } } + ] + }, + "rule": { + "type": "object", + "required": ["id", "match", "route_to"], + "properties": { + "id": { + "description": "Safe rule token surfaced in the x-lemonade-route header when matched.", + "type": "string", + "pattern": "^[A-Za-z0-9._-]+$" + }, + "match": { "$ref": "#/$defs/match_expr" }, + "route_to": { + "description": "Must name a candidate. The local/private/cloud route category is derived from the chosen component's recipe.", + "type": "string" + }, + "outputs": { + "description": "Engine-opaque pass-through bag, copied verbatim into the Decision (trust puts verdict here).", + "type": "object" + } + }, + "additionalProperties": false + }, + "match_expr": { + "description": "Nested any/all/not over leaf conditions; first-match-wins across rules. vLLM SR's flat single-operator form is the subset.", + "type": "object", + "minProperties": 1, + "properties": { + "any": { "type": "array", "items": { "$ref": "#/$defs/match_expr" } }, + "all": { "type": "array", "items": { "$ref": "#/$defs/match_expr" } }, + "not": { "$ref": "#/$defs/match_expr" }, + "classifier": { "type": "string" }, + "label": { "type": "string" }, + "min_score": { + "description": "Frozen v1 semantics: inclusive lower bound (score >= min_score). When neither min_score nor max_score is given, the band defaults to min_score: 0.5.", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_score": { + "description": "Frozen v1 semantics: inclusive upper bound (score <= max_score).", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "keywords_any": { + "description": "True if ANY keyword is present. Frozen v1 match: case-insensitive substring (ASCII case fold) over the input text. Token/BM25 matching is a future, separately named op, never a redefinition of this one.", + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "keywords_all": { + "description": "True if ALL keywords are present. Frozen v1 match: case-insensitive substring (ASCII case fold) over the input text.", + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "regex": { + "description": "True if the pattern matches the input. Frozen v1 dialect: ECMAScript (std::regex), case-sensitive. A different regex engine ships as a separately named op, never a redefinition of this one. Must be non-empty (an empty pattern matches everything; express an always-match rule by omitting the condition). Patterns with a nested unbounded quantifier (such as (X+)+) — the most common catastrophic-backtracking shape — are rejected at policy load as a safeguard on untrusted input; this mitigates but does not eliminate all backtracking risk, and a 1 MiB input cap bounds the worst case at evaluation time.", + "type": "string", + "minLength": 1 + }, + "min_chars": { + "description": "Inclusive lower bound on input length. Frozen v1 unit: UTF-8 bytes (not Unicode code points). Token-based length is a future min_tokens, never a redefinition of this one.", + "type": "integer", + "minimum": 0 + }, + "max_chars": { + "description": "Inclusive upper bound on input length. Frozen v1 unit: UTF-8 bytes (not Unicode code points).", + "type": "integer", + "minimum": 0 + }, + "min_total_chars": { + "description": "Inclusive lower bound on the total length of all text content in the request, not just the routing input: for chat `messages` the sum over every message of every role (system/user/assistant/tool), for a legacy `prompt` the prompt itself, and for a Responses `input` array the sum over all items (role-tagged or bare). Non-text content (images, audio) contributes nothing. Frozen v1 unit: UTF-8 bytes (not Unicode code points), the same unit as min_chars; token-based length is a future min_tokens, never a redefinition of this one. Authors sizing a bound against a context window can estimate ~4 bytes per token.", + "type": "integer", + "minimum": 0 + }, + "max_total_chars": { + "description": "Inclusive upper bound on the total length of all text content in the request. Same scope and frozen v1 unit (UTF-8 bytes) as min_total_chars.", + "type": "integer", + "minimum": 0 + }, + "has_tools": { "type": "boolean" }, + "has_images": { "type": "boolean" }, + "metadata": { "$ref": "#/$defs/metadata_match" } + }, + "additionalProperties": false + }, + "metadata_match": { + "description": "Deterministic leaf matching a caller-supplied OpenAI `metadata` key (e.g. task_class, consent), read verbatim from the request metadata map. Frozen v1 semantics: values are compared case-sensitively; a value is decoded into a token set by splitting on comma and trimming, so scalar and list-valued metadata match uniformly. A missing, empty, or whitespace-only value counts as absent (matches only `exists: false`). Exactly one comparator (equals / any / exists) must be present. A future comparator (regex, all, ...) ships as a new key, never a redefinition of these.", + "type": "object", + "required": ["key"], + "properties": { + "key": { + "description": "The metadata key to read. A missing, empty, or whitespace-only value matches only `exists: false`.", + "type": "string", + "minLength": 1 + }, + "equals": { + "description": "True if the raw metadata value equals this string exactly (case-sensitive).", + "type": "string" + }, + "any": { + "description": "True if the value's comma-decoded token set intersects this list (handles both scalar and comma-encoded list-valued metadata).", + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "exists": { + "description": "exists:true matches when the key is present and non-empty; exists:false matches when it is absent or empty.", + "type": "boolean" + } + }, + "oneOf": [ + { "required": ["equals"] }, + { "required": ["any"] }, + { "required": ["exists"] } + ], + "additionalProperties": false + } + } +} diff --git a/skills/lemonade-router-builder/scripts/route_policy.schema.source.json b/skills/lemonade-router-builder/scripts/route_policy.schema.source.json new file mode 100644 index 00000000..31c73485 --- /dev/null +++ b/skills/lemonade-router-builder/scripts/route_policy.schema.source.json @@ -0,0 +1,6 @@ +{ + "repo": "lemonade-sdk/lemonade", + "ref": "main", + "url": "https://raw.githubusercontent.com/lemonade-sdk/lemonade/main/src/cpp/resources/schemas/route_policy.schema.json", + "synced_at": "2026-10-27T22:38:54Z" +} diff --git a/skills/lemonade-router-builder/scripts/sync_schema.py b/skills/lemonade-router-builder/scripts/sync_schema.py new file mode 100644 index 00000000..e6268217 --- /dev/null +++ b/skills/lemonade-router-builder/scripts/sync_schema.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +Refresh the bundled route-policy schema from the Lemonade repository. + +scripts/validate.py reads its whole vocabulary - allowed keys at every level, +condition names, classifier types and their per-type required/forbidden +fields, value ranges - from scripts/route_policy.schema.json. That file is a +verbatim copy of the server's own schema +(src/cpp/resources/schemas/route_policy.schema.json in lemonade-sdk/lemonade), +so keeping the skill in step with the router is a matter of re-running this +script when Lemonade ships router changes. + +Pin --ref to the release your server runs (e.g. v11.8.1, v2026.43.3) so the +validator accepts exactly what that server accepts; `main` may carry features +no release has yet. + +Usage: + python3 scripts/sync_schema.py # fetch main, write if changed + python3 scripts/sync_schema.py --ref v11.8.1 # fetch a release tag + python3 scripts/sync_schema.py --check # exit 1 if bundled copy is stale + +Needs network access to raw.githubusercontent.com. No extra packages. +""" + +import argparse +import datetime +import json +import os +import sys +import urllib.request + +REPO = "lemonade-sdk/lemonade" +SCHEMA_REPO_PATH = "src/cpp/resources/schemas/route_policy.schema.json" +HERE = os.path.dirname(os.path.abspath(__file__)) +SCHEMA_PATH = os.path.join(HERE, "route_policy.schema.json") +SOURCE_PATH = os.path.join(HERE, "route_policy.schema.source.json") + +sys.path.insert(0, HERE) +from validate import load_vocab # noqa: E402 (sibling module) + + +def fetch(ref): + url = f"https://raw.githubusercontent.com/{REPO}/{ref}/{SCHEMA_REPO_PATH}" + with urllib.request.urlopen(url, timeout=30) as resp: + raw = resp.read().decode("utf-8") + schema = json.loads(raw) + if schema.get("properties", {}).get("recipe", {}).get("const") != "collection.router": + raise ValueError(f"{url} does not look like the collection.router schema") + return url, raw, schema + + +def vocab_diff(old_schema, new_schema): + """Human-readable lines for every vocabulary set that changed.""" + if old_schema is None: + return ["(no bundled schema yet)"] + old, new = load_vocab(old_schema), load_vocab(new_schema) + lines = [] + for name in sorted(new): + a, b = old.get(name), new[name] + if not isinstance(b, (set, frozenset)) or a == b: + continue + added, removed = sorted(b - (a or set())), sorted((a or set()) - b) + if added: + lines.append(f"{name}: + {', '.join(added)}") + if removed: + lines.append(f"{name}: - {', '.join(removed)}") + return lines + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--ref", default="main", help="branch or tag of lemonade-sdk/lemonade (default: main)") + p.add_argument("--check", action="store_true", help="report drift and exit 1 if stale; write nothing") + args = p.parse_args() + + try: + url, raw, schema = fetch(args.ref) + except Exception as e: # network, 404 on a bad ref, bad JSON + print(f"error: could not fetch schema at ref '{args.ref}': {e}", file=sys.stderr) + sys.exit(2) + + old_schema = None + if os.path.exists(SCHEMA_PATH): + with open(SCHEMA_PATH, encoding="utf-8") as f: + old_schema = json.load(f) + + if old_schema == schema: + print(f"Bundled schema is up to date with {REPO}@{args.ref}.") + sys.exit(0) + + changes = vocab_diff(old_schema, schema) + print(f"Bundled schema differs from {REPO}@{args.ref}:") + for line in changes or ["(descriptions/constraints only - no vocabulary change)"]: + print(f" {line}") + + if args.check: + sys.exit(1) + + with open(SCHEMA_PATH, "w", encoding="utf-8", newline="\n") as f: + f.write(raw) + with open(SOURCE_PATH, "w", encoding="utf-8", newline="\n") as f: + json.dump({ + "repo": REPO, + "ref": args.ref, + "url": url, + "synced_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + }, f, indent=2) + f.write("\n") + print(f"Wrote {os.path.relpath(SCHEMA_PATH)}. Update SKILL.md/reference.md for any " + "new conditions or classifier types listed above.") + + +if __name__ == "__main__": + main() diff --git a/skills/lemonade-router-builder/scripts/validate.py b/skills/lemonade-router-builder/scripts/validate.py index 6be230e9..e001d5d4 100644 --- a/skills/lemonade-router-builder/scripts/validate.py +++ b/skills/lemonade-router-builder/scripts/validate.py @@ -14,10 +14,17 @@ that needs GET /api/v1/models/{id} against a running lemond; see SKILL.md Step 8 for that optional live check. +The vocabulary - allowed keys at every level, condition names, classifier +types and their per-type required/forbidden fields, value ranges - is read +from route_policy.schema.json next to this script, a verbatim copy of the +server's schema. Run scripts/sync_schema.py to refresh it when Lemonade ships +router changes; nothing here hardcodes a key list. + Usage: python3 scripts/validate.py router.json python3 scripts/validate.py < router.json cat router.json | python3 scripts/validate.py - + python3 scripts/validate.py --schema other.schema.json router.json Exits 0 if no errors (warnings/advisories may still be present), 1 otherwise. JSON to stdout. @@ -25,23 +32,114 @@ import argparse import json +import os import re import sys -LEAF_KEYS = { - "keywords_any", "keywords_all", "regex", "min_chars", "max_chars", - "has_tools", "has_images", "classifier", "label", "min_score", - "max_score", "metadata", -} +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_SCHEMA_PATH = os.path.join(HERE, "route_policy.schema.json") +SCHEMA_SOURCE_PATH = os.path.join(HERE, "route_policy.schema.source.json") + LOGICAL_KEYS = {"all", "any", "not"} -CLASSIFIER_TYPES = {"classifier", "semantic_similarity", "llm"} -ON_ERROR_VALUES = {"match_false", "match_true"} +CLASSIFIER_BAND_KEYS = {"label", "min_score", "max_score"} +RESERVED_CLASSIFIER_TYPES = { + "pii_detection", "prompt_safety", "language_detection", + "domain_classification", "complexity", "sentiment", +} SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._-]+$") -ROOT_KEYS = {"version", "model_name", "recipe", "components", "models", "routing"} -ROUTING_KEYS = {"candidates", "default_model", "router", "rules", "classifiers"} -CLASSIFIER_KEYS = {"id", "type", "model", "labels", "default_label", "on_error", - "prompt", "reference_phrases"} +# JSON Schema keywords _check_value enforces, and ones that carry no +# constraint. Anything else found on a property is reported as unenforced so a +# schema refresh that introduces a new constraint kind is visible, not silent. +ENFORCED_KEYWORDS = { + "type", "const", "enum", "minimum", "maximum", "minLength", "pattern", + "minItems", "uniqueItems", "items", "minProperties", "additionalProperties", +} +ANNOTATION_KEYWORDS = {"description", "default", "title", "$comment", "examples"} + + +def _type_rules(classifier_def): + """Per-type field rules from the classifier's allOf if/then conditionals. + + Returns (by_type, gated, unparsed): by_type maps a type to the fields it + requires, forbids, and requires non-empty; gated maps a field to the types + allowed to carry it (e.g. on_truncation); unparsed counts conditionals of a + shape this function doesn't understand. + """ + by_type, gated, unparsed = {}, {}, 0 + for entry in classifier_def.get("allOf", []): + cond, then = entry.get("if", {}), entry.get("then", {}) + type_const = cond.get("properties", {}).get("type", {}).get("const") + if type_const: + negated = then.get("not", {}) + forbidden = set(negated.get("required", [])) + for alt in negated.get("anyOf", []): + forbidden |= set(alt.get("required", [])) + by_type[type_const] = { + "required": set(then.get("required", [])), + "forbidden": forbidden, + "non_empty": {k for k, v in then.get("properties", {}).items() + if v.get("minItems", 0) >= 1}, + } + elif cond.get("required") and "enum" in then.get("properties", {}).get("type", {}): + for field in cond["required"]: + gated[field] = set(then["properties"]["type"]["enum"]) + else: + unparsed += 1 + return by_type, gated, unparsed + + +def _unenforced_keywords(prop_schemas): + found = set() + for prop in prop_schemas.values(): + stack = [prop] + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + found |= set(node) - ENFORCED_KEYWORDS - ANNOTATION_KEYWORDS - {"$ref"} + stack.extend(v for k, v in node.items() if k in ("items", "additionalProperties")) + return found + + +def load_vocab(schema): + """Derive every key set and enum the validator needs from the schema.""" + defs = schema["$defs"] + clf = defs["classifier"] + by_type, gated, unparsed = _type_rules(clf) + all_types = set(clf["properties"]["type"]["enum"]) + leaf_props = defs["match_expr"]["properties"] + return { + "root_keys": set(schema["properties"]), + "routing_keys": set(defs["routing"]["properties"]), + "router_keys": set(defs["router_sugar"]["properties"]), + "classifier_keys": set(clf["properties"]), + "classifier_types": all_types - RESERVED_CLASSIFIER_TYPES, + "reserved_classifier_types": all_types & RESERVED_CLASSIFIER_TYPES, + "on_error_values": set(clf["properties"]["on_error"]["enum"]), + "rule_keys": set(defs["rule"]["properties"]), + "leaf_keys": set(leaf_props) - LOGICAL_KEYS, + "metadata_keys": set(defs["metadata_match"]["properties"]), + "type_gated_fields": set(gated), + # Not sets - consumed by the checks below, skipped by sync_schema's diff. + "_leaf_schemas": leaf_props, + "_classifier_schemas": clf["properties"], + "_type_rules": by_type, + "_gated": gated, + "_unparsed_conditionals": unparsed, + "_unenforced_keywords": _unenforced_keywords(leaf_props) | _unenforced_keywords(clf["properties"]), + } + + +def _load_schema(path): + with open(path, encoding="utf-8") as f: + return json.load(f) + + +# None only when the bundled schema is missing; main() reports that. Kept +# importable without it so sync_schema.py can bootstrap the first copy. +VOCAB = load_vocab(_load_schema(DEFAULT_SCHEMA_PATH)) if os.path.exists(DEFAULT_SCHEMA_PATH) else None + BAD_PROMPT_RE = re.compile( r"reply with only the exact model name|only the model name|" r"reply with only.{0,20}model name|" @@ -132,14 +230,73 @@ def _warn(issues, check, message, path=""): "message": f"{path}: {message}" if path else message}) -def _is_non_negative_int(v): - return isinstance(v, int) and not isinstance(v, bool) and v >= 0 - - def _is_score(v): return isinstance(v, (int, float)) and not isinstance(v, bool) and 0.0 <= v <= 1.0 +_JSON_TYPES = { + "string": lambda v: isinstance(v, str), + "integer": lambda v: isinstance(v, int) and not isinstance(v, bool), + "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool), + "boolean": lambda v: isinstance(v, bool), + "array": lambda v: isinstance(v, list), + "object": lambda v: isinstance(v, dict), +} + + +def _check_value(value, prop, name, issues, check, path): + """Check one value against a property schema from the bundled schema. + + Returns True when it conforms. Covers the JSON Schema keywords the route + policy schema uses on properties (see ENFORCED_KEYWORDS).""" + types = prop.get("type") + if types is not None: + types = types if isinstance(types, list) else [types] + if not any(_JSON_TYPES[t](value) for t in types if t in _JSON_TYPES): + _err(issues, check, f"'{name}' must be {' or '.join(types)}, got {value!r}", path) + return False + if "const" in prop and value != prop["const"]: + _err(issues, check, f"'{name}' must be {prop['const']!r}, got {value!r}", path) + return False + if "enum" in prop and value not in prop["enum"]: + _err(issues, check, f"'{name}' must be one of {prop['enum']}, got {value!r}", path) + return False + ok = True + if _JSON_TYPES["number"](value): + if "minimum" in prop and value < prop["minimum"]: + _err(issues, check, f"'{name}' must be >= {prop['minimum']}, got {value!r}", path) + ok = False + if "maximum" in prop and value > prop["maximum"]: + _err(issues, check, f"'{name}' must be <= {prop['maximum']}, got {value!r}", path) + ok = False + if isinstance(value, str): + if len(value) < prop.get("minLength", 0): + _err(issues, check, f"'{name}' must be a non-empty string", path) + ok = False + elif "pattern" in prop and not re.search(prop["pattern"], value): + _err(issues, check, f"'{name}' value {value!r} is blank or contains control characters " + f"(must match {prop['pattern']})", path) + ok = False + if isinstance(value, list): + if len(value) < prop.get("minItems", 0): + _err(issues, check, f"'{name}' must have at least {prop['minItems']} item(s)", path) + ok = False + if prop.get("uniqueItems") and len({json.dumps(x, sort_keys=True) for x in value}) != len(value): + _err(issues, check, f"'{name}' must not contain duplicates", path) + ok = False + if isinstance(prop.get("items"), dict): + for i, item in enumerate(value): + ok = _check_value(item, prop["items"], f"{name}[{i}]", issues, check, path) and ok + if isinstance(value, dict): + if len(value) < prop.get("minProperties", 0): + _err(issues, check, f"'{name}' must have at least {prop['minProperties']} entr(y/ies)", path) + ok = False + if isinstance(prop.get("additionalProperties"), dict): + for k, v in value.items(): + ok = _check_value(v, prop["additionalProperties"], f"{name}.{k}", issues, check, path) and ok + return ok + + def validate_match_expr(expr, classifiers, path, issues, depth=0): if depth > 64: _err(issues, "match_depth", "exceeds max nesting depth (64)", path) @@ -167,25 +324,24 @@ def validate_match_expr(expr, classifiers, path, issues, depth=0): return # Leaf node - unknown = set(expr.keys()) - LEAF_KEYS + unknown = set(expr.keys()) - VOCAB["leaf_keys"] if unknown: - _err(issues, "unknown_key", f"unknown condition key(s): {', '.join(sorted(unknown))}", path) - - condition_count = 0 - for key in ("keywords_any", "keywords_all"): - if key not in expr: + _err(issues, "unknown_key", + f"unknown condition key(s): {', '.join(sorted(unknown))} " + f"(known: {', '.join(sorted(VOCAB['leaf_keys']))})", path) + + # Every condition the schema defines except classifier/metadata (which + # need cross-references, below) is checked straight from its property + # schema, so a condition added upstream is validated after a schema sync + # with no code change here. + conditions = [k for k in expr if k in VOCAB["leaf_keys"] and k not in CLASSIFIER_BAND_KEYS] + for key in conditions: + if key in ("classifier", "metadata"): continue - condition_count += 1 - v = expr[key] - if not isinstance(v, list) or not v or not all(isinstance(x, str) and x for x in v): - _err(issues, "keywords", f"'{key}' must be a non-empty array of non-empty strings", path) - - if "regex" in expr: - condition_count += 1 - v = expr["regex"] - if not isinstance(v, str) or not v: - _err(issues, "regex", "'regex' must be a non-empty string", path) - else: + if not _check_value(expr[key], VOCAB["_leaf_schemas"][key], key, issues, key, path): + continue + if key == "regex": + v = expr["regex"] ecma_err, ecma_skip = _check_ecmascript_regex(v) if ecma_err: _err(issues, "regex_dialect", @@ -197,20 +353,7 @@ def validate_match_expr(expr, classifiers, path, issues, depth=0): "pattern contains nested unbounded quantifiers (e.g. (X+)+) which the " "server rejects at load time as a ReDoS safeguard", path) - for key in ("min_chars", "max_chars"): - if key in expr: - condition_count += 1 - if not _is_non_negative_int(expr[key]): - _err(issues, "char_count", f"'{key}' must be a non-negative integer, got {expr[key]!r}", path) - - for key in ("has_tools", "has_images"): - if key in expr: - condition_count += 1 - if not isinstance(expr[key], bool): - _err(issues, "boolean_condition", f"'{key}' must be a boolean, got {expr[key]!r}", path) - if "classifier" in expr: - condition_count += 1 cid = expr["classifier"] if not isinstance(cid, str) or not cid: _err(issues, "classifier_ref", "'classifier' must be a non-empty string", path) @@ -243,12 +386,11 @@ def validate_match_expr(expr, classifiers, path, issues, depth=0): _err(issues, "classifier_ref", "'label'/'min_score'/'max_score' require a 'classifier' key in the same leaf", path) if "metadata" in expr: - condition_count += 1 md = expr["metadata"] if not isinstance(md, dict) or not isinstance(md.get("key"), str) or not md.get("key"): _err(issues, "metadata", "requires a non-empty string 'key'", path) else: - unknown_md = set(md.keys()) - {"key", "equals", "any", "exists"} + unknown_md = set(md.keys()) - VOCAB["metadata_keys"] if unknown_md: _err(issues, "metadata", f"unknown key(s): {', '.join(sorted(unknown_md))}", path) comparators = [c for c in ("equals", "any", "exists") if c in md] @@ -263,7 +405,7 @@ def validate_match_expr(expr, classifiers, path, issues, depth=0): elif comparators[0] == "exists" and not isinstance(md["exists"], bool): _err(issues, "metadata", "'exists' must be a boolean", path) - if condition_count == 0: + if not conditions: _err(issues, "empty_leaf", "leaf has no recognized condition", path) @@ -274,7 +416,7 @@ def validate(policy): _err(issues, "root", "policy must be a JSON object") return issues - unknown_root = set(policy.keys()) - ROOT_KEYS + unknown_root = set(policy.keys()) - VOCAB["root_keys"] if unknown_root: _err(issues, "unknown_key", f"unknown root key(s): {', '.join(sorted(unknown_root))}") @@ -309,7 +451,7 @@ def validate(policy): _err(issues, "routing", "must be an object") return issues - unknown_routing = set(routing.keys()) - ROUTING_KEYS + unknown_routing = set(routing.keys()) - VOCAB["routing_keys"] if unknown_routing: _err(issues, "unknown_key", f"unknown routing key(s): {', '.join(sorted(unknown_routing))}") @@ -342,7 +484,7 @@ def validate(policy): if not isinstance(router, dict): _err(issues, "router", "routing.router must be an object") else: - unknown = set(router.keys()) - {"type", "model", "prompt"} + unknown = set(router.keys()) - VOCAB["router_keys"] if unknown: _err(issues, "router", f"unknown key(s): {', '.join(sorted(unknown))}") if router.get("type") != "llm": @@ -373,7 +515,7 @@ def validate(policy): if not isinstance(clf, dict): _err(issues, "classifier", "must be an object", p) continue - unknown_clf = set(clf.keys()) - CLASSIFIER_KEYS + unknown_clf = set(clf.keys()) - VOCAB["classifier_keys"] if unknown_clf: _err(issues, "unknown_key", f"unknown key(s): {', '.join(sorted(unknown_clf))}", p) @@ -387,25 +529,47 @@ def validate(policy): classifiers_by_id[cid] = clf ctype = clf.get("type") - if ctype not in CLASSIFIER_TYPES: - _err(issues, "classifier_type", f"type must be one of {sorted(CLASSIFIER_TYPES)}, got {ctype!r}", p) + if ctype in VOCAB["reserved_classifier_types"]: + _err(issues, "classifier_type", + f"type '{ctype}' is reserved in the schema but not implemented by the server " + f"yet (it rejects it at /pull); use one of {sorted(VOCAB['classifier_types'])}", p) + elif ctype not in VOCAB["classifier_types"]: + _err(issues, "classifier_type", + f"type must be one of {sorted(VOCAB['classifier_types'])}, got {ctype!r}", p) + else: + rules_for_type = VOCAB["_type_rules"].get(ctype, {}) + for field in sorted(rules_for_type.get("required", ())): + if field not in clf: + _err(issues, "classifier_fields", f"type '{ctype}' requires '{field}'", p) + for field in sorted(rules_for_type.get("forbidden", ()) & clf.keys()): + hint = (" - concept names in reference_phrases are the labels" + if (ctype, field) == ("semantic_similarity", "labels") else "") + _err(issues, "classifier_fields", f"type '{ctype}' must not declare '{field}'{hint}", p) + for field in sorted(rules_for_type.get("non_empty", ()) & clf.keys()): + if isinstance(clf[field], list) and not clf[field]: + _err(issues, "classifier_fields", f"type '{ctype}' requires a non-empty '{field}'", p) + for field in sorted(VOCAB["type_gated_fields"] & clf.keys()): + if ctype not in VOCAB["_gated"][field]: + _err(issues, "classifier_fields", + f"'{field}' is only accepted on types {sorted(VOCAB['_gated'][field])}", p) + + # Value shapes (on_error enum, labels items, reference_phrases + # structure, ...) straight from the schema's classifier properties. + for field in sorted(set(clf.keys()) & VOCAB["classifier_keys"] - {"id", "type"}): + _check_value(clf[field], VOCAB["_classifier_schemas"][field], field, issues, field, p) model = clf.get("model") - if not isinstance(model, str) or not model: + if "model" in clf and (not isinstance(model, str) or not model): _err(issues, "classifier_model", "model must be a non-empty string", p) - else: + elif model: needed_components.add(model) - on_error = clf.get("on_error") - if on_error is not None and on_error not in ON_ERROR_VALUES: - _err(issues, "on_error", f"must be one of {sorted(ON_ERROR_VALUES)}, got {on_error!r}", p) - labels = clf.get("labels") if ctype == "llm": prompt = clf.get("prompt") - if not isinstance(prompt, str) or not prompt.strip(): + if "prompt" in clf and (not isinstance(prompt, str) or not prompt.strip()): _err(issues, "llm_classifier", "requires a non-empty 'prompt'", p) - elif BAD_PROMPT_RE.search(prompt): + elif isinstance(prompt, str) and BAD_PROMPT_RE.search(prompt): _warn(issues, "llm_classifier_prompt_contract", "prompt tells the model to reply with a bare label, but the engine " "appends its own JSON {model, rationale} contract and the classifier " @@ -413,26 +577,19 @@ def validate(policy): "causes weaker models to output a bare string the parser rejects, " "making the rule silently never fire. Describe classification intent " "only; do not instruct the model how to format its reply.", p) - if not isinstance(labels, list) or not labels or not all(isinstance(x, str) and x for x in labels): - _err(issues, "llm_classifier", "requires a non-empty 'labels' array", p) - elif ctype == "semantic_similarity": - if "labels" in clf: - _err(issues, "semantic_similarity", - "must not declare 'labels' - concept names in reference_phrases are the labels", p) + if ctype == "semantic_similarity": rp = clf.get("reference_phrases") - if not isinstance(rp, dict) or not rp: - _err(issues, "semantic_similarity", "requires a non-empty 'reference_phrases' object", p) - rp = {} - else: - for concept, phrases in rp.items(): - if not isinstance(phrases, list) or not phrases or \ - not all(isinstance(x, str) and x for x in phrases): - _err(issues, "semantic_similarity", - f"concept '{concept}' needs a non-empty array of non-empty phrases", p) - labels = list(rp.keys()) - elif ctype == "classifier": - if labels is not None and (not isinstance(labels, list) or not all(isinstance(x, str) and x for x in labels)): - _err(issues, "classifier", "'labels', if present, must be an array of non-empty strings", p) + labels = list(rp.keys()) if isinstance(rp, dict) else [] + # Stricter than the schema (its phrase items carry no + # minLength): parse_reference_phrases rejects both at /pull. + for concept, phrases in (rp.items() if isinstance(rp, dict) else ()): + if not concept: + _err(issues, "reference_phrases", "concept names must be non-empty", p) + if isinstance(phrases, list) and any(x == "" for x in phrases): + _err(issues, "reference_phrases", + f"concept '{concept}' has an empty phrase; phrases must be non-empty strings", p) + elif not isinstance(labels, list): + labels = [] default_label = clf.get("default_label") if default_label is not None: @@ -454,7 +611,7 @@ def validate(policy): if not isinstance(rule, dict): _err(issues, "rule", "must be an object", p) continue - unknown = set(rule.keys()) - {"id", "match", "route_to", "outputs"} + unknown = set(rule.keys()) - VOCAB["rule_keys"] if unknown: _err(issues, "rule", f"unknown key(s): {', '.join(sorted(unknown))}", p) @@ -489,21 +646,43 @@ def validate(policy): def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("path", nargs="?", default="-", help="policy JSON file, or '-'/omitted for stdin") + p.add_argument("--schema", default=DEFAULT_SCHEMA_PATH, + help="route policy schema to validate against (default: the bundled copy)") args = p.parse_args() + global VOCAB + if not os.path.exists(args.schema): + print(f"error: schema not found at {args.schema}; run scripts/sync_schema.py", file=sys.stderr) + sys.exit(2) + if args.schema != DEFAULT_SCHEMA_PATH: + VOCAB = load_vocab(_load_schema(args.schema)) + schema_source = {"path": os.path.basename(args.schema)} + if args.schema == DEFAULT_SCHEMA_PATH and os.path.exists(SCHEMA_SOURCE_PATH): + with open(SCHEMA_SOURCE_PATH, encoding="utf-8") as f: + schema_source.update(json.load(f)) + raw = sys.stdin.read() if args.path == "-" else open(args.path, encoding="utf-8").read() try: policy = json.loads(raw) except json.JSONDecodeError as e: print(json.dumps({"ready": False, "errors": [{"check": "json", "severity": "error", "message": str(e)}], - "warnings": [], "advisories": []}, indent=2)) + "warnings": [], "advisories": [], "schema": schema_source}, indent=2)) sys.exit(1) issues = validate(policy) + # The schema grew a construct this validator can't read: say so rather than + # pass the policy on a check that silently didn't run. + if VOCAB["_unparsed_conditionals"] or VOCAB["_unenforced_keywords"]: + _warn(issues, "schema_drift", + "the schema uses constraints this validator does not enforce " + f"({VOCAB['_unparsed_conditionals']} unrecognized classifier conditional(s); " + f"keywords: {sorted(VOCAB['_unenforced_keywords']) or 'none'}) - confirm the policy " + "with POST /api/v1/routing/validate on the target server") errors = [i for i in issues if i["severity"] == "error"] warnings = [i for i in issues if i["severity"] == "warning"] advisories = [i for i in issues if i["severity"] == "advisory"] - result = {"ready": len(errors) == 0, "errors": errors, "warnings": warnings, "advisories": advisories} + result = {"ready": len(errors) == 0, "errors": errors, "warnings": warnings, "advisories": advisories, + "schema": schema_source} print(json.dumps(result, indent=2)) sys.exit(0 if len(errors) == 0 else 1)