Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 69 additions & 4 deletions scripts/summarize-provider-error.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@

import argparse
import json
from pathlib import Path
import os
import re
import stat
from typing import Any

ALLOWED_ERROR_NAMES = {
Expand Down Expand Up @@ -34,6 +36,60 @@
"usage_not_included",
}

MAX_INPUT_BYTES = 16 * 1024 * 1024
MAX_MESSAGE_CHARS = 65536
# Labels only: never return matched text, arbitrary codes, paths or identifiers.
CATEGORY_PATTERNS = {
"context_limit": r"context_length_exceeded|maximum context length|exceeds? (?:the )?context window",
"auth": r"invalid[ _]api[ _]key|authentication_error|unauthorized",
"quota": r"insufficient[ _](?:credits|quota)|quota exceeded",
"rate_limit": r"rate[ _]limit",
"model_unavailable": r"model_not_found|no endpoints found|model (?:is )?not (?:found|supported)",
"unsupported_parameter": r"unsupported (?:parameter|value)|does not support (?:temperature|top_p)",
"unsupported_api": r"UnsupportedApiError|unsupported API for",
"invalid_schema": r"invalid (?:JSON )?schema|invalid function name",
"local_transport": r"fetch failed|ECONNREFUSED|ENOTFOUND|ECONNRESET",
"local_io": r"\bENOENT\b|\bEACCES\b|\bEPERM\b",
"response_validation": r"(?:AI_)?TypeValidationError|InvalidResponseDataError",
}


def _categories(error: dict[str, Any]) -> set[str]:
categories: set[str] = set()

def inspect(value: Any, depth: int = 0) -> None:
if depth > 8:
return
if isinstance(value, dict):
for key in ("message", "responseBody", "data", "error", "cause"):
if key in value:
inspect(value[key], depth + 1)
elif isinstance(value, str) and len(value) <= MAX_MESSAGE_CHARS:
if value.lstrip().startswith("{"):
try:
inspect(json.loads(value), depth + 1)
return
except (ValueError, RecursionError):
pass
for category, pattern in CATEGORY_PATTERNS.items():
if re.search(pattern, value, re.IGNORECASE):
categories.add(category)

inspect(error)
return categories or {"unknown"}


def _read_private_input(path: str) -> str:
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(descriptor, "rb") as stream:
info = os.fstat(stream.fileno())
if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_INPUT_BYTES:
raise ValueError("unavailable")
data = stream.read(MAX_INPUT_BYTES + 1)
if len(data) > MAX_INPUT_BYTES:
raise ValueError("unavailable")
return data.decode("utf-8", errors="replace")


def _error_object(event: Any) -> dict[str, Any] | None:
if not isinstance(event, dict) or event.get("type") != "error":
Expand All @@ -57,20 +113,22 @@ def summarize(provider: str, stdout_text: str, stderr_text: str) -> dict[str, An
retryable_values: list[bool] = []
malformed_lines = 0
error_events = 0
categories: set[str] = set()

for line in stdout_text.splitlines():
if not line.strip():
continue
try:
event = json.loads(line)
except (json.JSONDecodeError, TypeError):
except (ValueError, TypeError, RecursionError):
malformed_lines += 1
continue

error = _error_object(event)
if error is None:
continue
error_events += 1
categories.update(_categories(error))

name = error.get("name")
if isinstance(name, str):
Expand Down Expand Up @@ -121,6 +179,7 @@ def summarize(provider: str, stdout_text: str, stderr_text: str) -> dict[str, An
"missing_fields": sorted(missing_fields),
"stdout_present": bool(stdout_text),
"stderr_present": bool(stderr_text),
"failure_categories": sorted(categories),
}


Expand All @@ -131,8 +190,14 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--stderr-file", required=True)
args = parser.parse_args(argv)

stdout_text = Path(args.stdout_file).read_text(encoding="utf-8", errors="replace")
stderr_text = Path(args.stderr_file).read_text(encoding="utf-8", errors="replace")
try:
stdout_text = _read_private_input(args.stdout_file)
stderr_text = _read_private_input(args.stderr_file)
except (OSError, ValueError):
metadata = summarize(args.provider, "", "")
metadata["failure_categories"] = ["input_unavailable"]
print(json.dumps(metadata, sort_keys=True, separators=(",", ":")))
return 0
print(json.dumps(summarize(args.provider, stdout_text, stderr_text), sort_keys=True, separators=(",", ":")))
return 0

Expand Down
111 changes: 111 additions & 0 deletions tests/test_provider_error_categories_cli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

SCRIPT = Path(__file__).parents[1] / "scripts/summarize-provider-error.py"
CANARY = "PRIVATE_PROMPT_TOKEN_BODY_CANARY"


class ProviderCategoryCliTests(unittest.TestCase):
def run_cli(self, stdout, stderr="", mutate=None):
with tempfile.TemporaryDirectory() as directory:
out, err = Path(directory) / "out", Path(directory) / "err"
out.write_text(stdout)
err.write_text(stderr)
if mutate:
mutate(out)
result = subprocess.run([sys.executable, str(SCRIPT), "--provider", "opencode",
"--stdout-file", str(out), "--stderr-file", str(err)],
capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0)
self.assertEqual(result.stderr, "")
self.assertNotIn(CANARY, result.stdout + result.stderr)
return json.loads(result.stdout)

def test_unknown_error_messages_have_finite_categories_without_content(self):
cases = {
"context_limit": "maximum context length exceeded",
"auth": "invalid API key",
"quota": "insufficient credits",
"rate_limit": "rate_limit_exceeded",
"model_unavailable": "No endpoints found for model",
"unsupported_parameter": "unsupported parameter temperature",
"unsupported_api": "UnsupportedApiError: Unsupported API for model",
"invalid_schema": "invalid schema for function",
"local_transport": "TypeError: fetch failed ECONNREFUSED",
"local_io": "ENOENT: no such file",
"response_validation": "AI_TypeValidationError",
"unknown": "opaque failure",
}
for category, message in cases.items():
with self.subTest(category=category):
event = {"type": "error", "error": {"name": "UnknownError", "data": {
"message": message + " " + CANARY, "responseHeaders": {"secret": CANARY}}}}
value = self.run_cli(json.dumps(event), CANARY)
self.assertEqual(value["failure_categories"], [category])
self.assertEqual(value["error_names"], ["UnknownError"])
self.assertEqual(value["status_codes"], [])
self.assertEqual(value["error_codes"], [])
self.assertIsNone(value["retryable"])

def test_nested_response_body_is_classified_without_exporting_nested_fields(self):
event = {"type": "error", "error": {"name": "UnknownError", "data": {
"responseBody": json.dumps({"error": {"message": "context_length_exceeded " + CANARY,
"code": CANARY, "statusCode": 401}})}}}
value = self.run_cli(json.dumps(event))
self.assertEqual(value["failure_categories"], ["context_limit"])
self.assertEqual(value["error_codes"], [])
self.assertEqual(value["status_codes"], [])

def test_prompt_tool_events_stderr_and_arbitrary_fields_are_not_classified(self):
events = [{"type": "text", "part": {"text": "context_length_exceeded " + CANARY}},
{"type": "tool_use", "error": {"message": "invalid API key " + CANARY}},
{"type": "error", "error": {"name": CANARY, "data": {"secret": "invalid schema " + CANARY}}}]
value = self.run_cli("\n".join(json.dumps(x) for x in events), "ENOENT " + CANARY)
self.assertEqual(value["failure_categories"], ["unknown"])
self.assertEqual(value["error_names"], ["unrecognized"])
self.assertEqual(value["structured_error_events"], 1)

def test_nested_arbitrary_fields_are_ignored(self):
event = {"type": "error", "error": {"name": "UnknownError", "data": {
"responseBody": json.dumps({"secret": "invalid API key " + CANARY})}}}
value = self.run_cli(json.dumps(event))
self.assertEqual(value["failure_categories"], ["unknown"])

def test_malformed_and_deep_json_are_fixed_metadata_only(self):
value = self.run_cli(CANARY + "\n" + "[" * 2000 + "]" * 2000)
self.assertEqual(value["failure_categories"], [])
# Decoder depth limits vary; a valid non-error array may be ignored.
self.assertIn(value["malformed_output_lines"], (1, 2))

def test_huge_numeric_json_never_emits_tracebacks(self):
value = self.run_cli('{"type":"error","error":{"data":{"code":' + "9" * 5000 + '}}}')
if hasattr(sys, "get_int_max_str_digits") and sys.get_int_max_str_digits():
self.assertEqual(value["malformed_output_lines"], 1)
self.assertEqual(value["failure_categories"], [])
else:
self.assertEqual(value["failure_categories"], ["unknown"])

def test_missing_oversized_and_symlink_inputs_are_unavailable(self):
for operation in ["missing", "oversized", "symlink"]:
def mutate(path):
if operation == "missing":
path.unlink()
elif operation == "oversized":
with path.open("wb") as stream:
stream.truncate(16 * 1024 * 1024 + 1)
else:
target = path.with_name("private")
target.write_text(CANARY)
path.unlink()
path.symlink_to(target)
with self.subTest(operation=operation):
value = self.run_cli("", mutate=mutate)
self.assertEqual(value["failure_categories"], ["input_unavailable"])


if __name__ == "__main__":
unittest.main()
Loading