diff --git a/scripts/summarize-provider-error.py b/scripts/summarize-provider-error.py index 268ea7f..9ef5268 100755 --- a/scripts/summarize-provider-error.py +++ b/scripts/summarize-provider-error.py @@ -5,7 +5,9 @@ import argparse import json -from pathlib import Path +import os +import re +import stat from typing import Any ALLOWED_ERROR_NAMES = { @@ -34,6 +36,60 @@ "usage_not_included", } +MAX_INPUT_BYTES = 16 * 1024 * 1024 +MAX_MESSAGE_CHARS = 65536 +# Labels only: never return matched text, arbitrary codes, paths or identifiers. +CATEGORY_PATTERNS = { + "context_limit": r"context_length_exceeded|maximum context length|exceeds? (?:the )?context window", + "auth": r"invalid[ _]api[ _]key|authentication_error|unauthorized", + "quota": r"insufficient[ _](?:credits|quota)|quota exceeded", + "rate_limit": r"rate[ _]limit", + "model_unavailable": r"model_not_found|no endpoints found|model (?:is )?not (?:found|supported)", + "unsupported_parameter": r"unsupported (?:parameter|value)|does not support (?:temperature|top_p)", + "unsupported_api": r"UnsupportedApiError|unsupported API for", + "invalid_schema": r"invalid (?:JSON )?schema|invalid function name", + "local_transport": r"fetch failed|ECONNREFUSED|ENOTFOUND|ECONNRESET", + "local_io": r"\bENOENT\b|\bEACCES\b|\bEPERM\b", + "response_validation": r"(?:AI_)?TypeValidationError|InvalidResponseDataError", +} + + +def _categories(error: dict[str, Any]) -> set[str]: + categories: set[str] = set() + + def inspect(value: Any, depth: int = 0) -> None: + if depth > 8: + return + if isinstance(value, dict): + for key in ("message", "responseBody", "data", "error", "cause"): + if key in value: + inspect(value[key], depth + 1) + elif isinstance(value, str) and len(value) <= MAX_MESSAGE_CHARS: + if value.lstrip().startswith("{"): + try: + inspect(json.loads(value), depth + 1) + return + except (ValueError, RecursionError): + pass + for category, pattern in CATEGORY_PATTERNS.items(): + if re.search(pattern, value, re.IGNORECASE): + categories.add(category) + + inspect(error) + return categories or {"unknown"} + + +def _read_private_input(path: str) -> str: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_INPUT_BYTES: + raise ValueError("unavailable") + data = stream.read(MAX_INPUT_BYTES + 1) + if len(data) > MAX_INPUT_BYTES: + raise ValueError("unavailable") + return data.decode("utf-8", errors="replace") + def _error_object(event: Any) -> dict[str, Any] | None: if not isinstance(event, dict) or event.get("type") != "error": @@ -57,13 +113,14 @@ def summarize(provider: str, stdout_text: str, stderr_text: str) -> dict[str, An retryable_values: list[bool] = [] malformed_lines = 0 error_events = 0 + categories: set[str] = set() for line in stdout_text.splitlines(): if not line.strip(): continue try: event = json.loads(line) - except (json.JSONDecodeError, TypeError): + except (ValueError, TypeError, RecursionError): malformed_lines += 1 continue @@ -71,6 +128,7 @@ def summarize(provider: str, stdout_text: str, stderr_text: str) -> dict[str, An if error is None: continue error_events += 1 + categories.update(_categories(error)) name = error.get("name") if isinstance(name, str): @@ -121,6 +179,7 @@ def summarize(provider: str, stdout_text: str, stderr_text: str) -> dict[str, An "missing_fields": sorted(missing_fields), "stdout_present": bool(stdout_text), "stderr_present": bool(stderr_text), + "failure_categories": sorted(categories), } @@ -131,8 +190,14 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--stderr-file", required=True) args = parser.parse_args(argv) - stdout_text = Path(args.stdout_file).read_text(encoding="utf-8", errors="replace") - stderr_text = Path(args.stderr_file).read_text(encoding="utf-8", errors="replace") + try: + stdout_text = _read_private_input(args.stdout_file) + stderr_text = _read_private_input(args.stderr_file) + except (OSError, ValueError): + metadata = summarize(args.provider, "", "") + metadata["failure_categories"] = ["input_unavailable"] + print(json.dumps(metadata, sort_keys=True, separators=(",", ":"))) + return 0 print(json.dumps(summarize(args.provider, stdout_text, stderr_text), sort_keys=True, separators=(",", ":"))) return 0 diff --git a/tests/test_provider_error_categories_cli.py b/tests/test_provider_error_categories_cli.py new file mode 100644 index 0000000..43e2469 --- /dev/null +++ b/tests/test_provider_error_categories_cli.py @@ -0,0 +1,111 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).parents[1] / "scripts/summarize-provider-error.py" +CANARY = "PRIVATE_PROMPT_TOKEN_BODY_CANARY" + + +class ProviderCategoryCliTests(unittest.TestCase): + def run_cli(self, stdout, stderr="", mutate=None): + with tempfile.TemporaryDirectory() as directory: + out, err = Path(directory) / "out", Path(directory) / "err" + out.write_text(stdout) + err.write_text(stderr) + if mutate: + mutate(out) + result = subprocess.run([sys.executable, str(SCRIPT), "--provider", "opencode", + "--stdout-file", str(out), "--stderr-file", str(err)], + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stderr, "") + self.assertNotIn(CANARY, result.stdout + result.stderr) + return json.loads(result.stdout) + + def test_unknown_error_messages_have_finite_categories_without_content(self): + cases = { + "context_limit": "maximum context length exceeded", + "auth": "invalid API key", + "quota": "insufficient credits", + "rate_limit": "rate_limit_exceeded", + "model_unavailable": "No endpoints found for model", + "unsupported_parameter": "unsupported parameter temperature", + "unsupported_api": "UnsupportedApiError: Unsupported API for model", + "invalid_schema": "invalid schema for function", + "local_transport": "TypeError: fetch failed ECONNREFUSED", + "local_io": "ENOENT: no such file", + "response_validation": "AI_TypeValidationError", + "unknown": "opaque failure", + } + for category, message in cases.items(): + with self.subTest(category=category): + event = {"type": "error", "error": {"name": "UnknownError", "data": { + "message": message + " " + CANARY, "responseHeaders": {"secret": CANARY}}}} + value = self.run_cli(json.dumps(event), CANARY) + self.assertEqual(value["failure_categories"], [category]) + self.assertEqual(value["error_names"], ["UnknownError"]) + self.assertEqual(value["status_codes"], []) + self.assertEqual(value["error_codes"], []) + self.assertIsNone(value["retryable"]) + + def test_nested_response_body_is_classified_without_exporting_nested_fields(self): + event = {"type": "error", "error": {"name": "UnknownError", "data": { + "responseBody": json.dumps({"error": {"message": "context_length_exceeded " + CANARY, + "code": CANARY, "statusCode": 401}})}}} + value = self.run_cli(json.dumps(event)) + self.assertEqual(value["failure_categories"], ["context_limit"]) + self.assertEqual(value["error_codes"], []) + self.assertEqual(value["status_codes"], []) + + def test_prompt_tool_events_stderr_and_arbitrary_fields_are_not_classified(self): + events = [{"type": "text", "part": {"text": "context_length_exceeded " + CANARY}}, + {"type": "tool_use", "error": {"message": "invalid API key " + CANARY}}, + {"type": "error", "error": {"name": CANARY, "data": {"secret": "invalid schema " + CANARY}}}] + value = self.run_cli("\n".join(json.dumps(x) for x in events), "ENOENT " + CANARY) + self.assertEqual(value["failure_categories"], ["unknown"]) + self.assertEqual(value["error_names"], ["unrecognized"]) + self.assertEqual(value["structured_error_events"], 1) + + def test_nested_arbitrary_fields_are_ignored(self): + event = {"type": "error", "error": {"name": "UnknownError", "data": { + "responseBody": json.dumps({"secret": "invalid API key " + CANARY})}}} + value = self.run_cli(json.dumps(event)) + self.assertEqual(value["failure_categories"], ["unknown"]) + + def test_malformed_and_deep_json_are_fixed_metadata_only(self): + value = self.run_cli(CANARY + "\n" + "[" * 2000 + "]" * 2000) + self.assertEqual(value["failure_categories"], []) + # Decoder depth limits vary; a valid non-error array may be ignored. + self.assertIn(value["malformed_output_lines"], (1, 2)) + + def test_huge_numeric_json_never_emits_tracebacks(self): + value = self.run_cli('{"type":"error","error":{"data":{"code":' + "9" * 5000 + '}}}') + if hasattr(sys, "get_int_max_str_digits") and sys.get_int_max_str_digits(): + self.assertEqual(value["malformed_output_lines"], 1) + self.assertEqual(value["failure_categories"], []) + else: + self.assertEqual(value["failure_categories"], ["unknown"]) + + def test_missing_oversized_and_symlink_inputs_are_unavailable(self): + for operation in ["missing", "oversized", "symlink"]: + def mutate(path): + if operation == "missing": + path.unlink() + elif operation == "oversized": + with path.open("wb") as stream: + stream.truncate(16 * 1024 * 1024 + 1) + else: + target = path.with_name("private") + target.write_text(CANARY) + path.unlink() + path.symlink_to(target) + with self.subTest(operation=operation): + value = self.run_cli("", mutate=mutate) + self.assertEqual(value["failure_categories"], ["input_unavailable"]) + + +if __name__ == "__main__": + unittest.main()