From 27c5c2008823527a49e91c03a83c22708a5c571d Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:13:20 -0700 Subject: [PATCH 1/5] feat(litellm): log every model fallback and count it (TASK-029) When router_settings.fallbacks answers a request from another group, the caller still sees the group it asked for. A community agent whose Nemotron route fails looks healthy while DeepSeek does the work (TASK-003). model_fallback_signal.py is a third LiteLLM callback. The v1.106.2 router calls log_success_fallback_event / log_failure_fallback_event on every CustomLogger at each fallback hop; each call prints one JSON line naming the requested group, the fallback group, the depth, the requested group's error and the key alias. No prompt, response or key hash. It never raises. The model_fallback log metric counts those lines from the litellm container so SRE can alert on it. Native hosted turns send disable_fallbacks (D81), so every count comes from the gateway, commonly-bot or llmService features. Verified against the real 1.106.2 Router with a local upstream returning OpenRouter's 404 for Nemotron: one served line when DeepSeek answers, one failed line when it does not, and the prompt in neither. Written by Rivet, a Commonly agent: https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- .../unit/helm/litellmDeepseekToolShim.test.js | 2 +- .../helm/litellmModelFallbackSignal.test.js | 55 +++++++++ .../litellm_model_fallback_signal_test.py | 111 ++++++++++++++++++ docs/development/LITELLM.md | 15 +++ .../files/litellm/model_fallback_signal.py | 86 ++++++++++++++ .../templates/agents/litellm-deployment.yaml | 3 + .../templates/configmaps/litellm-config.yaml | 7 +- k8s/monitoring/log-metrics.json | 6 + 8 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js create mode 100644 backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py create mode 100644 k8s/helm/commonly/files/litellm/model_fallback_signal.py diff --git a/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js b/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js index 7d6592b173..b7ea0ea81c 100644 --- a/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js +++ b/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js @@ -33,7 +33,7 @@ describe('LiteLLM DeepSeek tool shim', () => { for (const file of ['deepseek_tool_shim.py', 'deepseek_tool_shim_core.py']) { expect(cm).toContain(` ${file}: |\n {{- .Files.Get "files/litellm/${file}" | nindent 4 }}`); } - expect(cm).toContain('callbacks: ["rate_limit_signal.proxy_handler_instance", "deepseek_tool_shim.proxy_handler_instance"]'); + expect(cm).toMatch(/^ {6}callbacks: \[[^\]\n]*"deepseek_tool_shim\.proxy_handler_instance"[^\]\n]*\]$/m); }); test('both files are mounted next to config.yaml, where LiteLLM resolves callbacks', () => { diff --git a/backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js b/backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js new file mode 100644 index 0000000000..9fbc8cb26f --- /dev/null +++ b/backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js @@ -0,0 +1,55 @@ +/** + * The LiteLLM model-fallback signal (k8s/helm/commonly/files/litellm/ + * model_fallback_signal.py) prints one {"alarm":"model_fallback"} line each time + * the router answers a failed model group from router_settings.fallbacks + * (TASK-003). The `model_fallback` log metric counts those lines for SRE's alert. + * + * Its logic is tested in Python (litellm_model_fallback_signal_test.py, beside + * this file). This runs that suite and pins the wiring that makes the line + * reach the metric: the ConfigMap key, the mount next to config.yaml, the + * callbacks entry, and the log-metric filter on the litellm container. No helm + * render, because this jest job has no helm. + */ +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const repo = path.join(__dirname, '../../../..'); +const chart = path.join(repo, 'k8s/helm/commonly'); +const signal = path.join(chart, 'files/litellm/model_fallback_signal.py'); +const read = (rel) => fs.readFileSync(path.join(chart, rel), 'utf8'); + +describe('LiteLLM model fallback signal', () => { + test('python unit suite passes', () => { + const r = spawnSync('python3', ['-I', path.join(__dirname, 'litellm_model_fallback_signal_test.py'), signal], { + encoding: 'utf8', + timeout: 60000, + }); + if (r.error) throw r.error; + expect(`${r.stdout}${r.stderr}`).toMatch(/\nOK\s*$/); + expect(r.status).toBe(0); + }); + + test('the ConfigMap carries the module and registers it as a callback', () => { + const cm = read('templates/configmaps/litellm-config.yaml'); + expect(cm).toContain(' model_fallback_signal.py: |\n {{- .Files.Get "files/litellm/model_fallback_signal.py" | nindent 4 }}'); + expect(cm).toMatch(/^ {6}callbacks: \[[^\]\n]*"model_fallback_signal\.proxy_handler_instance"[^\]\n]*\]$/m); + }); + + test('the module is mounted next to config.yaml, where LiteLLM resolves callbacks', () => { + expect(read('templates/agents/litellm-deployment.yaml')).toContain([ + ' - name: litellm-config', + ' mountPath: /app/model_fallback_signal.py', + ' subPath: model_fallback_signal.py', + ].join('\n')); + }); + + test('the log metric counts the line from the litellm container', () => { + const metrics = JSON.parse(fs.readFileSync(path.join(repo, 'k8s/monitoring/log-metrics.json'), 'utf8')); + const metric = metrics.find((m) => m.name === 'model_fallback'); + expect(metric).toBeDefined(); + expect(metric.filter).toContain('resource.labels.container_name="litellm"'); + expect(metric.filter).toContain('jsonPayload.alarm="model_fallback"'); + expect(fs.readFileSync(signal, 'utf8')).toContain("'alarm': 'model_fallback'"); + }); +}); diff --git a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py new file mode 100644 index 0000000000..510c2a1ba4 --- /dev/null +++ b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py @@ -0,0 +1,111 @@ +"""Unit tests for k8s/helm/commonly/files/litellm/model_fallback_signal.py. + +Run by litellmModelFallbackSignal.test.js as `python3 -I `. +LiteLLM is not installed in CI, so a stub CustomLogger stands in for the one +module the signal imports. The kwargs mirror what the v1.106.2 router passes at +the fallback hop (router_utils/fallback_event_handlers.py): `model` is the group +tried instead, and the metadata carries original_model_group, model_group and +attempted_fallbacks. +""" +import asyncio +import contextlib +import importlib.util +import io +import json +import sys +import types +import unittest + +sys.dont_write_bytecode = True + +stub = types.ModuleType('litellm.integrations.custom_logger') +stub.CustomLogger = type('CustomLogger', (), {}) +for name in ('litellm', 'litellm.integrations'): + sys.modules.setdefault(name, types.ModuleType(name)) +sys.modules['litellm.integrations.custom_logger'] = stub + +SIGNAL_PATH = sys.argv.pop(1) +spec = importlib.util.spec_from_file_location('model_fallback_signal', SIGNAL_PATH) +signal = importlib.util.module_from_spec(spec) +spec.loader.exec_module(signal) + +NEMOTRON = 'openrouter/nvidia/nemotron-3-super-120b-a12b:free' +DEEPSEEK = 'deepseek-v4-flash' + + +class NotFoundError(Exception): + status_code = 404 + + +def hop_kwargs(metadata_name='metadata'): + return { + 'model': DEEPSEEK, + 'messages': [{'role': 'user', 'content': 'private prompt text'}], + metadata_name: { + 'original_model_group': NEMOTRON, + 'model_group': DEEPSEEK, + 'attempted_fallbacks': 1, + 'user_api_key_alias': 'community-agent-key', + 'user_api_key_hash': 'hash-should-not-appear', + }, + } + + +def captured_lines(hook, *args): + out = io.StringIO() + with contextlib.redirect_stdout(out): + asyncio.run(hook(*args)) + return [json.loads(line) for line in out.getvalue().splitlines() if line.strip()] + + +class FallbackLine(unittest.TestCase): + def test_a_served_fallback_names_both_groups_and_the_reason(self): + error = NotFoundError('OpenrouterException - No endpoints found for nvidia/nemotron-3-super-120b-a12b:free.') + lines = captured_lines(signal.proxy_handler_instance.log_success_fallback_event, NEMOTRON, hop_kwargs(), error) + self.assertEqual(lines, [{ + 'alarm': 'model_fallback', + 'outcome': 'served', + 'requested': NEMOTRON, + 'fallback': DEEPSEEK, + 'attempt': 1, + 'reason': 'NotFoundError 404: OpenrouterException - No endpoints found for nvidia/nemotron-3-super-120b-a12b:free.', + 'keyAlias': 'community-agent-key', + }]) + + def test_a_failed_fallback_is_reported_with_its_outcome(self): + lines = captured_lines(signal.proxy_handler_instance.log_failure_fallback_event, NEMOTRON, hop_kwargs(), NotFoundError('gone')) + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]['outcome'], 'failed') + self.assertEqual((lines[0]['requested'], lines[0]['fallback']), (NEMOTRON, DEEPSEEK)) + + def test_the_litellm_metadata_name_is_read_too(self): + line = signal.fallback_line('served', NEMOTRON, hop_kwargs('litellm_metadata'), None) + self.assertEqual((line['attempt'], line['keyAlias'], line['reason']), (1, 'community-agent-key', None)) + + def test_no_request_content_or_key_hash_is_logged(self): + out = json.dumps(signal.fallback_line('served', NEMOTRON, hop_kwargs(), NotFoundError('x'))) + self.assertNotIn('private prompt text', out) + self.assertNotIn('hash-should-not-appear', out) + + def test_the_reason_is_one_line_and_bounded(self): + line = signal.fallback_line('served', NEMOTRON, hop_kwargs(), RuntimeError('a\n' + 'b' * 1000)) + self.assertNotIn('\n', line['reason']) + self.assertLessEqual(len(line['reason']), len('RuntimeError: ') + signal.REASON_MAX) + + def test_a_dict_fallback_target_falls_back_to_the_metadata_group(self): + kwargs = hop_kwargs() + kwargs['model'] = {'model': DEEPSEEK} + self.assertEqual(signal.fallback_line('served', NEMOTRON, kwargs, None)['fallback'], DEEPSEEK) + + def test_malformed_events_still_print_one_line_and_never_raise(self): + class Unprintable: + def __str__(self): + raise ValueError('no str') + for args in ((None, None, None), (NEMOTRON, {'model': DEEPSEEK}, Unprintable())): + lines = captured_lines(signal.proxy_handler_instance.log_success_fallback_event, *args) + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]['alarm'], 'model_fallback') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/docs/development/LITELLM.md b/docs/development/LITELLM.md index 785b6deb40..e33a0f7342 100644 --- a/docs/development/LITELLM.md +++ b/docs/development/LITELLM.md @@ -123,6 +123,21 @@ a fake upstream. One quirk to expect: the `response.created` echo of the request tools still shows the flat names. To shim another DeepSeek group, add it to `SHIM_MODEL_GROUPS`. Tests are in `backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js`. +**Model fallback signal** (`/app/model_fallback_signal.py`, same source dir and +mount; the third callback). When `router_settings.fallbacks` answers a request +from another group, the caller still sees the group it asked for, so a +community agent whose Nemotron route fails looks healthy while DeepSeek does +the work (TASK-003). The callback prints one JSON line per fallback hop: +`{"alarm":"model_fallback","outcome":"served"|"failed","requested","fallback","attempt","reason","keyAlias"}`. +`reason` is the requested group's error, on one line and capped at 300 chars. +No prompt, response or key hash goes into the line. The `model_fallback` log +metric in `k8s/monitoring/log-metrics.json` counts these lines. Native hosted +turns send `disable_fallbacks` (D81), so every count comes from another caller: +the moltbot gateway, commonly-bot, or platform features through `llmService`. +Verified 2026-10-26 against the real 1.106.2 Router with a local upstream that +returns OpenRouter's 404 for Nemotron. Tests are in +`backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js`. + **Why this works:** LiteLLM's `Authenticator.get_access_token()` calls `_read_auth_file()` on every request — there's no in-memory token caching. So a `rename` of `auth.json` is picked up within ~1 request, no pod restart needed. diff --git a/k8s/helm/commonly/files/litellm/model_fallback_signal.py b/k8s/helm/commonly/files/litellm/model_fallback_signal.py new file mode 100644 index 0000000000..08817a28f9 --- /dev/null +++ b/k8s/helm/commonly/files/litellm/model_fallback_signal.py @@ -0,0 +1,86 @@ +"""LiteLLM callback: one JSON log line for every model fallback (TASK-003). + +When a model group fails, the router answers from its `router_settings.fallbacks` +chain, and the caller gets that answer with nothing to say another model wrote +it. Native hosted turns refuse fallbacks (`disable_fallbacks`, D81), but every +other caller, the moltbot gateway included, takes them silently. + +Every fallback passes through here: the router calls log_success_fallback_event +or log_failure_fallback_event on each CustomLogger in `litellm_settings.callbacks` +(litellm/router_utils/fallback_event_handlers.py, v1.106.2). Each event prints +one line to stdout, and GKE parses it into jsonPayload: + + {"alarm": "model_fallback", "outcome": "served" | "failed", + "requested": , "fallback": , + "attempt": , "reason": , + "keyAlias": } + +The log-based metric `model_fallback` (k8s/monitoring/log-metrics.json) counts +these lines. + +It logs no request content: only model group names, the depth, the error's class, +its status and a truncated message, and the key alias. It never raises, so a +malformed event can't cost a caller its answer. +""" +import json + +try: + from litellm.integrations.custom_logger import CustomLogger +except Exception: # LiteLLM itself is broken; stay importable. + CustomLogger = object + +REASON_MAX = 300 + + +def _metadata(kwargs): + # The router stamps the hop under whichever name the endpoint uses. + for name in ('metadata', 'litellm_metadata'): + value = kwargs.get(name) + if isinstance(value, dict) and ('original_model_group' in value or 'model_group' in value): + return value + return {} + + +def _reason(error): + if error is None: + return None + text = ' '.join(str(error).split())[:REASON_MAX] + status = getattr(error, 'status_code', None) + label = type(error).__name__ if status is None else f'{type(error).__name__} {status}' + return f'{label}: {text}' if text else label + + +def fallback_line(outcome, original_model_group, kwargs, original_exception): + kwargs = kwargs if isinstance(kwargs, dict) else {} + meta = _metadata(kwargs) + fallback = kwargs.get('model') + if not isinstance(fallback, str): + fallback = meta.get('model_group') + return { + 'alarm': 'model_fallback', + 'outcome': outcome, + 'requested': original_model_group or meta.get('original_model_group'), + 'fallback': fallback, + 'attempt': meta.get('attempted_fallbacks'), + 'reason': _reason(original_exception), + 'keyAlias': meta.get('user_api_key_alias'), + } + + +def emit(outcome, original_model_group, kwargs, original_exception): + try: + line = json.dumps(fallback_line(outcome, original_model_group, kwargs, original_exception), default=str) + except Exception as error: + line = json.dumps({'alarm': 'model_fallback', 'outcome': outcome, 'signalError': type(error).__name__}) + print(line, flush=True) + + +class ModelFallbackSignal(CustomLogger): + async def log_success_fallback_event(self, original_model_group, kwargs, original_exception): + emit('served', original_model_group, kwargs, original_exception) + + async def log_failure_fallback_event(self, original_model_group, kwargs, original_exception): + emit('failed', original_model_group, kwargs, original_exception) + + +proxy_handler_instance = ModelFallbackSignal() diff --git a/k8s/helm/commonly/templates/agents/litellm-deployment.yaml b/k8s/helm/commonly/templates/agents/litellm-deployment.yaml index 6a9eb38c91..b7e1f742d4 100644 --- a/k8s/helm/commonly/templates/agents/litellm-deployment.yaml +++ b/k8s/helm/commonly/templates/agents/litellm-deployment.yaml @@ -267,6 +267,9 @@ spec: - name: litellm-config mountPath: /app/deepseek_tool_shim_core.py subPath: deepseek_tool_shim_core.py + - name: litellm-config + mountPath: /app/model_fallback_signal.py + subPath: model_fallback_signal.py - name: chatgpt-auth mountPath: /chatgpt-auth resources: diff --git a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml index def020f59d..00a9aad6a0 100644 --- a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml +++ b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml @@ -331,7 +331,10 @@ data: # deepseek_tool_shim flattens codex's namespace tools into plain function # tools for the deepseek-flash-1017 group only, which DeepSeek requires; # source in files/litellm/deepseek_tool_shim.py, mounted from this ConfigMap. - callbacks: ["rate_limit_signal.proxy_handler_instance", "deepseek_tool_shim.proxy_handler_instance"] + # model_fallback_signal prints one {"alarm":"model_fallback"} line per + # fallback in router_settings.fallbacks, naming requested and fallback + # groups (TASK-003); the `model_fallback` log metric counts them. + callbacks: ["rate_limit_signal.proxy_handler_instance", "deepseek_tool_shim.proxy_handler_instance", "model_fallback_signal.proxy_handler_instance"] guardrails: # Anti-pattern content guardrails for the public-launch exposure. Both are @@ -401,4 +404,6 @@ data: {{- .Files.Get "files/litellm/deepseek_tool_shim.py" | nindent 4 }} deepseek_tool_shim_core.py: | {{- .Files.Get "files/litellm/deepseek_tool_shim_core.py" | nindent 4 }} + model_fallback_signal.py: | + {{- .Files.Get "files/litellm/model_fallback_signal.py" | nindent 4 }} {{- end }} diff --git a/k8s/monitoring/log-metrics.json b/k8s/monitoring/log-metrics.json index cd7d7f8c3c..185ce310b3 100644 --- a/k8s/monitoring/log-metrics.json +++ b/k8s/monitoring/log-metrics.json @@ -23,6 +23,12 @@ "filter": "resource.type=\"k8s_container\" AND resource.labels.namespace_name=\"commonly-dev\" AND resource.labels.container_name=\"litellm\" AND textPayload=~\"\\\"POST /chat/completions HTTP/1.1\\\" [45][0-9][0-9]\"", "metricDescriptor": { "metricKind": "DELTA", "valueType": "INT64", "unit": "1" } }, + { + "name": "model_fallback", + "description": "LiteLLM answered (outcome=served) or tried to answer (outcome=failed) a request from a model group other than the one requested, via router_settings.fallbacks (TASK-003). One line per fallback hop from k8s/helm/commonly/files/litellm/model_fallback_signal.py. The requested and fallback groups, the reason the requested group failed and the key alias are in the same log line. Native hosted turns refuse fallbacks (D81), so every count comes from another caller: the moltbot gateway, commonly-bot, or platform features.", + "filter": "resource.type=\"k8s_container\" AND resource.labels.namespace_name=\"commonly-dev\" AND resource.labels.container_name=\"litellm\" AND jsonPayload.alarm=\"model_fallback\"", + "metricDescriptor": { "metricKind": "DELTA", "valueType": "INT64", "unit": "1" } + }, { "name": "stranger_journey_failures", "description": "Stranger-journey runs that ended with result=fail (one JSON line per run from k8s/helm/commonly/files/stranger-journey.py). The failed step and its error are in the same log line.", From 27edcedbd5708e7087006e4b3b34fec7bc521387 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:31:23 -0700 Subject: [PATCH 2/5] fix(litellm): log only what each fallback hop proves (TASK-029) Three review findings on #2831. The reason carried the provider's error message, which can echo the request. It is now the error's class and status only. The key alias came out too. CodeQL traced user_api_key_alias into the print as sensitive data, and the line is useful without it. A two-hop chain (gpt-5.4 -> nemotron -> deepseek, as the chart has it) logged both hops as served, though nemotron failed. The router runs the inner hop inside the outer one, so the hop whose fallback answered logs first. The request's shared AttemptedFallbackTargets object is remembered by identity, in a bounded map, since the slots dataclass can be neither hashed nor weak-referenced. A later success hop of the same request now logs as passed. Each line also carries `from`, the group that failed at that hop, beside `requested`, the caller's group. Checked on the real 1.106.2 Router: the inner hop logs served and the outer passed; one hop and both-down are unchanged. Written by Rivet, a Commonly agent: https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- .../litellm_model_fallback_signal_test.py | 85 ++++++++++++++----- docs/development/LITELLM.md | 20 +++-- .../files/litellm/model_fallback_signal.py | 58 +++++++++---- k8s/monitoring/log-metrics.json | 2 +- 4 files changed, 122 insertions(+), 43 deletions(-) diff --git a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py index 510c2a1ba4..401550e67a 100644 --- a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py +++ b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py @@ -9,6 +9,7 @@ """ import asyncio import contextlib +import dataclasses import importlib.util import io import json @@ -37,14 +38,26 @@ class NotFoundError(Exception): status_code = 404 -def hop_kwargs(metadata_name='metadata'): +@dataclasses.dataclass(slots=True) +class Attempts: + """Stands in for AttemptedFallbackTargets, the object a request shares across + hops. Shaped like v1.106.2's: a slots dataclass, so unhashable and not + weak-referenceable, which a plain stub would hide.""" + keys: frozenset = frozenset() + + +GPT = 'gpt-5.4' + + +def hop_kwargs(metadata_name='metadata', attempts=None, requested=NEMOTRON, fallback=DEEPSEEK, depth=1): return { - 'model': DEEPSEEK, + 'model': fallback, 'messages': [{'role': 'user', 'content': 'private prompt text'}], + 'attempted_targets': attempts if attempts is not None else Attempts(), metadata_name: { - 'original_model_group': NEMOTRON, - 'model_group': DEEPSEEK, - 'attempted_fallbacks': 1, + 'original_model_group': requested, + 'model_group': fallback, + 'attempted_fallbacks': depth, 'user_api_key_alias': 'community-agent-key', 'user_api_key_hash': 'hash-should-not-appear', }, @@ -58,39 +71,71 @@ def captured_lines(hook, *args): return [json.loads(line) for line in out.getvalue().splitlines() if line.strip()] +served = signal.proxy_handler_instance.log_success_fallback_event +failed = signal.proxy_handler_instance.log_failure_fallback_event + + class FallbackLine(unittest.TestCase): def test_a_served_fallback_names_both_groups_and_the_reason(self): error = NotFoundError('OpenrouterException - No endpoints found for nvidia/nemotron-3-super-120b-a12b:free.') - lines = captured_lines(signal.proxy_handler_instance.log_success_fallback_event, NEMOTRON, hop_kwargs(), error) - self.assertEqual(lines, [{ + self.assertEqual(captured_lines(served, NEMOTRON, hop_kwargs(), error), [{ 'alarm': 'model_fallback', 'outcome': 'served', 'requested': NEMOTRON, + 'from': NEMOTRON, 'fallback': DEEPSEEK, 'attempt': 1, - 'reason': 'NotFoundError 404: OpenrouterException - No endpoints found for nvidia/nemotron-3-super-120b-a12b:free.', - 'keyAlias': 'community-agent-key', + 'reason': 'NotFoundError 404', }]) def test_a_failed_fallback_is_reported_with_its_outcome(self): - lines = captured_lines(signal.proxy_handler_instance.log_failure_fallback_event, NEMOTRON, hop_kwargs(), NotFoundError('gone')) + lines = captured_lines(failed, NEMOTRON, hop_kwargs(), NotFoundError('gone')) self.assertEqual(len(lines), 1) self.assertEqual(lines[0]['outcome'], 'failed') self.assertEqual((lines[0]['requested'], lines[0]['fallback']), (NEMOTRON, DEEPSEEK)) + def test_in_a_two_hop_chain_only_the_hop_that_answered_is_served(self): + # gpt-5.4 -> nemotron -> deepseek, as the chart chains them. The router + # runs the inner hop inside the outer one, so the inner hop logs first. + attempts = Attempts() + inner = captured_lines(served, NEMOTRON, hop_kwargs(attempts=attempts, requested=GPT, depth=2), NotFoundError('x')) + outer = captured_lines(served, GPT, hop_kwargs(attempts=attempts, requested=GPT, fallback=NEMOTRON), RuntimeError('x')) + self.assertEqual([(l['outcome'], l['requested'], l['from'], l['fallback']) for l in inner + outer], [ + ('served', GPT, NEMOTRON, DEEPSEEK), + ('passed', GPT, GPT, NEMOTRON), + ]) + # Another request starts fresh. + self.assertEqual(captured_lines(served, NEMOTRON, hop_kwargs(), None)[0]['outcome'], 'served') + + def test_requests_are_remembered_up_to_a_bound(self): + first = Attempts() + captured_lines(served, NEMOTRON, hop_kwargs(attempts=first), None) + for _ in range(signal.ANSWERED_MAX): + captured_lines(served, NEMOTRON, hop_kwargs(), None) + self.assertEqual(len(signal._ANSWERED), signal.ANSWERED_MAX) + self.assertNotIn(id(first), signal._ANSWERED) + + def test_a_hop_without_its_request_object_is_taken_at_its_word(self): + kwargs = hop_kwargs() + del kwargs['attempted_targets'] + for _ in range(2): + self.assertEqual(captured_lines(served, NEMOTRON, kwargs, None)[0]['outcome'], 'served') + def test_the_litellm_metadata_name_is_read_too(self): line = signal.fallback_line('served', NEMOTRON, hop_kwargs('litellm_metadata'), None) - self.assertEqual((line['attempt'], line['keyAlias'], line['reason']), (1, 'community-agent-key', None)) + self.assertEqual((line['attempt'], line['requested'], line['reason']), (1, NEMOTRON, None)) - def test_no_request_content_or_key_hash_is_logged(self): + def test_no_request_content_key_or_alias_is_logged(self): out = json.dumps(signal.fallback_line('served', NEMOTRON, hop_kwargs(), NotFoundError('x'))) - self.assertNotIn('private prompt text', out) - self.assertNotIn('hash-should-not-appear', out) + for secret in ('private prompt text', 'hash-should-not-appear', 'community-agent-key'): + self.assertNotIn(secret, out) - def test_the_reason_is_one_line_and_bounded(self): - line = signal.fallback_line('served', NEMOTRON, hop_kwargs(), RuntimeError('a\n' + 'b' * 1000)) - self.assertNotIn('\n', line['reason']) - self.assertLessEqual(len(line['reason']), len('RuntimeError: ') + signal.REASON_MAX) + def test_the_reason_never_carries_the_error_message(self): + # A provider can echo the request into its error, prompt included. + error = NotFoundError('400 invalid request: messages[0].content="private prompt text"') + line = signal.fallback_line('served', NEMOTRON, hop_kwargs(), error) + self.assertEqual(line['reason'], 'NotFoundError 404') + self.assertEqual(signal.fallback_line('served', NEMOTRON, hop_kwargs(), RuntimeError('a\nb'))['reason'], 'RuntimeError') def test_a_dict_fallback_target_falls_back_to_the_metadata_group(self): kwargs = hop_kwargs() @@ -101,8 +146,8 @@ def test_malformed_events_still_print_one_line_and_never_raise(self): class Unprintable: def __str__(self): raise ValueError('no str') - for args in ((None, None, None), (NEMOTRON, {'model': DEEPSEEK}, Unprintable())): - lines = captured_lines(signal.proxy_handler_instance.log_success_fallback_event, *args) + for args in ((None, None, None), (NEMOTRON, {'model': DEEPSEEK, 'attempted_targets': []}, Unprintable())): + lines = captured_lines(served, *args) self.assertEqual(len(lines), 1) self.assertEqual(lines[0]['alarm'], 'model_fallback') diff --git a/docs/development/LITELLM.md b/docs/development/LITELLM.md index e33a0f7342..71e483aaff 100644 --- a/docs/development/LITELLM.md +++ b/docs/development/LITELLM.md @@ -128,14 +128,20 @@ mount; the third callback). When `router_settings.fallbacks` answers a request from another group, the caller still sees the group it asked for, so a community agent whose Nemotron route fails looks healthy while DeepSeek does the work (TASK-003). The callback prints one JSON line per fallback hop: -`{"alarm":"model_fallback","outcome":"served"|"failed","requested","fallback","attempt","reason","keyAlias"}`. -`reason` is the requested group's error, on one line and capped at 300 chars. -No prompt, response or key hash goes into the line. The `model_fallback` log -metric in `k8s/monitoring/log-metrics.json` counts these lines. Native hosted -turns send `disable_fallbacks` (D81), so every count comes from another caller: -the moltbot gateway, commonly-bot, or platform features through `llmService`. +`{"alarm":"model_fallback","outcome","requested","from","fallback","attempt","reason"}`. +`requested` is the group the caller asked for; `from` is the group that failed +at this hop. The chart chains two hops (gpt-5.4 → nemotron → deepseek), and the +router runs the inner hop inside the outer one, so the hop whose fallback +answered logs first: it is `served`, and the outer hop of the same request is +`passed`. `failed` means the fallback did not answer. `reason` is the failure's +class and status only. The error message stays out, because a provider can +echo the request into it, and so do the key and its alias. The `model_fallback` log +metric in `k8s/monitoring/log-metrics.json` counts these lines, one per hop. +Native hosted turns send `disable_fallbacks` (D81), so every count comes from +another caller: the moltbot gateway, commonly-bot, or platform features +through `llmService`. Verified 2026-10-26 against the real 1.106.2 Router with a local upstream that -returns OpenRouter's 404 for Nemotron. Tests are in +returns OpenRouter's 404 for Nemotron, one hop and two. Tests are in `backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js`. **Why this works:** LiteLLM's `Authenticator.get_access_token()` calls `_read_auth_file()` diff --git a/k8s/helm/commonly/files/litellm/model_fallback_signal.py b/k8s/helm/commonly/files/litellm/model_fallback_signal.py index 08817a28f9..c099338fc5 100644 --- a/k8s/helm/commonly/files/litellm/model_fallback_signal.py +++ b/k8s/helm/commonly/files/litellm/model_fallback_signal.py @@ -1,4 +1,4 @@ -"""LiteLLM callback: one JSON log line for every model fallback (TASK-003). +"""LiteLLM callback: one JSON log line for every model fallback hop (TASK-003). When a model group fails, the router answers from its `router_settings.fallbacks` chain, and the caller gets that answer with nothing to say another model wrote @@ -10,26 +10,39 @@ (litellm/router_utils/fallback_event_handlers.py, v1.106.2). Each event prints one line to stdout, and GKE parses it into jsonPayload: - {"alarm": "model_fallback", "outcome": "served" | "failed", - "requested": , "fallback": , - "attempt": , "reason": , - "keyAlias": } + {"alarm": "model_fallback", "outcome": "served" | "passed" | "failed", + "requested": , "from": , + "fallback": , "attempt": , + "reason": } + +A chain can have two hops (gpt-5.4 -> nemotron -> deepseek-v4-flash). The router +runs the inner hop inside the outer one, so the hop whose fallback answered logs +first. `served` marks that hop. `passed` marks an outer hop of the same request, +whose fallback failed as well. `failed` means the fallback did not answer. The log-based metric `model_fallback` (k8s/monitoring/log-metrics.json) counts these lines. -It logs no request content: only model group names, the depth, the error's class, -its status and a truncated message, and the key alias. It never raises, so a -malformed event can't cost a caller its answer. +It logs only model group names, the depth, and the error's class and status. It +never logs the error's message, because a provider can echo the request into it, +and never logs the key or its alias. It never raises, so a malformed event can't +cost a caller its answer. """ import json +from collections import OrderedDict try: from litellm.integrations.custom_logger import CustomLogger except Exception: # LiteLLM itself is broken; stay importable. CustomLogger = object -REASON_MAX = 300 +# The router shares one AttemptedFallbackTargets object across every hop of a +# request (kwargs['attempted_targets']). A request is in here once one of its +# hops has logged `served`. That object is a slots dataclass, neither hashable +# nor weak-referenceable, so it is held by identity, and the oldest entries go +# first. A request's hops log within one call, far inside the bound. +_ANSWERED = OrderedDict() +ANSWERED_MAX = 1024 def _metadata(kwargs): @@ -44,10 +57,21 @@ def _metadata(kwargs): def _reason(error): if error is None: return None - text = ' '.join(str(error).split())[:REASON_MAX] status = getattr(error, 'status_code', None) - label = type(error).__name__ if status is None else f'{type(error).__name__} {status}' - return f'{label}: {text}' if text else label + return type(error).__name__ if status is None else f'{type(error).__name__} {status}' + + +def _first_answer(kwargs): + """True for the first success hop of a request: the hop whose fallback answered.""" + attempts = kwargs.get('attempted_targets') if isinstance(kwargs, dict) else None + if attempts is None: + return True # No per-request object to key on, so take the hop at its word. + if _ANSWERED.get(id(attempts)) is attempts: + return False + _ANSWERED[id(attempts)] = attempts + while len(_ANSWERED) > ANSWERED_MAX: + _ANSWERED.popitem(last=False) + return True def fallback_line(outcome, original_model_group, kwargs, original_exception): @@ -59,11 +83,11 @@ def fallback_line(outcome, original_model_group, kwargs, original_exception): return { 'alarm': 'model_fallback', 'outcome': outcome, - 'requested': original_model_group or meta.get('original_model_group'), + 'requested': meta.get('original_model_group') or original_model_group, + 'from': original_model_group, 'fallback': fallback, 'attempt': meta.get('attempted_fallbacks'), 'reason': _reason(original_exception), - 'keyAlias': meta.get('user_api_key_alias'), } @@ -77,7 +101,11 @@ def emit(outcome, original_model_group, kwargs, original_exception): class ModelFallbackSignal(CustomLogger): async def log_success_fallback_event(self, original_model_group, kwargs, original_exception): - emit('served', original_model_group, kwargs, original_exception) + try: + outcome = 'served' if _first_answer(kwargs) else 'passed' + except Exception: + outcome = 'served' + emit(outcome, original_model_group, kwargs, original_exception) async def log_failure_fallback_event(self, original_model_group, kwargs, original_exception): emit('failed', original_model_group, kwargs, original_exception) diff --git a/k8s/monitoring/log-metrics.json b/k8s/monitoring/log-metrics.json index 185ce310b3..0b5467dddf 100644 --- a/k8s/monitoring/log-metrics.json +++ b/k8s/monitoring/log-metrics.json @@ -25,7 +25,7 @@ }, { "name": "model_fallback", - "description": "LiteLLM answered (outcome=served) or tried to answer (outcome=failed) a request from a model group other than the one requested, via router_settings.fallbacks (TASK-003). One line per fallback hop from k8s/helm/commonly/files/litellm/model_fallback_signal.py. The requested and fallback groups, the reason the requested group failed and the key alias are in the same log line. Native hosted turns refuse fallbacks (D81), so every count comes from another caller: the moltbot gateway, commonly-bot, or platform features.", + "description": "LiteLLM answered (outcome=served) or tried to answer (outcome=failed) a request from a model group other than the one requested, via router_settings.fallbacks (TASK-003). One line per fallback hop from k8s/helm/commonly/files/litellm/model_fallback_signal.py; on a two-hop chain the hop that answered is served and the outer hop is passed. The requested group, the group that failed, the fallback group and the failure's class and status are in the same log line. Native hosted turns refuse fallbacks (D81), so every count comes from another caller: the moltbot gateway, commonly-bot, or platform features.", "filter": "resource.type=\"k8s_container\" AND resource.labels.namespace_name=\"commonly-dev\" AND resource.labels.container_name=\"litellm\" AND jsonPayload.alarm=\"model_fallback\"", "metricDescriptor": { "metricKind": "DELTA", "valueType": "INT64", "unit": "1" } }, From 0ef97c9515efc31d6e4d8bc3f769e8935e20d6d4 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:32:19 -0700 Subject: [PATCH 3/5] fix(litellm): a closed stdout can't raise from the fallback signal (TASK-029) Sable's gate note on #2831: print sat outside the try. Written by Rivet, a Commonly agent: https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- .../unit/helm/litellm_model_fallback_signal_test.py | 7 +++++++ k8s/helm/commonly/files/litellm/model_fallback_signal.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py index 401550e67a..7a7dfb18b9 100644 --- a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py +++ b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py @@ -142,6 +142,13 @@ def test_a_dict_fallback_target_falls_back_to_the_metadata_group(self): kwargs['model'] = {'model': DEEPSEEK} self.assertEqual(signal.fallback_line('served', NEMOTRON, kwargs, None)['fallback'], DEEPSEEK) + def test_a_closed_stdout_does_not_raise_into_the_request(self): + closed = io.StringIO() + closed.close() + with contextlib.redirect_stdout(closed): + asyncio.run(served(NEMOTRON, hop_kwargs(), NotFoundError('x'))) + asyncio.run(failed(NEMOTRON, hop_kwargs(), NotFoundError('x'))) + def test_malformed_events_still_print_one_line_and_never_raise(self): class Unprintable: def __str__(self): diff --git a/k8s/helm/commonly/files/litellm/model_fallback_signal.py b/k8s/helm/commonly/files/litellm/model_fallback_signal.py index c099338fc5..464437bab8 100644 --- a/k8s/helm/commonly/files/litellm/model_fallback_signal.py +++ b/k8s/helm/commonly/files/litellm/model_fallback_signal.py @@ -96,7 +96,10 @@ def emit(outcome, original_model_group, kwargs, original_exception): line = json.dumps(fallback_line(outcome, original_model_group, kwargs, original_exception), default=str) except Exception as error: line = json.dumps({'alarm': 'model_fallback', 'outcome': outcome, 'signalError': type(error).__name__}) - print(line, flush=True) + try: + print(line, flush=True) + except Exception: + pass # No stdout to write to; the caller's answer comes first. class ModelFallbackSignal(CustomLogger): From c78b6b7d7edae0d1682d6b7e8d2041984a21c879 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:00:54 -0700 Subject: [PATCH 4/5] fix(litellm): read the fallback hop from LiteLLM's own metadata bucket (TASK-029) The callback picked the first bucket holding model-group fields, so on an endpoint with litellm_metadata, a caller's own metadata.original_model_group could reach `requested`. Choose the bucket the way LiteLLM does (get_metadata_variable_name_from_kwargs, core_helpers.py:319 in 1.106.2): litellm_metadata whenever the request has one, else metadata. The new test fails on the old selector and passes here. Written by Rivet, a Commonly agent Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- .../unit/helm/litellm_model_fallback_signal_test.py | 7 +++++++ .../commonly/files/litellm/model_fallback_signal.py | 12 ++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py index 7a7dfb18b9..deaf78054d 100644 --- a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py +++ b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py @@ -125,6 +125,13 @@ def test_the_litellm_metadata_name_is_read_too(self): line = signal.fallback_line('served', NEMOTRON, hop_kwargs('litellm_metadata'), None) self.assertEqual((line['attempt'], line['requested'], line['reason']), (1, NEMOTRON, None)) + def test_a_callers_own_metadata_never_stands_in_for_the_routers(self): + kwargs = hop_kwargs('litellm_metadata', requested=GPT) + kwargs['metadata'] = {'original_model_group': 'caller text', 'model_group': 'caller text', 'attempted_fallbacks': 9} + line = signal.fallback_line('served', NEMOTRON, kwargs, None) + self.assertEqual((line['requested'], line['attempt']), (GPT, 1)) + self.assertNotIn('caller text', json.dumps(line)) + def test_no_request_content_key_or_alias_is_logged(self): out = json.dumps(signal.fallback_line('served', NEMOTRON, hop_kwargs(), NotFoundError('x'))) for secret in ('private prompt text', 'hash-should-not-appear', 'community-agent-key'): diff --git a/k8s/helm/commonly/files/litellm/model_fallback_signal.py b/k8s/helm/commonly/files/litellm/model_fallback_signal.py index 464437bab8..f974f8d33a 100644 --- a/k8s/helm/commonly/files/litellm/model_fallback_signal.py +++ b/k8s/helm/commonly/files/litellm/model_fallback_signal.py @@ -46,12 +46,12 @@ def _metadata(kwargs): - # The router stamps the hop under whichever name the endpoint uses. - for name in ('metadata', 'litellm_metadata'): - value = kwargs.get(name) - if isinstance(value, dict) and ('original_model_group' in value or 'model_group' in value): - return value - return {} + # The bucket the router stamps the hop into, chosen the way LiteLLM chooses + # it: litellm_metadata whenever the request has one, else metadata + # (get_metadata_variable_name_from_kwargs). Then `metadata` is the caller's + # own, so picking a bucket by its contents would log a caller's text. + value = kwargs.get('litellm_metadata' if 'litellm_metadata' in kwargs else 'metadata') + return value if isinstance(value, dict) else {} def _reason(error): From e3a10c8636f2d526b6e9447586e3b27351cdf4ec Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:47:15 -0700 Subject: [PATCH 5/5] fix(litellm): read the hop's bucket by the router function, as LiteLLM stamps it (TASK-029) run_async_fallback stamps a hop into the bucket get_router_metadata_variable_name names for the router function (fallback_event_handlers.py:717, v1.106.2), not the one get_metadata_variable_name_from_kwargs picks. A chat completion that carried a litellm_metadata had its hop stamped into `metadata`, and the callback read the unstamped bucket: on a real 1.106.2 Router the line logged attempt: null. It now calls LiteLLM's own helper, with a mirror of the 1.106.2 rule when that isn't importable (the stubbed test harness), and logs attempt: 1. The new chat-completion test fails on the previous head. Written by Rivet, a Commonly agent Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- .../litellm_model_fallback_signal_test.py | 29 +++++++++++++++++++ .../files/litellm/model_fallback_signal.py | 22 ++++++++++---- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py index deaf78054d..1c60cbfa66 100644 --- a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py +++ b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py @@ -49,9 +49,24 @@ class Attempts: GPT = 'gpt-5.4' +def router_function(name): + def function(): + pass + function.__name__ = name + return function + + +# The v1.106.2 router function behind each bucket: a chat completion stamps +# `metadata`, and /v1/messages takes the generic path, which stamps +# `litellm_metadata`. +CHAT = router_function('_acompletion') +MESSAGES = router_function('_ageneric_api_call_with_fallbacks_anthropic_messages_attempt') + + def hop_kwargs(metadata_name='metadata', attempts=None, requested=NEMOTRON, fallback=DEEPSEEK, depth=1): return { 'model': fallback, + 'original_function': MESSAGES if metadata_name == 'litellm_metadata' else CHAT, 'messages': [{'role': 'user', 'content': 'private prompt text'}], 'attempted_targets': attempts if attempts is not None else Attempts(), metadata_name: { @@ -132,6 +147,20 @@ def test_a_callers_own_metadata_never_stands_in_for_the_routers(self): self.assertEqual((line['requested'], line['attempt']), (GPT, 1)) self.assertNotIn('caller text', json.dumps(line)) + def test_a_chat_completion_reads_metadata_even_beside_a_litellm_metadata(self): + # The router stamps a chat completion's hop into `metadata` and leaves a + # litellm_metadata the request carried as it was. + kwargs = hop_kwargs(requested=GPT, depth=2) + kwargs['litellm_metadata'] = {'model_group': 'unstamped'} + line = signal.fallback_line('served', NEMOTRON, kwargs, None) + self.assertEqual((line['requested'], line['attempt']), (GPT, 2)) + self.assertNotIn('unstamped', json.dumps(line)) + + def test_without_a_router_function_the_hop_is_read_from_metadata(self): + kwargs = hop_kwargs(requested=GPT) + del kwargs['original_function'] + self.assertEqual(signal.fallback_line('served', NEMOTRON, kwargs, None)['requested'], GPT) + def test_no_request_content_key_or_alias_is_logged(self): out = json.dumps(signal.fallback_line('served', NEMOTRON, hop_kwargs(), NotFoundError('x'))) for secret in ('private prompt text', 'hash-should-not-appear', 'community-agent-key'): diff --git a/k8s/helm/commonly/files/litellm/model_fallback_signal.py b/k8s/helm/commonly/files/litellm/model_fallback_signal.py index f974f8d33a..5336716117 100644 --- a/k8s/helm/commonly/files/litellm/model_fallback_signal.py +++ b/k8s/helm/commonly/files/litellm/model_fallback_signal.py @@ -36,6 +36,15 @@ except Exception: # LiteLLM itself is broken; stay importable. CustomLogger = object +try: + from litellm.router_utils.batch_utils import get_router_metadata_variable_name +except Exception: # Not importable here; mirror v1.106.2's rule (batch_utils.py:155). + def get_router_metadata_variable_name(function_name): + markers = ('batch', 'generic_api_call', '_acreate_batch', 'file', '_ageneric_api_call_with_fallbacks') + if function_name and any(marker in function_name for marker in markers): + return 'litellm_metadata' + return 'metadata' + # The router shares one AttemptedFallbackTargets object across every hop of a # request (kwargs['attempted_targets']). A request is in here once one of its # hops has logged `served`. That object is a slots dataclass, neither hashable @@ -46,11 +55,14 @@ def _metadata(kwargs): - # The bucket the router stamps the hop into, chosen the way LiteLLM chooses - # it: litellm_metadata whenever the request has one, else metadata - # (get_metadata_variable_name_from_kwargs). Then `metadata` is the caller's - # own, so picking a bucket by its contents would log a caller's text. - value = kwargs.get('litellm_metadata' if 'litellm_metadata' in kwargs else 'metadata') + # The bucket the router stamped this hop into. run_async_fallback picks it by + # the router function's name, not by which buckets the request carries + # (fallback_event_handlers.py:717): litellm_metadata for batch, file and + # generic API calls such as /v1/messages, metadata for the rest, chat + # completions included. Choosing by contents could read a bucket the router + # never stamped, the caller's own among them. + function_name = getattr(kwargs.get('original_function'), '__name__', None) + value = kwargs.get(get_router_metadata_variable_name(function_name)) return value if isinstance(value, dict) else {}