diff --git a/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js b/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js index 7d6592b173..b7ea0ea81c 100644 --- a/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js +++ b/backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js @@ -33,7 +33,7 @@ describe('LiteLLM DeepSeek tool shim', () => { for (const file of ['deepseek_tool_shim.py', 'deepseek_tool_shim_core.py']) { expect(cm).toContain(` ${file}: |\n {{- .Files.Get "files/litellm/${file}" | nindent 4 }}`); } - expect(cm).toContain('callbacks: ["rate_limit_signal.proxy_handler_instance", "deepseek_tool_shim.proxy_handler_instance"]'); + expect(cm).toMatch(/^ {6}callbacks: \[[^\]\n]*"deepseek_tool_shim\.proxy_handler_instance"[^\]\n]*\]$/m); }); test('both files are mounted next to config.yaml, where LiteLLM resolves callbacks', () => { diff --git a/backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js b/backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js new file mode 100644 index 0000000000..9fbc8cb26f --- /dev/null +++ b/backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js @@ -0,0 +1,55 @@ +/** + * The LiteLLM model-fallback signal (k8s/helm/commonly/files/litellm/ + * model_fallback_signal.py) prints one {"alarm":"model_fallback"} line each time + * the router answers a failed model group from router_settings.fallbacks + * (TASK-003). The `model_fallback` log metric counts those lines for SRE's alert. + * + * Its logic is tested in Python (litellm_model_fallback_signal_test.py, beside + * this file). This runs that suite and pins the wiring that makes the line + * reach the metric: the ConfigMap key, the mount next to config.yaml, the + * callbacks entry, and the log-metric filter on the litellm container. No helm + * render, because this jest job has no helm. + */ +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const repo = path.join(__dirname, '../../../..'); +const chart = path.join(repo, 'k8s/helm/commonly'); +const signal = path.join(chart, 'files/litellm/model_fallback_signal.py'); +const read = (rel) => fs.readFileSync(path.join(chart, rel), 'utf8'); + +describe('LiteLLM model fallback signal', () => { + test('python unit suite passes', () => { + const r = spawnSync('python3', ['-I', path.join(__dirname, 'litellm_model_fallback_signal_test.py'), signal], { + encoding: 'utf8', + timeout: 60000, + }); + if (r.error) throw r.error; + expect(`${r.stdout}${r.stderr}`).toMatch(/\nOK\s*$/); + expect(r.status).toBe(0); + }); + + test('the ConfigMap carries the module and registers it as a callback', () => { + const cm = read('templates/configmaps/litellm-config.yaml'); + expect(cm).toContain(' model_fallback_signal.py: |\n {{- .Files.Get "files/litellm/model_fallback_signal.py" | nindent 4 }}'); + expect(cm).toMatch(/^ {6}callbacks: \[[^\]\n]*"model_fallback_signal\.proxy_handler_instance"[^\]\n]*\]$/m); + }); + + test('the module is mounted next to config.yaml, where LiteLLM resolves callbacks', () => { + expect(read('templates/agents/litellm-deployment.yaml')).toContain([ + ' - name: litellm-config', + ' mountPath: /app/model_fallback_signal.py', + ' subPath: model_fallback_signal.py', + ].join('\n')); + }); + + test('the log metric counts the line from the litellm container', () => { + const metrics = JSON.parse(fs.readFileSync(path.join(repo, 'k8s/monitoring/log-metrics.json'), 'utf8')); + const metric = metrics.find((m) => m.name === 'model_fallback'); + expect(metric).toBeDefined(); + expect(metric.filter).toContain('resource.labels.container_name="litellm"'); + expect(metric.filter).toContain('jsonPayload.alarm="model_fallback"'); + expect(fs.readFileSync(signal, 'utf8')).toContain("'alarm': 'model_fallback'"); + }); +}); diff --git a/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py new file mode 100644 index 0000000000..1c60cbfa66 --- /dev/null +++ b/backend/__tests__/unit/helm/litellm_model_fallback_signal_test.py @@ -0,0 +1,199 @@ +"""Unit tests for k8s/helm/commonly/files/litellm/model_fallback_signal.py. + +Run by litellmModelFallbackSignal.test.js as `python3 -I `. +LiteLLM is not installed in CI, so a stub CustomLogger stands in for the one +module the signal imports. The kwargs mirror what the v1.106.2 router passes at +the fallback hop (router_utils/fallback_event_handlers.py): `model` is the group +tried instead, and the metadata carries original_model_group, model_group and +attempted_fallbacks. +""" +import asyncio +import contextlib +import dataclasses +import importlib.util +import io +import json +import sys +import types +import unittest + +sys.dont_write_bytecode = True + +stub = types.ModuleType('litellm.integrations.custom_logger') +stub.CustomLogger = type('CustomLogger', (), {}) +for name in ('litellm', 'litellm.integrations'): + sys.modules.setdefault(name, types.ModuleType(name)) +sys.modules['litellm.integrations.custom_logger'] = stub + +SIGNAL_PATH = sys.argv.pop(1) +spec = importlib.util.spec_from_file_location('model_fallback_signal', SIGNAL_PATH) +signal = importlib.util.module_from_spec(spec) +spec.loader.exec_module(signal) + +NEMOTRON = 'openrouter/nvidia/nemotron-3-super-120b-a12b:free' +DEEPSEEK = 'deepseek-v4-flash' + + +class NotFoundError(Exception): + status_code = 404 + + +@dataclasses.dataclass(slots=True) +class Attempts: + """Stands in for AttemptedFallbackTargets, the object a request shares across + hops. Shaped like v1.106.2's: a slots dataclass, so unhashable and not + weak-referenceable, which a plain stub would hide.""" + keys: frozenset = frozenset() + + +GPT = 'gpt-5.4' + + +def router_function(name): + def function(): + pass + function.__name__ = name + return function + + +# The v1.106.2 router function behind each bucket: a chat completion stamps +# `metadata`, and /v1/messages takes the generic path, which stamps +# `litellm_metadata`. +CHAT = router_function('_acompletion') +MESSAGES = router_function('_ageneric_api_call_with_fallbacks_anthropic_messages_attempt') + + +def hop_kwargs(metadata_name='metadata', attempts=None, requested=NEMOTRON, fallback=DEEPSEEK, depth=1): + return { + 'model': fallback, + 'original_function': MESSAGES if metadata_name == 'litellm_metadata' else CHAT, + 'messages': [{'role': 'user', 'content': 'private prompt text'}], + 'attempted_targets': attempts if attempts is not None else Attempts(), + metadata_name: { + 'original_model_group': requested, + 'model_group': fallback, + 'attempted_fallbacks': depth, + 'user_api_key_alias': 'community-agent-key', + 'user_api_key_hash': 'hash-should-not-appear', + }, + } + + +def captured_lines(hook, *args): + out = io.StringIO() + with contextlib.redirect_stdout(out): + asyncio.run(hook(*args)) + return [json.loads(line) for line in out.getvalue().splitlines() if line.strip()] + + +served = signal.proxy_handler_instance.log_success_fallback_event +failed = signal.proxy_handler_instance.log_failure_fallback_event + + +class FallbackLine(unittest.TestCase): + def test_a_served_fallback_names_both_groups_and_the_reason(self): + error = NotFoundError('OpenrouterException - No endpoints found for nvidia/nemotron-3-super-120b-a12b:free.') + self.assertEqual(captured_lines(served, NEMOTRON, hop_kwargs(), error), [{ + 'alarm': 'model_fallback', + 'outcome': 'served', + 'requested': NEMOTRON, + 'from': NEMOTRON, + 'fallback': DEEPSEEK, + 'attempt': 1, + 'reason': 'NotFoundError 404', + }]) + + def test_a_failed_fallback_is_reported_with_its_outcome(self): + lines = captured_lines(failed, NEMOTRON, hop_kwargs(), NotFoundError('gone')) + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]['outcome'], 'failed') + self.assertEqual((lines[0]['requested'], lines[0]['fallback']), (NEMOTRON, DEEPSEEK)) + + def test_in_a_two_hop_chain_only_the_hop_that_answered_is_served(self): + # gpt-5.4 -> nemotron -> deepseek, as the chart chains them. The router + # runs the inner hop inside the outer one, so the inner hop logs first. + attempts = Attempts() + inner = captured_lines(served, NEMOTRON, hop_kwargs(attempts=attempts, requested=GPT, depth=2), NotFoundError('x')) + outer = captured_lines(served, GPT, hop_kwargs(attempts=attempts, requested=GPT, fallback=NEMOTRON), RuntimeError('x')) + self.assertEqual([(l['outcome'], l['requested'], l['from'], l['fallback']) for l in inner + outer], [ + ('served', GPT, NEMOTRON, DEEPSEEK), + ('passed', GPT, GPT, NEMOTRON), + ]) + # Another request starts fresh. + self.assertEqual(captured_lines(served, NEMOTRON, hop_kwargs(), None)[0]['outcome'], 'served') + + def test_requests_are_remembered_up_to_a_bound(self): + first = Attempts() + captured_lines(served, NEMOTRON, hop_kwargs(attempts=first), None) + for _ in range(signal.ANSWERED_MAX): + captured_lines(served, NEMOTRON, hop_kwargs(), None) + self.assertEqual(len(signal._ANSWERED), signal.ANSWERED_MAX) + self.assertNotIn(id(first), signal._ANSWERED) + + def test_a_hop_without_its_request_object_is_taken_at_its_word(self): + kwargs = hop_kwargs() + del kwargs['attempted_targets'] + for _ in range(2): + self.assertEqual(captured_lines(served, NEMOTRON, kwargs, None)[0]['outcome'], 'served') + + def test_the_litellm_metadata_name_is_read_too(self): + line = signal.fallback_line('served', NEMOTRON, hop_kwargs('litellm_metadata'), None) + self.assertEqual((line['attempt'], line['requested'], line['reason']), (1, NEMOTRON, None)) + + def test_a_callers_own_metadata_never_stands_in_for_the_routers(self): + kwargs = hop_kwargs('litellm_metadata', requested=GPT) + kwargs['metadata'] = {'original_model_group': 'caller text', 'model_group': 'caller text', 'attempted_fallbacks': 9} + line = signal.fallback_line('served', NEMOTRON, kwargs, None) + self.assertEqual((line['requested'], line['attempt']), (GPT, 1)) + self.assertNotIn('caller text', json.dumps(line)) + + def test_a_chat_completion_reads_metadata_even_beside_a_litellm_metadata(self): + # The router stamps a chat completion's hop into `metadata` and leaves a + # litellm_metadata the request carried as it was. + kwargs = hop_kwargs(requested=GPT, depth=2) + kwargs['litellm_metadata'] = {'model_group': 'unstamped'} + line = signal.fallback_line('served', NEMOTRON, kwargs, None) + self.assertEqual((line['requested'], line['attempt']), (GPT, 2)) + self.assertNotIn('unstamped', json.dumps(line)) + + def test_without_a_router_function_the_hop_is_read_from_metadata(self): + kwargs = hop_kwargs(requested=GPT) + del kwargs['original_function'] + self.assertEqual(signal.fallback_line('served', NEMOTRON, kwargs, None)['requested'], GPT) + + def test_no_request_content_key_or_alias_is_logged(self): + out = json.dumps(signal.fallback_line('served', NEMOTRON, hop_kwargs(), NotFoundError('x'))) + for secret in ('private prompt text', 'hash-should-not-appear', 'community-agent-key'): + self.assertNotIn(secret, out) + + def test_the_reason_never_carries_the_error_message(self): + # A provider can echo the request into its error, prompt included. + error = NotFoundError('400 invalid request: messages[0].content="private prompt text"') + line = signal.fallback_line('served', NEMOTRON, hop_kwargs(), error) + self.assertEqual(line['reason'], 'NotFoundError 404') + self.assertEqual(signal.fallback_line('served', NEMOTRON, hop_kwargs(), RuntimeError('a\nb'))['reason'], 'RuntimeError') + + def test_a_dict_fallback_target_falls_back_to_the_metadata_group(self): + kwargs = hop_kwargs() + kwargs['model'] = {'model': DEEPSEEK} + self.assertEqual(signal.fallback_line('served', NEMOTRON, kwargs, None)['fallback'], DEEPSEEK) + + def test_a_closed_stdout_does_not_raise_into_the_request(self): + closed = io.StringIO() + closed.close() + with contextlib.redirect_stdout(closed): + asyncio.run(served(NEMOTRON, hop_kwargs(), NotFoundError('x'))) + asyncio.run(failed(NEMOTRON, hop_kwargs(), NotFoundError('x'))) + + def test_malformed_events_still_print_one_line_and_never_raise(self): + class Unprintable: + def __str__(self): + raise ValueError('no str') + for args in ((None, None, None), (NEMOTRON, {'model': DEEPSEEK, 'attempted_targets': []}, Unprintable())): + lines = captured_lines(served, *args) + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]['alarm'], 'model_fallback') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/docs/development/LITELLM.md b/docs/development/LITELLM.md index 785b6deb40..71e483aaff 100644 --- a/docs/development/LITELLM.md +++ b/docs/development/LITELLM.md @@ -123,6 +123,27 @@ a fake upstream. One quirk to expect: the `response.created` echo of the request tools still shows the flat names. To shim another DeepSeek group, add it to `SHIM_MODEL_GROUPS`. Tests are in `backend/__tests__/unit/helm/litellmDeepseekToolShim.test.js`. +**Model fallback signal** (`/app/model_fallback_signal.py`, same source dir and +mount; the third callback). When `router_settings.fallbacks` answers a request +from another group, the caller still sees the group it asked for, so a +community agent whose Nemotron route fails looks healthy while DeepSeek does +the work (TASK-003). The callback prints one JSON line per fallback hop: +`{"alarm":"model_fallback","outcome","requested","from","fallback","attempt","reason"}`. +`requested` is the group the caller asked for; `from` is the group that failed +at this hop. The chart chains two hops (gpt-5.4 → nemotron → deepseek), and the +router runs the inner hop inside the outer one, so the hop whose fallback +answered logs first: it is `served`, and the outer hop of the same request is +`passed`. `failed` means the fallback did not answer. `reason` is the failure's +class and status only. The error message stays out, because a provider can +echo the request into it, and so do the key and its alias. The `model_fallback` log +metric in `k8s/monitoring/log-metrics.json` counts these lines, one per hop. +Native hosted turns send `disable_fallbacks` (D81), so every count comes from +another caller: the moltbot gateway, commonly-bot, or platform features +through `llmService`. +Verified 2026-10-26 against the real 1.106.2 Router with a local upstream that +returns OpenRouter's 404 for Nemotron, one hop and two. Tests are in +`backend/__tests__/unit/helm/litellmModelFallbackSignal.test.js`. + **Why this works:** LiteLLM's `Authenticator.get_access_token()` calls `_read_auth_file()` on every request — there's no in-memory token caching. So a `rename` of `auth.json` is picked up within ~1 request, no pod restart needed. diff --git a/k8s/helm/commonly/files/litellm/model_fallback_signal.py b/k8s/helm/commonly/files/litellm/model_fallback_signal.py new file mode 100644 index 0000000000..5336716117 --- /dev/null +++ b/k8s/helm/commonly/files/litellm/model_fallback_signal.py @@ -0,0 +1,129 @@ +"""LiteLLM callback: one JSON log line for every model fallback hop (TASK-003). + +When a model group fails, the router answers from its `router_settings.fallbacks` +chain, and the caller gets that answer with nothing to say another model wrote +it. Native hosted turns refuse fallbacks (`disable_fallbacks`, D81), but every +other caller, the moltbot gateway included, takes them silently. + +Every fallback passes through here: the router calls log_success_fallback_event +or log_failure_fallback_event on each CustomLogger in `litellm_settings.callbacks` +(litellm/router_utils/fallback_event_handlers.py, v1.106.2). Each event prints +one line to stdout, and GKE parses it into jsonPayload: + + {"alarm": "model_fallback", "outcome": "served" | "passed" | "failed", + "requested": , "from": , + "fallback": , "attempt": , + "reason": } + +A chain can have two hops (gpt-5.4 -> nemotron -> deepseek-v4-flash). The router +runs the inner hop inside the outer one, so the hop whose fallback answered logs +first. `served` marks that hop. `passed` marks an outer hop of the same request, +whose fallback failed as well. `failed` means the fallback did not answer. + +The log-based metric `model_fallback` (k8s/monitoring/log-metrics.json) counts +these lines. + +It logs only model group names, the depth, and the error's class and status. It +never logs the error's message, because a provider can echo the request into it, +and never logs the key or its alias. It never raises, so a malformed event can't +cost a caller its answer. +""" +import json +from collections import OrderedDict + +try: + from litellm.integrations.custom_logger import CustomLogger +except Exception: # LiteLLM itself is broken; stay importable. + CustomLogger = object + +try: + from litellm.router_utils.batch_utils import get_router_metadata_variable_name +except Exception: # Not importable here; mirror v1.106.2's rule (batch_utils.py:155). + def get_router_metadata_variable_name(function_name): + markers = ('batch', 'generic_api_call', '_acreate_batch', 'file', '_ageneric_api_call_with_fallbacks') + if function_name and any(marker in function_name for marker in markers): + return 'litellm_metadata' + return 'metadata' + +# The router shares one AttemptedFallbackTargets object across every hop of a +# request (kwargs['attempted_targets']). A request is in here once one of its +# hops has logged `served`. That object is a slots dataclass, neither hashable +# nor weak-referenceable, so it is held by identity, and the oldest entries go +# first. A request's hops log within one call, far inside the bound. +_ANSWERED = OrderedDict() +ANSWERED_MAX = 1024 + + +def _metadata(kwargs): + # The bucket the router stamped this hop into. run_async_fallback picks it by + # the router function's name, not by which buckets the request carries + # (fallback_event_handlers.py:717): litellm_metadata for batch, file and + # generic API calls such as /v1/messages, metadata for the rest, chat + # completions included. Choosing by contents could read a bucket the router + # never stamped, the caller's own among them. + function_name = getattr(kwargs.get('original_function'), '__name__', None) + value = kwargs.get(get_router_metadata_variable_name(function_name)) + return value if isinstance(value, dict) else {} + + +def _reason(error): + if error is None: + return None + status = getattr(error, 'status_code', None) + return type(error).__name__ if status is None else f'{type(error).__name__} {status}' + + +def _first_answer(kwargs): + """True for the first success hop of a request: the hop whose fallback answered.""" + attempts = kwargs.get('attempted_targets') if isinstance(kwargs, dict) else None + if attempts is None: + return True # No per-request object to key on, so take the hop at its word. + if _ANSWERED.get(id(attempts)) is attempts: + return False + _ANSWERED[id(attempts)] = attempts + while len(_ANSWERED) > ANSWERED_MAX: + _ANSWERED.popitem(last=False) + return True + + +def fallback_line(outcome, original_model_group, kwargs, original_exception): + kwargs = kwargs if isinstance(kwargs, dict) else {} + meta = _metadata(kwargs) + fallback = kwargs.get('model') + if not isinstance(fallback, str): + fallback = meta.get('model_group') + return { + 'alarm': 'model_fallback', + 'outcome': outcome, + 'requested': meta.get('original_model_group') or original_model_group, + 'from': original_model_group, + 'fallback': fallback, + 'attempt': meta.get('attempted_fallbacks'), + 'reason': _reason(original_exception), + } + + +def emit(outcome, original_model_group, kwargs, original_exception): + try: + line = json.dumps(fallback_line(outcome, original_model_group, kwargs, original_exception), default=str) + except Exception as error: + line = json.dumps({'alarm': 'model_fallback', 'outcome': outcome, 'signalError': type(error).__name__}) + try: + print(line, flush=True) + except Exception: + pass # No stdout to write to; the caller's answer comes first. + + +class ModelFallbackSignal(CustomLogger): + async def log_success_fallback_event(self, original_model_group, kwargs, original_exception): + try: + outcome = 'served' if _first_answer(kwargs) else 'passed' + except Exception: + outcome = 'served' + emit(outcome, original_model_group, kwargs, original_exception) + + async def log_failure_fallback_event(self, original_model_group, kwargs, original_exception): + emit('failed', original_model_group, kwargs, original_exception) + + +proxy_handler_instance = ModelFallbackSignal() diff --git a/k8s/helm/commonly/templates/agents/litellm-deployment.yaml b/k8s/helm/commonly/templates/agents/litellm-deployment.yaml index 6a9eb38c91..b7e1f742d4 100644 --- a/k8s/helm/commonly/templates/agents/litellm-deployment.yaml +++ b/k8s/helm/commonly/templates/agents/litellm-deployment.yaml @@ -267,6 +267,9 @@ spec: - name: litellm-config mountPath: /app/deepseek_tool_shim_core.py subPath: deepseek_tool_shim_core.py + - name: litellm-config + mountPath: /app/model_fallback_signal.py + subPath: model_fallback_signal.py - name: chatgpt-auth mountPath: /chatgpt-auth resources: diff --git a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml index def020f59d..00a9aad6a0 100644 --- a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml +++ b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml @@ -331,7 +331,10 @@ data: # deepseek_tool_shim flattens codex's namespace tools into plain function # tools for the deepseek-flash-1017 group only, which DeepSeek requires; # source in files/litellm/deepseek_tool_shim.py, mounted from this ConfigMap. - callbacks: ["rate_limit_signal.proxy_handler_instance", "deepseek_tool_shim.proxy_handler_instance"] + # model_fallback_signal prints one {"alarm":"model_fallback"} line per + # fallback in router_settings.fallbacks, naming requested and fallback + # groups (TASK-003); the `model_fallback` log metric counts them. + callbacks: ["rate_limit_signal.proxy_handler_instance", "deepseek_tool_shim.proxy_handler_instance", "model_fallback_signal.proxy_handler_instance"] guardrails: # Anti-pattern content guardrails for the public-launch exposure. Both are @@ -401,4 +404,6 @@ data: {{- .Files.Get "files/litellm/deepseek_tool_shim.py" | nindent 4 }} deepseek_tool_shim_core.py: | {{- .Files.Get "files/litellm/deepseek_tool_shim_core.py" | nindent 4 }} + model_fallback_signal.py: | + {{- .Files.Get "files/litellm/model_fallback_signal.py" | nindent 4 }} {{- end }} diff --git a/k8s/monitoring/log-metrics.json b/k8s/monitoring/log-metrics.json index cd7d7f8c3c..0b5467dddf 100644 --- a/k8s/monitoring/log-metrics.json +++ b/k8s/monitoring/log-metrics.json @@ -23,6 +23,12 @@ "filter": "resource.type=\"k8s_container\" AND resource.labels.namespace_name=\"commonly-dev\" AND resource.labels.container_name=\"litellm\" AND textPayload=~\"\\\"POST /chat/completions HTTP/1.1\\\" [45][0-9][0-9]\"", "metricDescriptor": { "metricKind": "DELTA", "valueType": "INT64", "unit": "1" } }, + { + "name": "model_fallback", + "description": "LiteLLM answered (outcome=served) or tried to answer (outcome=failed) a request from a model group other than the one requested, via router_settings.fallbacks (TASK-003). One line per fallback hop from k8s/helm/commonly/files/litellm/model_fallback_signal.py; on a two-hop chain the hop that answered is served and the outer hop is passed. The requested group, the group that failed, the fallback group and the failure's class and status are in the same log line. Native hosted turns refuse fallbacks (D81), so every count comes from another caller: the moltbot gateway, commonly-bot, or platform features.", + "filter": "resource.type=\"k8s_container\" AND resource.labels.namespace_name=\"commonly-dev\" AND resource.labels.container_name=\"litellm\" AND jsonPayload.alarm=\"model_fallback\"", + "metricDescriptor": { "metricKind": "DELTA", "valueType": "INT64", "unit": "1" } + }, { "name": "stranger_journey_failures", "description": "Stranger-journey runs that ended with result=fail (one JSON line per run from k8s/helm/commonly/files/stranger-journey.py). The failed step and its error are in the same log line.",