From 912b7847b6936f0f72bd098fd1d5f4aaa1884f2d Mon Sep 17 00:00:00 2001 From: Wrisa Date: Thu, 16 Apr 2026 13:18:31 -0700 Subject: [PATCH 01/27] Add workflow and refactor LLM for langchain --- .../examples/workflow/main.py | 115 +++++++ .../examples/workflow/requirements.txt | 8 + .../langchain/callback_handler.py | 45 +-- .../tests/test_workflow_chain.py | 307 ++++++++++++++++++ 4 files changed, 448 insertions(+), 27 deletions(-) create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py new file mode 100644 index 00000000000..d74f5efa73a --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py @@ -0,0 +1,115 @@ +""" +LangGraph StateGraph example with an LLM node. + +Similar to the manual example (../manual/main.py) but uses LangGraph's StateGraph +with a node that calls ChatOpenAI. OpenTelemetry LangChain instrumentation traces +the LLM calls made from within the graph node. +""" + +from typing import Annotated + + +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI +from langgraph.graph import END, START, StateGraph +from langgraph.graph.message import add_messages +from typing_extensions import TypedDict + +from opentelemetry import _logs, metrics, trace +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + + +# Configure tracing +trace.set_tracer_provider(TracerProvider()) +span_processor = BatchSpanProcessor(OTLPSpanExporter()) +trace.get_tracer_provider().add_span_processor(span_processor) + +# Configure logging +_logs.set_logger_provider(LoggerProvider()) +_logs.get_logger_provider().add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter()) +) + +# Configure metrics +metrics.set_meter_provider( + MeterProvider( + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(), + ), + ] + ) +) + + +class GraphState(TypedDict): + """State for the graph; messages are accumulated with add_messages.""" + + messages: Annotated[list, add_messages] + + +def build_graph(llm: ChatOpenAI): + """Build a StateGraph with a single LLM node.""" + + def llm_node(state: GraphState) -> dict: + """Node that invokes the LLM with the current messages.""" + response = llm.invoke(state["messages"]) + return {"messages": [response]} + + builder = StateGraph(GraphState) + builder.add_node("llm", llm_node) + builder.add_edge(START, "llm") + builder.add_edge("llm", END) + return builder.compile() + + +def main(): + # Set up instrumentation (traces LLM calls from within graph nodes) + LangChainInstrumentor().instrument() + + # ChatOpenAI setup + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + frequency_penalty=0.5, + presence_penalty=0.5, + stop_sequences=["\n", "Human:", "AI:"], + seed=100, + ) + + graph = build_graph(llm) + + initial_messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + result = graph.invoke({"messages": initial_messages}) + + print("LangGraph output (messages):") + for msg in result.get("messages", []): + print(f" {type(msg).__name__}: {msg.content}") + + # Un-instrument after use + LangChainInstrumentor().uninstrument() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt new file mode 100644 index 00000000000..f27cb4a3c1b --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt @@ -0,0 +1,8 @@ +langchain==0.3.21 +langchain_openai +langgraph +opentelemetry-sdk>=1.39.0 +opentelemetry-exporter-otlp-proto-grpc>=1.39.0 + +# Uncomment after langchain instrumentation is released +# opentelemetry-instrumentation-langchain~=2.0b0.dev \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 5235af3c7f1..42ba85bd8b3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -25,10 +25,9 @@ _InvocationManager, ) from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import InferenceInvocation from opentelemetry.util.genai.types import ( - Error, InputMessage, - LLMInvocation, # TODO: migrate to InferenceInvocation MessagePart, OutputMessage, Text, @@ -140,25 +139,22 @@ def on_chat_model_start( ) ) - llm_invocation = LLMInvocation( + llm_invocation = self._telemetry_handler.start_inference( + provider, request_model=request_model, - input_messages=input_messages, - provider=provider, - top_p=top_p, - frequency_penalty=frequency_penalty, - presence_penalty=presence_penalty, - stop_sequences=stop_sequences, - seed=seed, - temperature=temperature, - max_tokens=max_tokens, - ) - llm_invocation = self._telemetry_handler.start_llm( - invocation=llm_invocation ) + llm_invocation.input_messages = input_messages + llm_invocation.top_p = top_p + llm_invocation.frequency_penalty = frequency_penalty + llm_invocation.presence_penalty = presence_penalty + llm_invocation.stop_sequences = stop_sequences + llm_invocation.seed = seed + llm_invocation.temperature = temperature + llm_invocation.max_tokens = max_tokens self._invocation_manager.add_invocation_state( run_id=run_id, parent_run_id=parent_run_id, - invocation=llm_invocation, # pyright: ignore[reportArgumentType] + invocation=llm_invocation, ) def on_llm_end( @@ -172,7 +168,7 @@ def on_llm_end( llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) if llm_invocation is None or not isinstance( llm_invocation, - LLMInvocation, + InferenceInvocation, ): # If the invocation does not exist, we cannot set attributes or end it return @@ -247,10 +243,8 @@ def on_llm_end( if response_id is not None: llm_invocation.response_id = str(response_id) - llm_invocation = self._telemetry_handler.stop_llm( - invocation=llm_invocation - ) - if llm_invocation.span and not llm_invocation.span.is_recording(): + llm_invocation.stop() + if not llm_invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id=run_id) def on_llm_error( @@ -264,14 +258,11 @@ def on_llm_error( llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) if llm_invocation is None or not isinstance( llm_invocation, - LLMInvocation, + InferenceInvocation, ): # If the invocation does not exist, we cannot set attributes or end it return - error_otel = Error(message=str(error), type=type(error)) - llm_invocation = self._telemetry_handler.fail_llm( - invocation=llm_invocation, error=error_otel - ) - if llm_invocation.span and not llm_invocation.span.is_recording(): + llm_invocation.fail(error) + if not llm_invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id=run_id) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py new file mode 100644 index 00000000000..d407378e7f6 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py @@ -0,0 +1,307 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests verifying CSA propagation through the LangChain callback handler. + +These tests exercise the OpenTelemetryLangChainCallbackHandler directly using +mock LangChain callback payloads, so they do not require live models or VCR +cassettes. The key behaviour under test: + + on_chain_start(parent_run_id=None) + → WorkflowInvocation created + → TelemetryHandler.start() called + → gen_ai.workflow.name written to context-scoped attributes (CSA) + + on_chat_model_start(...) + → LLMInvocation created + → TelemetryHandler.start() called + → gen_ai.workflow.name read from CSA and stamped on the LLM invocation + + on_llm_end(...) + → LLM span closed with gen_ai.workflow.name attribute set + + on_chain_end(...) + → workflow span closed, CSA scope ends +""" + +from __future__ import annotations + +import os +import uuid +from unittest import TestCase +from unittest.mock import patch + +from langchain_core.outputs import ChatGeneration, LLMResult +from langchain_core.messages import AIMessage + +from opentelemetry import baggage +from opentelemetry import context as otel_context +from opentelemetry.instrumentation.langchain.callback_handler import ( + OpenTelemetryLangChainCallbackHandler, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import SpanKind +from opentelemetry.util.genai.context_attributes import ( + get_context_scoped_attributes, +) +from opentelemetry.util.genai.handler import TelemetryHandler + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + +def _make_serialized(name: str) -> dict: + """Minimal serialized dict that on_chain_start / on_chat_model_start expect.""" + return {"name": name} + + +def _make_llm_result(content: str = "hello") -> LLMResult: + """Minimal LLMResult with one generation.""" + msg = AIMessage(content=content) + gen = ChatGeneration(message=msg, text=content) + gen.generation_info = {"finish_reason": "stop"} + return LLMResult(generations=[[gen]], llm_output={"model_name": "gpt-3.5-turbo"}) + + +def _make_chat_invocation_params(model_name: str = "gpt-3.5-turbo") -> dict: + """kwargs dict that on_chat_model_start receives for a ChatOpenAI call.""" + return {"invocation_params": {"model_name": model_name, "params": {"model_name": model_name}}} + + +# --------------------------------------------------------------------------- +# Base test class +# --------------------------------------------------------------------------- + +class _CallbackHandlerTestBase(TestCase): + def setUp(self) -> None: + self.span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + SimpleSpanProcessor(self.span_exporter) + ) + telemetry_handler = TelemetryHandler(tracer_provider=tracer_provider) + self.handler = OpenTelemetryLangChainCallbackHandler( + telemetry_handler=telemetry_handler + ) + + def _finished_spans(self): + return self.span_exporter.get_finished_spans() + + def _spans_by_kind(self, kind: SpanKind): + return [s for s in self._finished_spans() if s.kind == kind] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestWorkflowSpanCreation(_CallbackHandlerTestBase): + """Verify that a workflow span is created for top-level chains.""" + + def test_workflow_span_created_for_top_level_chain(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("MyChain"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_end( + outputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + self.assertEqual(internal_spans[0].name, "invoke_workflow MyChain") + + def test_no_workflow_span_for_nested_chain(self) -> None: + """Chains with a parent_run_id are nested — no extra workflow span.""" + parent_run_id = uuid.uuid4() + child_run_id = uuid.uuid4() + + # Start parent (top-level) + self.handler.on_chain_start( + serialized=_make_serialized("ParentChain"), + inputs={}, + run_id=parent_run_id, + parent_run_id=None, + ) + # Start child (nested — should NOT create a workflow span) + self.handler.on_chain_start( + serialized=_make_serialized("ChildChain"), + inputs={}, + run_id=child_run_id, + parent_run_id=parent_run_id, + ) + self.handler.on_chain_end( + outputs={}, + run_id=child_run_id, + parent_run_id=parent_run_id, + ) + self.handler.on_chain_end( + outputs={}, + run_id=parent_run_id, + parent_run_id=None, + ) + + # Only one INTERNAL (workflow) span — for the parent, not the child + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + self.assertEqual(internal_spans[0].name, "invoke_workflow ParentChain") + + +class TestLLMSpanGetsWorkflowName(_CallbackHandlerTestBase): + """Verify gen_ai.workflow.name is propagated to the LLM span via CSA.""" + + def test_llm_span_inside_chain_gets_workflow_name(self) -> None: + chain_run_id = uuid.uuid4() + llm_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("MyPipeline"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chat_model_start( + serialized=_make_serialized("ChatOpenAI"), + messages=[[AIMessage(content="hi")]], + run_id=llm_run_id, + parent_run_id=chain_run_id, + metadata={"ls_provider": "openai"}, + **_make_chat_invocation_params("gpt-3.5-turbo"), + ) + self.handler.on_llm_end( + response=_make_llm_result(), + run_id=llm_run_id, + parent_run_id=chain_run_id, + ) + self.handler.on_chain_end( + outputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + + client_spans = self._spans_by_kind(SpanKind.CLIENT) + self.assertEqual(len(client_spans), 1) + self.assertEqual( + client_spans[0].attributes.get("gen_ai.workflow.name"), + "MyPipeline", + ) + + def test_workflow_name_from_metadata_override(self) -> None: + """metadata['workflow_name'] overrides the serialized chain name.""" + chain_run_id = uuid.uuid4() + llm_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("InternalChainName"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + metadata={"workflow_name": "my_custom_wf"}, + ) + self.handler.on_chat_model_start( + serialized=_make_serialized("ChatOpenAI"), + messages=[[AIMessage(content="hi")]], + run_id=llm_run_id, + parent_run_id=chain_run_id, + metadata={"ls_provider": "openai"}, + **_make_chat_invocation_params("gpt-3.5-turbo"), + ) + self.handler.on_llm_end( + response=_make_llm_result(), + run_id=llm_run_id, + parent_run_id=chain_run_id, + ) + self.handler.on_chain_end( + outputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + self.assertEqual(internal_spans[0].name, "invoke_workflow my_custom_wf") + + client_spans = self._spans_by_kind(SpanKind.CLIENT) + self.assertEqual(len(client_spans), 1) + self.assertEqual( + client_spans[0].attributes.get("gen_ai.workflow.name"), + "my_custom_wf", + ) + + +class TestCSANotLeakedToBaggage(_CallbackHandlerTestBase): + """Verify that gen_ai.workflow.name is NOT written to W3C Baggage by default.""" + + def test_csa_not_leaked_to_baggage(self) -> None: + env = {k: v for k, v in os.environ.items() if k != "OTEL_PYTHON_GENAI_CAPTURE_BAGGAGE"} + with patch.dict(os.environ, env, clear=True): + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("BaggageTestChain"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + try: + # While workflow is active, baggage should NOT contain the workflow name + baggage_value = baggage.get_baggage("gen_ai.workflow.name") + self.assertIsNone( + baggage_value, + "gen_ai.workflow.name must not be leaked to W3C Baggage by default", + ) + finally: + self.handler.on_chain_end( + outputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + + +class TestCSAScopeEndsAfterChain(_CallbackHandlerTestBase): + """Verify that the CSA is no longer visible after the chain ends.""" + + def test_csa_not_visible_outside_workflow_scope(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("ScopedChain"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_end( + outputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + + # After on_chain_end the context token is detached — CSA should be gone + attrs = get_context_scoped_attributes() + self.assertIsNone( + attrs.get("gen_ai.workflow.name"), + "gen_ai.workflow.name should not be visible after workflow scope ends", + ) From 1ce4cf5ee9e2b86bbc41b0c98f7ee51bc7e09188 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Thu, 16 Apr 2026 14:09:45 -0700 Subject: [PATCH 02/27] add workflow support and genai dependancy --- .../pyproject.toml | 3 +- .../langchain/callback_handler.py | 71 +++++ .../tests/test_workflow_chain.py | 252 ++++++++++++++++++ 3 files changed, 325 insertions(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml index 80a406b7dae..a87b82f9992 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml @@ -25,7 +25,8 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "opentelemetry-instrumentation ~= 0.57b0", + "opentelemetry-instrumentation ~= 0.60b0", + "opentelemetry-util-genai >= 0.4b0.dev", ] [project.optional-dependencies] diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 42ba85bd8b3..7ac8decd9b1 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -26,6 +26,7 @@ ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import InferenceInvocation +from opentelemetry.util.genai.invocation import WorkflowInvocation from opentelemetry.util.genai.types import ( InputMessage, MessagePart, @@ -44,6 +45,75 @@ def __init__(self, telemetry_handler: TelemetryHandler) -> None: self._telemetry_handler = telemetry_handler self._invocation_manager = _InvocationManager() + def on_chain_start( + self, + serialized: dict[str, Any], + inputs: dict[str, Any], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + metadata: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> Any: + payload = serialized or {} + name_source = ( + payload.get("name") + or payload.get("id") + or kwargs.get("name") + or (metadata.get("langgraph_node") if metadata else None) + ) + name = str(name_source or "chain") + + if parent_run_id is None: + workflow_name_override = metadata.get("workflow_name") if metadata else None + wf = self._telemetry_handler.start_workflow(name=workflow_name_override or name) + self._invocation_manager.add_invocation_state(run_id, None, wf) + return + else: + self._invocation_manager.add_invocation_state(run_id, parent_run_id) + + + def on_chain_end( + self, + outputs: dict[str, Any], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + invocation = self._invocation_manager.get_invocation(run_id=run_id) + if invocation is None or not isinstance( + invocation, WorkflowInvocation + ): + # If the invocation does not exist, we cannot set attributes or end it + return + + invocation.stop() + + if not invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id) + + + def on_chain_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + invocation = self._invocation_manager.get_invocation(run_id=run_id) + if invocation is None or not isinstance( + invocation, WorkflowInvocation + ): + # If the invocation does not exist, we cannot set attributes or end it + return + + invocation.fail(error) + if not invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id=run_id) + def on_chat_model_start( self, serialized: dict[str, Any], @@ -266,3 +336,4 @@ def on_llm_error( llm_invocation.fail(error) if not llm_invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id=run_id) + diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py index d407378e7f6..86c76eb6852 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py @@ -57,6 +57,7 @@ InMemorySpanExporter, ) from opentelemetry.trace import SpanKind +from opentelemetry.trace.status import StatusCode from opentelemetry.util.genai.context_attributes import ( get_context_scoped_attributes, ) @@ -305,3 +306,254 @@ def test_csa_not_visible_outside_workflow_scope(self) -> None: attrs.get("gen_ai.workflow.name"), "gen_ai.workflow.name should not be visible after workflow scope ends", ) + + +class TestWorkflowErrorPath(_CallbackHandlerTestBase): + """Verify on_chain_error records error status and cleans up state.""" + + def test_workflow_span_has_error_status_on_chain_error(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("FailingChain"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_error( + error=ValueError("something went wrong"), + run_id=chain_run_id, + parent_run_id=None, + ) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + span = internal_spans[0] + self.assertEqual(span.name, "invoke_workflow FailingChain") + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertIn("something went wrong", span.status.description) + + def test_workflow_span_error_type_attribute(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("FailingChain"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_error( + error=RuntimeError("boom"), + run_id=chain_run_id, + parent_run_id=None, + ) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + self.assertEqual( + internal_spans[0].attributes.get("error.type"), "RuntimeError" + ) + + def test_chain_error_cleans_up_invocation_state(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("FailingChain"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_error( + error=ValueError("oops"), + run_id=chain_run_id, + parent_run_id=None, + ) + + # Invocation should have been removed — a second error call is a no-op + self.handler.on_chain_error( + error=ValueError("duplicate"), + run_id=chain_run_id, + parent_run_id=None, + ) + # Only one span — the second call was silently ignored + self.assertEqual(len(self._finished_spans()), 1) + + def test_chain_error_csa_scope_ends(self) -> None: + """CSA should be gone after on_chain_error, same as on_chain_end.""" + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("FailingChain"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_error( + error=ValueError("oops"), + run_id=chain_run_id, + parent_run_id=None, + ) + + attrs = get_context_scoped_attributes() + self.assertIsNone( + attrs.get("gen_ai.workflow.name"), + "gen_ai.workflow.name should not be visible after workflow error", + ) + + def test_chain_error_unknown_run_id_is_noop(self) -> None: + """on_chain_error with an unknown run_id must not raise.""" + self.handler.on_chain_error( + error=ValueError("no matching invocation"), + run_id=uuid.uuid4(), + parent_run_id=None, + ) + self.assertEqual(len(self._finished_spans()), 0) + + def test_chain_end_unknown_run_id_is_noop(self) -> None: + """on_chain_end with an unknown run_id must not raise.""" + self.handler.on_chain_end( + outputs={}, + run_id=uuid.uuid4(), + parent_run_id=None, + ) + self.assertEqual(len(self._finished_spans()), 0) + + +class TestLLMErrorInsideWorkflow(_CallbackHandlerTestBase): + """Verify on_llm_error inside a workflow doesn't break the parent workflow span.""" + + def test_llm_error_inside_workflow_records_error_on_llm_span(self) -> None: + chain_run_id = uuid.uuid4() + llm_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("MyPipeline"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chat_model_start( + serialized=_make_serialized("ChatOpenAI"), + messages=[[AIMessage(content="hi")]], + run_id=llm_run_id, + parent_run_id=chain_run_id, + metadata={"ls_provider": "openai"}, + **_make_chat_invocation_params("gpt-3.5-turbo"), + ) + self.handler.on_llm_error( + error=RuntimeError("model timeout"), + run_id=llm_run_id, + parent_run_id=chain_run_id, + ) + self.handler.on_chain_end( + outputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + + client_spans = self._spans_by_kind(SpanKind.CLIENT) + self.assertEqual(len(client_spans), 1) + self.assertEqual(client_spans[0].status.status_code, StatusCode.ERROR) + + # Workflow span still finishes (not in error state) + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + self.assertNotEqual(internal_spans[0].status.status_code, StatusCode.ERROR) + + def test_llm_error_inside_workflow_llm_span_is_child_of_workflow(self) -> None: + chain_run_id = uuid.uuid4() + llm_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized=_make_serialized("MyPipeline"), + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chat_model_start( + serialized=_make_serialized("ChatOpenAI"), + messages=[[AIMessage(content="hi")]], + run_id=llm_run_id, + parent_run_id=chain_run_id, + metadata={"ls_provider": "openai"}, + **_make_chat_invocation_params("gpt-3.5-turbo"), + ) + self.handler.on_llm_end( + response=_make_llm_result(), + run_id=llm_run_id, + parent_run_id=chain_run_id, + ) + self.handler.on_chain_end( + outputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + client_spans = self._spans_by_kind(SpanKind.CLIENT) + self.assertEqual(len(internal_spans), 1) + self.assertEqual(len(client_spans), 1) + + workflow_span = internal_spans[0] + llm_span = client_spans[0] + self.assertEqual( + llm_span.context.trace_id, + workflow_span.context.trace_id, + "LLM span and workflow span must share the same trace", + ) + self.assertEqual( + llm_span.parent.span_id, + workflow_span.context.span_id, + "LLM span must be a child of the workflow span", + ) + + +class TestWorkflowNameFallback(_CallbackHandlerTestBase): + """Verify the name resolution fallback chain in on_chain_start.""" + + def test_name_falls_back_to_id_list(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized={"id": ["pkg", "mod", "MyRunnableClass"]}, + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_end(outputs={}, run_id=chain_run_id, parent_run_id=None) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + # id is a list — _safe_str(list) produces a string; just verify a span was created + self.assertTrue(internal_spans[0].name.startswith("invoke_workflow ")) + + def test_name_falls_back_to_langgraph_node(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized={}, + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + metadata={"langgraph_node": "my_node"}, + ) + self.handler.on_chain_end(outputs={}, run_id=chain_run_id, parent_run_id=None) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + self.assertEqual(internal_spans[0].name, "invoke_workflow my_node") + + def test_name_defaults_to_chain_when_nothing_provided(self) -> None: + chain_run_id = uuid.uuid4() + + self.handler.on_chain_start( + serialized={}, + inputs={}, + run_id=chain_run_id, + parent_run_id=None, + ) + self.handler.on_chain_end(outputs={}, run_id=chain_run_id, parent_run_id=None) + + internal_spans = self._spans_by_kind(SpanKind.INTERNAL) + self.assertEqual(len(internal_spans), 1) + self.assertEqual(internal_spans[0].name, "invoke_workflow chain") From e9fff126f51d3630f9f6a8c8d854a23527facfda Mon Sep 17 00:00:00 2001 From: wrisa Date: Mon, 20 Apr 2026 11:27:36 -0700 Subject: [PATCH 03/27] fixed errors --- .../examples/workflow/main.py | 4 +- .../langchain/callback_handler.py | 29 ++++++----- .../tests/test_workflow_chain.py | 48 ++++++++++++++----- uv.lock | 2 + 4 files changed, 56 insertions(+), 27 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py index d74f5efa73a..b48865ee21b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py @@ -8,7 +8,6 @@ from typing import Annotated - from langchain_core.messages import HumanMessage, SystemMessage from langchain_openai import ChatOpenAI from langgraph.graph import END, START, StateGraph @@ -33,7 +32,6 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor - # Configure tracing trace.set_tracer_provider(TracerProvider()) span_processor = BatchSpanProcessor(OTLPSpanExporter()) @@ -112,4 +110,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 7ac8decd9b1..c55b93481a2 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -25,8 +25,10 @@ _InvocationManager, ) from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.invocation import InferenceInvocation -from opentelemetry.util.genai.invocation import WorkflowInvocation +from opentelemetry.util.genai.invocation import ( + InferenceInvocation, + WorkflowInvocation, +) from opentelemetry.util.genai.types import ( InputMessage, MessagePart, @@ -58,21 +60,26 @@ def on_chain_start( ) -> Any: payload = serialized or {} name_source = ( - payload.get("name") - or payload.get("id") - or kwargs.get("name") - or (metadata.get("langgraph_node") if metadata else None) + payload.get("name") + or payload.get("id") + or kwargs.get("name") + or (metadata.get("langgraph_node") if metadata else None) ) name = str(name_source or "chain") if parent_run_id is None: - workflow_name_override = metadata.get("workflow_name") if metadata else None - wf = self._telemetry_handler.start_workflow(name=workflow_name_override or name) + workflow_name_override = ( + metadata.get("workflow_name") if metadata else None + ) + wf = self._telemetry_handler.start_workflow( + name=workflow_name_override or name + ) self._invocation_manager.add_invocation_state(run_id, None, wf) return else: - self._invocation_manager.add_invocation_state(run_id, parent_run_id) - + self._invocation_manager.add_invocation_state( + run_id, parent_run_id + ) def on_chain_end( self, @@ -94,7 +101,6 @@ def on_chain_end( if not invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id) - def on_chain_error( self, error: BaseException, @@ -336,4 +342,3 @@ def on_llm_error( llm_invocation.fail(error) if not llm_invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id=run_id) - diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py index 86c76eb6852..e1ce91a564b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py @@ -43,11 +43,10 @@ from unittest import TestCase from unittest.mock import patch -from langchain_core.outputs import ChatGeneration, LLMResult from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, LLMResult from opentelemetry import baggage -from opentelemetry import context as otel_context from opentelemetry.instrumentation.langchain.callback_handler import ( OpenTelemetryLangChainCallbackHandler, ) @@ -63,11 +62,11 @@ ) from opentelemetry.util.genai.handler import TelemetryHandler - # --------------------------------------------------------------------------- # Shared helpers # --------------------------------------------------------------------------- + def _make_serialized(name: str) -> dict: """Minimal serialized dict that on_chain_start / on_chat_model_start expect.""" return {"name": name} @@ -78,18 +77,26 @@ def _make_llm_result(content: str = "hello") -> LLMResult: msg = AIMessage(content=content) gen = ChatGeneration(message=msg, text=content) gen.generation_info = {"finish_reason": "stop"} - return LLMResult(generations=[[gen]], llm_output={"model_name": "gpt-3.5-turbo"}) + return LLMResult( + generations=[[gen]], llm_output={"model_name": "gpt-3.5-turbo"} + ) def _make_chat_invocation_params(model_name: str = "gpt-3.5-turbo") -> dict: """kwargs dict that on_chat_model_start receives for a ChatOpenAI call.""" - return {"invocation_params": {"model_name": model_name, "params": {"model_name": model_name}}} + return { + "invocation_params": { + "model_name": model_name, + "params": {"model_name": model_name}, + } + } # --------------------------------------------------------------------------- # Base test class # --------------------------------------------------------------------------- + class _CallbackHandlerTestBase(TestCase): def setUp(self) -> None: self.span_exporter = InMemorySpanExporter() @@ -113,6 +120,7 @@ def _spans_by_kind(self, kind: SpanKind): # Tests # --------------------------------------------------------------------------- + class TestWorkflowSpanCreation(_CallbackHandlerTestBase): """Verify that a workflow span is created for top-level chains.""" @@ -243,7 +251,9 @@ def test_workflow_name_from_metadata_override(self) -> None: internal_spans = self._spans_by_kind(SpanKind.INTERNAL) self.assertEqual(len(internal_spans), 1) - self.assertEqual(internal_spans[0].name, "invoke_workflow my_custom_wf") + self.assertEqual( + internal_spans[0].name, "invoke_workflow my_custom_wf" + ) client_spans = self._spans_by_kind(SpanKind.CLIENT) self.assertEqual(len(client_spans), 1) @@ -257,7 +267,11 @@ class TestCSANotLeakedToBaggage(_CallbackHandlerTestBase): """Verify that gen_ai.workflow.name is NOT written to W3C Baggage by default.""" def test_csa_not_leaked_to_baggage(self) -> None: - env = {k: v for k, v in os.environ.items() if k != "OTEL_PYTHON_GENAI_CAPTURE_BAGGAGE"} + env = { + k: v + for k, v in os.environ.items() + if k != "OTEL_PYTHON_GENAI_CAPTURE_BAGGAGE" + } with patch.dict(os.environ, env, clear=True): chain_run_id = uuid.uuid4() @@ -458,9 +472,13 @@ def test_llm_error_inside_workflow_records_error_on_llm_span(self) -> None: # Workflow span still finishes (not in error state) internal_spans = self._spans_by_kind(SpanKind.INTERNAL) self.assertEqual(len(internal_spans), 1) - self.assertNotEqual(internal_spans[0].status.status_code, StatusCode.ERROR) + self.assertNotEqual( + internal_spans[0].status.status_code, StatusCode.ERROR + ) - def test_llm_error_inside_workflow_llm_span_is_child_of_workflow(self) -> None: + def test_llm_error_inside_workflow_llm_span_is_child_of_workflow( + self, + ) -> None: chain_run_id = uuid.uuid4() llm_run_id = uuid.uuid4() @@ -520,7 +538,9 @@ def test_name_falls_back_to_id_list(self) -> None: run_id=chain_run_id, parent_run_id=None, ) - self.handler.on_chain_end(outputs={}, run_id=chain_run_id, parent_run_id=None) + self.handler.on_chain_end( + outputs={}, run_id=chain_run_id, parent_run_id=None + ) internal_spans = self._spans_by_kind(SpanKind.INTERNAL) self.assertEqual(len(internal_spans), 1) @@ -537,7 +557,9 @@ def test_name_falls_back_to_langgraph_node(self) -> None: parent_run_id=None, metadata={"langgraph_node": "my_node"}, ) - self.handler.on_chain_end(outputs={}, run_id=chain_run_id, parent_run_id=None) + self.handler.on_chain_end( + outputs={}, run_id=chain_run_id, parent_run_id=None + ) internal_spans = self._spans_by_kind(SpanKind.INTERNAL) self.assertEqual(len(internal_spans), 1) @@ -552,7 +574,9 @@ def test_name_defaults_to_chain_when_nothing_provided(self) -> None: run_id=chain_run_id, parent_run_id=None, ) - self.handler.on_chain_end(outputs={}, run_id=chain_run_id, parent_run_id=None) + self.handler.on_chain_end( + outputs={}, run_id=chain_run_id, parent_run_id=None + ) internal_spans = self._spans_by_kind(SpanKind.INTERNAL) self.assertEqual(len(internal_spans), 1) diff --git a/uv.lock b/uv.lock index 14d5ea35b6f..b532c37e778 100644 --- a/uv.lock +++ b/uv.lock @@ -3530,6 +3530,7 @@ name = "opentelemetry-instrumentation-langchain" source = { editable = "instrumentation-genai/opentelemetry-instrumentation-langchain" } dependencies = [ { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-util-genai" }, ] [package.optional-dependencies] @@ -3541,6 +3542,7 @@ instruments = [ requires-dist = [ { name = "langchain", marker = "extra == 'instruments'", specifier = ">=0.3.21" }, { name = "opentelemetry-instrumentation", editable = "opentelemetry-instrumentation" }, + { name = "opentelemetry-util-genai", editable = "util/opentelemetry-util-genai" }, ] provides-extras = ["instruments"] From 6e3462025413841076b957d419050bd08dbdddf0 Mon Sep 17 00:00:00 2001 From: wrisa Date: Mon, 20 Apr 2026 11:41:52 -0700 Subject: [PATCH 04/27] fixed changelog --- .../opentelemetry-instrumentation-langchain/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md index 767dfcc7ed6..a23d8b33d5c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Add LangChain workflow span support and refactor LLM invocation + ([#4449](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4449)) - Fix compatibility with wrapt 2.x by using positional arguments in `wrap_function_wrapper()` calls ([#4445](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4445)) - Added span support for genAI langchain llm invocation. From c17607d97883f5b1d7a8857791addff18f3383c6 Mon Sep 17 00:00:00 2001 From: wrisa Date: Mon, 20 Apr 2026 11:55:18 -0700 Subject: [PATCH 05/27] fixed error --- .../instrumentation/langchain/callback_handler.py | 3 ++- .../instrumentation/langchain/invocation_manager.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index c55b93481a2..f99a91a4be2 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -77,8 +77,9 @@ def on_chain_start( self._invocation_manager.add_invocation_state(run_id, None, wf) return else: + # TODO: For agent invocation self._invocation_manager.add_invocation_state( - run_id, parent_run_id + run_id, parent_run_id, ) def on_chain_end( diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index e8d2293baea..0431934f1ed 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -39,7 +39,7 @@ def add_invocation_state( self, run_id: UUID, parent_run_id: Optional[UUID], - invocation: GenAIInvocation, + invocation: GenAIInvocation = None, ): invocation_state = _InvocationState(invocation=invocation) self._invocations[run_id] = invocation_state From a63f271dd81f6de1863ba97c2dec62a24582e65d Mon Sep 17 00:00:00 2001 From: wrisa Date: Mon, 20 Apr 2026 12:02:55 -0700 Subject: [PATCH 06/27] fixed type error --- .../instrumentation/langchain/callback_handler.py | 3 ++- .../instrumentation/langchain/invocation_manager.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index f99a91a4be2..31a7db329d4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -79,7 +79,8 @@ def on_chain_start( else: # TODO: For agent invocation self._invocation_manager.add_invocation_state( - run_id, parent_run_id, + run_id, + parent_run_id, ) def on_chain_end( diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index 0431934f1ed..1383c861e82 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -23,7 +23,7 @@ @dataclass class _InvocationState: - invocation: GenAIInvocation + invocation: Optional[GenAIInvocation] children: List[UUID] = field(default_factory=lambda: list()) @@ -39,7 +39,7 @@ def add_invocation_state( self, run_id: UUID, parent_run_id: Optional[UUID], - invocation: GenAIInvocation = None, + invocation: Optional[GenAIInvocation] = None, ): invocation_state = _InvocationState(invocation=invocation) self._invocations[run_id] = invocation_state From e8be64160a552f041f627ba1efa3ce0d91c6ad46 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Tue, 21 Apr 2026 09:22:05 -0700 Subject: [PATCH 07/27] removed optional --- .../opentelemetry/instrumentation/langchain/callback_handler.py | 1 + .../instrumentation/langchain/invocation_manager.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 31a7db329d4..8b000ee7c63 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -81,6 +81,7 @@ def on_chain_start( self._invocation_manager.add_invocation_state( run_id, parent_run_id, + None ) def on_chain_end( diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index 1383c861e82..139e9f4a6ce 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -39,7 +39,7 @@ def add_invocation_state( self, run_id: UUID, parent_run_id: Optional[UUID], - invocation: Optional[GenAIInvocation] = None, + invocation: GenAIInvocation, ): invocation_state = _InvocationState(invocation=invocation) self._invocations[run_id] = invocation_state From a9dc1885b14387ec4a345f0265cf95fd568fc4c8 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Tue, 21 Apr 2026 11:49:07 -0700 Subject: [PATCH 08/27] fixed error --- .../instrumentation/langchain/callback_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 8b000ee7c63..d04654a73ae 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -79,9 +79,7 @@ def on_chain_start( else: # TODO: For agent invocation self._invocation_manager.add_invocation_state( - run_id, - parent_run_id, - None + run_id, parent_run_id, None ) def on_chain_end( From 36056317051d40ead8aec371aef49425e32f74d2 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Tue, 21 Apr 2026 12:05:54 -0700 Subject: [PATCH 09/27] ignore --- .../opentelemetry/instrumentation/langchain/callback_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index d04654a73ae..0223467643d 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -79,7 +79,7 @@ def on_chain_start( else: # TODO: For agent invocation self._invocation_manager.add_invocation_state( - run_id, parent_run_id, None + run_id, parent_run_id, None # type: ignore[arg-type] ) def on_chain_end( From 4b7808fdbc5cb00461e5990f70354c45fdf9186d Mon Sep 17 00:00:00 2001 From: Wrisa Date: Tue, 21 Apr 2026 12:26:30 -0700 Subject: [PATCH 10/27] fixed requests --- .../instrumentation/langchain/callback_handler.py | 4 +++- tests/opentelemetry-docker-tests/tests/test-requirements.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 0223467643d..3c214d9e65b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -79,7 +79,9 @@ def on_chain_start( else: # TODO: For agent invocation self._invocation_manager.add_invocation_state( - run_id, parent_run_id, None # type: ignore[arg-type] + run_id, + parent_run_id, + None, # type: ignore[arg-type] ) def on_chain_end( diff --git a/tests/opentelemetry-docker-tests/tests/test-requirements.txt b/tests/opentelemetry-docker-tests/tests/test-requirements.txt index 434f44d5975..50dfa259c9e 100644 --- a/tests/opentelemetry-docker-tests/tests/test-requirements.txt +++ b/tests/opentelemetry-docker-tests/tests/test-requirements.txt @@ -58,7 +58,7 @@ python-dotenv==0.21.1 pytz==2024.1 PyYAML==5.3.1 redis==5.0.1 -requests==2.25.0 +requests==2.32.4 six==1.16.0 SQLAlchemy==1.4.52 texttable==1.7.0 From 1b5f2a30e6714d95d987c9ae2eed99a92cf6447b Mon Sep 17 00:00:00 2001 From: Wrisa Date: Fri, 24 Apr 2026 13:16:34 -0700 Subject: [PATCH 11/27] Added delete and removed optional --- .../examples/workflow/main.py | 2 +- .../instrumentation/langchain/callback_handler.py | 6 ++++++ .../instrumentation/langchain/invocation_manager.py | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py index b48865ee21b..961d33b33ee 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py @@ -88,7 +88,7 @@ def main(): top_p=0.9, frequency_penalty=0.5, presence_penalty=0.5, - stop_sequences=["\n", "Human:", "AI:"], + stop=["\n", "Human:", "AI:"], seed=100, ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 3c214d9e65b..a9e9c91aa08 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -96,6 +96,12 @@ def on_chain_end( if invocation is None or not isinstance( invocation, WorkflowInvocation ): + # In on_chain_start, invocation state with None invocation will be added for intermediate chains and agents. + # If the invocation is None, it means on_chain_start was called but we are not tracking this chain (e.g. it's an agent or intermediate chain), + # so we should not attempt to stop it. Instead, we just clean up the invocation state since we won't be tracking it. + if self._invocation_manager.get_invocation_state(run_id): + self._invocation_manager.delete_invocation_state(run_id) + # If the invocation does not exist, we cannot set attributes or end it return diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index 139e9f4a6ce..6db8c923cdb 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -23,7 +23,7 @@ @dataclass class _InvocationState: - invocation: Optional[GenAIInvocation] + invocation: GenAIInvocation children: List[UUID] = field(default_factory=lambda: list()) @@ -47,6 +47,10 @@ def add_invocation_state( if parent_run_id is not None and parent_run_id in self._invocations: parent_invocation_state = self._invocations[parent_run_id] parent_invocation_state.children.append(run_id) + + def get_invocation_state(self, run_id: UUID) -> Optional[_InvocationState]: + invocation_state = self._invocations.get(run_id) + return invocation_state def get_invocation(self, run_id: UUID) -> Optional[GenAIInvocation]: invocation_state = self._invocations.get(run_id) From 93e806b02f779f84247c801575fa0f7d44da0688 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Fri, 24 Apr 2026 14:12:13 -0700 Subject: [PATCH 12/27] removed non workflow --- .../langchain/callback_handler.py | 15 +------- .../langchain/invocation_manager.py | 6 +--- .../tests/test_workflow_chain.py | 35 ------------------- 3 files changed, 2 insertions(+), 54 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index a9e9c91aa08..5d7b47f741b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -75,14 +75,7 @@ def on_chain_start( name=workflow_name_override or name ) self._invocation_manager.add_invocation_state(run_id, None, wf) - return - else: - # TODO: For agent invocation - self._invocation_manager.add_invocation_state( - run_id, - parent_run_id, - None, # type: ignore[arg-type] - ) + # TODO: handle non-workflow chains (e.g. agent sub-chains) in the future def on_chain_end( self, @@ -96,12 +89,6 @@ def on_chain_end( if invocation is None or not isinstance( invocation, WorkflowInvocation ): - # In on_chain_start, invocation state with None invocation will be added for intermediate chains and agents. - # If the invocation is None, it means on_chain_start was called but we are not tracking this chain (e.g. it's an agent or intermediate chain), - # so we should not attempt to stop it. Instead, we just clean up the invocation state since we won't be tracking it. - if self._invocation_manager.get_invocation_state(run_id): - self._invocation_manager.delete_invocation_state(run_id) - # If the invocation does not exist, we cannot set attributes or end it return diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index 6db8c923cdb..98176ec1e3f 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -40,17 +40,13 @@ def add_invocation_state( run_id: UUID, parent_run_id: Optional[UUID], invocation: GenAIInvocation, - ): + ) -> None: invocation_state = _InvocationState(invocation=invocation) self._invocations[run_id] = invocation_state if parent_run_id is not None and parent_run_id in self._invocations: parent_invocation_state = self._invocations[parent_run_id] parent_invocation_state.children.append(run_id) - - def get_invocation_state(self, run_id: UUID) -> Optional[_InvocationState]: - invocation_state = self._invocations.get(run_id) - return invocation_state def get_invocation(self, run_id: UUID) -> Optional[GenAIInvocation]: invocation_state = self._invocations.get(run_id) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py index e1ce91a564b..736116655ce 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py @@ -143,41 +143,6 @@ def test_workflow_span_created_for_top_level_chain(self) -> None: self.assertEqual(len(internal_spans), 1) self.assertEqual(internal_spans[0].name, "invoke_workflow MyChain") - def test_no_workflow_span_for_nested_chain(self) -> None: - """Chains with a parent_run_id are nested — no extra workflow span.""" - parent_run_id = uuid.uuid4() - child_run_id = uuid.uuid4() - - # Start parent (top-level) - self.handler.on_chain_start( - serialized=_make_serialized("ParentChain"), - inputs={}, - run_id=parent_run_id, - parent_run_id=None, - ) - # Start child (nested — should NOT create a workflow span) - self.handler.on_chain_start( - serialized=_make_serialized("ChildChain"), - inputs={}, - run_id=child_run_id, - parent_run_id=parent_run_id, - ) - self.handler.on_chain_end( - outputs={}, - run_id=child_run_id, - parent_run_id=parent_run_id, - ) - self.handler.on_chain_end( - outputs={}, - run_id=parent_run_id, - parent_run_id=None, - ) - - # Only one INTERNAL (workflow) span — for the parent, not the child - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - self.assertEqual(internal_spans[0].name, "invoke_workflow ParentChain") - class TestLLMSpanGetsWorkflowName(_CallbackHandlerTestBase): """Verify gen_ai.workflow.name is propagated to the LLM span via CSA.""" From 16ee93ab203a8570d8c13f0dd72bdd1dfa9cace1 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 27 Apr 2026 10:09:01 -0700 Subject: [PATCH 13/27] removed line --- .../examples/workflow/requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt index f27cb4a3c1b..88f3320d424 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt @@ -2,7 +2,4 @@ langchain==0.3.21 langchain_openai langgraph opentelemetry-sdk>=1.39.0 -opentelemetry-exporter-otlp-proto-grpc>=1.39.0 - -# Uncomment after langchain instrumentation is released -# opentelemetry-instrumentation-langchain~=2.0b0.dev \ No newline at end of file +opentelemetry-exporter-otlp-proto-grpc>=1.39.0 \ No newline at end of file From eee467a6310ee9e41d505b8c5c7fbe72b350b17a Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 27 Apr 2026 10:10:04 -0700 Subject: [PATCH 14/27] added new line --- .../examples/workflow/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt index 88f3320d424..656212ca312 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/requirements.txt @@ -2,4 +2,4 @@ langchain==0.3.21 langchain_openai langgraph opentelemetry-sdk>=1.39.0 -opentelemetry-exporter-otlp-proto-grpc>=1.39.0 \ No newline at end of file +opentelemetry-exporter-otlp-proto-grpc>=1.39.0 From 86d3a64f735ff16a2c7b59fd979083477725a94d Mon Sep 17 00:00:00 2001 From: Wrisa Date: Fri, 1 May 2026 09:57:52 -0700 Subject: [PATCH 15/27] classify operation --- .../langchain/callback_handler.py | 31 ++- .../langchain/operation_mapping.py | 256 ++++++++++++++++++ 2 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 5d7b47f741b..00c70d67d52 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -35,6 +35,11 @@ OutputMessage, Text, ) +from opentelemetry.instrumentation.langchain.operation_mapping import ( + OperationName, + classify_chain_run, + resolve_agent_name, +) class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): @@ -58,24 +63,30 @@ def on_chain_start( metadata: Optional[dict[str, Any]] = None, **kwargs: Any, ) -> Any: - payload = serialized or {} - name_source = ( - payload.get("name") - or payload.get("id") - or kwargs.get("name") - or (metadata.get("langgraph_node") if metadata else None) + operation = classify_chain_run( + serialized, metadata, kwargs, parent_run_id ) - name = str(name_source or "chain") - if parent_run_id is None: + if operation == OperationName.INVOKE_WORKFLOW: + workflow_name = kwargs.get("name") or serialized.get("name") workflow_name_override = ( metadata.get("workflow_name") if metadata else None ) wf = self._telemetry_handler.start_workflow( - name=workflow_name_override or name + name=workflow_name_override or workflow_name ) self._invocation_manager.add_invocation_state(run_id, None, wf) - # TODO: handle non-workflow chains (e.g. agent sub-chains) in the future + elif operation == OperationName.INVOKE_AGENT: + agent_name = resolve_agent_name(serialized, metadata, kwargs) + + if metadata: + agent_id = metadata.get("agent_id") + agent_desc = metadata.get("agent_description") + + for key in ("thread_id", "session_id", "conversation_id"): + conv_id = metadata.get(key) + + provider = metadata.get("ls_provider", "unknown") def on_chain_end( self, diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py new file mode 100644 index 00000000000..56177a58813 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py @@ -0,0 +1,256 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Callback-to-semconv operation mapping for LangChain callbacks. + +Maps each LangChain callback to the correct GenAI semantic convention +operation name. Direct callbacks (``on_chat_model_start``, +``on_llm_start``, ``on_tool_start``, ``on_retriever_start``) have a +fixed 1-to-1 mapping. ``on_chain_start`` requires heuristic +classification because LangChain emits this callback for agents, +workflows, and internal plumbing alike. +""" + +from __future__ import annotations + +from typing import Any, Optional +from uuid import UUID + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + +__all__ = [ + "OperationName", + "classify_chain_run", + "resolve_agent_name", + "should_ignore_chain", +] + +# --------------------------------------------------------------------------- +# Operation name constants (sourced from the GenAI semconv enum where +# available, with string fallbacks for values not yet in the enum). +# --------------------------------------------------------------------------- + + +class OperationName: + """Canonical GenAI semantic convention operation names.""" + + CHAT: str = GenAI.GenAiOperationNameValues.CHAT.value + TEXT_COMPLETION: str = GenAI.GenAiOperationNameValues.TEXT_COMPLETION.value + INVOKE_AGENT: str = GenAI.GenAiOperationNameValues.INVOKE_AGENT.value + EXECUTE_TOOL: str = GenAI.GenAiOperationNameValues.EXECUTE_TOOL.value + # invoke_workflow is not yet in the semconv enum; use the expected + # string value so the mapping is forward-compatible. + INVOKE_WORKFLOW: str = "invoke_workflow" + + +# --------------------------------------------------------------------------- +# LangGraph markers – names and prefixes produced by LangGraph that must +# be recognized when classifying ``on_chain_start`` callbacks. +# --------------------------------------------------------------------------- + +LANGGRAPH_NODE_KEY = "langgraph_node" +LANGGRAPH_START_NODE = "__start__" +MIDDLEWARE_PREFIX = "Middleware." +LANGGRAPH_IDENTIFIER = "LangGraph" + +# Metadata keys used by callers to override classification. +_META_AGENT_SPAN = "otel_agent_span" +_META_WORKFLOW_SPAN = "otel_workflow_span" +_META_AGENT_NAME = "agent_name" +_META_AGENT_TYPE = "agent_type" +_META_OTEL_TRACE = "otel_trace" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def resolve_agent_name( + serialized: dict[str, Any], + metadata: Optional[dict[str, Any]], + kwargs: dict[str, Any], +) -> Optional[str]: + """Derive the best-effort agent name from callback arguments. + + Checks (in priority order): + 1. ``metadata["agent_name"]`` + 2. ``kwargs["name"]`` + 3. ``serialized["name"]`` + 4. ``metadata["langgraph_node"]`` (if present and not a start node) + """ + if metadata: + name = metadata.get(_META_AGENT_NAME) + if name: + return str(name) + + name = kwargs.get("name") + if name: + return str(name) + + name = serialized.get("name") + if name: + return str(name) + + if metadata: + node = metadata.get(LANGGRAPH_NODE_KEY) + if node and node != LANGGRAPH_START_NODE: + return str(node) + + return None + + +def _has_agent_signals(metadata: Optional[dict[str, Any]]) -> bool: + """Return True when metadata contains any signal that the chain is an agent.""" + if not metadata: + return False + return bool( + metadata.get(_META_AGENT_SPAN) + or metadata.get(_META_AGENT_NAME) + or metadata.get(_META_AGENT_TYPE) + ) + + +def _is_langgraph_agent_node( + serialized: dict[str, Any], + metadata: Optional[dict[str, Any]], + kwargs: dict[str, Any], +) -> bool: + """Detect a LangGraph agent node that is not a start/middleware node.""" + if not metadata: + return False + + node = metadata.get(LANGGRAPH_NODE_KEY) + if not node: + return False + + # Exclude start and middleware nodes. + if node == LANGGRAPH_START_NODE: + return False + + name = resolve_agent_name(serialized, metadata, kwargs) + if name and name.startswith(MIDDLEWARE_PREFIX): + return False + + return True + + +def _looks_like_workflow( + serialized: dict[str, Any], + metadata: Optional[dict[str, Any]], + parent_run_id: Optional[UUID], +) -> bool: + """Return True if the chain looks like a top-level workflow/graph.""" + if parent_run_id is not None: + return False + + # An explicit workflow override is authoritative. + if metadata and metadata.get(_META_WORKFLOW_SPAN): + return True + + # Heuristic: check for LangGraph identifier in the serialized repr. + name = serialized.get("name", "") + graph_id = ( + serialized.get("graph", {}).get("id", "") + if isinstance(serialized.get("graph"), dict) + else "" + ) + return LANGGRAPH_IDENTIFIER in name or LANGGRAPH_IDENTIFIER in graph_id + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def should_ignore_chain( + metadata: Optional[dict[str, Any]], + agent_name: Optional[str], + parent_run_id: Optional[UUID], + kwargs: dict[str, Any], +) -> bool: + """Return True if the chain callback should be silently suppressed. + + Suppression happens when: + * The node is the LangGraph ``__start__`` node. + * The name carries the ``Middleware.`` prefix. + * ``metadata["otel_trace"]`` is explicitly ``False``. + * ``metadata["otel_agent_span"]`` is explicitly ``False`` and no other + agent signals are present. + """ + if metadata: + node = metadata.get(LANGGRAPH_NODE_KEY) + if node == LANGGRAPH_START_NODE: + return True + + if metadata.get(_META_OTEL_TRACE) is False: + return True + + if ( + metadata.get(_META_AGENT_SPAN) is False + and not metadata.get(_META_AGENT_NAME) + and not metadata.get(_META_AGENT_TYPE) + ): + return True + + if agent_name and agent_name.startswith(MIDDLEWARE_PREFIX): + return True + + name_from_kwargs = kwargs.get("name", "") + if isinstance(name_from_kwargs, str) and name_from_kwargs.startswith( + MIDDLEWARE_PREFIX + ): + return True + + return False + + +def classify_chain_run( + serialized: dict[str, Any], + metadata: Optional[dict[str, Any]], + kwargs: dict[str, Any], + parent_run_id: Optional[UUID] = None, +) -> Optional[str]: + """Classify a ``on_chain_start`` callback into a semconv operation. + + Returns one of the :class:`OperationName` constants, or ``None`` when + the chain should be suppressed (no span emitted). + + Classification order: + 1. Check for explicit suppression signals. + 2. Check for agent signals → ``invoke_agent``. + 3. Check for workflow signals → ``invoke_workflow``. + 4. Default: ``None`` (suppress – unclassified chains are not emitted). + """ + agent_name = resolve_agent_name(serialized, metadata, kwargs) + + # 1. Suppress known noise. + if should_ignore_chain(metadata, agent_name, parent_run_id, kwargs): + return None + + # 2. Agent detection. + if _has_agent_signals(metadata): + return OperationName.INVOKE_AGENT + + if _is_langgraph_agent_node(serialized, metadata, kwargs): + return OperationName.INVOKE_AGENT + + # 3. Workflow / orchestration detection. + if _looks_like_workflow(serialized, metadata, parent_run_id): + return OperationName.INVOKE_WORKFLOW + + # 4. Default: suppress unclassified chains. + return None \ No newline at end of file From 06e438afc051405c7fe53fd6d4314e8b5f68863f Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 14:32:55 -0700 Subject: [PATCH 16/27] add invoke agent --- .../langchain/callback_handler.py | 102 +++- .../langchain/invocation_manager.py | 32 +- .../langchain/operation_mapping.py | 54 +- .../tests/conftest.py | 5 +- .../tests/test_invocation_manager.py | 133 ++++- .../tests/test_workflow_chain.py | 548 ------------------ 6 files changed, 244 insertions(+), 630 deletions(-) delete mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 00c70d67d52..7b65e133471 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -24,8 +24,14 @@ from opentelemetry.instrumentation.langchain.invocation_manager import ( _InvocationManager, ) +from opentelemetry.instrumentation.langchain.operation_mapping import ( + OperationName, + classify_chain_run, + resolve_agent_name, +) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( + AgentInvocation, InferenceInvocation, WorkflowInvocation, ) @@ -35,11 +41,6 @@ OutputMessage, Text, ) -from opentelemetry.instrumentation.langchain.operation_mapping import ( - OperationName, - classify_chain_run, - resolve_agent_name, -) class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): @@ -72,21 +73,73 @@ def on_chain_start( workflow_name_override = ( metadata.get("workflow_name") if metadata else None ) - wf = self._telemetry_handler.start_workflow( + workflow = self._telemetry_handler.start_workflow( name=workflow_name_override or workflow_name ) - self._invocation_manager.add_invocation_state(run_id, None, wf) + self._invocation_manager.add_invocation_state( + run_id, parent_run_id, workflow + ) elif operation == OperationName.INVOKE_AGENT: - agent_name = resolve_agent_name(serialized, metadata, kwargs) - - if metadata: - agent_id = metadata.get("agent_id") - agent_desc = metadata.get("agent_description") + # agent name passed by the user + suggested_agent_name = resolve_agent_name( + serialized, metadata, kwargs + ) + # find if there is an agent already + agent_invocation = self._find_nearest_agent(parent_run_id) + agent_invocation_name = ( + agent_invocation.agent_name if agent_invocation else None + ) + if suggested_agent_name: + suggested_agent_name_lower = suggested_agent_name.lower() + agent_invocation_name_lower = ( + agent_invocation_name.lower() + if agent_invocation_name + else None + ) + if suggested_agent_name_lower != agent_invocation_name_lower: + agent = self._telemetry_handler.start_invoke_local_agent( + provider=metadata.get("ls_provider", "unknown") + if metadata + else "unknown", + ) + agent.agent_name = suggested_agent_name - for key in ("thread_id", "session_id", "conversation_id"): - conv_id = metadata.get(key) + if metadata: + agent.agent_id = metadata.get("agent_id") + agent.agent_description = metadata.get( + "agent_description" + ) - provider = metadata.get("ls_provider", "unknown") + for key in ( + "thread_id", + "session_id", + "conversation_id", + ): + conv_id = metadata.get(key) + if conv_id: + agent.conversation_id = conv_id + break + + self._invocation_manager.add_invocation_state( + run_id, parent_run_id, agent + ) + else: + # We create invoke_agent span for the initial chain for agent. All follow-up chains invoked for agent invocation will not create agent span. + self._invocation_manager.add_invocation_state( + run_id, parent_run_id, None + ) + else: + # No agent name could be resolved; still register the run_id so that + # parent-child traversal (e.g. _find_nearest_agent) is not broken for + # any children of this node. + self._invocation_manager.add_invocation_state( + run_id, parent_run_id, None + ) + else: + # For unclassified chains, we still want to track them in the invocation manager to maintain the parent-child relationships, even though we won't create spans for them. + self._invocation_manager.add_invocation_state( + run_id, parent_run_id, None + ) def on_chain_end( self, @@ -98,9 +151,10 @@ def on_chain_end( ) -> Any: invocation = self._invocation_manager.get_invocation(run_id=run_id) if invocation is None or not isinstance( - invocation, WorkflowInvocation + invocation, (WorkflowInvocation, AgentInvocation) ): # If the invocation does not exist, we cannot set attributes or end it + self._invocation_manager.delete_invocation_state(run_id) return invocation.stop() @@ -118,9 +172,10 @@ def on_chain_error( ) -> Any: invocation = self._invocation_manager.get_invocation(run_id=run_id) if invocation is None or not isinstance( - invocation, WorkflowInvocation + invocation, (WorkflowInvocation, AgentInvocation) ): # If the invocation does not exist, we cannot set attributes or end it + self._invocation_manager.delete_invocation_state(run_id) return invocation.fail(error) @@ -349,3 +404,16 @@ def on_llm_error( llm_invocation.fail(error) if not llm_invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id=run_id) + + def _find_nearest_agent( + self, run_id: Optional[UUID] + ) -> Optional[AgentInvocation]: + current = run_id + visited = set() + while current is not None and current not in visited: + visited.add(current) + entity = self._invocation_manager.get_invocation(current) + if isinstance(entity, AgentInvocation): + return entity + current = self._invocation_manager.get_parent_run_id(current) + return None diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index 98176ec1e3f..7736dc491c3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -25,6 +25,8 @@ class _InvocationState: invocation: GenAIInvocation children: List[UUID] = field(default_factory=lambda: list()) + parent_run_id: Optional[UUID] = None + ended: bool = False class _InvocationManager: @@ -39,23 +41,45 @@ def add_invocation_state( self, run_id: UUID, parent_run_id: Optional[UUID], - invocation: GenAIInvocation, + invocation: Optional[GenAIInvocation], ) -> None: invocation_state = _InvocationState(invocation=invocation) - self._invocations[run_id] = invocation_state if parent_run_id is not None and parent_run_id in self._invocations: + invocation_state.parent_run_id = parent_run_id + parent_invocation_state = self._invocations[parent_run_id] parent_invocation_state.children.append(run_id) + self._invocations[run_id] = invocation_state + def get_invocation(self, run_id: UUID) -> Optional[GenAIInvocation]: invocation_state = self._invocations.get(run_id) return invocation_state.invocation if invocation_state else None + def get_parent_run_id(self, run_id: UUID) -> Optional[UUID]: + invocation_state = self._invocations.get(run_id) + return invocation_state.parent_run_id if invocation_state else None + def delete_invocation_state(self, run_id: UUID) -> None: invocation_state = self._invocations.get(run_id) if not invocation_state: return - for child_id in list(invocation_state.children): - self._invocations.pop(child_id, None) + + invocation_state.ended = True + + # Defer removal if any children are still live, so upward traversal + # (e.g. _find_nearest_agent) can still walk through this node. + if any(c in self._invocations for c in invocation_state.children): + return + self._invocations.pop(run_id, None) + + # Propagate cleanup upward: if the parent has already ended and has no + # more live children, it can now be removed too. + if invocation_state.parent_run_id: + parent_state = self._invocations.get( + invocation_state.parent_run_id + ) + if parent_state is not None and parent_state.ended: + self.delete_invocation_state(invocation_state.parent_run_id) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py index 56177a58813..cebaeac59a4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py @@ -47,10 +47,7 @@ class OperationName: """Canonical GenAI semantic convention operation names.""" - CHAT: str = GenAI.GenAiOperationNameValues.CHAT.value - TEXT_COMPLETION: str = GenAI.GenAiOperationNameValues.TEXT_COMPLETION.value INVOKE_AGENT: str = GenAI.GenAiOperationNameValues.INVOKE_AGENT.value - EXECUTE_TOOL: str = GenAI.GenAiOperationNameValues.EXECUTE_TOOL.value # invoke_workflow is not yet in the semconv enum; use the expected # string value so the mapping is forward-compatible. INVOKE_WORKFLOW: str = "invoke_workflow" @@ -101,7 +98,7 @@ def resolve_agent_name( if name: return str(name) - name = serialized.get("name") + name = serialized.get("name") if serialized else None if name: return str(name) @@ -124,30 +121,6 @@ def _has_agent_signals(metadata: Optional[dict[str, Any]]) -> bool: ) -def _is_langgraph_agent_node( - serialized: dict[str, Any], - metadata: Optional[dict[str, Any]], - kwargs: dict[str, Any], -) -> bool: - """Detect a LangGraph agent node that is not a start/middleware node.""" - if not metadata: - return False - - node = metadata.get(LANGGRAPH_NODE_KEY) - if not node: - return False - - # Exclude start and middleware nodes. - if node == LANGGRAPH_START_NODE: - return False - - name = resolve_agent_name(serialized, metadata, kwargs) - if name and name.startswith(MIDDLEWARE_PREFIX): - return False - - return True - - def _looks_like_workflow( serialized: dict[str, Any], metadata: Optional[dict[str, Any]], @@ -162,13 +135,16 @@ def _looks_like_workflow( return True # Heuristic: check for LangGraph identifier in the serialized repr. - name = serialized.get("name", "") - graph_id = ( - serialized.get("graph", {}).get("id", "") - if isinstance(serialized.get("graph"), dict) - else "" - ) - return LANGGRAPH_IDENTIFIER in name or LANGGRAPH_IDENTIFIER in graph_id + if serialized: + name = serialized.get("name", "") + graph_id = ( + serialized.get("graph", {}).get("id", "") + if isinstance(serialized.get("graph"), dict) + else "" + ) + return LANGGRAPH_IDENTIFIER in name or LANGGRAPH_IDENTIFIER in graph_id + + return True # --------------------------------------------------------------------------- @@ -179,7 +155,6 @@ def _looks_like_workflow( def should_ignore_chain( metadata: Optional[dict[str, Any]], agent_name: Optional[str], - parent_run_id: Optional[UUID], kwargs: dict[str, Any], ) -> bool: """Return True if the chain callback should be silently suppressed. @@ -238,19 +213,16 @@ def classify_chain_run( agent_name = resolve_agent_name(serialized, metadata, kwargs) # 1. Suppress known noise. - if should_ignore_chain(metadata, agent_name, parent_run_id, kwargs): + if should_ignore_chain(metadata, agent_name, kwargs): return None # 2. Agent detection. if _has_agent_signals(metadata): return OperationName.INVOKE_AGENT - if _is_langgraph_agent_node(serialized, metadata, kwargs): - return OperationName.INVOKE_AGENT - # 3. Workflow / orchestration detection. if _looks_like_workflow(serialized, metadata, parent_run_id): return OperationName.INVOKE_WORKFLOW # 4. Default: suppress unclassified chains. - return None \ No newline at end of file + return None diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py index 7d608ea8b36..a75b4840c55 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -208,9 +208,10 @@ def deserialize(cassette_string): return yaml.load(cassette_string, Loader=yaml.Loader) -@pytest.fixture(scope="module", autouse=True) +@pytest.fixture(scope="function", autouse=True) def fixture_vcr(vcr): - vcr.register_serializer("yaml", PrettyPrintJSONBody) + if vcr is not None: + vcr.register_serializer("yaml", PrettyPrintJSONBody) return vcr diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py index cf57a226bab..261bc327c45 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py @@ -101,7 +101,73 @@ def test_delete_invocation_state(invocation_manager, mock_invocation): assert run_id not in invocation_manager._invocations -def test_delete_invocation_state_with_children(invocation_manager): +def test_delete_invocation_state_deferred_while_children_live( + invocation_manager, +): + """Deleting a parent while children are still live defers its removal.""" + parent_id = uuid.uuid4() + child_id = uuid.uuid4() + + parent_invocation = mock.Mock(spec=GenAIInvocation) + child_invocation = mock.Mock(spec=GenAIInvocation) + + invocation_manager.add_invocation_state( + run_id=parent_id, parent_run_id=None, invocation=parent_invocation + ) + invocation_manager.add_invocation_state( + run_id=child_id, parent_run_id=parent_id, invocation=child_invocation + ) + + # Delete the parent while the child is still live + invocation_manager.delete_invocation_state(parent_id) + + # Parent should still be present (deferred) because child is live + assert parent_id in invocation_manager._invocations + assert invocation_manager._invocations[parent_id].ended is True + + # After the child is deleted, the parent should also be cleaned up + invocation_manager.delete_invocation_state(child_id) + + assert child_id not in invocation_manager._invocations + assert parent_id not in invocation_manager._invocations + + +def test_delete_invocation_state_propagates_upward(invocation_manager): + """When the last child is removed, an already-ended parent is cleaned up.""" + grandparent_id = uuid.uuid4() + parent_id = uuid.uuid4() + child_id = uuid.uuid4() + + for run_id, parent in [ + (grandparent_id, None), + (parent_id, grandparent_id), + (child_id, parent_id), + ]: + invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=parent, + invocation=mock.Mock(spec=GenAIInvocation), + ) + + # Mark grandparent and parent as ended (deferred) + invocation_manager.delete_invocation_state(grandparent_id) + invocation_manager.delete_invocation_state(parent_id) + + assert grandparent_id in invocation_manager._invocations # deferred + assert parent_id in invocation_manager._invocations # deferred + + # Removing the last live node should cascade upward + invocation_manager.delete_invocation_state(child_id) + + assert child_id not in invocation_manager._invocations + assert parent_id not in invocation_manager._invocations + assert grandparent_id not in invocation_manager._invocations + + +def test_delete_invocation_state_with_multiple_children_defers_until_last( + invocation_manager, +): + """Parent removal is deferred until all children are gone.""" parent_id = uuid.uuid4() child1_id = uuid.uuid4() child2_id = uuid.uuid4() @@ -110,32 +176,63 @@ def test_delete_invocation_state_with_children(invocation_manager): child1_invocation = mock.Mock(spec=GenAIInvocation) child2_invocation = mock.Mock(spec=GenAIInvocation) - # Add parent and children + invocation_manager.add_invocation_state( + run_id=parent_id, parent_run_id=None, invocation=parent_invocation + ) + invocation_manager.add_invocation_state( + run_id=child1_id, parent_run_id=parent_id, invocation=child1_invocation + ) + invocation_manager.add_invocation_state( + run_id=child2_id, parent_run_id=parent_id, invocation=child2_invocation + ) + + # Delete parent while both children live → deferred + invocation_manager.delete_invocation_state(parent_id) + assert parent_id in invocation_manager._invocations + + # Remove first child → parent still deferred (child2 is live) + invocation_manager.delete_invocation_state(child1_id) + assert parent_id in invocation_manager._invocations + + # Remove last child → parent is now cleaned up + invocation_manager.delete_invocation_state(child2_id) + assert parent_id not in invocation_manager._invocations + + +def test_get_parent_run_id_returns_none_for_unknown(invocation_manager): + assert invocation_manager.get_parent_run_id(uuid.uuid4()) is None + + +def test_get_parent_run_id_returns_registered_parent(invocation_manager): + parent_id = uuid.uuid4() + child_id = uuid.uuid4() + invocation_manager.add_invocation_state( run_id=parent_id, parent_run_id=None, - invocation=parent_invocation, + invocation=mock.Mock(spec=GenAIInvocation), ) invocation_manager.add_invocation_state( - run_id=child1_id, + run_id=child_id, parent_run_id=parent_id, - invocation=child1_invocation, + invocation=mock.Mock(spec=GenAIInvocation), ) + + assert invocation_manager.get_parent_run_id(child_id) == parent_id + assert invocation_manager.get_parent_run_id(parent_id) is None + + +def test_none_invocation_can_be_stored_and_retrieved(invocation_manager): + """Nodes with no associated span (None invocation) must still be tracked.""" + run_id = uuid.uuid4() + invocation_manager.add_invocation_state( - run_id=child2_id, - parent_run_id=parent_id, - invocation=child2_invocation, + run_id=run_id, parent_run_id=None, invocation=None ) - # Verify initial state - assert len(invocation_manager._invocations) == 3 - assert len(invocation_manager._invocations[parent_id].children) == 2 + assert run_id in invocation_manager._invocations + assert invocation_manager.get_invocation(run_id) is None - # Delete parent - invocation_manager.delete_invocation_state(parent_id) - # Verify parent and all children were removed - assert parent_id not in invocation_manager._invocations - assert child1_id not in invocation_manager._invocations - assert child2_id not in invocation_manager._invocations - assert len(invocation_manager._invocations) == 0 +def test_delete_nonexistent_run_id_does_not_raise(invocation_manager): + invocation_manager.delete_invocation_state(uuid.uuid4()) # must not raise diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py deleted file mode 100644 index 736116655ce..00000000000 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_workflow_chain.py +++ /dev/null @@ -1,548 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Unit tests verifying CSA propagation through the LangChain callback handler. - -These tests exercise the OpenTelemetryLangChainCallbackHandler directly using -mock LangChain callback payloads, so they do not require live models or VCR -cassettes. The key behaviour under test: - - on_chain_start(parent_run_id=None) - → WorkflowInvocation created - → TelemetryHandler.start() called - → gen_ai.workflow.name written to context-scoped attributes (CSA) - - on_chat_model_start(...) - → LLMInvocation created - → TelemetryHandler.start() called - → gen_ai.workflow.name read from CSA and stamped on the LLM invocation - - on_llm_end(...) - → LLM span closed with gen_ai.workflow.name attribute set - - on_chain_end(...) - → workflow span closed, CSA scope ends -""" - -from __future__ import annotations - -import os -import uuid -from unittest import TestCase -from unittest.mock import patch - -from langchain_core.messages import AIMessage -from langchain_core.outputs import ChatGeneration, LLMResult - -from opentelemetry import baggage -from opentelemetry.instrumentation.langchain.callback_handler import ( - OpenTelemetryLangChainCallbackHandler, -) -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, -) -from opentelemetry.trace import SpanKind -from opentelemetry.trace.status import StatusCode -from opentelemetry.util.genai.context_attributes import ( - get_context_scoped_attributes, -) -from opentelemetry.util.genai.handler import TelemetryHandler - -# --------------------------------------------------------------------------- -# Shared helpers -# --------------------------------------------------------------------------- - - -def _make_serialized(name: str) -> dict: - """Minimal serialized dict that on_chain_start / on_chat_model_start expect.""" - return {"name": name} - - -def _make_llm_result(content: str = "hello") -> LLMResult: - """Minimal LLMResult with one generation.""" - msg = AIMessage(content=content) - gen = ChatGeneration(message=msg, text=content) - gen.generation_info = {"finish_reason": "stop"} - return LLMResult( - generations=[[gen]], llm_output={"model_name": "gpt-3.5-turbo"} - ) - - -def _make_chat_invocation_params(model_name: str = "gpt-3.5-turbo") -> dict: - """kwargs dict that on_chat_model_start receives for a ChatOpenAI call.""" - return { - "invocation_params": { - "model_name": model_name, - "params": {"model_name": model_name}, - } - } - - -# --------------------------------------------------------------------------- -# Base test class -# --------------------------------------------------------------------------- - - -class _CallbackHandlerTestBase(TestCase): - def setUp(self) -> None: - self.span_exporter = InMemorySpanExporter() - tracer_provider = TracerProvider() - tracer_provider.add_span_processor( - SimpleSpanProcessor(self.span_exporter) - ) - telemetry_handler = TelemetryHandler(tracer_provider=tracer_provider) - self.handler = OpenTelemetryLangChainCallbackHandler( - telemetry_handler=telemetry_handler - ) - - def _finished_spans(self): - return self.span_exporter.get_finished_spans() - - def _spans_by_kind(self, kind: SpanKind): - return [s for s in self._finished_spans() if s.kind == kind] - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -class TestWorkflowSpanCreation(_CallbackHandlerTestBase): - """Verify that a workflow span is created for top-level chains.""" - - def test_workflow_span_created_for_top_level_chain(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("MyChain"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_end( - outputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - self.assertEqual(internal_spans[0].name, "invoke_workflow MyChain") - - -class TestLLMSpanGetsWorkflowName(_CallbackHandlerTestBase): - """Verify gen_ai.workflow.name is propagated to the LLM span via CSA.""" - - def test_llm_span_inside_chain_gets_workflow_name(self) -> None: - chain_run_id = uuid.uuid4() - llm_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("MyPipeline"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chat_model_start( - serialized=_make_serialized("ChatOpenAI"), - messages=[[AIMessage(content="hi")]], - run_id=llm_run_id, - parent_run_id=chain_run_id, - metadata={"ls_provider": "openai"}, - **_make_chat_invocation_params("gpt-3.5-turbo"), - ) - self.handler.on_llm_end( - response=_make_llm_result(), - run_id=llm_run_id, - parent_run_id=chain_run_id, - ) - self.handler.on_chain_end( - outputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - - client_spans = self._spans_by_kind(SpanKind.CLIENT) - self.assertEqual(len(client_spans), 1) - self.assertEqual( - client_spans[0].attributes.get("gen_ai.workflow.name"), - "MyPipeline", - ) - - def test_workflow_name_from_metadata_override(self) -> None: - """metadata['workflow_name'] overrides the serialized chain name.""" - chain_run_id = uuid.uuid4() - llm_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("InternalChainName"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - metadata={"workflow_name": "my_custom_wf"}, - ) - self.handler.on_chat_model_start( - serialized=_make_serialized("ChatOpenAI"), - messages=[[AIMessage(content="hi")]], - run_id=llm_run_id, - parent_run_id=chain_run_id, - metadata={"ls_provider": "openai"}, - **_make_chat_invocation_params("gpt-3.5-turbo"), - ) - self.handler.on_llm_end( - response=_make_llm_result(), - run_id=llm_run_id, - parent_run_id=chain_run_id, - ) - self.handler.on_chain_end( - outputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - self.assertEqual( - internal_spans[0].name, "invoke_workflow my_custom_wf" - ) - - client_spans = self._spans_by_kind(SpanKind.CLIENT) - self.assertEqual(len(client_spans), 1) - self.assertEqual( - client_spans[0].attributes.get("gen_ai.workflow.name"), - "my_custom_wf", - ) - - -class TestCSANotLeakedToBaggage(_CallbackHandlerTestBase): - """Verify that gen_ai.workflow.name is NOT written to W3C Baggage by default.""" - - def test_csa_not_leaked_to_baggage(self) -> None: - env = { - k: v - for k, v in os.environ.items() - if k != "OTEL_PYTHON_GENAI_CAPTURE_BAGGAGE" - } - with patch.dict(os.environ, env, clear=True): - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("BaggageTestChain"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - try: - # While workflow is active, baggage should NOT contain the workflow name - baggage_value = baggage.get_baggage("gen_ai.workflow.name") - self.assertIsNone( - baggage_value, - "gen_ai.workflow.name must not be leaked to W3C Baggage by default", - ) - finally: - self.handler.on_chain_end( - outputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - - -class TestCSAScopeEndsAfterChain(_CallbackHandlerTestBase): - """Verify that the CSA is no longer visible after the chain ends.""" - - def test_csa_not_visible_outside_workflow_scope(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("ScopedChain"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_end( - outputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - - # After on_chain_end the context token is detached — CSA should be gone - attrs = get_context_scoped_attributes() - self.assertIsNone( - attrs.get("gen_ai.workflow.name"), - "gen_ai.workflow.name should not be visible after workflow scope ends", - ) - - -class TestWorkflowErrorPath(_CallbackHandlerTestBase): - """Verify on_chain_error records error status and cleans up state.""" - - def test_workflow_span_has_error_status_on_chain_error(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("FailingChain"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_error( - error=ValueError("something went wrong"), - run_id=chain_run_id, - parent_run_id=None, - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - span = internal_spans[0] - self.assertEqual(span.name, "invoke_workflow FailingChain") - self.assertEqual(span.status.status_code, StatusCode.ERROR) - self.assertIn("something went wrong", span.status.description) - - def test_workflow_span_error_type_attribute(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("FailingChain"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_error( - error=RuntimeError("boom"), - run_id=chain_run_id, - parent_run_id=None, - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - self.assertEqual( - internal_spans[0].attributes.get("error.type"), "RuntimeError" - ) - - def test_chain_error_cleans_up_invocation_state(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("FailingChain"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_error( - error=ValueError("oops"), - run_id=chain_run_id, - parent_run_id=None, - ) - - # Invocation should have been removed — a second error call is a no-op - self.handler.on_chain_error( - error=ValueError("duplicate"), - run_id=chain_run_id, - parent_run_id=None, - ) - # Only one span — the second call was silently ignored - self.assertEqual(len(self._finished_spans()), 1) - - def test_chain_error_csa_scope_ends(self) -> None: - """CSA should be gone after on_chain_error, same as on_chain_end.""" - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("FailingChain"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_error( - error=ValueError("oops"), - run_id=chain_run_id, - parent_run_id=None, - ) - - attrs = get_context_scoped_attributes() - self.assertIsNone( - attrs.get("gen_ai.workflow.name"), - "gen_ai.workflow.name should not be visible after workflow error", - ) - - def test_chain_error_unknown_run_id_is_noop(self) -> None: - """on_chain_error with an unknown run_id must not raise.""" - self.handler.on_chain_error( - error=ValueError("no matching invocation"), - run_id=uuid.uuid4(), - parent_run_id=None, - ) - self.assertEqual(len(self._finished_spans()), 0) - - def test_chain_end_unknown_run_id_is_noop(self) -> None: - """on_chain_end with an unknown run_id must not raise.""" - self.handler.on_chain_end( - outputs={}, - run_id=uuid.uuid4(), - parent_run_id=None, - ) - self.assertEqual(len(self._finished_spans()), 0) - - -class TestLLMErrorInsideWorkflow(_CallbackHandlerTestBase): - """Verify on_llm_error inside a workflow doesn't break the parent workflow span.""" - - def test_llm_error_inside_workflow_records_error_on_llm_span(self) -> None: - chain_run_id = uuid.uuid4() - llm_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("MyPipeline"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chat_model_start( - serialized=_make_serialized("ChatOpenAI"), - messages=[[AIMessage(content="hi")]], - run_id=llm_run_id, - parent_run_id=chain_run_id, - metadata={"ls_provider": "openai"}, - **_make_chat_invocation_params("gpt-3.5-turbo"), - ) - self.handler.on_llm_error( - error=RuntimeError("model timeout"), - run_id=llm_run_id, - parent_run_id=chain_run_id, - ) - self.handler.on_chain_end( - outputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - - client_spans = self._spans_by_kind(SpanKind.CLIENT) - self.assertEqual(len(client_spans), 1) - self.assertEqual(client_spans[0].status.status_code, StatusCode.ERROR) - - # Workflow span still finishes (not in error state) - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - self.assertNotEqual( - internal_spans[0].status.status_code, StatusCode.ERROR - ) - - def test_llm_error_inside_workflow_llm_span_is_child_of_workflow( - self, - ) -> None: - chain_run_id = uuid.uuid4() - llm_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized=_make_serialized("MyPipeline"), - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chat_model_start( - serialized=_make_serialized("ChatOpenAI"), - messages=[[AIMessage(content="hi")]], - run_id=llm_run_id, - parent_run_id=chain_run_id, - metadata={"ls_provider": "openai"}, - **_make_chat_invocation_params("gpt-3.5-turbo"), - ) - self.handler.on_llm_end( - response=_make_llm_result(), - run_id=llm_run_id, - parent_run_id=chain_run_id, - ) - self.handler.on_chain_end( - outputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - client_spans = self._spans_by_kind(SpanKind.CLIENT) - self.assertEqual(len(internal_spans), 1) - self.assertEqual(len(client_spans), 1) - - workflow_span = internal_spans[0] - llm_span = client_spans[0] - self.assertEqual( - llm_span.context.trace_id, - workflow_span.context.trace_id, - "LLM span and workflow span must share the same trace", - ) - self.assertEqual( - llm_span.parent.span_id, - workflow_span.context.span_id, - "LLM span must be a child of the workflow span", - ) - - -class TestWorkflowNameFallback(_CallbackHandlerTestBase): - """Verify the name resolution fallback chain in on_chain_start.""" - - def test_name_falls_back_to_id_list(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized={"id": ["pkg", "mod", "MyRunnableClass"]}, - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_end( - outputs={}, run_id=chain_run_id, parent_run_id=None - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - # id is a list — _safe_str(list) produces a string; just verify a span was created - self.assertTrue(internal_spans[0].name.startswith("invoke_workflow ")) - - def test_name_falls_back_to_langgraph_node(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized={}, - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - metadata={"langgraph_node": "my_node"}, - ) - self.handler.on_chain_end( - outputs={}, run_id=chain_run_id, parent_run_id=None - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - self.assertEqual(internal_spans[0].name, "invoke_workflow my_node") - - def test_name_defaults_to_chain_when_nothing_provided(self) -> None: - chain_run_id = uuid.uuid4() - - self.handler.on_chain_start( - serialized={}, - inputs={}, - run_id=chain_run_id, - parent_run_id=None, - ) - self.handler.on_chain_end( - outputs={}, run_id=chain_run_id, parent_run_id=None - ) - - internal_spans = self._spans_by_kind(SpanKind.INTERNAL) - self.assertEqual(len(internal_spans), 1) - self.assertEqual(internal_spans[0].name, "invoke_workflow chain") From 5594a721bf72f14a00fb0086f1b0e0e869b7fac0 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 16:17:59 -0700 Subject: [PATCH 17/27] fixed errors and added tests --- .../examples/agent/main.py | 103 ++++ .../examples/agent/requirements.txt | 5 + .../examples/agent/single_node_agent.py | 108 ++++ .../langchain/callback_handler.py | 2 +- .../langchain/invocation_manager.py | 2 +- .../tests/test_callback_handler.py | 555 ++++++++++++++++++ 6 files changed, 773 insertions(+), 2 deletions(-) create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/requirements.txt create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py new file mode 100644 index 00000000000..ed66c246fd5 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py @@ -0,0 +1,103 @@ +""" +ReAct agent example built with LangGraph. + +Uses LangGraph's prebuilt create_react_agent with simple calculator tools. +OpenTelemetry LangChain instrumentation traces the LLM calls made by the agent. +""" + +from uuid import uuid4 + +from langchain_core.messages import HumanMessage +from langchain_core.tools import tool +from langchain_openai import ChatOpenAI +from langchain.agents import create_agent + +from opentelemetry import _logs, metrics, trace +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +# Configure tracing +trace.set_tracer_provider(TracerProvider()) +span_processor = BatchSpanProcessor(OTLPSpanExporter()) +trace.get_tracer_provider().add_span_processor(span_processor) + +# Configure logging +_logs.set_logger_provider(LoggerProvider()) +_logs.get_logger_provider().add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter()) +) + +# Configure metrics +metrics.set_meter_provider( + MeterProvider( + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(), + ), + ] + ) +) + + +@tool +def multiply(a: float, b: float) -> float: + """Multiply two numbers together.""" + return a * b + + +@tool +def add(a: float, b: float) -> float: + """Add two numbers together.""" + return a + b + + +def main(): + LangChainInstrumentor().instrument() + + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + seed=100, + ) + + session_id = str(uuid4()) + agent = create_agent( + llm, tools=[multiply, add], name="coordinator" + ).with_config( + { + "metadata": { + "agent_name": "coordinator", + "session_id": session_id, + }, + } + ) + + result = agent.invoke( + {"messages": [HumanMessage(content="What is (3 * 4) + 7?")]} + ) + + print("Agent output:") + for msg in result["messages"]: + print(f" {type(msg).__name__}: {msg.content}") + + LangChainInstrumentor().uninstrument() + + +if __name__ == "__main__": + main() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/requirements.txt new file mode 100644 index 00000000000..88f3320d424 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/requirements.txt @@ -0,0 +1,5 @@ +langchain==0.3.21 +langchain_openai +langgraph +opentelemetry-sdk>=1.39.0 +opentelemetry-exporter-otlp-proto-grpc>=1.39.0 \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py new file mode 100644 index 00000000000..571bfe945f0 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py @@ -0,0 +1,108 @@ +""" +Single-node agent example built with StateGraph. + +A single ReAct agent answers arithmetic questions using calculator tools. +OpenTelemetry LangChain instrumentation traces all LLM calls. +""" + +from langchain_core.messages import HumanMessage +from langchain_core.tools import tool +from langchain_openai import ChatOpenAI +from langgraph.graph import END, START, MessagesState, StateGraph +from langchain.agents import create_agent + +from opentelemetry import _logs, metrics, trace +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +# Configure tracing +trace.set_tracer_provider(TracerProvider()) +trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + +# Configure logging +_logs.set_logger_provider(LoggerProvider()) +_logs.get_logger_provider().add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter()) +) + +# Configure metrics +metrics.set_meter_provider( + MeterProvider( + metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())] + ) +) + + +# --- Tools ---------------------------------------------------------------- + +@tool +def multiply(a: float, b: float) -> float: + """Multiply two numbers.""" + return a * b + + +@tool +def add(a: float, b: float) -> float: + """Add two numbers.""" + return a + b + + +# --- Graph ---------------------------------------------------------------- + +def build_single_node_graph(llm: ChatOpenAI): + from uuid import uuid4 + session_id = str(uuid4()) + + agent = create_agent( + llm, tools=[multiply, add], name="math_agent" + ).with_config( + { + "metadata": { + "agent_name": "math_agent", + "session_id": session_id, + }, + } + ) + + def run_agent(state: MessagesState) -> dict: + result = agent.invoke({"messages": state["messages"]}) + return {"messages": result["messages"]} + + builder = StateGraph(MessagesState) + builder.add_node("math_agent", run_agent) + builder.add_edge(START, "math_agent") + builder.add_edge("math_agent", END) + + return builder.compile() + + +def main(): + LangChainInstrumentor().instrument() + + llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0.1, seed=100) + graph = build_single_node_graph(llm) + + questions = [ + "What is 12 multiplied by 7?", + "What is 15 plus 27?", + ] + + for question in questions: + print(f"\nQuestion: {question}") + result = graph.invoke({"messages": [HumanMessage(content=question)]}) + last = result["messages"][-1] + print(f" Answer: {last.content}") + + LangChainInstrumentor().uninstrument() + + +if __name__ == "__main__": + main() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 7b65e133471..f5be48160c7 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -409,7 +409,7 @@ def _find_nearest_agent( self, run_id: Optional[UUID] ) -> Optional[AgentInvocation]: current = run_id - visited = set() + visited: set[UUID] = set() while current is not None and current not in visited: visited.add(current) entity = self._invocation_manager.get_invocation(current) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index 7736dc491c3..d8536ba77ce 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -23,7 +23,7 @@ @dataclass class _InvocationState: - invocation: GenAIInvocation + invocation: Optional[GenAIInvocation] children: List[UUID] = field(default_factory=lambda: list()) parent_run_id: Optional[UUID] = None ended: bool = False diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py new file mode 100644 index 00000000000..7ba4a2cc8c5 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py @@ -0,0 +1,555 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for on_chain_start / on_chain_end / on_chain_error in +OpenTelemetryLangChainCallbackHandler. + +All TelemetryHandler interactions are mocked so that these tests exercise only +the callback-handler logic and the invocation-manager bookkeeping. +""" + +from __future__ import annotations + +import uuid +from unittest import mock + +import pytest + +from opentelemetry.instrumentation.langchain.callback_handler import ( + OpenTelemetryLangChainCallbackHandler, +) +from opentelemetry.util.genai.invocation import ( + AgentInvocation, + WorkflowInvocation, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_handler(): + """Return a handler wired to a MagicMock TelemetryHandler.""" + telemetry = mock.MagicMock() + + # start_workflow returns a mock WorkflowInvocation + workflow_inv = mock.MagicMock(spec=WorkflowInvocation) + workflow_inv.span = mock.MagicMock() + workflow_inv.span.is_recording.return_value = False + telemetry.start_workflow.return_value = workflow_inv + + # start_invoke_local_agent returns a mock AgentInvocation + agent_inv = mock.MagicMock(spec=AgentInvocation) + agent_inv.span = mock.MagicMock() + agent_inv.span.is_recording.return_value = False + telemetry.start_invoke_local_agent.return_value = agent_inv + + handler = OpenTelemetryLangChainCallbackHandler(telemetry) + return handler, telemetry, workflow_inv, agent_inv + + +def _run_id(): + return uuid.uuid4() + + +# --------------------------------------------------------------------------- +# on_chain_start – INVOKE_WORKFLOW +# --------------------------------------------------------------------------- + +class TestOnChainStartWorkflow: + def test_workflow_span_created(self): + handler, telemetry, workflow_inv, _ = _make_handler() + run_id = _run_id() + + # LangGraph graph serialized dict triggers workflow classification + handler.on_chain_start( + serialized={"name": "LangGraph", "id": ["langgraph"]}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + telemetry.start_workflow.assert_called_once() + assert handler._invocation_manager.get_invocation(run_id) is workflow_inv + + def test_workflow_name_from_serialized(self): + handler, telemetry, _, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "MyLangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + telemetry.start_workflow.assert_called_once_with(name="MyLangGraph") + + def test_workflow_name_overridden_by_metadata(self): + handler, telemetry, _, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "MyLangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={"workflow_name": "custom_workflow"}, + ) + + telemetry.start_workflow.assert_called_once_with(name="custom_workflow") + + def test_workflow_registered_in_invocation_manager(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + assert handler._invocation_manager.get_invocation(run_id) is workflow_inv + + +# --------------------------------------------------------------------------- +# on_chain_start – INVOKE_AGENT +# --------------------------------------------------------------------------- + +class TestOnChainStartAgent: + def test_new_agent_span_created(self): + handler, telemetry, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent", "ls_provider": "openai"}, + ) + + telemetry.start_invoke_local_agent.assert_called_once_with(provider="openai") + assert agent_inv.agent_name == "math_agent" + assert handler._invocation_manager.get_invocation(run_id) is agent_inv + + def test_agent_metadata_set(self): + handler, _, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={ + "agent_name": "math_agent", + "agent_id": "agent-123", + "agent_description": "does math", + "thread_id": "thread-abc", + }, + ) + + assert agent_inv.agent_id == "agent-123" + assert agent_inv.agent_description == "does math" + assert agent_inv.conversation_id == "thread-abc" + + def test_conversation_id_prefers_thread_id_over_session_id(self): + handler, _, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={ + "agent_name": "math_agent", + "thread_id": "t1", + "session_id": "s1", + }, + ) + + assert agent_inv.conversation_id == "t1" + + def test_duplicate_agent_name_does_not_create_new_span(self): + """When the nearest ancestor already has the same agent name, no new + AgentInvocation span is created; the run is still tracked with None.""" + handler, telemetry, _, agent_inv = _make_handler() + parent_run_id = _run_id() + child_run_id = _run_id() + + # Register the parent agent + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=parent_run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + telemetry.start_invoke_local_agent.reset_mock() + + # A child chain with the same agent name should NOT create a new span + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=child_run_id, + parent_run_id=parent_run_id, + metadata={"agent_name": "math_agent"}, + ) + + telemetry.start_invoke_local_agent.assert_not_called() + assert handler._invocation_manager.get_invocation(child_run_id) is None + + def test_different_agent_name_creates_new_span(self): + """A child chain with a different agent name creates a new AgentInvocation.""" + handler, telemetry, _, _ = _make_handler() + parent_run_id = _run_id() + child_run_id = _run_id() + + # First agent + first_agent_inv = mock.MagicMock(spec=AgentInvocation) + first_agent_inv.span = mock.MagicMock() + first_agent_inv.span.is_recording.return_value = False + telemetry.start_invoke_local_agent.return_value = first_agent_inv + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=parent_run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + # Second agent with a different name + second_agent_inv = mock.MagicMock(spec=AgentInvocation) + second_agent_inv.span = mock.MagicMock() + second_agent_inv.span.is_recording.return_value = False + telemetry.start_invoke_local_agent.return_value = second_agent_inv + + handler.on_chain_start( + serialized={"name": "weather_agent"}, + inputs={}, + run_id=child_run_id, + parent_run_id=parent_run_id, + metadata={"agent_name": "weather_agent"}, + ) + + assert handler._invocation_manager.get_invocation(child_run_id) is second_agent_inv + assert second_agent_inv.agent_name == "weather_agent" + + def test_agent_name_comparison_is_case_insensitive(self): + handler, telemetry, _, _ = _make_handler() + parent_run_id = _run_id() + child_run_id = _run_id() + + parent_agent_inv = mock.MagicMock(spec=AgentInvocation) + parent_agent_inv.span = mock.MagicMock() + parent_agent_inv.span.is_recording.return_value = False + telemetry.start_invoke_local_agent.return_value = parent_agent_inv + + handler.on_chain_start( + serialized={"name": "Math_Agent"}, + inputs={}, + run_id=parent_run_id, + parent_run_id=None, + metadata={"agent_name": "Math_Agent"}, + ) + telemetry.start_invoke_local_agent.reset_mock() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=child_run_id, + parent_run_id=parent_run_id, + metadata={"agent_name": "math_agent"}, + ) + + # Same name (case-insensitive) → no new span + telemetry.start_invoke_local_agent.assert_not_called() + + def test_no_agent_name_registers_none_invocation(self): + """When resolve_agent_name returns None the run_id must still be + registered so that child traversal works.""" + handler, telemetry, _, _ = _make_handler() + run_id = _run_id() + + # metadata has otel_agent_span=True so classify_chain_run → INVOKE_AGENT, + # but no agent_name / kwargs name / serialized name, so resolve_agent_name + # returns None. + handler.on_chain_start( + serialized={}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={"otel_agent_span": True}, + ) + + telemetry.start_invoke_local_agent.assert_not_called() + # run_id must still be registered (with None invocation) so traversal works + assert run_id in handler._invocation_manager._invocations + assert handler._invocation_manager.get_invocation(run_id) is None + + def test_no_agent_name_child_can_still_find_ancestor_agent(self): + """Even when an intermediate node has no agent name, a deeper child + must still be able to walk up and find a grandparent AgentInvocation.""" + handler, telemetry, _, agent_inv = _make_handler() + grandparent_id = _run_id() + parent_id = _run_id() + child_id = _run_id() + + # Grandparent: a known agent + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=grandparent_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + # Parent: INVOKE_AGENT but no resolvable name → registers None + telemetry.start_invoke_local_agent.reset_mock() + handler.on_chain_start( + serialized={}, + inputs={}, + run_id=parent_id, + parent_run_id=grandparent_id, + metadata={"otel_agent_span": True}, + ) + + # Child: should find the grandparent agent via _find_nearest_agent + found = handler._find_nearest_agent(parent_id) + assert found is agent_inv + + +# --------------------------------------------------------------------------- +# on_chain_start – unclassified +# --------------------------------------------------------------------------- + +class TestOnChainStartUnclassified: + def test_unclassified_chain_registers_none_and_no_span(self): + handler, telemetry, _, _ = _make_handler() + run_id = _run_id() + parent_run_id = _run_id() + + # Register the parent first so that the child links correctly + handler._invocation_manager.add_invocation_state( + parent_run_id, None, None + ) + + handler.on_chain_start( + serialized={"name": "SomeInternalChain"}, + inputs={}, + run_id=run_id, + parent_run_id=parent_run_id, + ) + + telemetry.start_workflow.assert_not_called() + telemetry.start_invoke_local_agent.assert_not_called() + assert run_id in handler._invocation_manager._invocations + assert handler._invocation_manager.get_invocation(run_id) is None + + +# --------------------------------------------------------------------------- +# on_chain_end +# --------------------------------------------------------------------------- + +class TestOnChainEnd: + def test_workflow_invocation_stopped_on_chain_end(self): + handler, telemetry, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + handler.on_chain_end(outputs={}, run_id=run_id) + + workflow_inv.stop.assert_called_once() + + def test_agent_invocation_stopped_on_chain_end(self): + handler, telemetry, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + handler.on_chain_end(outputs={}, run_id=run_id) + + agent_inv.stop.assert_called_once() + + def test_none_invocation_on_chain_end_does_not_raise(self): + """on_chain_end for a run registered with None invocation (unclassified + or duplicate agent) must silently do nothing.""" + handler, _, _, _ = _make_handler() + run_id = _run_id() + + handler._invocation_manager.add_invocation_state(run_id, None, None) + + # Must not raise + handler.on_chain_end(outputs={}, run_id=run_id) + + def test_unknown_run_id_on_chain_end_does_not_raise(self): + handler, _, _, _ = _make_handler() + handler.on_chain_end(outputs={}, run_id=_run_id()) + + def test_invocation_state_cleaned_up_after_chain_end(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + # span.is_recording() returns False → should be cleaned up + workflow_inv.span.is_recording.return_value = False + handler.on_chain_end(outputs={}, run_id=run_id) + + assert run_id not in handler._invocation_manager._invocations + + +# --------------------------------------------------------------------------- +# on_chain_error +# --------------------------------------------------------------------------- + +class TestOnChainError: + def test_workflow_invocation_failed_on_chain_error(self): + handler, telemetry, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + err = RuntimeError("something went wrong") + handler.on_chain_error(error=err, run_id=run_id) + + workflow_inv.fail.assert_called_once_with(err) + + def test_agent_invocation_failed_on_chain_error(self): + handler, telemetry, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + err = ValueError("agent failed") + handler.on_chain_error(error=err, run_id=run_id) + + agent_inv.fail.assert_called_once_with(err) + + def test_none_invocation_on_chain_error_does_not_raise(self): + handler, _, _, _ = _make_handler() + run_id = _run_id() + + handler._invocation_manager.add_invocation_state(run_id, None, None) + + handler.on_chain_error(error=RuntimeError("boom"), run_id=run_id) + + def test_unknown_run_id_on_chain_error_does_not_raise(self): + handler, _, _, _ = _make_handler() + handler.on_chain_error(error=RuntimeError("boom"), run_id=_run_id()) + + def test_invocation_state_cleaned_up_after_chain_error(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + workflow_inv.span.is_recording.return_value = False + handler.on_chain_error(error=RuntimeError("boom"), run_id=run_id) + + assert run_id not in handler._invocation_manager._invocations + + +# --------------------------------------------------------------------------- +# _find_nearest_agent +# --------------------------------------------------------------------------- + +class TestFindNearestAgent: + def test_returns_none_when_no_agent_in_ancestry(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + assert handler._find_nearest_agent(run_id) is None + + def test_finds_direct_parent_agent(self): + handler, telemetry, _, agent_inv = _make_handler() + parent_id = _run_id() + child_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=parent_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + # Register the child as unclassified so it links to the parent + handler._invocation_manager.add_invocation_state(child_id, parent_id, None) + + found = handler._find_nearest_agent(child_id) + assert found is agent_inv + + def test_finds_grandparent_agent(self): + handler, telemetry, _, agent_inv = _make_handler() + grandparent_id = _run_id() + parent_id = _run_id() + child_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=grandparent_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + handler._invocation_manager.add_invocation_state(parent_id, grandparent_id, None) + handler._invocation_manager.add_invocation_state(child_id, parent_id, None) + + found = handler._find_nearest_agent(child_id) + assert found is agent_inv From 53dbd7b7c042c7a72587f40269f4586a234af3eb Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 16:23:30 -0700 Subject: [PATCH 18/27] fixed precommit error --- .../examples/agent/single_node_agent.py | 5 +++-- .../tests/test_callback_handler.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py index 571bfe945f0..28ba2337775 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py @@ -5,11 +5,13 @@ OpenTelemetry LangChain instrumentation traces all LLM calls. """ +from uuid import uuid4 + +from langchain.agents import create_agent from langchain_core.messages import HumanMessage from langchain_core.tools import tool from langchain_openai import ChatOpenAI from langgraph.graph import END, START, MessagesState, StateGraph -from langchain.agents import create_agent from opentelemetry import _logs, metrics, trace from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter @@ -58,7 +60,6 @@ def add(a: float, b: float) -> float: # --- Graph ---------------------------------------------------------------- def build_single_node_graph(llm: ChatOpenAI): - from uuid import uuid4 session_id = str(uuid4()) agent = create_agent( diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py index 7ba4a2cc8c5..0da974046cf 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py @@ -308,7 +308,6 @@ def test_no_agent_name_child_can_still_find_ancestor_agent(self): handler, telemetry, _, agent_inv = _make_handler() grandparent_id = _run_id() parent_id = _run_id() - child_id = _run_id() # Grandparent: a known agent handler.on_chain_start( From 0cd6b287d4b8b780c5f37c4da48f337190447252 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 16:28:26 -0700 Subject: [PATCH 19/27] fixed error --- .../examples/agent/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py index ed66c246fd5..daf1f34250e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py @@ -7,10 +7,10 @@ from uuid import uuid4 +from langchain.agents import create_agent from langchain_core.messages import HumanMessage from langchain_core.tools import tool from langchain_openai import ChatOpenAI -from langchain.agents import create_agent from opentelemetry import _logs, metrics, trace from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( From 5ede64d691cab881e1131ad930e410631792eadd Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 16:47:32 -0700 Subject: [PATCH 20/27] fixed test and example --- .../examples/agent/single_node_agent.py | 16 ++++++++++++---- .../tests/test_callback_handler.py | 2 -- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py index 28ba2337775..c42dbceb712 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py @@ -14,9 +14,15 @@ from langgraph.graph import END, START, MessagesState, StateGraph from opentelemetry import _logs, metrics, trace -from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter -from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) from opentelemetry.instrumentation.langchain import LangChainInstrumentor from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor @@ -27,7 +33,9 @@ # Configure tracing trace.set_tracer_provider(TracerProvider()) -trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) +trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) +) # Configure logging _logs.set_logger_provider(LoggerProvider()) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py index 0da974046cf..f0b8343d5d0 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py @@ -24,8 +24,6 @@ import uuid from unittest import mock -import pytest - from opentelemetry.instrumentation.langchain.callback_handler import ( OpenTelemetryLangChainCallbackHandler, ) From 3844025dd6603834db2d9a24aae31823866b7b04 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 17:10:27 -0700 Subject: [PATCH 21/27] removed unused --- .../examples/agent/single_node_agent.py | 2 ++ .../tests/test_callback_handler.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py index c42dbceb712..94c2695605e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py @@ -53,6 +53,7 @@ # --- Tools ---------------------------------------------------------------- + @tool def multiply(a: float, b: float) -> float: """Multiply two numbers.""" @@ -67,6 +68,7 @@ def add(a: float, b: float) -> float: # --- Graph ---------------------------------------------------------------- + def build_single_node_graph(llm: ChatOpenAI): session_id = str(uuid4()) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py index f0b8343d5d0..591803fe448 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py @@ -19,8 +19,6 @@ the callback-handler logic and the invocation-manager bookkeeping. """ -from __future__ import annotations - import uuid from unittest import mock From 546eab842fc462a021fd15442a6e69c628c29216 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 17:21:13 -0700 Subject: [PATCH 22/27] fixed test --- .../tests/test_callback_handler.py | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py index 591803fe448..2b605ddade4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py @@ -1,17 +1,3 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - """Unit tests for on_chain_start / on_chain_end / on_chain_error in OpenTelemetryLangChainCallbackHandler. @@ -30,11 +16,11 @@ WorkflowInvocation, ) - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _make_handler(): """Return a handler wired to a MagicMock TelemetryHandler.""" telemetry = mock.MagicMock() @@ -63,6 +49,7 @@ def _run_id(): # on_chain_start – INVOKE_WORKFLOW # --------------------------------------------------------------------------- + class TestOnChainStartWorkflow: def test_workflow_span_created(self): handler, telemetry, workflow_inv, _ = _make_handler() @@ -77,7 +64,9 @@ def test_workflow_span_created(self): ) telemetry.start_workflow.assert_called_once() - assert handler._invocation_manager.get_invocation(run_id) is workflow_inv + assert ( + handler._invocation_manager.get_invocation(run_id) is workflow_inv + ) def test_workflow_name_from_serialized(self): handler, telemetry, _, _ = _make_handler() @@ -104,7 +93,9 @@ def test_workflow_name_overridden_by_metadata(self): metadata={"workflow_name": "custom_workflow"}, ) - telemetry.start_workflow.assert_called_once_with(name="custom_workflow") + telemetry.start_workflow.assert_called_once_with( + name="custom_workflow" + ) def test_workflow_registered_in_invocation_manager(self): handler, _, workflow_inv, _ = _make_handler() @@ -117,13 +108,16 @@ def test_workflow_registered_in_invocation_manager(self): parent_run_id=None, ) - assert handler._invocation_manager.get_invocation(run_id) is workflow_inv + assert ( + handler._invocation_manager.get_invocation(run_id) is workflow_inv + ) # --------------------------------------------------------------------------- # on_chain_start – INVOKE_AGENT # --------------------------------------------------------------------------- + class TestOnChainStartAgent: def test_new_agent_span_created(self): handler, telemetry, _, agent_inv = _make_handler() @@ -137,7 +131,9 @@ def test_new_agent_span_created(self): metadata={"agent_name": "math_agent", "ls_provider": "openai"}, ) - telemetry.start_invoke_local_agent.assert_called_once_with(provider="openai") + telemetry.start_invoke_local_agent.assert_called_once_with( + provider="openai" + ) assert agent_inv.agent_name == "math_agent" assert handler._invocation_manager.get_invocation(run_id) is agent_inv @@ -243,7 +239,10 @@ def test_different_agent_name_creates_new_span(self): metadata={"agent_name": "weather_agent"}, ) - assert handler._invocation_manager.get_invocation(child_run_id) is second_agent_inv + assert ( + handler._invocation_manager.get_invocation(child_run_id) + is second_agent_inv + ) assert second_agent_inv.agent_name == "weather_agent" def test_agent_name_comparison_is_case_insensitive(self): @@ -333,6 +332,7 @@ def test_no_agent_name_child_can_still_find_ancestor_agent(self): # on_chain_start – unclassified # --------------------------------------------------------------------------- + class TestOnChainStartUnclassified: def test_unclassified_chain_registers_none_and_no_span(self): handler, telemetry, _, _ = _make_handler() @@ -361,6 +361,7 @@ def test_unclassified_chain_registers_none_and_no_span(self): # on_chain_end # --------------------------------------------------------------------------- + class TestOnChainEnd: def test_workflow_invocation_stopped_on_chain_end(self): handler, telemetry, workflow_inv, _ = _make_handler() @@ -430,6 +431,7 @@ def test_invocation_state_cleaned_up_after_chain_end(self): # on_chain_error # --------------------------------------------------------------------------- + class TestOnChainError: def test_workflow_invocation_failed_on_chain_error(self): handler, telemetry, workflow_inv, _ = _make_handler() @@ -497,6 +499,7 @@ def test_invocation_state_cleaned_up_after_chain_error(self): # _find_nearest_agent # --------------------------------------------------------------------------- + class TestFindNearestAgent: def test_returns_none_when_no_agent_in_ancestry(self): handler, _, workflow_inv, _ = _make_handler() @@ -525,7 +528,9 @@ def test_finds_direct_parent_agent(self): ) # Register the child as unclassified so it links to the parent - handler._invocation_manager.add_invocation_state(child_id, parent_id, None) + handler._invocation_manager.add_invocation_state( + child_id, parent_id, None + ) found = handler._find_nearest_agent(child_id) assert found is agent_inv @@ -543,8 +548,12 @@ def test_finds_grandparent_agent(self): parent_run_id=None, metadata={"agent_name": "math_agent"}, ) - handler._invocation_manager.add_invocation_state(parent_id, grandparent_id, None) - handler._invocation_manager.add_invocation_state(child_id, parent_id, None) + handler._invocation_manager.add_invocation_state( + parent_id, grandparent_id, None + ) + handler._invocation_manager.add_invocation_state( + child_id, parent_id, None + ) found = handler._find_nearest_agent(child_id) assert found is agent_inv From e6b9b68d2c88a44f5b1132c59b4e9a4ecb5ef925 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 4 May 2026 17:27:51 -0700 Subject: [PATCH 23/27] updated changelog --- .../opentelemetry-instrumentation-langchain/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md index a23d8b33d5c..9878790377b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased -- Add LangChain workflow span support and refactor LLM invocation +- Add LangChain workflow and agent span support. Also refactor LLM invocation. ([#4449](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4449)) - Fix compatibility with wrapt 2.x by using positional arguments in `wrap_function_wrapper()` calls ([#4445](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4445)) From c5478138081d110b9bfd06cb9104a59446ed454b Mon Sep 17 00:00:00 2001 From: Wrisa Date: Thu, 7 May 2026 16:46:19 -0700 Subject: [PATCH 24/27] added SPDX license --- .../examples/agent/main.py | 3 +++ .../examples/agent/single_node_agent.py | 3 +++ .../examples/workflow/main.py | 3 +++ .../instrumentation/langchain/operation_mapping.py | 13 +------------ .../tests/test_callback_handler.py | 3 +++ 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py index daf1f34250e..c2d578e05bc 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + """ ReAct agent example built with LangGraph. diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py index 94c2695605e..72a9b5dcb76 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/single_node_agent.py @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + """ Single-node agent example built with StateGraph. diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py index 961d33b33ee..a4b4d18c118 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + """ LangGraph StateGraph example with an LLM node. diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py index cebaeac59a4..45f0a879fc2 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/operation_mapping.py @@ -1,16 +1,5 @@ # Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 """Callback-to-semconv operation mapping for LangChain callbacks. diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py index 2b605ddade4..a58a3189e18 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + """Unit tests for on_chain_start / on_chain_end / on_chain_error in OpenTelemetryLangChainCallbackHandler. From 3067e5fdda442b65b7d485238f9fd078172e9bbd Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 11 May 2026 17:09:14 -0700 Subject: [PATCH 25/27] input output messages and updated example --- .../examples/agent/main.py | 6 +- .../examples/workflow/main.py | 102 +++-- .../pyproject.toml | 2 +- .../langchain/callback_handler.py | 9 + .../instrumentation/langchain/utils.py | 93 ++++ .../tests/test_callback_handler.py | 433 ++++++++++++++++++ 6 files changed, 603 insertions(+), 42 deletions(-) create mode 100644 instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py index c2d578e05bc..bbbedf5056e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/agent/main.py @@ -10,10 +10,10 @@ from uuid import uuid4 -from langchain.agents import create_agent from langchain_core.messages import HumanMessage from langchain_core.tools import tool from langchain_openai import ChatOpenAI +from langgraph.prebuilt import create_react_agent from opentelemetry import _logs, metrics, trace from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( @@ -80,9 +80,7 @@ def main(): ) session_id = str(uuid4()) - agent = create_agent( - llm, tools=[multiply, add], name="coordinator" - ).with_config( + agent = create_react_agent(llm, tools=[multiply, add]).with_config( { "metadata": { "agent_name": "coordinator", diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py index a4b4d18c118..b7de4c3b599 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/workflow/main.py @@ -2,11 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 """ -LangGraph StateGraph example with an LLM node. +LangGraph StateGraph example with two LLM nodes. -Similar to the manual example (../manual/main.py) but uses LangGraph's StateGraph -with a node that calls ChatOpenAI. OpenTelemetry LangChain instrumentation traces -the LLM calls made from within the graph node. +Graph topology: + + START → researcher → summariser → END + +Steps: + 1. *researcher* – gathers factual background on the user's question. + 2. *summariser* – condenses the researcher's output into a concise answer. + +OpenTelemetry LangChain instrumentation traces both LLM calls. """ from typing import Annotated @@ -37,8 +43,9 @@ # Configure tracing trace.set_tracer_provider(TracerProvider()) -span_processor = BatchSpanProcessor(OTLPSpanExporter()) -trace.get_tracer_provider().add_span_processor(span_processor) +trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) +) # Configure logging _logs.set_logger_provider(LoggerProvider()) @@ -49,66 +56,87 @@ # Configure metrics metrics.set_meter_provider( MeterProvider( - metric_readers=[ - PeriodicExportingMetricReader( - OTLPMetricExporter(), - ), - ] + metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())] ) ) class GraphState(TypedDict): - """State for the graph; messages are accumulated with add_messages.""" + """State shared across all graph nodes.""" messages: Annotated[list, add_messages] + research: str def build_graph(llm: ChatOpenAI): - """Build a StateGraph with a single LLM node.""" - - def llm_node(state: GraphState) -> dict: - """Node that invokes the LLM with the current messages.""" - response = llm.invoke(state["messages"]) + """Build a StateGraph with a researcher node and a summariser node.""" + + def researcher(state: GraphState) -> dict: + """Gather factual background on the last user message.""" + response = llm.invoke( + [ + SystemMessage( + content="You are a research assistant. Provide 2-3 factual sentences." + ), + HumanMessage(content=state["messages"][-1].content), + ] + ) + return { + "research": response.content, + "messages": [response], + } + + def summariser(state: GraphState) -> dict: + """Condense the researcher's output into one concise sentence.""" + response = llm.invoke( + [ + SystemMessage( + content="You are an expert summariser. Condense the text below into one clear sentence." + ), + HumanMessage(content=state["research"]), + ] + ) return {"messages": [response]} builder = StateGraph(GraphState) - builder.add_node("llm", llm_node) - builder.add_edge(START, "llm") - builder.add_edge("llm", END) + builder.add_node("researcher", researcher) + builder.add_node("summariser", summariser) + + builder.add_edge(START, "researcher") + builder.add_edge("researcher", "summariser") + builder.add_edge("summariser", END) + return builder.compile() def main(): - # Set up instrumentation (traces LLM calls from within graph nodes) LangChainInstrumentor().instrument() - # ChatOpenAI setup llm = ChatOpenAI( model="gpt-3.5-turbo", temperature=0.1, - max_tokens=100, - top_p=0.9, - frequency_penalty=0.5, - presence_penalty=0.5, - stop=["\n", "Human:", "AI:"], - seed=100, + max_tokens=200, + seed=42, ) graph = build_graph(llm) - initial_messages = [ - SystemMessage(content="You are a helpful assistant!"), - HumanMessage(content="What is the capital of France?"), - ] + question = "What is the capital of France?" + print(f"Question: {question}\n") + + result = graph.invoke( + { + "messages": [HumanMessage(content=question)], + "research": "", + } + ) - result = graph.invoke({"messages": initial_messages}) + print("Research output:") + print(f" {result['research']}\n") - print("LangGraph output (messages):") - for msg in result.get("messages", []): - print(f" {type(msg).__name__}: {msg.content}") + print("Final summary:") + print(f" {result['messages'][-1].content}") - # Un-instrument after use LangChainInstrumentor().uninstrument() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml index a87b82f9992..a6c61cc0cec 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ "opentelemetry-instrumentation ~= 0.60b0", - "opentelemetry-util-genai >= 0.4b0.dev", + "opentelemetry-util-genai >= 0.4b0", ] [project.optional-dependencies] diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index f5ea66e343f..633b3eefad1 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -18,6 +18,10 @@ classify_chain_run, resolve_agent_name, ) +from opentelemetry.instrumentation.langchain.utils import ( + make_input_message, + make_last_output_message, +) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( AgentInvocation, @@ -65,6 +69,7 @@ def on_chain_start( workflow = self._telemetry_handler.start_workflow( name=workflow_name_override or workflow_name ) + workflow.input_messages = make_input_message(inputs) self._invocation_manager.add_invocation_state( run_id, parent_run_id, workflow ) @@ -92,6 +97,7 @@ def on_chain_start( else "unknown", ) agent.agent_name = suggested_agent_name + agent.input_messages = make_input_message(inputs) if metadata: agent.agent_id = metadata.get("agent_id") @@ -146,6 +152,9 @@ def on_chain_end( self._invocation_manager.delete_invocation_state(run_id) return + if isinstance(invocation, (WorkflowInvocation, AgentInvocation)): + invocation.output_messages = make_last_output_message(outputs) + invocation.stop() if not invocation.span.is_recording(): diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py new file mode 100644 index 00000000000..2a0f67b7fcd --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py @@ -0,0 +1,93 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import json +from typing import Any, Optional + +from langchain_core.messages import AIMessage + +from opentelemetry.util.genai.types import ( + InputMessage, + OutputMessage, + Text, +) + + +def make_input_message(data: Any) -> list[InputMessage]: + """Create structured input message with full data as JSON.""" + if not isinstance(data, dict): + return [] + input_messages: list[InputMessage] = [] + messages = data.get("messages") + if messages is not None: + for msg in messages: + content = getattr(msg, "content", "") + if content: + input_message = InputMessage( + role="user", parts=[Text(content)] + ) + input_messages.append(input_message) + return input_messages + # Fallback: serialize non-message state fields as input. + # Common in LangGraph where nodes use structured state fields + # (e.g., user_query) rather than a message list. + exclude_keys = {"messages", "intermediate_steps"} + input_data = { + k: v + for k, v in data.items() + if k not in exclude_keys and v is not None + } + if input_data: + serialized = serialize(input_data) + if serialized: + return [InputMessage(role="user", parts=[Text(serialized)])] + return input_messages + + +def make_output_message(data: dict[str, Any]) -> list[OutputMessage]: + """Create structured output message with full data as JSON.""" + if not isinstance(data, dict): + return [] + output_messages: list[OutputMessage] = [] + messages = data.get("messages") + if messages is None: + return [] + for msg in messages: + content = getattr(msg, "content", "") + if content: + if isinstance(msg, AIMessage): + output_message = OutputMessage( + role="assistant", + parts=[Text(msg.content)], + finish_reason="stop", + ) + output_messages.append(output_message) + return output_messages + + +def make_last_output_message(data: dict[str, Any]) -> list[OutputMessage]: + """Extract only the last AI message as the output. + + For Workflow and AgentInvocation spans, the final AI message best represents + the actual output. Intermediate AI messages (e.g., tool-call decisions) are + already captured in child LLM invocation spans. + """ + all_messages = make_output_message(data) + if all_messages: + return [all_messages[-1]] + return [] + + +def serialize(obj: Any) -> Optional[str]: + """Serialize object to JSON string. + + Uses default=str to handle non-JSON-serializable objects (like LangChain + message objects) by converting them to their string representation while + keeping the overall structure as valid JSON. + """ + if obj is None: + return None + try: + return json.dumps(obj, ensure_ascii=False, default=str) + except (TypeError, ValueError): + return None diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py index a58a3189e18..26bbcffd7e3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler.py @@ -11,13 +11,22 @@ import uuid from unittest import mock +from langchain_core.messages import AIMessage, HumanMessage + from opentelemetry.instrumentation.langchain.callback_handler import ( OpenTelemetryLangChainCallbackHandler, ) +from opentelemetry.instrumentation.langchain.utils import ( + make_input_message, + make_last_output_message, + make_output_message, + serialize, +) from opentelemetry.util.genai.invocation import ( AgentInvocation, WorkflowInvocation, ) +from opentelemetry.util.genai.types import InputMessage, OutputMessage, Text # --------------------------------------------------------------------------- # Helpers @@ -560,3 +569,427 @@ def test_finds_grandparent_agent(self): found = handler._find_nearest_agent(child_id) assert found is agent_inv + + +# --------------------------------------------------------------------------- +# utils.make_input_message +# --------------------------------------------------------------------------- + + +class TestMakeInputMessage: + def test_returns_empty_list_for_non_dict(self): + assert make_input_message("not a dict") == [] + assert make_input_message(None) == [] + assert make_input_message(42) == [] + + def test_empty_dict_returns_empty_list(self): + assert make_input_message({}) == [] + + def test_messages_key_with_human_message(self): + msg = HumanMessage(content="Hello") + result = make_input_message({"messages": [msg]}) + + assert len(result) == 1 + assert isinstance(result[0], InputMessage) + assert result[0].role == "user" + assert len(result[0].parts) == 1 + assert isinstance(result[0].parts[0], Text) + assert result[0].parts[0].content == "Hello" + + def test_messages_key_skips_empty_content(self): + msg_empty = HumanMessage(content="") + msg_valid = HumanMessage(content="Hi") + result = make_input_message({"messages": [msg_empty, msg_valid]}) + + assert len(result) == 1 + assert result[0].parts[0].content == "Hi" + + def test_messages_key_multiple_messages(self): + msgs = [HumanMessage(content="First"), HumanMessage(content="Second")] + result = make_input_message({"messages": msgs}) + + assert len(result) == 2 + assert result[0].parts[0].content == "First" + assert result[1].parts[0].content == "Second" + + def test_messages_key_takes_priority_over_other_fields(self): + msg = HumanMessage(content="hello") + result = make_input_message( + {"messages": [msg], "user_query": "should be ignored"} + ) + + assert len(result) == 1 + assert result[0].parts[0].content == "hello" + + def test_fallback_serializes_non_message_state_fields(self): + result = make_input_message({"user_query": "what is 2+2?"}) + + assert len(result) == 1 + assert result[0].role == "user" + # The content should be a JSON serialization of the dict + assert "user_query" in result[0].parts[0].content + assert "what is 2+2?" in result[0].parts[0].content + + def test_fallback_excludes_intermediate_steps_key(self): + # messages key absent → fallback path runs; intermediate_steps excluded + result = make_input_message( + { + "user_query": "hi", + "intermediate_steps": [("tool", "result")], + } + ) + + assert len(result) == 1 + content = result[0].parts[0].content + assert "intermediate_steps" not in content + assert "user_query" in content + + def test_messages_key_present_skips_fallback_even_with_other_fields(self): + # messages key present (even empty list) → early return, fallback not reached + result = make_input_message( + { + "user_query": "ignored", + "messages": [], + "intermediate_steps": [("tool", "result")], + } + ) + + assert result == [] + + def test_fallback_returns_empty_when_all_values_are_none(self): + result = make_input_message({"user_query": None, "context": None}) + assert result == [] + + def test_fallback_returns_empty_when_only_excluded_keys(self): + result = make_input_message( + {"messages": None, "intermediate_steps": None} + ) + assert result == [] + + def test_messages_key_with_empty_list(self): + # messages key present but empty → return empty list (no fallback) + result = make_input_message({"messages": [], "user_query": "ignored"}) + assert result == [] + + +# --------------------------------------------------------------------------- +# utils.make_output_message / make_last_output_message +# --------------------------------------------------------------------------- + + +class TestMakeOutputMessage: + def test_returns_empty_list_for_non_dict(self): + assert make_output_message("not a dict") == [] + assert make_output_message(None) == [] + + def test_returns_empty_list_when_no_messages_key(self): + assert make_output_message({"output": "hi"}) == [] + + def test_returns_empty_list_when_messages_is_none(self): + assert make_output_message({"messages": None}) == [] + + def test_ai_message_produces_assistant_output(self): + ai_msg = AIMessage(content="The answer is 42") + result = make_output_message({"messages": [ai_msg]}) + + assert len(result) == 1 + assert isinstance(result[0], OutputMessage) + assert result[0].role == "assistant" + assert result[0].finish_reason == "stop" + assert result[0].parts[0].content == "The answer is 42" + + def test_non_ai_message_skipped(self): + human_msg = HumanMessage(content="Hello") + result = make_output_message({"messages": [human_msg]}) + assert result == [] + + def test_ai_message_with_empty_content_skipped(self): + ai_msg = AIMessage(content="") + result = make_output_message({"messages": [ai_msg]}) + assert result == [] + + def test_multiple_ai_messages_all_returned(self): + msgs = [ + AIMessage(content="First response"), + AIMessage(content="Second response"), + ] + result = make_output_message({"messages": msgs}) + + assert len(result) == 2 + assert result[0].parts[0].content == "First response" + assert result[1].parts[0].content == "Second response" + + def test_mixed_messages_only_ai_returned(self): + msgs = [ + HumanMessage(content="question"), + AIMessage(content="answer"), + HumanMessage(content="follow-up"), + ] + result = make_output_message({"messages": msgs}) + + assert len(result) == 1 + assert result[0].parts[0].content == "answer" + + +class TestMakeLastOutputMessage: + def test_returns_only_last_ai_message(self): + msgs = [ + AIMessage(content="intermediate"), + AIMessage(content="final answer"), + ] + result = make_last_output_message({"messages": msgs}) + + assert len(result) == 1 + assert result[0].parts[0].content == "final answer" + + def test_returns_empty_when_no_ai_messages(self): + result = make_last_output_message( + {"messages": [HumanMessage(content="hi")]} + ) + assert result == [] + + def test_returns_empty_for_empty_outputs(self): + assert make_last_output_message({}) == [] + + def test_single_ai_message_returned(self): + ai_msg = AIMessage(content="only response") + result = make_last_output_message({"messages": [ai_msg]}) + + assert len(result) == 1 + assert result[0].parts[0].content == "only response" + + +# --------------------------------------------------------------------------- +# utils.serialize +# --------------------------------------------------------------------------- + + +class TestSerialize: + def test_none_returns_none(self): + assert serialize(None) is None + + def test_dict_serialized_to_json(self): + result = serialize({"key": "value"}) + assert result == '{"key": "value"}' + + def test_list_serialized_to_json(self): + result = serialize([1, 2, 3]) + assert result == "[1, 2, 3]" + + def test_non_serializable_falls_back_to_str(self): + class Custom: + def __str__(self): + return "custom_repr" + + result = serialize({"obj": Custom()}) + assert result is not None + assert "custom_repr" in result + + def test_string_value(self): + result = serialize("hello") + assert result == '"hello"' + + +# --------------------------------------------------------------------------- +# input_messages / output_messages set on invocations via callback handler +# --------------------------------------------------------------------------- + + +class TestInputMessagesOnInvocations: + def test_workflow_input_messages_set_from_messages_key(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + msg = HumanMessage(content="What is the weather?") + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={"messages": [msg]}, + run_id=run_id, + parent_run_id=None, + ) + + assigned = workflow_inv.input_messages + assert len(assigned) == 1 + assert assigned[0].role == "user" + assert assigned[0].parts[0].content == "What is the weather?" + + def test_workflow_input_messages_set_from_state_fallback(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={"user_query": "plan a trip"}, + run_id=run_id, + parent_run_id=None, + ) + + assigned = workflow_inv.input_messages + assert len(assigned) == 1 + assert "user_query" in assigned[0].parts[0].content + assert "plan a trip" in assigned[0].parts[0].content + + def test_workflow_input_messages_empty_for_empty_inputs(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + assert workflow_inv.input_messages == [] + + def test_agent_input_messages_set_from_messages_key(self): + handler, _, _, agent_inv = _make_handler() + run_id = _run_id() + msg = HumanMessage(content="Solve x+2=5") + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={"messages": [msg]}, + run_id=run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + assigned = agent_inv.input_messages + assert len(assigned) == 1 + assert assigned[0].parts[0].content == "Solve x+2=5" + + def test_agent_input_messages_set_from_state_fallback(self): + handler, _, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={"problem": "integrate x^2"}, + run_id=run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + assigned = agent_inv.input_messages + assert len(assigned) == 1 + assert "integrate x^2" in assigned[0].parts[0].content + + +class TestOutputMessagesOnInvocations: + def test_workflow_output_messages_set_on_chain_end(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + ai_msg = AIMessage(content="The final answer is 42") + handler.on_chain_end( + outputs={"messages": [ai_msg]}, + run_id=run_id, + ) + + assigned = workflow_inv.output_messages + assert len(assigned) == 1 + assert assigned[0].role == "assistant" + assert assigned[0].parts[0].content == "The final answer is 42" + + def test_workflow_output_messages_only_last_ai_message(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + msgs = [ + AIMessage(content="intermediate tool call"), + AIMessage(content="final answer"), + ] + handler.on_chain_end(outputs={"messages": msgs}, run_id=run_id) + + assigned = workflow_inv.output_messages + assert len(assigned) == 1 + assert assigned[0].parts[0].content == "final answer" + + def test_workflow_output_messages_empty_when_no_ai_messages(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + handler.on_chain_end( + outputs={"messages": [HumanMessage(content="hi")]}, + run_id=run_id, + ) + + assert workflow_inv.output_messages == [] + + def test_workflow_output_messages_empty_for_empty_outputs(self): + handler, _, workflow_inv, _ = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "LangGraph"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + ) + + handler.on_chain_end(outputs={}, run_id=run_id) + + assert workflow_inv.output_messages == [] + + def test_agent_output_messages_set_on_chain_end(self): + handler, _, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + ai_msg = AIMessage(content="x = 3") + handler.on_chain_end(outputs={"messages": [ai_msg]}, run_id=run_id) + + assigned = agent_inv.output_messages + assert len(assigned) == 1 + assert assigned[0].parts[0].content == "x = 3" + + def test_agent_output_messages_only_last_ai_message(self): + handler, _, _, agent_inv = _make_handler() + run_id = _run_id() + + handler.on_chain_start( + serialized={"name": "math_agent"}, + inputs={}, + run_id=run_id, + parent_run_id=None, + metadata={"agent_name": "math_agent"}, + ) + + msgs = [ + AIMessage(content="let me think..."), + AIMessage(content="the answer is 7"), + ] + handler.on_chain_end(outputs={"messages": msgs}, run_id=run_id) + + assigned = agent_inv.output_messages + assert len(assigned) == 1 + assert assigned[0].parts[0].content == "the answer is 7" From 0f3dce219db4f556ac07896f3b35b2a16b78cdd6 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 11 May 2026 17:41:53 -0700 Subject: [PATCH 26/27] fixed typecheck --- .../langchain/callback_handler.py | 3 +- .../instrumentation/langchain/utils.py | 32 +++++++++---------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 633b3eefad1..e181739c645 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -152,8 +152,7 @@ def on_chain_end( self._invocation_manager.delete_invocation_state(run_id) return - if isinstance(invocation, (WorkflowInvocation, AgentInvocation)): - invocation.output_messages = make_last_output_message(outputs) + invocation.output_messages = make_last_output_message(outputs) invocation.stop() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py index 2a0f67b7fcd..1d3d64dbd77 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py @@ -17,12 +17,13 @@ def make_input_message(data: Any) -> list[InputMessage]: """Create structured input message with full data as JSON.""" if not isinstance(data, dict): return [] + data_dict: dict[str, Any] = data input_messages: list[InputMessage] = [] - messages = data.get("messages") + messages: Any = data_dict.get("messages") if messages is not None: for msg in messages: - content = getattr(msg, "content", "") - if content: + content: Any = getattr(msg, "content", "") + if content and isinstance(content, str): input_message = InputMessage( role="user", parts=[Text(content)] ) @@ -32,9 +33,9 @@ def make_input_message(data: Any) -> list[InputMessage]: # Common in LangGraph where nodes use structured state fields # (e.g., user_query) rather than a message list. exclude_keys = {"messages", "intermediate_steps"} - input_data = { + input_data: dict[str, Any] = { k: v - for k, v in data.items() + for k, v in data_dict.items() if k not in exclude_keys and v is not None } if input_data: @@ -46,22 +47,19 @@ def make_input_message(data: Any) -> list[InputMessage]: def make_output_message(data: dict[str, Any]) -> list[OutputMessage]: """Create structured output message with full data as JSON.""" - if not isinstance(data, dict): - return [] output_messages: list[OutputMessage] = [] - messages = data.get("messages") + messages: Any = data.get("messages") if messages is None: return [] for msg in messages: - content = getattr(msg, "content", "") - if content: - if isinstance(msg, AIMessage): - output_message = OutputMessage( - role="assistant", - parts=[Text(msg.content)], - finish_reason="stop", - ) - output_messages.append(output_message) + content: Any = getattr(msg, "content", "") + if content and isinstance(msg, AIMessage) and isinstance(content, str): + output_message = OutputMessage( + role="assistant", + parts=[Text(content)], + finish_reason="stop", + ) + output_messages.append(output_message) return output_messages From debafbeba01f1c2550a6a431d20839633f025840 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 11 May 2026 17:50:00 -0700 Subject: [PATCH 27/27] add cast --- .../src/opentelemetry/instrumentation/langchain/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py index 1d3d64dbd77..b5de9323bfe 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import json -from typing import Any, Optional +from typing import Any, Optional, cast from langchain_core.messages import AIMessage @@ -17,7 +17,7 @@ def make_input_message(data: Any) -> list[InputMessage]: """Create structured input message with full data as JSON.""" if not isinstance(data, dict): return [] - data_dict: dict[str, Any] = data + data_dict = cast(dict[str, Any], data) input_messages: list[InputMessage] = [] messages: Any = data_dict.get("messages") if messages is not None: