diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 171d36f716b..8aba00e1c21 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -252,6 +252,25 @@ jobs: - name: Run tests run: tox -e py314-test-instrumentation-openai-v2-latest -- -ra + py313-test-instrumentation-openai-v2-pydantic1_ubuntu-latest: + name: instrumentation-openai-v2-pydantic1 3.13 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py313-test-instrumentation-openai-v2-pydantic1 -- -ra + pypy3-test-instrumentation-openai-v2-oldest_ubuntu-latest: name: instrumentation-openai-v2-oldest pypy-3.9 Ubuntu runs-on: ubuntu-latest diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md index 76d2deabc45..6fc545df24c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md @@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#4280](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4280)) - Add async wrappers for OpenAI Responses API streams and response stream managers ([#4325](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4325)) +- Add strongly typed Responses API extractors with validation and content + extraction improvements + ([#4337](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4337)) ## Version 2.3b0 (2025-12-24) diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_extractors.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_extractors.py new file mode 100644 index 00000000000..2d5e8702b19 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_extractors.py @@ -0,0 +1,419 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, List, Optional, TypeVar, Union + +from pydantic import BaseModel, Field, StrictInt, StrictStr, ValidationError + +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) + +_PYDANTIC_V2 = hasattr(BaseModel, "model_validate") + +if _PYDANTIC_V2: + from pydantic import ConfigDict +else: + ConfigDict = None + +if TYPE_CHECKING: + from opentelemetry.util.genai.types import ( + InputMessage, + LLMInvocation, + OutputMessage, + Text, + ) + +try: + from opentelemetry.util.genai.types import ( + InputMessage, + OutputMessage, + Text, + ) +except ImportError: + InputMessage = None + OutputMessage = None + Text = None + +GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read.input_tokens" +GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = ( + "gen_ai.usage.cache_creation.input_tokens" +) + +_logger = logging.getLogger(__name__) +ModelT = TypeVar("ModelT", bound=BaseModel) + + +class _ExtractorModel(BaseModel): + if _PYDANTIC_V2: + model_config = ConfigDict(extra="ignore", from_attributes=True) + else: + + class Config: + extra = "ignore" + orm_mode = True + + +class _ResponseTextFormatModel(_ExtractorModel): + type: Optional[StrictStr] = None + + +class _ResponseTextConfigModel(_ExtractorModel): + format: Optional[_ResponseTextFormatModel] = None + + +class _ResponseInputContentModel(_ExtractorModel): + text: Optional[StrictStr] = None + + +class _ResponseInputItemModel(_ExtractorModel): + role: Optional[StrictStr] = None + content: Optional[Union[StrictStr, List[_ResponseInputContentModel]]] = ( + None + ) + + +class _ResponsesRequestModel(_ExtractorModel): + instructions: Optional[StrictStr] = None + input: Optional[Union[StrictStr, List[_ResponseInputItemModel]]] = None + text: Optional[_ResponseTextConfigModel] = None + + +class _ResponseOutputContentModel(_ExtractorModel): + type: Optional[StrictStr] = None + text: Optional[StrictStr] = None + refusal: Optional[StrictStr] = None + + +class _ResponseOutputItemModel(_ExtractorModel): + type: Optional[StrictStr] = None + role: Optional[StrictStr] = None + status: Optional[StrictStr] = None + content: List[_ResponseOutputContentModel] = Field(default_factory=list) + + +class _UsageDetailsModel(_ExtractorModel): + cached_tokens: Optional[StrictInt] = None + cache_creation_input_tokens: Optional[StrictInt] = None + + +class _UsageModel(_ExtractorModel): + input_tokens: Optional[StrictInt] = None + output_tokens: Optional[StrictInt] = None + prompt_tokens: Optional[StrictInt] = None + completion_tokens: Optional[StrictInt] = None + input_tokens_details: Optional[_UsageDetailsModel] = None + prompt_tokens_details: Optional[_UsageDetailsModel] = None + + +class _ResponsesResultModel(_ExtractorModel): + # Responses can be partial or otherwise omit `output`. Treat that as "no + # terminal message items available yet" so downstream extraction helpers + # naturally return empty lists instead of raising. + output: List[_ResponseOutputItemModel] = Field(default_factory=list) + model: Optional[StrictStr] = None + id: Optional[StrictStr] = None + service_tier: Optional[StrictStr] = None + usage: Optional[_UsageModel] = None + + +def _rebuild_model(model_type: type[BaseModel]) -> None: + if _PYDANTIC_V2: + model_type.model_rebuild(_types_namespace=globals()) + else: + model_type.update_forward_refs(**globals()) + + +for _model_type in ( + _ResponseTextFormatModel, + _ResponseTextConfigModel, + _ResponseInputContentModel, + _ResponseInputItemModel, + _ResponsesRequestModel, + _ResponseOutputContentModel, + _ResponseOutputItemModel, + _UsageDetailsModel, + _UsageModel, + _ResponsesResultModel, +): + _rebuild_model(_model_type) + + +def _validate_model( + model_type: type[ModelT], value: object, context: str +) -> ModelT | None: + try: + if _PYDANTIC_V2: + return model_type.model_validate(value) + if isinstance(value, Mapping): + return model_type.parse_obj(value) + return model_type.from_orm(value) + except ValidationError: + _logger.debug( + "OpenAI responses extractor validation failed for %s", + context, + exc_info=True, + ) + return None + + +def _validate_request_kwargs( + kwargs: Mapping[str, object], +) -> _ResponsesRequestModel | None: + return _validate_model( + _ResponsesRequestModel, kwargs, "request parameters" + ) + + +def _validate_response_result(result: object) -> _ResponsesResultModel | None: + return _validate_model(_ResponsesResultModel, result, "response payload") + + +def _coerce_response_result( + result: object | _ResponsesResultModel | None, +) -> _ResponsesResultModel | None: + if result is None: + return None + + if isinstance(result, _ResponsesResultModel): + return result + + return _validate_response_result(result) + + +def _extract_system_instruction(kwargs: Mapping[str, object]) -> list["Text"]: + """Extract system instruction from the ``instructions`` parameter.""" + if Text is None: + return [] + + request = _validate_request_kwargs(kwargs) + if request is None or request.instructions is None: + return [] + + return [Text(content=request.instructions)] + + +def _extract_input_messages( + kwargs: Mapping[str, object], +) -> list["InputMessage"]: + """Extract input messages from Responses API kwargs.""" + if InputMessage is None or Text is None: + return [] + + request = _validate_request_kwargs(kwargs) + if request is None or request.input is None: + return [] + + if isinstance(request.input, str): + return [InputMessage(role="user", parts=[Text(content=request.input)])] + + messages: list[InputMessage] = [] + for item in request.input: + if item.role is None: + continue + + if isinstance(item.content, str): + messages.append( + InputMessage( + role=item.role, parts=[Text(content=item.content)] + ) + ) + continue + + if item.content is None: + continue + + parts = [ + Text(content=part.text) + for part in item.content + if part.text is not None + ] + if parts: + messages.append(InputMessage(role=item.role, parts=parts)) + + return messages + + +def _extract_output_parts( + content_blocks: Sequence[_ResponseOutputContentModel], +) -> list["Text"]: + if Text is None: + return [] + + parts: list[Text] = [] + for block in content_blocks: + if block.type == "output_text" and block.text is not None: + parts.append(Text(content=block.text)) + elif block.type == "refusal" and block.refusal is not None: + parts.append(Text(content=block.refusal)) + return parts + + +def _finish_reason_from_status(status: str | None) -> str | None: + # Responses API output items expose lifecycle statuses rather than finish + # reasons. We map the normal terminal state to the GenAI "stop" reason, + # preserve the other terminal statuses verbatim, and drop non-terminal or + # unknown states from the finish_reasons attribute. + if status == "completed": + return "stop" + if status in {"failed", "cancelled", "incomplete"}: + return status + return None + + +def _extract_output_messages_from_model( + result: _ResponsesResultModel, +) -> list["OutputMessage"]: + if OutputMessage is None or Text is None: + return [] + + messages: list[OutputMessage] = [] + for item in result.output: + if item.type != "message": + continue + finish_reason = _finish_reason_from_status(item.status) + if finish_reason is None: + continue + + messages.append( + OutputMessage( + role=item.role if item.role is not None else "assistant", + parts=_extract_output_parts(item.content), + finish_reason=finish_reason, + ) + ) + + return messages + + +def _extract_output_messages( + result: object | _ResponsesResultModel | None, +) -> list["OutputMessage"]: + """Extract output messages from a Responses API result.""" + validated_result = _coerce_response_result(result) + if validated_result is None: + return [] + + return _extract_output_messages_from_model(validated_result) + + +def _extract_finish_reasons_from_model( + result: _ResponsesResultModel, +) -> list[str]: + finish_reasons: list[str] = [] + for item in result.output: + if item.type != "message": + continue + finish_reason = _finish_reason_from_status(item.status) + if finish_reason is not None: + finish_reasons.append(finish_reason) + return finish_reasons + + +def _extract_finish_reasons( + result: object | _ResponsesResultModel | None, +) -> list[str]: + """Extract finish reasons from Responses API output items.""" + validated_result = _coerce_response_result(result) + if validated_result is None: + return [] + + return _extract_finish_reasons_from_model(validated_result) + + +def _extract_output_type(kwargs: Mapping[str, object]) -> str | None: + """Extract output type from Responses API request text.format.""" + request = _validate_request_kwargs(kwargs) + if request is None or request.text is None or request.text.format is None: + return None + + if request.text.format.type == "json_schema": + return "json" + return request.text.format.type + + +def _set_invocation_usage_attributes( + invocation: "LLMInvocation", usage: _UsageModel +) -> None: + if usage.input_tokens is not None: + invocation.input_tokens = usage.input_tokens + else: + invocation.input_tokens = usage.prompt_tokens + + if usage.output_tokens is not None: + invocation.output_tokens = usage.output_tokens + else: + invocation.output_tokens = usage.completion_tokens + + details = ( + usage.input_tokens_details + if usage.input_tokens_details is not None + else usage.prompt_tokens_details + ) + if details is None: + return + + if details.cached_tokens is not None: + invocation.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = ( + details.cached_tokens + ) + + if details.cache_creation_input_tokens is not None: + invocation.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] = ( + details.cache_creation_input_tokens + ) + + +def _set_invocation_response_attributes( + invocation: "LLMInvocation", + result: object | None, + capture_content: bool, +) -> None: + # This helper sits at the raw response boundary for instrumentation. Unlike + # the extractor helpers above, it intentionally validates exactly once and + # then fans out to the `_from_model` helpers rather than accepting an + # already-coerced `_ResponsesResultModel`. + if result is None: + return + + validated_result = _validate_response_result(result) + if validated_result is None: + return + + if validated_result.model is not None: + invocation.response_model_name = validated_result.model + + if validated_result.id is not None: + invocation.response_id = validated_result.id + + if validated_result.service_tier is not None: + invocation.attributes[ + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER + ] = validated_result.service_tier + + if validated_result.usage is not None: + _set_invocation_usage_attributes(invocation, validated_result.usage) + + finish_reasons = _extract_finish_reasons_from_model(validated_result) + if finish_reasons: + invocation.finish_reasons = finish_reasons + + if capture_content: + output_messages = _extract_output_messages_from_model(validated_result) + if output_messages: + invocation.output_messages = output_messages diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.pydantic1.txt b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.pydantic1.txt new file mode 100644 index 00000000000..9ca166184dc --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.pydantic1.txt @@ -0,0 +1,55 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ******************************** +# WARNING: NOT HERMETIC !!!!!!!!!! +# ******************************** +# +# This "requirements.txt" is installed in conjunction +# with multiple other dependencies in the top-level "tox.ini" +# file. In particular, please see: +# +# openai-pydantic1: {[testenv]test_deps} +# openai-pydantic1: -r {toxinidir}/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.pydantic1.txt +# +# This provides additional dependencies, namely: +# +# opentelemetry-api +# opentelemetry-sdk +# opentelemetry-semantic-conventions +# +# ... with a "dev" version based on the latest distribution. + + +# This variant of the requirements aims to test the system using +# the newest supported version of external dependencies with Pydantic 1.x. + +openai==1.109.1 +pydantic==1.10.24 +httpx==0.27.2 +# older jiter is required for PyPy < 3.11 +jiter==0.11.1 +Deprecated==1.2.14 +importlib-metadata==6.11.0 +packaging==24.0 +pytest==7.4.4 +pytest-vcr==1.0.2 +pytest-asyncio==0.21.0 +wrapt==1.16.0 +# test with the latest version of opentelemetry-api, sdk, and semantic conventions + +-e opentelemetry-instrumentation +-e instrumentation-genai/opentelemetry-instrumentation-openai-v2 +-e util/opentelemetry-util-genai diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_response_extractors.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_response_extractors.py new file mode 100644 index 00000000000..1cf9a1a70e1 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_response_extractors.py @@ -0,0 +1,365 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from types import SimpleNamespace +from unittest import mock + +import pytest + +from opentelemetry.instrumentation.openai_v2 import response_extractors +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) +from opentelemetry.util.genai.types import LLMInvocation + + +def _validate_compat_model(loaded_module, model_type, value): + return loaded_module._validate_model(model_type, value, "test") + + +@pytest.fixture(scope="module", name="loaded_module") +def _loaded_module_fixture(): + return response_extractors + + +@pytest.fixture( + scope="module", name="gen_ai_usage_cache_creation_input_tokens" +) +def _gen_ai_usage_cache_creation_input_tokens_fixture(loaded_module): + return loaded_module.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + + +@pytest.fixture(scope="module", name="gen_ai_usage_cache_read_input_tokens") +def _gen_ai_usage_cache_read_input_tokens_fixture(loaded_module): + return loaded_module.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS + + +def test_extract_system_instruction_returns_text_for_string(loaded_module): + instructions = loaded_module._extract_system_instruction( + {"instructions": "Be concise"} + ) + + assert [part.content for part in instructions] == ["Be concise"] + + +def test_extract_input_messages_supports_string_and_mixed_message_content( + loaded_module, +): + from_string = loaded_module._extract_input_messages({"input": "Hello"}) + from_list = loaded_module._extract_input_messages( + { + "input": [ + {"role": "user", "content": "First"}, + SimpleNamespace( + role="assistant", + content=[ + {"text": "Second"}, + SimpleNamespace(text="Third"), + {"type": "input_image", "image_url": "ignored"}, + ], + ), + {"role": None, "content": "ignored"}, + ] + } + ) + + assert [ + (msg.role, [part.content for part in msg.parts]) for msg in from_string + ] == [("user", ["Hello"])] + assert [ + (msg.role, [part.content for part in msg.parts]) for msg in from_list + ] == [ + ("user", ["First"]), + ("assistant", ["Second", "Third"]), + ] + + +def test_extract_output_messages_maps_parts_and_finish_reasons(loaded_module): + result = SimpleNamespace( + output=[ + SimpleNamespace( + type="message", + role="assistant", + status="completed", + content=[ + SimpleNamespace(type="output_text", text="Done"), + SimpleNamespace(type="refusal", refusal="Cannot comply"), + SimpleNamespace(type="summary", text="ignored"), + ], + ), + SimpleNamespace( + type="message", + status="incomplete", + content=[SimpleNamespace(type="output_text", text="Partial")], + ), + SimpleNamespace( + type="message", + status="queued", + content=[SimpleNamespace(type="output_text", text="Pending")], + ), + SimpleNamespace(type="tool_call", status="completed", content=[]), + ] + ) + + messages = loaded_module._extract_output_messages(result) + + assert [(msg.role, msg.finish_reason) for msg in messages] == [ + ("assistant", "stop"), + ("assistant", "incomplete"), + ] + assert [[part.content for part in msg.parts] for msg in messages] == [ + ["Done", "Cannot comply"], + ["Partial"], + ] + + +def test_extract_finish_reasons_only_reads_message_items(loaded_module): + result = SimpleNamespace( + output=[ + SimpleNamespace(type="message", status="completed"), + SimpleNamespace(type="message", status=None), + SimpleNamespace(type="message", status="in_progress"), + SimpleNamespace(type="tool_call", status="incomplete"), + ] + ) + + assert loaded_module._extract_finish_reasons(result) == ["stop"] + + +def test_extract_output_type_handles_text_format_mapping(loaded_module): + assert ( + loaded_module._extract_output_type( + {"text": {"format": {"type": "json_schema"}}} + ) + == "json" + ) + assert ( + loaded_module._extract_output_type( + {"text": {"format": {"type": "text"}}} + ) + == "text" + ) + assert ( + loaded_module._extract_output_type( + { + "text": SimpleNamespace( + format=SimpleNamespace(type="json_schema") + ) + } + ) + == "json" + ) + assert ( + loaded_module._extract_output_type( + {"text": SimpleNamespace(format=SimpleNamespace(type="text"))} + ) + == "text" + ) + # Invalid request shapes should degrade to no extracted output type rather + # than surfacing validation errors from instrumentation. + assert ( + loaded_module._extract_output_type({"text": {"format": "plain"}}) + is None + ) + assert loaded_module._extract_output_type({"text": "plain"}) is None + + +def test_extractors_handle_missing_genai_types_import(loaded_module): + with ( + mock.patch.object(loaded_module, "Text", None), + mock.patch.object(loaded_module, "InputMessage", None), + mock.patch.object(loaded_module, "OutputMessage", None), + ): + assert ( + loaded_module._extract_system_instruction({"instructions": "hi"}) + == [] + ) + assert loaded_module._extract_input_messages({"input": "hi"}) == [] + assert ( + loaded_module._extract_output_messages( + SimpleNamespace( + output=[SimpleNamespace(type="message", content=[])] + ) + ) + == [] + ) + + +def test_set_invocation_response_attributes_populates_usage_and_metadata( + loaded_module, + gen_ai_usage_cache_creation_input_tokens, + gen_ai_usage_cache_read_input_tokens, +): + invocation = LLMInvocation(request_model="gpt-4o-mini") + result = SimpleNamespace( + model="gpt-4.1", + id="resp_123", + service_tier="scale", + usage=SimpleNamespace( + prompt_tokens=11, + completion_tokens=7, + prompt_tokens_details=SimpleNamespace( + cached_tokens=3, + cache_creation_input_tokens=5, + ), + ), + ) + + loaded_module._set_invocation_response_attributes( + invocation, result, capture_content=False + ) + + assert invocation.response_model_name == "gpt-4.1" + assert invocation.response_id == "resp_123" + assert invocation.input_tokens == 11 + assert invocation.output_tokens == 7 + assert invocation.attributes == { + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER: "scale", + gen_ai_usage_cache_read_input_tokens: 3, + gen_ai_usage_cache_creation_input_tokens: 5, + } + + +def test_set_invocation_response_attributes_accepts_mapping_usage( + loaded_module, + gen_ai_usage_cache_creation_input_tokens, + gen_ai_usage_cache_read_input_tokens, +): + invocation = LLMInvocation(request_model="gpt-4o-mini") + result = SimpleNamespace( + usage={ + "input_tokens": 13, + "output_tokens": 8, + "input_tokens_details": { + "cached_tokens": 2, + "cache_creation_input_tokens": 4, + }, + } + ) + + loaded_module._set_invocation_response_attributes( + invocation, result, capture_content=False + ) + + assert invocation.input_tokens == 13 + assert invocation.output_tokens == 8 + assert invocation.attributes == { + gen_ai_usage_cache_read_input_tokens: 2, + gen_ai_usage_cache_creation_input_tokens: 4, + } + + +def test_set_invocation_response_attributes_populates_output_messages( + loaded_module, +): + invocation = LLMInvocation(request_model="gpt-4o-mini") + result = SimpleNamespace( + output=[ + SimpleNamespace( + type="message", + role="assistant", + status="completed", + content=[SimpleNamespace(type="output_text", text="Done")], + ) + ] + ) + + loaded_module._set_invocation_response_attributes( + invocation, result, capture_content=True + ) + + assert invocation.finish_reasons == ["stop"] + assert [ + (message.role, message.finish_reason) + for message in invocation.output_messages + ] == [("assistant", "stop")] + assert [ + [part.content for part in message.parts] + for message in invocation.output_messages + ] == [["Done"]] + + +def test_prevalidated_response_model_skips_revalidation( + loaded_module, monkeypatch +): + validated_result = _validate_compat_model( + loaded_module, + loaded_module._ResponsesResultModel, + SimpleNamespace( + output=[ + SimpleNamespace( + type="message", + status="completed", + content=[SimpleNamespace(type="output_text", text="Done")], + ) + ] + ), + ) + assert validated_result is not None + + def _unexpected_validation(_result): + raise AssertionError("unexpected response revalidation") + + monkeypatch.setattr( + loaded_module, "_validate_response_result", _unexpected_validation + ) + + assert loaded_module._extract_finish_reasons(validated_result) == ["stop"] + messages = loaded_module._extract_output_messages(validated_result) + assert [part.content for part in messages[0].parts] == ["Done"] + + +@pytest.mark.parametrize( + ("kwargs", "extractor_name"), + [ + ({"instructions": ["not-a-string"]}, "_extract_system_instruction"), + ({"input": 42}, "_extract_input_messages"), + ({"text": {"format": {"type": 42}}}, "_extract_output_type"), + ], +) +def test_request_validation_errors_are_logged_and_ignored( + loaded_module, caplog, kwargs, extractor_name +): + caplog.set_level(logging.DEBUG, logger=loaded_module.__name__) + extractor = getattr(loaded_module, extractor_name) + + result = extractor(kwargs) + + assert result in ([], None) + assert "OpenAI responses extractor validation failed" in caplog.text + + +def test_response_validation_errors_are_logged_and_ignored( + loaded_module, caplog +): + caplog.set_level(logging.DEBUG, logger=loaded_module.__name__) + invocation = LLMInvocation(request_model="gpt-4o-mini") + invalid_result = SimpleNamespace(output=42, usage=42) + + assert loaded_module._extract_output_messages(invalid_result) == [] + assert loaded_module._extract_finish_reasons(invalid_result) == [] + + loaded_module._set_invocation_response_attributes( + invocation, invalid_result, capture_content=True + ) + + assert invocation.response_model_name is None + assert invocation.response_id is None + assert invocation.input_tokens is None + assert invocation.output_tokens is None + assert invocation.finish_reasons is None + assert not invocation.output_messages + assert not invocation.attributes + assert "OpenAI responses extractor validation failed" in caplog.text diff --git a/tox.ini b/tox.ini index b351bf13eee..5b13ed9365a 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ envlist = ; instrumentation-openai py3{9,10,11,12,13,14}-test-instrumentation-openai-v2-{oldest,latest} + py313-test-instrumentation-openai-v2-pydantic1 pypy3-test-instrumentation-openai-v2-{oldest,latest} lint-instrumentation-openai-v2 @@ -482,6 +483,8 @@ deps = # and the latest version of OTel API and SDK openai-latest: {[testenv]test_deps} openai-latest: -r {toxinidir}/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt + openai-pydantic1: {[testenv]test_deps} + openai-pydantic1: -r {toxinidir}/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.pydantic1.txt lint-instrumentation-openai-v2: -r {toxinidir}/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.oldest.txt openai_agents-oldest: -r {toxinidir}/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/requirements.oldest.txt openai_agents-latest: {[testenv]test_deps}