diff --git a/CHANGELOG.md b/CHANGELOG.md index 344b1341321..fd04db5d99f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +**Features**: + +- Transform deprecated `gen_ai.request.messages`, `gen_ai.response.text`, and `gen_ai.response.tool_calls` attributes into their normalized replacements (`gen_ai.input.messages` and `gen_ai.output.messages`). ([#6201](https://github.com/getsentry/relay/pull/6201)) + ## 26.7.0 **Features**: diff --git a/relay-conventions/build/attributes.rs b/relay-conventions/build/attributes.rs index 34f56e201d5..5022217a64f 100644 --- a/relay-conventions/build/attributes.rs +++ b/relay-conventions/build/attributes.rs @@ -40,6 +40,12 @@ pub enum DeprecationStatus { Backfill, /// Only write the replacement name. Normalize, + /// Move the value to the replacement name and apply a value transformation. + /// + /// For write behavior purposes, this produces `CurrentName` — the generic attribute + /// renaming logic leaves these attributes alone. Dedicated transformation code handles + /// the full move-and-reshape. + Transform, } /// Information about an attribute's deprecation. @@ -126,6 +132,7 @@ fn format_write_behavior(deprecation: Option<&Deprecation>) -> String { DeprecationStatus::Normalize => { format!("WriteBehavior::NewName({name})") } + DeprecationStatus::Transform => "WriteBehavior::CurrentName".to_owned(), } } diff --git a/relay-conventions/sentry-conventions b/relay-conventions/sentry-conventions index 1602b1dde00..eff36f9095d 160000 --- a/relay-conventions/sentry-conventions +++ b/relay-conventions/sentry-conventions @@ -1 +1 @@ -Subproject commit 1602b1dde007194b98811adf0bef98c95f69642e +Subproject commit eff36f9095d28e3c5557861b61c00c4ebd1d7a2c diff --git a/relay-event-normalization/src/eap/ai.rs b/relay-event-normalization/src/eap/ai.rs index 2d875a2f05c..1b5ba2cffe9 100644 --- a/relay-event-normalization/src/eap/ai.rs +++ b/relay-event-normalization/src/eap/ai.rs @@ -34,6 +34,7 @@ pub fn normalize_ai( return; } + crate::eap::gen_ai_transform::transform_gen_ai(attributes); normalize_model(attributes); normalize_ai_type(attributes); normalize_total_tokens(attributes); diff --git a/relay-event-normalization/src/eap/gen_ai_transform.rs b/relay-event-normalization/src/eap/gen_ai_transform.rs new file mode 100644 index 00000000000..f5380a00f6e --- /dev/null +++ b/relay-event-normalization/src/eap/gen_ai_transform.rs @@ -0,0 +1,558 @@ +//! Transforms deprecated gen_ai attributes into their normalized replacements. +//! +//! These transformations go beyond simple renaming — they reshape attribute values +//! to match the new schema. For the transformation specifications, see: +//! +//! +//! Attributes with `_status: "transform"` in sentry-conventions produce +//! `WriteBehavior::CurrentName`, so [`super::normalize_attribute_names`] leaves +//! them alone — the full move-and-reshape is handled here. +//! +//! The functions are generic over [`AttributesLike`] so they work on both +//! [`Attributes`](relay_event_schema::protocol::Attributes) (SpanV2) and +//! [`SpanData`](relay_event_schema::protocol::SpanData) (SpanV1). + +// This module intentionally reads deprecated attribute keys to transform their values. +#![allow(deprecated)] + +use relay_conventions::attributes::*; +use serde::{Deserialize, Serialize}; + +use super::attribute_like::{AttributeLike, AttributesLike}; + +// --- Input models (what SDKs send) --- + +/// A message in the old `gen_ai.request.messages` format. +#[derive(Deserialize)] +struct OldMessage { + #[serde(default)] + content: Option, + #[serde(flatten)] + rest: serde_json::Map, +} + +/// The `content` field of an old message — either a string or an array of parts. +#[derive(Deserialize)] +#[serde(untagged)] +enum OldContent { + String(String), + Parts(Vec>), +} + +/// The `gen_ai.response.text` attribute — can be various shapes. +#[derive(Deserialize)] +#[serde(untagged)] +enum ResponseText { + /// A plain JSON string. + String(String), + /// A JSON array (of strings or objects). + Array(Vec), + /// A JSON object with a `content` field. + Object { content: serde_json::Value }, +} + +/// An item in a `gen_ai.response.text` array. +#[derive(Deserialize)] +#[serde(untagged)] +enum ResponseTextItem { + String(String), + Object { content: serde_json::Value }, +} + +// --- Output models (what we produce) --- + +/// A message in the new `gen_ai.input.messages` / `gen_ai.output.messages` format. +#[derive(Serialize)] +struct NewMessage { + #[serde(skip_serializing_if = "Option::is_none")] + role: Option, + parts: Vec, + #[serde(flatten)] + rest: serde_json::Map, +} + +/// A tool call entry from `gen_ai.response.tool_calls`. +#[derive(Deserialize)] +struct ToolCall { + #[serde(flatten)] + fields: serde_json::Map, +} + +// --- Transformation logic --- + +/// Applies gen_ai attribute transformations. +pub(crate) fn transform_gen_ai(attributes: &mut T) { + transform_request_messages(attributes); + transform_response_to_output_messages(attributes); +} + +/// Transforms `gen_ai.request.messages` → `gen_ai.input.messages`. +fn transform_request_messages(attributes: &mut T) { + if attributes.contains_key(GEN_AI__INPUT__MESSAGES) { + attributes.remove(GEN_AI__REQUEST__MESSAGES); + return; + } + + let Some(raw) = get_str(attributes, GEN_AI__REQUEST__MESSAGES) else { + return; + }; + + let Ok(messages) = serde_json::from_str::>(&raw) else { + // Not a valid message array — move as-is (just a key rename). + if let Some(attr) = attributes.remove(GEN_AI__REQUEST__MESSAGES) { + attributes.insert(GEN_AI__INPUT__MESSAGES.to_owned(), attr); + } + return; + }; + + let new_messages: Vec = messages + .into_iter() + .map(|msg| { + let OldMessage { content, mut rest } = msg; + let role = rest + .remove("role") + .and_then(|v| v.as_str().map(String::from)); + + let parts = match content { + Some(OldContent::String(s)) => { + vec![serde_json::json!({"type": "text", "content": s})] + } + Some(OldContent::Parts(items)) => items + .into_iter() + .map(|mut part| { + if !part.contains_key("content") { + if let Some(text) = part.get("text").cloned() { + part.insert("content".to_owned(), text); + } + } + serde_json::Value::Object(part) + }) + .collect(), + None => { + // No content field — put everything back and skip. + if let Some(role) = role { + rest.insert("role".to_owned(), serde_json::Value::String(role)); + } + // Return the original object unchanged since there's nothing to transform. + // We still need to return a NewMessage, so we use an empty parts vec + // and let the rest carry the original fields. + return NewMessage { + role: None, + parts: vec![], + rest, + }; + } + }; + + NewMessage { role, parts, rest } + }) + .collect(); + + if let Ok(json) = serde_json::to_string(&new_messages) { + attributes.insert( + GEN_AI__INPUT__MESSAGES.to_owned(), + T::Value::from(json).into(), + ); + } + attributes.remove(GEN_AI__REQUEST__MESSAGES); +} + +/// Transforms `gen_ai.response.text` + `gen_ai.response.tool_calls` → `gen_ai.output.messages`. +fn transform_response_to_output_messages(attributes: &mut T) { + if attributes.contains_key(GEN_AI__OUTPUT__MESSAGES) { + attributes.remove(GEN_AI__RESPONSE__TEXT); + attributes.remove(GEN_AI__RESPONSE__TOOL_CALLS); + return; + } + + let response_text = get_str(attributes, GEN_AI__RESPONSE__TEXT); + let tool_calls_raw = get_str(attributes, GEN_AI__RESPONSE__TOOL_CALLS); + + if response_text.is_none() && tool_calls_raw.is_none() { + return; + } + + let mut parts = Vec::new(); + + if let Some(ref text) = response_text { + extract_text_parts(text, &mut parts); + } + + if let Some(ref raw) = tool_calls_raw { + extract_tool_call_parts(raw, &mut parts); + } + + // Always clean up deprecated keys. + attributes.remove(GEN_AI__RESPONSE__TEXT); + attributes.remove(GEN_AI__RESPONSE__TOOL_CALLS); + + if parts.is_empty() { + return; + } + + let output = [NewMessage { + role: Some("assistant".to_owned()), + parts, + rest: Default::default(), + }]; + if let Ok(json) = serde_json::to_string(&output) { + attributes.insert( + GEN_AI__OUTPUT__MESSAGES.to_owned(), + T::Value::from(json).into(), + ); + } +} + +/// Extracts text parts from a `gen_ai.response.text` value. +fn extract_text_parts(raw: &str, parts: &mut Vec) { + let Ok(parsed) = serde_json::from_str::(raw) else { + // Not valid JSON — treat the entire raw string as plain text. + parts.push(serde_json::json!({"type": "text", "content": raw})); + return; + }; + + match parsed { + ResponseText::String(s) => { + parts.push(serde_json::json!({"type": "text", "content": s})); + } + ResponseText::Array(items) => { + for item in items { + match item { + ResponseTextItem::String(s) => { + parts.push(serde_json::json!({"type": "text", "content": s})); + } + ResponseTextItem::Object { content } => { + parts.push(serde_json::json!({"type": "text", "content": content})); + } + } + } + } + ResponseText::Object { content } => { + parts.push(serde_json::json!({"type": "text", "content": content})); + } + } +} + +/// Extracts tool_call parts from a `gen_ai.response.tool_calls` value. +fn extract_tool_call_parts(raw: &str, parts: &mut Vec) { + let Ok(tool_calls) = serde_json::from_str::>(raw) else { + return; + }; + for tool_call in tool_calls { + let mut fields = tool_call.fields; + fields.insert( + "type".to_owned(), + serde_json::Value::String("tool_call".to_owned()), + ); + parts.push(serde_json::Value::Object(fields)); + } +} + +/// Gets a string attribute value as an owned String. +fn get_str(attributes: &T, key: &str) -> Option { + let annotated = attributes.as_object().get(key)?; + let value = annotated.value()?; + value.as_str().map(|s| s.to_owned()) +} + +#[cfg(test)] +mod tests { + use relay_event_schema::protocol::Attributes; + + use super::*; + + fn make_attributes(pairs: &[(&str, &str)]) -> Attributes { + let mut attrs = Attributes::new(); + for (key, value) in pairs { + attrs.insert(key.to_string(), value.to_string()); + } + attrs + } + + fn get_string(attributes: &Attributes, key: &str) -> Option { + attributes.get_value(key)?.as_str().map(|s| s.to_owned()) + } + + fn parse_json(s: &str) -> serde_json::Value { + serde_json::from_str(s).unwrap() + } + + mod request_messages { + use super::*; + + #[test] + fn string_content_to_parts() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"user","content":"hello"}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "user", "parts": [{"type": "text", "content": "hello"}]}]), + ); + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + } + + #[test] + fn array_content_to_parts() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"user","content":[{"type":"text","text":"hello"}]}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "user", "parts": [{"type": "text", "text": "hello", "content": "hello"}]}]), + ); + } + + #[test] + fn does_not_overwrite_existing() { + let existing = r#"[{"role":"user","parts":[{"type":"text","content":"existing"}]}]"#; + let mut attrs = make_attributes(&[ + (GEN_AI__INPUT__MESSAGES, existing), + ( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"user","content":"ignored"}]"#, + ), + ]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!(parse_json(&result), parse_json(existing)); + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + } + + #[test] + fn preserves_extra_fields() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"system","content":"be helpful","metadata":{"key":"value"}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + let parsed = parse_json(&result); + assert_eq!(parsed[0]["role"], "system"); + assert_eq!(parsed[0]["metadata"]["key"], "value"); + assert!(parsed[0].get("content").is_none()); + } + + #[test] + fn messages_with_metadata() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"system","content":"You are a helpful assistant.","response_metadata":{}},{"role":"user","content":"What is the capital of France?","response_metadata":{}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + let parsed = parse_json(&result); + assert_eq!(parsed[0]["role"], "system"); + assert_eq!(parsed[0]["response_metadata"], serde_json::json!({})); + assert_eq!( + parsed[0]["parts"], + serde_json::json!([{"type": "text", "content": "You are a helpful assistant."}]), + ); + assert!(parsed[0].get("content").is_none()); + } + + #[test] + fn preserves_unconvertible_content() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"tool","content":{"result":"ok"}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + // Object content doesn't match OldContent variants, so the message + // is moved as-is since the whole array fails to parse as Vec. + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "tool", "content": {"result": "ok"}}]), + ); + } + + #[test] + fn no_messages_is_noop() { + let mut attrs = make_attributes(&[]); + transform_gen_ai(&mut attrs); + assert!(get_string(&attrs, GEN_AI__INPUT__MESSAGES).is_none()); + } + + #[test] + fn invalid_json_moves_as_is() { + let mut attrs = make_attributes(&[(GEN_AI__REQUEST__MESSAGES, "not json")]); + + transform_gen_ai(&mut attrs); + + assert_eq!( + get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(), + "not json" + ); + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + } + } + + mod response_to_output { + use super::*; + + #[test] + fn plain_text() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, "hello")]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"} + ]}]), + ); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + } + + #[test] + fn json_string() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, r#""hello world""#)]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello world"} + ]}]), + ); + } + + #[test] + fn array_of_strings() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, r#"["hello","world"]"#)]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"}, + {"type": "text", "content": "world"} + ]}]), + ); + } + + #[test] + fn object_with_content() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, r#"{"content":"hello"}"#)]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"} + ]}]), + ); + } + + #[test] + fn array_of_objects_with_content() { + let mut attrs = make_attributes(&[( + GEN_AI__RESPONSE__TEXT, + r#"[{"content":"hello"},{"content":"world"}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"}, + {"type": "text", "content": "world"} + ]}]), + ); + } + + #[test] + fn text_and_tool_calls() { + let mut attrs = make_attributes(&[ + (GEN_AI__RESPONSE__TEXT, "hello"), + ( + GEN_AI__RESPONSE__TOOL_CALLS, + r#"[{"id":"call_1","name":"weather","arguments":{}}]"#, + ), + ]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"}, + {"id": "call_1", "name": "weather", "arguments": {}, "type": "tool_call"} + ]}]), + ); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TOOL_CALLS).is_none()); + } + + #[test] + fn tool_calls_only() { + let mut attrs = make_attributes(&[( + GEN_AI__RESPONSE__TOOL_CALLS, + r#"[{"id":"call_1","name":"weather","arguments":{}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"id": "call_1", "name": "weather", "arguments": {}, "type": "tool_call"} + ]}]), + ); + } + + #[test] + fn does_not_overwrite_existing() { + let existing = + r#"[{"role":"assistant","parts":[{"type":"text","content":"existing"}]}]"#; + let mut attrs = make_attributes(&[ + (GEN_AI__OUTPUT__MESSAGES, existing), + (GEN_AI__RESPONSE__TEXT, "ignored"), + ]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!(parse_json(&result), parse_json(existing)); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + } + + #[test] + fn no_response_attributes_is_noop() { + let mut attrs = make_attributes(&[]); + transform_gen_ai(&mut attrs); + assert!(get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).is_none()); + } + } +} diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index 363b0c17a3b..acd0396bfaf 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -30,6 +30,7 @@ use crate::{ mod ai; mod attribute_like; +pub(crate) mod gen_ai_transform; mod mobile; mod size; pub mod time; diff --git a/relay-event-normalization/src/normalize/span/ai.rs b/relay-event-normalization/src/normalize/span/ai.rs index 2e5f74be9a8..be02d6559f2 100644 --- a/relay-event-normalization/src/normalize/span/ai.rs +++ b/relay-event-normalization/src/normalize/span/ai.rs @@ -369,6 +369,7 @@ fn enrich_ai_span_data( let data = span_data.get_or_insert_with(SpanData::default); + crate::eap::gen_ai_transform::transform_gen_ai(data); map_ai_measurements_to_data(data, measurements.value()); set_total_tokens(data); diff --git a/tests/integration/test_ai.py b/tests/integration/test_ai.py index b90b9636cc2..d7334c65e91 100644 --- a/tests/integration/test_ai.py +++ b/tests/integration/test_ai.py @@ -1,6 +1,8 @@ +import json from datetime import datetime, timezone from sentry_relay.consts import DataCategory +from sentry_sdk.envelope import Envelope, Item, PayloadRef from .asserts import matches_any, time_within_delta @@ -397,9 +399,10 @@ def test_ai_spans_example_transaction( "gen_ai.response.model": {"type": "string", "value": "gpt-4o"}, "gen_ai.output.messages": { "type": "string", - "value": "True. \n\n- London: 61°F \n- San Francisco: 13°C", + # Transformed from gen_ai.response.text into parts format. + # Exact JSON verified in test_gen_ai_transform_response_to_output_v1. + "value": matches_any(), }, - "gen_ai.response.text": None, "gen_ai.response.tokens_per_second": {"type": "double", "value": 130.0}, "gen_ai.usage.input_tokens": {"type": "integer", "value": 245}, "gen_ai.usage.output_tokens": {"type": "integer", "value": 65}, @@ -504,7 +507,6 @@ def test_ai_spans_example_transaction( "value": "Another weather prompt", }, "gen_ai.request.available_tools": None, - "gen_ai.request.messages": None, "gen_ai.request.model": {"type": "string", "value": "gpt-4o"}, "gen_ai.response.finish_reasons": { "type": "string", @@ -519,9 +521,8 @@ def test_ai_spans_example_transaction( "value": "gpt-4o-2024-08-06", }, "gen_ai.response.tokens_per_second": {"type": "double", "value": 92.0}, - "gen_ai.response.tool_calls": None, "gen_ai.system": None, - "gen_ai.output.messages": { + "gen_ai.response.tool_calls": { "type": "string", "value": "some_tool_calls", }, @@ -1059,7 +1060,6 @@ def test_ai_spans_example_transaction( "value": "Some AI Prompt about " "the Wheather", }, "gen_ai.request.available_tools": None, - "gen_ai.request.messages": None, "gen_ai.request.model": {"type": "string", "value": "gpt-4o"}, "gen_ai.response.finish_reasons": { "type": "string", @@ -1075,12 +1075,9 @@ def test_ai_spans_example_transaction( }, "gen_ai.output.messages": { "type": "string", - "value": "True. \n" - "\n" - "- London: 61°F \n" - "- San Francisco: 13°C", + # Transformed from gen_ai.response.text into parts format. + "value": matches_any(), }, - "gen_ai.response.text": None, "gen_ai.response.tokens_per_second": {"type": "double", "value": 38.0}, "gen_ai.system": None, "gen_ai.provider.name": {"type": "string", "value": "openai.responses"}, @@ -1341,3 +1338,276 @@ def test_ai_spans_example_transaction( "quantity": 10, }, ] + + +TEST_CONFIG = { + "outcomes": { + "emit_outcomes": True, + } +} + + +def envelope_with_spans(*payloads: dict, trace_info=None, metadata=None) -> Envelope: + envelope = Envelope() + envelope.add_item( + Item( + type="span", + payload=PayloadRef(json={"items": payloads, **(metadata or {})}), + content_type="application/vnd.sentry.items.span.v2+json", + headers={"item_count": len(payloads)}, + ) + ) + envelope.headers["trace"] = trace_info + return envelope + + +def test_gen_ai_transform_request_messages_v1( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV1 (transaction) path: gen_ai.request.messages with JSON content + gets transformed to gen_ai.input.messages with parts format. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing()) + + ts = datetime.now(timezone.utc) + messages = json.dumps([{"role": "user", "content": "What is the weather?"}]) + + relay.send_transaction( + project_id, + { + "contexts": { + "trace": { + "span_id": "aaaa000000000001", + "trace_id": "a9351cd574f092f6acad48e250981f11", + "op": "gen_ai.generate_text", + "origin": "manual", + "status": "ok", + }, + }, + "spans": [ + { + "span_id": "bbbb000000000001", + "trace_id": "a9351cd574f092f6acad48e250981f11", + "parent_span_id": "aaaa000000000001", + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "status": "ok", + "op": "gen_ai.generate_text", + "data": { + "gen_ai.request.messages": messages, + "gen_ai.request.model": "gpt-4o", + }, + }, + ], + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "transaction": "test-transform", + "type": "transaction", + "transaction_info": {"source": "custom"}, + "platform": "python", + }, + ) + + spans = spans_consumer.get_spans(n=2) + ai_span = next(s for s in spans if s["span_id"] == "bbbb000000000001") + attrs = ai_span["attributes"] + + result = json.loads(attrs["gen_ai.input.messages"]["value"]) + assert result == [ + {"role": "user", "parts": [{"type": "text", "content": "What is the weather?"}]} + ] + assert "gen_ai.request.messages" not in attrs + + +def test_gen_ai_transform_response_to_output_v1( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV1 (transaction) path: gen_ai.response.text gets transformed into + gen_ai.output.messages with assistant parts format. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing()) + + ts = datetime.now(timezone.utc) + relay.send_transaction( + project_id, + { + "contexts": { + "trace": { + "span_id": "aaaa000000000002", + "trace_id": "b9351cd574f092f6acad48e250981f11", + "op": "gen_ai.generate_text", + "origin": "manual", + "status": "ok", + }, + }, + "spans": [ + { + "span_id": "bbbb000000000002", + "trace_id": "b9351cd574f092f6acad48e250981f11", + "parent_span_id": "aaaa000000000002", + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "status": "ok", + "op": "gen_ai.generate_text", + "data": { + "gen_ai.response.text": "The weather is sunny.", + "gen_ai.request.model": "gpt-4o", + }, + }, + ], + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "transaction": "test-transform", + "type": "transaction", + "transaction_info": {"source": "custom"}, + "platform": "python", + }, + ) + + spans = spans_consumer.get_spans(n=2) + ai_span = next(s for s in spans if s["span_id"] == "bbbb000000000002") + attrs = ai_span["attributes"] + + result = json.loads(attrs["gen_ai.output.messages"]["value"]) + assert result == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "The weather is sunny."}], + } + ] + assert "gen_ai.response.text" not in attrs + + +def test_gen_ai_transform_request_messages_v2( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV2 (direct span ingestion) path: gen_ai.request.messages with JSON content + gets transformed to gen_ai.input.messages with parts format. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + + ts = datetime.now(timezone.utc) + messages = json.dumps([{"role": "user", "content": "What is the weather?"}]) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp(), + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "c9351cd574f092f6acad48e250981f11", + "span_id": "cccc000000000001", + "is_segment": True, + "name": "gen_ai.generate_text", + "status": "ok", + "attributes": { + "sentry.op": {"value": "gen_ai.generate_text", "type": "string"}, + "gen_ai.request.messages": {"value": messages, "type": "string"}, + "gen_ai.request.model": {"value": "gpt-4o", "type": "string"}, + }, + }, + trace_info={ + "trace_id": "c9351cd574f092f6acad48e250981f11", + "public_key": mini_sentry.get_dsn_public_key(project_id), + }, + ) + + relay.send_envelope(project_id, envelope) + + spans = spans_consumer.get_spans(n=1) + attrs = spans[0]["attributes"] + + result = json.loads(attrs["gen_ai.input.messages"]["value"]) + assert result == [ + {"role": "user", "parts": [{"type": "text", "content": "What is the weather?"}]} + ] + assert "gen_ai.request.messages" not in attrs + + +def test_gen_ai_transform_response_text_and_tool_calls_v2( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV2 (direct span ingestion) path: both gen_ai.response.text and + gen_ai.response.tool_calls get combined into gen_ai.output.messages. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + + ts = datetime.now(timezone.utc) + tool_calls = json.dumps( + [{"id": "call_1", "name": "get_weather", "arguments": {"city": "London"}}] + ) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp(), + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "d9351cd574f092f6acad48e250981f11", + "span_id": "dddd000000000001", + "is_segment": True, + "name": "gen_ai.generate_text", + "status": "ok", + "attributes": { + "sentry.op": {"value": "gen_ai.generate_text", "type": "string"}, + "gen_ai.response.text": {"value": "Let me check.", "type": "string"}, + "gen_ai.response.tool_calls": {"value": tool_calls, "type": "string"}, + "gen_ai.request.model": {"value": "gpt-4o", "type": "string"}, + }, + }, + trace_info={ + "trace_id": "d9351cd574f092f6acad48e250981f11", + "public_key": mini_sentry.get_dsn_public_key(project_id), + }, + ) + + relay.send_envelope(project_id, envelope) + + spans = spans_consumer.get_spans(n=1) + attrs = spans[0]["attributes"] + + result = json.loads(attrs["gen_ai.output.messages"]["value"]) + assert result == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": "Let me check."}, + { + "id": "call_1", + "name": "get_weather", + "arguments": {"city": "London"}, + "type": "tool_call", + }, + ], + } + ] + assert "gen_ai.response.text" not in attrs + assert "gen_ai.response.tool_calls" not in attrs