From bcf6ea491b3c9e7464cc918f6509e1766f0e9683 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 21 Sep 2026 02:13:13 -0600 Subject: [PATCH] feat: add span customizers (SDK-316) --- README.md | 36 +++ lib/braintrust.rb | 5 +- lib/braintrust/config.rb | 11 +- lib/braintrust/span_customizer.rb | 17 ++ lib/braintrust/state.rb | 6 +- lib/braintrust/trace.rb | 3 +- lib/braintrust/trace/span_exporter.rb | 78 ++++++- test/braintrust/trace/span_customizer_test.rb | 219 ++++++++++++++++++ 8 files changed, 361 insertions(+), 14 deletions(-) create mode 100644 lib/braintrust/span_customizer.rb create mode 100644 test/braintrust/trace/span_customizer_test.rb diff --git a/README.md b/README.md index e335c149..1136d62a 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ This is the official Ruby SDK for [Braintrust](https://www.braintrust.dev), for - [Supported providers](#supported-providers) - [Manually applying instrumentation](#manually-applying-instrumentation) - [Creating custom spans](#creating-custom-spans) + - [Span customizers](#span-customizers) - [Attachments](#attachments) - [Viewing traces](#viewing-traces) - [Evals](#evals) @@ -120,6 +121,7 @@ Braintrust.init | `filter_ai_spans` | `ENV['BRAINTRUST_OTEL_FILTER_AI_SPANS']` | Only export AI-related spans | | `org_name` | `ENV['BRAINTRUST_ORG_NAME']` | Organization name | | `set_global` | `true` | Set as global state. Set to `false` for isolated instances | +| `span_customizers` | `[]` | Ordered objects with optional synchronous export hooks (see [Span customizers](#span-customizers)) | **Example with options:** @@ -192,6 +194,40 @@ tracer.in_span("process-request") do |span| end ``` +### Span customizers + +Register customizers before creating spans to redact or transform outgoing telemetry: + +```ruby +require "braintrust" + +class RedactContent < Braintrust::SpanCustomizer + def on_span_export(span) + %w[braintrust.input_json braintrust.output_json].each do |key| + span.attributes[key] = JSON.generate("[redacted]") if span.attributes.key?(key) + end + span + end +end + +Braintrust.init( + default_project: "my-project", + span_customizers: [RedactContent.new] +) +``` + +`Braintrust::SpanCustomizer` is an extensible base class with a no-op `on_span_export`. Any object may be registered; an omitted hook is also a no-op. `Braintrust::Config.new` / `.from_env`, `Braintrust::State.from_env`, and a directly constructed `Braintrust::Trace::SpanExporter` also accept `span_customizers:`. Registration is programmatic, not environment-based. Configuration and exporters retain frozen copies of the ordered list, not copies of the customizer objects. + +Hooks run synchronously in registration order, each receiving its predecessor's result. The argument is a completed `OpenTelemetry::SDK::Trace::SpanData` snapshot, after Braintrust origin metadata is added but before destination grouping or OTLP serialization. Hooks apply to **all completed spans reaching the Braintrust exporter**, including manual, evaluation, and instrumented spans that pass any configured span filters. An unrelated exporter supplied through `exporter:` is not wrapped with these hooks. + +You may mutate and return the snapshot or return a replacement `SpanData`; replacement is not an implicit merge. Nested span data is copied so customization does not modify application/provider return values, live spans, or another exporter's data. OTel resource objects retain their normal API; replace resources with `OpenTelemetry::SDK::Resources::Resource.create(...)` when changing resource attributes. Added attributes, events, and links have their recorded counts adjusted to prevent negative OTLP dropped counts. + +Always return valid, serializable span data, never `nil` to drop a span. Preserve `trace_id`, `span_id`, and `parent_span_id`; the exporter checks these after every hook. You may change `braintrust.parent` to route the result to another project or experiment. Existing destination/header behavior otherwise remains unchanged. + +Customization is **fail-closed for the whole batch**: an exception, invalid return, changed protected ID, or serialization failure returns an export failure and sends none of that batch. The SDK logs the failure without falling back to unredacted originals. Hook failures do not affect application return values. + +Keep hooks fast and avoid blocking I/O; they may run on a background export thread. OTLP transport retries reuse the already customized bytes. Submitting the batch through `export` again invokes the hooks again on fresh snapshots, so do not assume one invocation per logical span. Without customizers, the existing export path is unchanged. + ### Attachments Log binary data (images, PDFs, audio) in your traces: diff --git a/lib/braintrust.rb b/lib/braintrust.rb index 496fc226..fa0ac05d 100644 --- a/lib/braintrust.rb +++ b/lib/braintrust.rb @@ -2,6 +2,7 @@ require_relative "braintrust/version" require_relative "braintrust/config" +require_relative "braintrust/span_customizer" require_relative "braintrust/state" require_relative "braintrust/trace" require_relative "braintrust/api" @@ -42,6 +43,7 @@ class Error < StandardError; end # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use instead of creating one # @param filter_ai_spans [Boolean, nil] Enable AI span filtering (overrides BRAINTRUST_OTEL_FILTER_AI_SPANS env var) # @param span_filter_funcs [Array, nil] Custom span filter functions + # @param span_customizers [Array, nil] Ordered synchronous export customizers # @param exporter [Exporter, nil] Optional exporter override (for testing) # @param auto_instrument [Boolean, Hash, nil] Auto-instrumentation config: # - nil (default): use BRAINTRUST_AUTO_INSTRUMENT env var, default true if not set @@ -49,7 +51,7 @@ class Error < StandardError; end # - false: explicitly disable # - Hash with :only or :except keys for filtering # @return [State] the created state - def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, set_global: true, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, exporter: nil, auto_instrument: nil) + def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, set_global: true, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil, exporter: nil, auto_instrument: nil) state = State.from_env( api_key: api_key, org_name: org_name, @@ -61,6 +63,7 @@ def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, a tracer_provider: tracer_provider, filter_ai_spans: filter_ai_spans, span_filter_funcs: span_filter_funcs, + span_customizers: span_customizers, exporter: exporter ) diff --git a/lib/braintrust/config.rb b/lib/braintrust/config.rb index 1aa2a1a0..b28168af 100644 --- a/lib/braintrust/config.rb +++ b/lib/braintrust/config.rb @@ -8,10 +8,10 @@ module Braintrust # and allows overriding with explicit options class Config attr_reader :api_key, :org_name, :default_project, :app_url, :api_url, - :filter_ai_spans, :span_filter_funcs + :filter_ai_spans, :span_filter_funcs, :span_customizers def initialize(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, - filter_ai_spans: nil, span_filter_funcs: nil) + filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil) @api_key = api_key @org_name = org_name @default_project = default_project @@ -19,6 +19,7 @@ def initialize(api_key: nil, org_name: nil, default_project: nil, app_url: nil, @api_url = api_url @filter_ai_spans = filter_ai_spans @span_filter_funcs = span_filter_funcs || [] + @span_customizers = (span_customizers || []).dup.freeze end # Create a Config from environment variables, with option overrides @@ -30,9 +31,10 @@ def initialize(api_key: nil, org_name: nil, default_project: nil, app_url: nil, # @param api_url [String, nil] API URL (overrides BRAINTRUST_API_URL env var) # @param filter_ai_spans [Boolean, nil] Enable AI span filtering (overrides BRAINTRUST_OTEL_FILTER_AI_SPANS env var) # @param span_filter_funcs [Array, nil] Custom span filter functions + # @param span_customizers [Array, nil] Ordered export customizers (copied and frozen) # @return [Config] the created config def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, - filter_ai_spans: nil, span_filter_funcs: nil) + filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil) # Parse filter_ai_spans from ENV if not explicitly provided env_filter_ai_spans = ENV["BRAINTRUST_OTEL_FILTER_AI_SPANS"] filter_ai_spans_value = if filter_ai_spans.nil? @@ -48,7 +50,8 @@ def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: ni app_url: app_url || ENV["BRAINTRUST_APP_URL"] || "https://www.braintrust.dev", api_url: api_url || ENV["BRAINTRUST_API_URL"] || "https://api.braintrust.dev", filter_ai_spans: filter_ai_spans_value, - span_filter_funcs: span_filter_funcs + span_filter_funcs: span_filter_funcs, + span_customizers: span_customizers ) end end diff --git a/lib/braintrust/span_customizer.rb b/lib/braintrust/span_customizer.rb new file mode 100644 index 00000000..77de7001 --- /dev/null +++ b/lib/braintrust/span_customizer.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Braintrust + # Optional synchronous hooks that transform outgoing telemetry, not live spans. + # Subclass this class or supply any object implementing the desired hooks. + class SpanCustomizer + # Transform a completed, isolated OpenTelemetry SpanData snapshot. + # Return this snapshot or a replacement SpanData, never nil. Preserve its + # trace_id, span_id and parent_span_id. Exceptions fail the entire batch. + # + # @param span [OpenTelemetry::SDK::Trace::SpanData] + # @return [OpenTelemetry::SDK::Trace::SpanData] + def on_span_export(span) + span + end + end +end diff --git a/lib/braintrust/state.rb b/lib/braintrust/state.rb index f95e5fd9..423ce2ab 100644 --- a/lib/braintrust/state.rb +++ b/lib/braintrust/state.rb @@ -24,9 +24,10 @@ class MissingAPIKeyError < ArgumentError; end # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use # @param filter_ai_spans [Boolean, nil] Enable AI span filtering # @param span_filter_funcs [Array, nil] Custom span filter functions + # @param span_customizers [Array, nil] Ordered synchronous export customizers # @param exporter [Exporter, nil] Optional exporter override (for testing) # @return [State] the created state - def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, exporter: nil) + def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil, exporter: nil) require_relative "config" config = Config.from_env( api_key: api_key, @@ -35,7 +36,8 @@ def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: ni app_url: app_url, api_url: api_url, filter_ai_spans: filter_ai_spans, - span_filter_funcs: span_filter_funcs + span_filter_funcs: span_filter_funcs, + span_customizers: span_customizers ) new( api_key: config.api_key, diff --git a/lib/braintrust/trace.rb b/lib/braintrust/trace.rb index 2c6563fe..63254e18 100644 --- a/lib/braintrust/trace.rb +++ b/lib/braintrust/trace.rb @@ -91,7 +91,8 @@ def self.enable(tracer_provider, state: nil, exporter: nil, config: nil) # Create OTLP HTTP exporter unless override provided exporter ||= SpanExporter.new( endpoint: "#{state.api_url}/otel/v1/traces", - api_key: state.api_key + api_key: state.api_key, + span_customizers: config&.span_customizers ) # Use SimpleSpanProcessor for InMemorySpanExporter (testing), BatchSpanProcessor for production diff --git a/lib/braintrust/trace/span_exporter.rb b/lib/braintrust/trace/span_exporter.rb index 32ab2705..aa4bf91c 100644 --- a/lib/braintrust/trace/span_exporter.rb +++ b/lib/braintrust/trace/span_exporter.rb @@ -3,41 +3,107 @@ require "opentelemetry/exporter/otlp" require_relative "../state" require_relative "span_origin" +require_relative "../logger" module Braintrust module Trace # Custom OTLP exporter for the Braintrust backend. On export it: - # - stamps span origin provenance onto each SpanData (via the prepended SpanOrigin behavior) + # - stamps span origin provenance onto each SpanData + # - runs optional customizers on isolated export snapshots # - groups spans by braintrust.parent and sets the x-bt-parent header per group, # so the backend routes them to the correct experiment/project # # Thread safety: BatchSpanProcessor serializes export() calls via its # @export_mutex, so @headers mutation here is safe. class SpanExporter < OpenTelemetry::Exporter::OTLP::Exporter - prepend SpanOrigin - PARENT_ATTR_KEY = SpanProcessor::PARENT_ATTR_KEY PARENT_HEADER = "x-bt-parent" SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE - def initialize(endpoint:, api_key:) + def initialize(endpoint:, api_key:, span_customizers: nil) raise State::MissingAPIKeyError, "api_key is required" if api_key.nil? || api_key.empty? + @span_customizers = (span_customizers || []).dup.freeze + super(endpoint: endpoint, headers: {"Authorization" => "Bearer #{api_key}"}) end def export(span_data, timeout: nil) + customizers = @span_customizers + customize = customizers && !customizers.empty? + environment = Internal::Env.detect_environment + if customize + return FAILURE if @shutdown + + begin + span_data = span_data.map do |span| + # SpanData includes nested mutable strings, attributes, events, + # links and resources. A shallow dup would leak hook mutations to + # application data or another exporter. Only SDK-owned data is + # marshaled here; no externally supplied serialized bytes are read. + # Resources cache an Enumerator, which cannot be marshaled. + snapshot = span.dup + snapshot.resource = span.resource.attribute_enumerator.to_h + snapshot = Marshal.load(Marshal.dump(snapshot)) + snapshot.resource = OpenTelemetry::SDK::Resources::Resource.create(snapshot.resource) + snapshot = SpanOrigin.enrich(snapshot, environment: environment) + snapshot.attributes = snapshot.attributes.dup + customize_span(snapshot, customizers) + end + groups = span_data.group_by { |sd| sd.attributes&.[](PARENT_ATTR_KEY) } + # Validate serialization for the entire batch before any group can + # leave the process. Reuse these bytes rather than encoding twice. + encoded_groups = groups.transform_values { |spans| encode(spans) } + raise TypeError, "Customized spans must be OTLP serializable" if encoded_groups.any? { |_, bytes| bytes.nil? } + rescue => e + Log.error("Failed to customize spans for export: #{e.class}") + return FAILURE + end + else + span_data = span_data.map { |span| SpanOrigin.enrich(span, environment: environment) } + groups = span_data.group_by { |sd| sd.attributes&.[](PARENT_ATTR_KEY) } + end + failed = false - span_data.group_by { |sd| sd.attributes&.[](PARENT_ATTR_KEY) }.each do |parent_value, spans| + groups.each do |parent_value, spans| @headers[PARENT_HEADER] = parent_value if parent_value - failed = true unless super(spans, timeout: timeout) == SUCCESS + result = if customize + send_bytes(encoded_groups.fetch(parent_value), timeout: timeout) + else + super(spans, timeout: timeout) + end + failed = true unless result == SUCCESS ensure @headers.delete(PARENT_HEADER) end failed ? FAILURE : SUCCESS end + + private + + def customize_span(span, customizers) + trace_id = span.trace_id.dup + span_id = span.span_id.dup + parent_span_id = span.parent_span_id.dup + customizers.each do |customizer| + next unless customizer.respond_to?(:on_span_export) + + span = customizer.on_span_export(span) + unless span.is_a?(OpenTelemetry::SDK::Trace::SpanData) + raise TypeError, "SpanCustomizer#on_span_export must return SpanData" + end + unless span.trace_id == trace_id && span.span_id == span_id && span.parent_span_id == parent_span_id + raise ArgumentError, "SpanCustomizer#on_span_export must preserve trace, span and parent IDs" + end + end + # Added entries cannot have negative dropped counts in OTLP. + span.total_recorded_attributes = [span.total_recorded_attributes, span.attributes&.size.to_i].max + span.total_recorded_events = [span.total_recorded_events, span.events&.size.to_i].max + span.total_recorded_links = [span.total_recorded_links, span.links&.size.to_i].max + span + end end end end diff --git a/test/braintrust/trace/span_customizer_test.rb b/test/braintrust/trace/span_customizer_test.rb new file mode 100644 index 00000000..d1f24428 --- /dev/null +++ b/test/braintrust/trace/span_customizer_test.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +require "test_helper" + +class Braintrust::Trace::SpanCustomizerTest < Minitest::Test + SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS + FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE + ENDPOINT = "https://api.ruby-sdk-fixture.com/otel/v1/traces" + + def setup + @requests = [] + @providers = [] + @exporters = [] + stub_request(:post, ENDPOINT).to_return do |request| + body = (request.headers["Content-Encoding"] == "gzip") ? Zlib.gunzip(request.body) : request.body + decoded = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(body) + @requests << {headers: request.headers, spans: decoded.resource_spans.flat_map { |resource| resource.scope_spans.flat_map { |scope| scope.spans.to_a } }} + {status: 200, body: ""} + end + end + + def teardown + @providers.each(&:shutdown) + @exporters.each(&:shutdown) + end + + def test_order_replacement_routing_and_snapshot_registration_through_init + first = customizer do |span| + origin = JSON.parse(span.attributes.fetch("braintrust.context_json")) + replacement = span.dup + replacement.name = "#{origin.fetch("span_origin").fetch("name")}:redacted" + replacement.attributes = {"braintrust.parent" => "project_name:redacted", "braintrust.input_json" => '"[redacted]"'} + replacement + end + second = customizer do |span| + span.name += ":second" + span.attributes["customized"] = true + span + end + customizers = [Object.new, Braintrust::SpanCustomizer.new, first, second] + provider = make_provider + state = Braintrust.init( + api_key: "test-api-key", default_project: "original", + blocking_login: true, set_global: false, auto_instrument: false, + tracer_provider: provider, span_customizers: customizers + ) + customizers.clear + assert_raises(FrozenError) { state.config.span_customizers.clear } + provider.tracer("manual").start_span("private", attributes: {"braintrust.input_json" => '"secret"'}).finish + provider.force_flush + + assert_equal 1, @requests.size + request = @requests.fetch(0) + assert_equal "project_name:redacted", request[:headers]["X-Bt-Parent"] + assert_equal "Bearer test-api-key", request[:headers]["Authorization"] + span = request[:spans].fetch(0) + assert_equal "braintrust.sdk.ruby:redacted:second", span.name + assert_equal '"[redacted]"', attributes(span).fetch("braintrust.input_json").string_value + assert attributes(span).fetch("customized").bool_value + refute attributes(span).key?("braintrust.context_json"), "replacement must not implicitly merge removed fields" + end + + def test_customization_does_not_mutate_source_or_other_exporters + provider = make_provider + memory = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new + provider.add_span_processor(OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(memory)) + input = +"private" + source_span = provider.tracer("app").start_span("original", attributes: {"input" => [input]}) + source_span.add_event("event", attributes: {"value" => +"private"}) + source_span.finish + source = memory.finished_spans.fetch(0) + hook = customizer do |span| + span.name.replace("redacted") + span.attributes.fetch("input").fetch(0).replace("redacted") + span.events.fetch(0).attributes.fetch("value").replace("redacted") + span + end + + assert_equal SUCCESS, make_exporter([hook]).export([source]) + exported = @requests.fetch(0)[:spans].fetch(0) + assert_equal "redacted", exported.name + assert_equal "redacted", attributes(exported).fetch("input").array_value.values.fetch(0).string_value + assert_equal "redacted", exported.events.fetch(0).attributes.find { |attr| attr.key == "value" }.value.string_value + assert_equal "private", input + assert_equal "original", source.name + assert_equal ["private"], source.attributes.fetch("input") + assert_equal "private", source.events.fetch(0).attributes.fetch("value") + refute source.attributes.key?("braintrust.context_json") + assert_equal source.trace_id, exported.trace_id + assert_equal source.span_id, exported.span_id + end + + def test_exception_after_mutation_sends_none_of_batch_and_preserves_source + spans = two_destinations + hook = customizer do |span| + if span.name == "second" + span.attributes.fetch("input").replace("changed") + raise "redaction failed" + end + span + end + + assert_equal FAILURE, make_exporter([hook]).export(spans) + assert_empty @requests + assert_equal "private", spans.last.attributes.fetch("input") + end + + def test_nil_and_non_span_returns_fail_whole_batch + [nil, {}].each do |invalid| + hook = customizer { |span| (span.name == "second") ? invalid : span } + assert_equal FAILURE, make_exporter([hook]).export(two_destinations) + assert_empty @requests + end + end + + def test_unserializable_replacement_fails_before_first_destination_is_sent + hook = customizer do |span| + span.start_timestamp = "invalid" if span.name == "second" + span + end + + assert_equal FAILURE, make_exporter([hook]).export(two_destinations) + assert_empty @requests + end + + def test_each_hook_must_preserve_all_ids_even_if_later_hook_would_restore_them + [:trace_id, :span_id, :parent_span_id].each do |field| + spans = two_destinations + original = spans.last.public_send(field).dup + later_called = false + mutate = customizer do |span| + span.public_send(field).replace("x" * original.bytesize) if span.name == "second" + span + end + restore = customizer do |span| + if span.name == "second" + later_called = true + span.public_send("#{field}=", original) + end + span + end + + assert_equal FAILURE, make_exporter([mutate, restore]).export(spans) + assert_empty @requests + refute later_called + assert_equal original, spans.last.public_send(field) + end + end + + def test_parent_identity_survives_successful_replacement + provider = make_provider + tracer = provider.tracer("app") + parent = tracer.start_span("parent") + context = OpenTelemetry::Trace.context_with_span(parent) + child = tracer.start_span("child", with_parent: context) + child.finish + parent.finish + source = child.to_span_data + hook = customizer do |span| + replacement = span.dup + replacement.name = "replacement" + replacement + end + + assert_equal SUCCESS, make_exporter([hook]).export([source]) + exported = @requests.fetch(0)[:spans].fetch(0) + assert_equal source.trace_id, exported.trace_id + assert_equal source.span_id, exported.span_id + assert_equal parent.context.span_id, exported.parent_span_id + end + + def test_resubmission_customizes_fresh_snapshot_each_time + calls = 0 + hook = customizer do |span| + calls += 1 + span.name += ":customized" + span + end + customizers = [hook] + exporter = make_exporter(customizers) + customizers.clear + source = make_span("original") + + 2.times { assert_equal SUCCESS, exporter.export([source]) } + assert_equal 2, calls + assert_equal ["original:customized", "original:customized"], @requests.map { |request| request[:spans].fetch(0).name } + assert_equal "original", source.name + end + + private + + def customizer(&block) + Object.new.tap { |object| object.define_singleton_method(:on_span_export, &block) } + end + + def make_provider + OpenTelemetry::SDK::Trace::TracerProvider.new.tap { |provider| @providers << provider } + end + + def make_exporter(customizers) + Braintrust::Trace::SpanExporter.new(endpoint: ENDPOINT, api_key: "test-key", span_customizers: customizers).tap do |exporter| + @exporters << exporter + end + end + + def make_span(name, parent: "project_name:original") + span = make_provider.tracer("app").start_span(name, attributes: {"braintrust.parent" => parent, "input" => +"private"}) + span.finish + span.to_span_data + end + + def two_destinations + [make_span("first", parent: "project_name:first"), make_span("second", parent: "project_name:second")] + end + + def attributes(span) + span.attributes.to_h { |attribute| [attribute.key, attribute.value] } + end +end