diff --git a/lib/braintrust/dataset.rb b/lib/braintrust/dataset.rb index ee65f23..a7804b4 100644 --- a/lib/braintrust/dataset.rb +++ b/lib/braintrust/dataset.rb @@ -166,14 +166,14 @@ def build_record(raw, dataset_id) record end - # Build origin JSON for tracing/linking + # Build origin pointer for tracing/linking # @param raw [Hash] Raw record from API # @param dataset_id [String] Dataset ID (fallback if not in record) - # @return [String, nil] JSON-serialized origin, or nil if record lacks required fields + # @return [Hash, nil] Origin pointer, or nil if record lacks required fields def build_origin(raw, dataset_id) return nil unless raw["id"] && raw["_xact_id"] - Internal::Origin.to_json( + Internal::Origin.build( object_type: "dataset", object_id: raw["dataset_id"] || dataset_id, id: raw["id"], diff --git a/lib/braintrust/eval.rb b/lib/braintrust/eval.rb index ac27b48..cc6b887 100644 --- a/lib/braintrust/eval.rb +++ b/lib/braintrust/eval.rb @@ -336,7 +336,7 @@ def resolve_dataset(dataset, project, state) # Use pinned version if available, otherwise compute from max(_xact_id) version = dataset_obj.version version ||= cases - .filter_map { |c| c[:origin] && JSON.parse(c[:origin])["_xact_id"] } + .filter_map { |c| c[:origin]&.dig("_xact_id") } .max {cases: cases, dataset_id: dataset_obj.id, dataset_version: version} diff --git a/lib/braintrust/eval/runner.rb b/lib/braintrust/eval/runner.rb index 262c389..c7c1d37 100644 --- a/lib/braintrust/eval/runner.rb +++ b/lib/braintrust/eval/runner.rb @@ -95,7 +95,7 @@ def run_eval_case(kase, errors) set_json_attr(eval_span, "braintrust.expected_json", kase.expected) unless kase.expected.nil? set_json_attr(eval_span, "braintrust.metadata", kase.metadata) if kase.metadata eval_span.set_attribute("braintrust.tags", kase.tags) if kase.tags - eval_span.set_attribute("braintrust.origin", kase.origin) if kase.origin + set_json_attr(eval_span, "braintrust.origin", kase.origin) if kase.origin # Run task begin @@ -275,9 +275,7 @@ def build_case_context(eval_case) def report_progress(eval_span, kase, **fields) return unless eval_context.on_progress progress = {"id" => eval_span.context.hex_span_id}.merge(fields.transform_keys(&:to_s)) - if kase.origin - progress["origin"] = kase.origin.is_a?(String) ? JSON.parse(kase.origin) : kase.origin - end + progress["origin"] = kase.origin if kase.origin eval_context.on_progress.call(progress) rescue => e Braintrust.logger.warn("on_progress callback error: #{e.message}") diff --git a/lib/braintrust/internal/origin.rb b/lib/braintrust/internal/origin.rb index 935da8b..a98a7f4 100644 --- a/lib/braintrust/internal/origin.rb +++ b/lib/braintrust/internal/origin.rb @@ -1,27 +1,27 @@ # frozen_string_literal: true -require "json" - module Braintrust module Internal - # Origin provides serialization for source object pointers in Braintrust. - # Used internally to link spans back to their source records (e.g., dataset rows). + # Origin builds source object pointers, which link spans back to the record + # they came from (e.g., a dataset row). Pointers stay Hashes internally and + # are serialized once at the boundary that needs them, so that inbound + # pointers from the wire and ones we build here have the same shape. module Origin - # Serialize an origin pointer to JSON + # Build an origin pointer # @param object_type [String] Type of source object (e.g., "dataset", "playground_logs") # @param object_id [String] ID of the source object # @param id [String] ID of the specific record within the source # @param xact_id [String] Transaction ID # @param created [String, nil] Creation timestamp - # @return [String] JSON-serialized origin - def self.to_json(object_type:, object_id:, id:, xact_id:, created:) - JSON.dump({ - object_type: object_type, - object_id: object_id, - id: id, - _xact_id: xact_id, - created: created - }) + # @return [Hash] Origin pointer with string keys, matching the wire format + def self.build(object_type:, object_id:, id:, xact_id:, created:) + { + "object_type" => object_type, + "object_id" => object_id, + "id" => id, + "_xact_id" => xact_id, + "created" => created + } end end end diff --git a/lib/braintrust/server/services/eval_service.rb b/lib/braintrust/server/services/eval_service.rb index 593d68d..6f056b4 100644 --- a/lib/braintrust/server/services/eval_service.rb +++ b/lib/braintrust/server/services/eval_service.rb @@ -2,12 +2,19 @@ require "json" +require_relative "../../eval/case" + module Braintrust module Server module Services # Framework-agnostic service for running evaluations and streaming SSE results. # Must be long-lived (not per-request) to preserve the @state_cache across requests. class Eval + # Row fields this SDK understands, derived from the Case contract so that + # adding a field there carries it through here. Anything else on an + # inbound row is a field this version has no code for, and is ignored. + CASE_FIELDS = Braintrust::Eval::Case.members.map(&:to_s).freeze + def initialize(evaluators) @evaluators = evaluators @state_mutex = Mutex.new @@ -177,9 +184,10 @@ def resolve_parameters(raw_params, evaluator) # Returns [cases, dataset] where exactly one is non-nil. def resolve_data_source(data) if data.key?("data") - cases = data["data"].map do |d| - {input: d["input"], expected: d["expected"]} - end + # Rows arrive inline from the Playground carrying tags, metadata and + # an origin pointer back to the row they came from. Carry them all: + # the Playground matches streamed results to its grid by origin. + cases = data["data"].map { |row| row.slice(*CASE_FIELDS).transform_keys(&:to_sym) } [cases, nil] elsif data.key?("dataset_id") [nil, Braintrust::Dataset::ID.new(id: data["dataset_id"])] diff --git a/test/braintrust/dataset_test.rb b/test/braintrust/dataset_test.rb index 15b2310..7cc789f 100644 --- a/test/braintrust/dataset_test.rb +++ b/test/braintrust/dataset_test.rb @@ -90,7 +90,7 @@ def test_dataset_is_enumerable # Origin generation tests # ============================================ - def test_build_origin_creates_valid_json + def test_build_origin_creates_pointer state = mock_state dataset = Braintrust::Dataset.new(id: "dataset-123", state: state) @@ -105,11 +105,10 @@ def test_build_origin_creates_valid_json origin = dataset.send(:build_origin, raw_record, "dataset-123") assert origin - parsed = JSON.parse(origin) - assert_equal "dataset", parsed["object_type"] - assert_equal "dataset-123", parsed["object_id"] - assert_equal "record-456", parsed["id"] - assert_equal "1000196022104685824", parsed["_xact_id"] + assert_equal "dataset", origin["object_type"] + assert_equal "dataset-123", origin["object_id"] + assert_equal "record-456", origin["id"] + assert_equal "1000196022104685824", origin["_xact_id"] end def test_build_origin_uses_fallback_dataset_id @@ -124,8 +123,7 @@ def test_build_origin_uses_fallback_dataset_id origin = dataset.send(:build_origin, raw_record, "fallback-id") - parsed = JSON.parse(origin) - assert_equal "fallback-id", parsed["object_id"] + assert_equal "fallback-id", origin["object_id"] end def test_build_origin_returns_nil_when_missing_required_fields @@ -177,7 +175,7 @@ def test_fetch_all_returns_records_with_origin assert record[:origin], "Record should have origin" # Verify origin structure - origin = JSON.parse(record[:origin]) + origin = record[:origin] assert_equal "dataset", origin["object_type"] end end diff --git a/test/braintrust/eval_test.rb b/test/braintrust/eval_test.rb index 8a42314..5c72f6e 100644 --- a/test/braintrust/eval_test.rb +++ b/test/braintrust/eval_test.rb @@ -805,6 +805,39 @@ def test_runner_does_not_set_origin_when_case_has_no_origin assert_nil eval_span.attributes["braintrust.origin"] end + # Origin pointers travel as Hashes and are serialized once, here, at the span + # boundary. Handing OpenTelemetry a Hash does not raise: it logs and drops the + # attribute, so the dataset link would be lost silently. + def test_runner_serializes_hash_origin_onto_eval_span + rig = setup_otel_test_rig + origin = { + "object_type" => "dataset", + "object_id" => "ds-abc", + "id" => "row-1", + "_xact_id" => "1000196022104685824" + } + + task = ->(input:) { input.upcase } + scorer = Braintrust::Scorer.new("exact") { |expected:, output:| (output == expected) ? 1.0 : 0.0 } + + run_test_eval( + experiment_id: "test-exp-123", + experiment_name: "test-hash-origin", + project_id: "test-proj-123", + project_name: "test-project", + cases: [{input: "hello", expected: "HELLO", origin: origin}], + task: task, + scorers: [scorer], + state: rig.state, + tracer_provider: rig.tracer_provider + ) + + eval_span = rig.drain.find { |s| s.name == "eval" } + + assert eval_span, "Expected eval span" + assert_equal origin, JSON.parse(eval_span.attributes["braintrust.origin"]) + end + # Integration test: verify real API dataset records result in correct origin on spans # Note: Dataset is not deleted after test - relies on idempotent create (same pattern as other dataset tests) def test_eval_with_remote_dataset_sets_origin_from_api_response diff --git a/test/braintrust/internal/origin_test.rb b/test/braintrust/internal/origin_test.rb index 0053484..1af21e0 100644 --- a/test/braintrust/internal/origin_test.rb +++ b/test/braintrust/internal/origin_test.rb @@ -4,8 +4,8 @@ require "braintrust/internal/origin" class Braintrust::Internal::OriginTest < Minitest::Test - def test_to_json_serializes_all_fields - result = Braintrust::Internal::Origin.to_json( + def test_build_includes_all_fields + origin = Braintrust::Internal::Origin.build( object_type: "dataset", object_id: "dataset-123", id: "record-456", @@ -13,17 +13,15 @@ def test_to_json_serializes_all_fields created: "2025-10-24T15:29:18.118Z" ) - parsed = JSON.parse(result) - - assert_equal "dataset", parsed["object_type"] - assert_equal "dataset-123", parsed["object_id"] - assert_equal "record-456", parsed["id"] - assert_equal "1000196022104685824", parsed["_xact_id"] - assert_equal "2025-10-24T15:29:18.118Z", parsed["created"] + assert_equal "dataset", origin["object_type"] + assert_equal "dataset-123", origin["object_id"] + assert_equal "record-456", origin["id"] + assert_equal "1000196022104685824", origin["_xact_id"] + assert_equal "2025-10-24T15:29:18.118Z", origin["created"] end - def test_to_json_handles_nil_created - result = Braintrust::Internal::Origin.to_json( + def test_build_handles_nil_created + origin = Braintrust::Internal::Origin.build( object_type: "dataset", object_id: "dataset-123", id: "record-456", @@ -31,17 +29,14 @@ def test_to_json_handles_nil_created created: nil ) - parsed = JSON.parse(result) - - assert_equal "dataset", parsed["object_type"] - assert_equal "dataset-123", parsed["object_id"] - assert_equal "record-456", parsed["id"] - assert_equal "1000196022104685824", parsed["_xact_id"] - assert_nil parsed["created"] + assert_nil origin["created"] + assert_equal "record-456", origin["id"] end - def test_to_json_returns_valid_json_string - result = Braintrust::Internal::Origin.to_json( + # Pointers we build must be shaped like the ones the Playground sends, so both + # sources flow through the SDK identically. + def test_build_uses_string_keys_matching_the_wire_format + origin = Braintrust::Internal::Origin.build( object_type: "dataset", object_id: "abc-123", id: "def-456", @@ -49,13 +44,12 @@ def test_to_json_returns_valid_json_string created: "2025-01-01T00:00:00Z" ) - assert_instance_of String, result - # Should not raise - JSON.parse(result) + assert_instance_of Hash, origin + assert_equal %w[object_type object_id id _xact_id created].sort, origin.keys.sort end - def test_to_json_with_playground_logs_type - result = Braintrust::Internal::Origin.to_json( + def test_build_with_playground_logs_type + origin = Braintrust::Internal::Origin.build( object_type: "playground_logs", object_id: "playground-123", id: "log-456", @@ -63,7 +57,6 @@ def test_to_json_with_playground_logs_type created: "2025-01-01T00:00:00Z" ) - parsed = JSON.parse(result) - assert_equal "playground_logs", parsed["object_type"] + assert_equal "playground_logs", origin["object_type"] end end diff --git a/test/braintrust/server/rack/eval_endpoint_test.rb b/test/braintrust/server/rack/eval_endpoint_test.rb index 8e943ff..73fa03c 100644 --- a/test/braintrust/server/rack/eval_endpoint_test.rb +++ b/test/braintrust/server/rack/eval_endpoint_test.rb @@ -69,6 +69,29 @@ def test_progress_events_contain_output assert_equal "HELLO", JSON.parse(data["data"]) end + # End-to-end over the real HTTP path, driven by the request body a + # Playground actually posts (captured from a live remote eval run). + # Every progress event must echo the origin its row arrived with, or the + # Playground cannot match results to its grid rows and spins forever. + def test_progress_events_echo_origin_from_captured_playground_request + body = load_json_fixture("playground/eval_request_inline") + @evaluators[body["name"]] = test_evaluator( + task: ->(input:) { input.to_s.upcase }, scorers: [noop_scorer] + ) + + post_json "/eval", body + + assert_equal 200, last_response.status + events = parse_sse_events(last_response.body) + progress = events.select { |e| e[:event] == "progress" }.map { |e| JSON.parse(e[:data]) } + + expected_origins = body.dig("data", "data").map { |row| row["origin"] } + refute_empty progress + progress.each do |p| + assert_includes expected_origins, p["origin"], "progress event missing or wrong origin" + end + end + def test_summary_event_contains_scores scorer = Braintrust::Scorer.new("exact") { |expected:, output:| (output == expected) ? 1.0 : 0.0 } @evaluators["scored-eval"] = test_evaluator( diff --git a/test/braintrust/server/services/eval_service_test.rb b/test/braintrust/server/services/eval_service_test.rb index 4f5a32d..a753125 100644 --- a/test/braintrust/server/services/eval_service_test.rb +++ b/test/braintrust/server/services/eval_service_test.rb @@ -89,6 +89,72 @@ def test_validate_accepts_dataset_name assert_equal({name: "my-dataset", project: "my-project"}, result[:dataset]) end + # --- validate: inline Playground rows --- + + # The request body a Playground actually posts, captured from a live + # remote eval run. Rows arrive detached from their dataset and carry an + # origin pointer back to it, which the Playground uses to match streamed + # results to its grid rows. + def playground_rows + load_json_fixture("playground/eval_request_inline").dig("data", "data") + end + + def test_validate_preserves_origin_from_inline_rows + @evaluators["test-eval"] = test_evaluator(task: ->(input) { input }) + result = service.validate({ + "name" => "test-eval", + "data" => {"data" => playground_rows} + }) + + kase = result[:cases].first + assert_equal "Grilled chicken breast", kase[:input] + assert_equal "protein", kase[:expected] + assert_equal playground_rows.first["origin"]["id"], kase[:origin]["id"] + assert_equal "dataset", kase[:origin]["object_type"] + assert_equal playground_rows.first["origin"]["object_id"], kase[:origin]["object_id"] + end + + def test_validate_preserves_tags_and_metadata_from_inline_rows + @evaluators["test-eval"] = test_evaluator(task: ->(input) { input }) + result = service.validate({ + "name" => "test-eval", + "data" => {"data" => [ + playground_rows.first.merge("tags" => ["smoke"], "metadata" => {"source" => "playground"}) + ]} + }) + + kase = result[:cases].first + assert_equal ["smoke"], kase[:tags] + assert_equal({"source" => "playground"}, kase[:metadata]) + end + + # Real rows carry id, _xact_id, created and upsert_id, none of which this + # SDK models. They must be ignored rather than raising or being mistaken + # for Case fields, so that a new protocol field cannot break the runner. + def test_validate_ignores_unrecognized_inline_row_fields + @evaluators["test-eval"] = test_evaluator(task: ->(input) { input }) + result = service.validate({ + "name" => "test-eval", + "data" => {"data" => [playground_rows.first.merge("some_future_field" => "x")]} + }) + + kase = result[:cases].first + assert_equal "Grilled chicken breast", kase[:input] + assert_equal %i[input expected metadata origin].sort, kase.keys.sort + refute kase.key?(:upsert_id) + refute kase.key?(:some_future_field) + end + + def test_validate_handles_inline_rows_without_origin + @evaluators["test-eval"] = test_evaluator(task: ->(input) { input }) + result = service.validate({ + "name" => "test-eval", + "data" => {"data" => [{"input" => "x"}]} + }) + + assert_equal({input: "x"}, result[:cases].first) + end + # --- stream --- def test_stream_emits_progress_and_done_events @@ -107,6 +173,31 @@ def test_stream_emits_progress_and_done_events assert_equal "done", events.last[:event] end + # The Playground matches streamed results to its grid rows by origin. + # Without it, results arrive unmatched and the UI spins forever. + def test_stream_emits_origin_on_every_progress_event + @evaluators["upcase-eval"] = test_evaluator( + task: ->(input) { input.to_s.upcase }, scorers: [noop_scorer] + ) + s = service + validated = s.validate({ + "name" => "upcase-eval", + "data" => {"data" => playground_rows}, + "experiment_name" => "exp" + }) + + events = collect_streamed_events(s, validated) + progress = events.select { |e| e[:event] == "progress" }.map { |e| JSON.parse(e[:data]) } + + expected_origins = playground_rows.map { |row| row["origin"] } + refute_empty progress + progress.each do |p| + assert_includes expected_origins, p["origin"], "progress event missing or wrong origin" + end + assert_equal expected_origins.length, progress.map { |p| p["origin"] }.uniq.length, + "every row should be represented in the progress stream" + end + def test_stream_emits_summary_with_scores scorer = Braintrust::Eval.scorer("exact") { |_i, e, o| (o == e) ? 1.0 : 0.0 } @evaluators["scored-eval"] = test_evaluator( diff --git a/test/fixtures/playground/eval_request_inline.json b/test/fixtures/playground/eval_request_inline.json new file mode 100644 index 0000000..33d1d63 --- /dev/null +++ b/test/fixtures/playground/eval_request_inline.json @@ -0,0 +1,52 @@ +{ + "name": "food-classifier", + "parameters": {}, + "data": { + "data": [ + { + "expected": "protein", + "input": "Grilled chicken breast", + "metadata": null, + "id": "35e83320-06d2-47c9-8c30-731db7cef92a", + "_xact_id": "1000197637647737942", + "created": "1785970989574", + "origin": { + "object_type": "dataset", + "object_id": "c2fc45d0-85bc-441c-9b81-be613a864e67", + "id": "35e83320-06d2-47c9-8c30-731db7cef92a", + "_xact_id": "1000197637647737942", + "created": "1785970989574" + }, + "upsert_id": "0a1a7065-a191-44d2-9df5-9c58fa15e951" + }, + { + "expected": "vegetable", + "input": "Romaine lettuce", + "metadata": null, + "id": "2d91c914-b83c-44a0-a5de-6a59b4449047", + "_xact_id": "1000197637647737942", + "created": "1785970989575", + "origin": { + "object_type": "dataset", + "object_id": "c2fc45d0-85bc-441c-9b81-be613a864e67", + "id": "2d91c914-b83c-44a0-a5de-6a59b4449047", + "_xact_id": "1000197637647737942", + "created": "1785970989575" + }, + "upsert_id": "54a9408e-8937-429a-adab-cc5eb033c0e8" + } + ] + }, + "scores": [], + "project_id": "fc70fc0f-fd38-4890-bb9f-9de80132f5de", + "parent": { + "object_type": "playground_logs", + "object_id": "56b0678a-5c0a-44c3-b100-232bd4aa9673", + "propagated_event": { + "span_attributes": { + "generation": "0cd4601a-f36a-4951-b4a7-94f9743d8920" + } + } + }, + "stream": true +} diff --git a/test/support/fixture_helper.rb b/test/support/fixture_helper.rb index 5971769..faff180 100644 --- a/test/support/fixture_helper.rb +++ b/test/support/fixture_helper.rb @@ -1,3 +1,5 @@ +require "json" + module Test module Support module FixtureHelper @@ -22,6 +24,14 @@ def with_png_file(data: PNG_DATA, filename: "test_image", extension: ".png", &bl with_tmp_file(data: data, filename: filename, extension: extension, binary: true, &block) end + # Load a JSON fixture from test/fixtures. + # @param name [String] path under test/fixtures, without the .json extension + # @return [Hash, Array] the parsed fixture + def load_json_fixture(name) + path = File.expand_path("../fixtures/#{name}.json", __dir__) + JSON.parse(File.read(path)) + end + # Create a temporary file and yield to the block # File is automatically cleaned up after the block # @param data [String] content to write to the file (default: empty)