Add native Ruby eval support

README.md

@@ -44,12 +44,12 @@ jobs:
 | Input                  | Required | Description                                                                                                                                                 |
 | ---------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `api_key`              | Yes      | Your [Braintrust API key](https://www.braintrust.dev/app/settings/api-keys).                                                                                |
-| `runtime`              | Yes      | The runtime to use: `node`, `python`, or `go`.                                                                                                              |
+| `runtime`              | Yes      | The runtime to use: `node`, `python`, `go`, or `ruby`.                                                                                                      |
 | `root`                 | No       | Root directory containing your evals. Defaults to `.`.                                                                                                      |
-| `paths`                | No       | Paths or glob patterns, relative to `root`, containing evals to run. Defaults to `.`.                                                                       |
-| `package_manager`      | No       | `npm` or `pnpm` for Node; `pip` or `uv` for Python; `go` for Go. Can be omitted for the default package manager.                                            |
+| `paths`                | No       | Paths or glob patterns, relative to `root`, containing evals to run. For Ruby, this must be one entrypoint file. Defaults to `.`.                            |
+| `package_manager`      | No       | `npm` or `pnpm` for Node; `pip` or `uv` for Python; `go` for Go; `bundler` for Ruby. Can be omitted for the default package manager.                         |
 | `use_proxy`            | No       | Set to `true` to use the Braintrust proxy at `https://braintrustproxy.com/v1`, which can cache repetitive LLM calls and speed up evals. Defaults to `true`. |
-| `terminate_on_failure` | No       | Set to `true` to stop the eval process when an error occurs. Defaults to `false`. Ignored for Go evals.                                                     |
+| `terminate_on_failure` | No       | Set to `true` to stop the eval process when an error occurs. Defaults to `false`. Ignored for Go and Ruby evals.                                            |
 | `report_scores`        | No       | Comma- or newline-separated score names to include in the PR comment. Defaults to all available scores.                                                     |
 | `report_metrics`       | No       | Comma- or newline-separated metric names to include in the PR comment. Defaults to all available metrics.                                                  |
 | `github_token`         | No       | GitHub token used to create or update PR comments. Defaults to `${{ github.token }}`.                                                                       |
@@ -110,6 +110,7 @@ For more fully configured workflows, see the `examples` directory:
 - [`python with pip`](examples/python/pip.yml)
 - [`python with uv`](examples/python/uv.yml)
 - [`go`](examples/go/go.yml)
+- [`ruby with Bundler`](examples/ruby/ruby.yml)
 
 ## Runtime behavior
 
@@ -130,6 +131,33 @@ For more fully configured workflows, see the `examples` directory:
   }
   fmt.Println(string(b))
   ```
+- **Ruby:** the action runs `ruby <entrypoint>` or
+  `bundle exec ruby <entrypoint>` from `root`. `paths` is one required file,
+  passed as one argument even when its name contains spaces. The entrypoint is
+  responsible for running one or more evals and printing one summary JSON object
+  per line. Ruby 3.2 or newer and the application's installed bundle are required.
+
+  The Ruby SDK does not currently emit the action's server-backed comparison
+  payload directly. Copy
+  [`braintrust_report.rb`](examples/ruby/scripts/braintrust_report.rb) into the
+  application and call `BraintrustCI.report(result)` after each `Eval.run`. The
+  helper flushes the configured SDK tracer, reads the public experiment summary
+  API, retries bounded ingestion delays, and prints compatible JSONL. See the
+  [complete Ruby example](examples/ruby/ruby.yml) and
+  [entrypoint](examples/ruby/evals/run.rb).
+
+  For deterministic comparisons, set `BRAINTRUST_BASE_EXPERIMENT_ID` to an
+  experiment ID from the same project's main-branch run. Without it, the API
+  selects its normal fallback baseline. The helper reports evaluator errors,
+  while the entrypoint decides whether they should fail CI; uncomment the
+  example's `exit(1) if result.failed?` policy to make them fatal. Score quality
+  thresholds remain an application policy. `terminate_on_failure` is logged and
+  ignored because the Ruby SDK has no corresponding mid-eval option.
+
+  An optional follow-up is to move this reporting helper into the Ruby SDK. That
+  path would centralize summary fetching, retries, endpoint selection, and JSONL
+  serialization, after which applications would no longer need to copy the
+  helper. Native action support in this release does not depend on that SDK work.
 
 The action creates or updates a single PR comment with a Braintrust link and a
 result table with score and metric sections. To show only selected results, set

action.yml

@@ -17,16 +17,19 @@ inputs:
     required: false
     default: "."
   paths:
-    description: "A list of paths (and glob patterns) to include in the evals"
+    description:
+      "Paths or glob patterns to include in the evals. For Ruby, this must be
+      one entrypoint file."
     required: false
     default: "."
   runtime:
-    description: "The runtime to use for evals. Valid values: node, python, go."
+    description:
+      "The runtime to use for evals. Valid values: node, python, go, ruby."
     required: true
   package_manager:
     description:
       "The package manager to use for evals. Valid values: npm or pnpm for node,
-      pip or uv for python, or go for Go."
+      pip or uv for python, go for Go, or bundler for Ruby."
     required: false
     default: ""
   use_proxy:

eval/src/braintrust.test.ts

@@ -0,0 +1,68 @@
+import { mkdtempSync, mkdirSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import path from "path";
+import { describe, expect, it } from "vite-plus/test";
+
+import { Params } from "./main";
+import { buildRubyCommand } from "./braintrust";
+
+function rubyParams(overrides: Partial<Params> = {}): Params {
+  return {
+    api_key: "test-key",
+    root: ".",
+    paths: ".",
+    runtime: "ruby",
+    package_manager: "",
+    use_proxy: false,
+    terminate_on_failure: false,
+    report_scores: [],
+    report_metrics: [],
+    ...overrides,
+  };
+}
+
+describe("buildRubyCommand", () => {
+  it("passes an entrypoint containing spaces as one Bundler argument", () => {
+    const root = mkdtempSync(path.join(tmpdir(), "eval-action-ruby-"));
+    mkdirSync(path.join(root, "eval files"));
+    writeFileSync(path.join(root, "eval files", "run eval.rb"), "");
+
+    expect(
+      buildRubyCommand(
+        rubyParams({
+          package_manager: "bundler",
+          root,
+          paths: "eval files/run eval.rb",
+        }),
+      ),
+    ).toEqual({
+      command: "bundle",
+      args: ["exec", "ruby", "eval files/run eval.rb"],
+    });
+  });
+
+  it("uses Ruby directly when the package manager is omitted", () => {
+    const root = mkdtempSync(path.join(tmpdir(), "eval-action-ruby-"));
+    writeFileSync(path.join(root, "run.rb"), "");
+
+    expect(buildRubyCommand(rubyParams({ root, paths: "run.rb" }))).toEqual({
+      command: "ruby",
+      args: ["run.rb"],
+    });
+  });
+
+  it("rejects the default path, a missing path, and a directory", () => {
+    const root = mkdtempSync(path.join(tmpdir(), "eval-action-ruby-"));
+    mkdirSync(path.join(root, "evals"));
+
+    expect(() => buildRubyCommand(rubyParams({ root, paths: "." }))).toThrow(
+      /one entrypoint file/,
+    );
+    expect(() =>
+      buildRubyCommand(rubyParams({ root, paths: "missing.rb" })),
+    ).toThrow(/does not exist: missing\.rb/);
+    expect(() =>
+      buildRubyCommand(rubyParams({ root, paths: "evals" })),
+    ).toThrow(/not a file: evals/);
+  });
+});

eval/src/braintrust.ts

@@ -1,4 +1,5 @@
 import path from "path";
+import fs from "fs";
 import * as core from "@actions/core";
 import { spawn } from "child_process";
 
@@ -79,10 +80,24 @@ function parseSummaryLine(line: string) {
   }
 }
 
-async function runCommand(command: string, onSummary: OnSummaryFn) {
-  core.info(`> $ ${command}`);
+interface RubyCommand {
+  command: string;
+  args: string[];
+}
+
+async function runCommand(
+  command: string,
+  onSummary: OnSummaryFn,
+  args?: string[],
+) {
+  const display = args
+    ? [command, ...args.map(arg => JSON.stringify(arg))].join(" ")
+    : command;
+  core.info(`> $ ${display}`);
   return new Promise((resolve, reject) => {
-    const process = spawn(command, { shell: true });
+    const process = args
+      ? spawn(command, args)
+      : spawn(command, { shell: true });
     let stdoutBuffer = "";
 
     const handleStdoutLine = (line: string) => {
@@ -107,6 +122,16 @@ async function runCommand(command: string, onSummary: OnSummaryFn) {
       core.info(data.toString()); // Outputs the stderr of the command
     });
 
+    process.on("error", error => {
+      const hint =
+        command === "bundle"
+          ? " Ensure Bundler is installed and available on PATH."
+          : command === "ruby"
+            ? " Ensure Ruby is installed and available on PATH."
+            : "";
+      reject(new Error(`Failed to start ${command}: ${error.message}.${hint}`));
+    });
+
     process.on("close", code => {
       if (stdoutBuffer.length > 0) {
         handleStdoutLine(stdoutBuffer);
@@ -122,6 +147,46 @@ async function runCommand(command: string, onSummary: OnSummaryFn) {
   });
 }
 
+function validateRubyEntrypoint(root: string, entrypoint: string) {
+  if (entrypoint.trim() === "" || entrypoint === ".") {
+    throw new Error(
+      "Ruby evals require paths to name one entrypoint file (for example, evals/run.rb)",
+    );
+  }
+
+  const resolved = path.resolve(root, entrypoint);
+  let stat: fs.Stats;
+  try {
+    stat = fs.statSync(resolved);
+  } catch {
+    throw new Error(`Ruby eval entrypoint does not exist: ${entrypoint}`);
+  }
+  if (!stat.isFile()) {
+    throw new Error(`Ruby eval entrypoint is not a file: ${entrypoint}`);
+  }
+}
+
+export function buildRubyCommand(args: Params): RubyCommand {
+  validateRubyEntrypoint(args.root, args.paths);
+  if (args.terminate_on_failure) {
+    core.info("Ignoring terminate_on_failure for Ruby evals");
+  }
+  switch ((args.package_manager || "").toLowerCase().trim()) {
+    case "":
+      return {
+        command: "ruby",
+        args: [args.paths],
+      };
+    case "bundler":
+      return {
+        command: "bundle",
+        args: ["exec", "ruby", args.paths],
+      };
+    default:
+      throw new Error(`Unsupported package manager: ${args.package_manager}`);
+  }
+}
+
 export async function runEval(args: Params, onSummary: OnSummaryFn) {
   const { api_key, root, paths, terminate_on_failure } = args;
 
@@ -136,6 +201,11 @@ export async function runEval(args: Params, onSummary: OnSummaryFn) {
     core.exportVariable("OPENAI_BASE_URL", "https://braintrustproxy.com/v1");
   }
 
+  const rubyCommand =
+    args.runtime.toLowerCase().trim() === "ruby"
+      ? buildRubyCommand(args)
+      : undefined;
+
   // Change working directory
   process.chdir(path.resolve(root));
 
@@ -188,10 +258,12 @@ export async function runEval(args: Params, onSummary: OnSummaryFn) {
               `Unsupported package manager: ${args.package_manager}`,
             );
         }
+      case "ruby":
+        return rubyCommand!.command;
       default:
         throw new Error(`Unsupported runtime: ${args.runtime}`);
     }
   })();
 
-  await runCommand(command, onSummary);
+  await runCommand(command, onSummary, rubyCommand?.args);
 }

eval/src/main.test.ts

@@ -1,7 +1,7 @@
 import type { ExperimentSummary } from "braintrust";
 import { describe, expect, it } from "vite-plus/test";
 
-import { formatSummary, parseReportNames } from "./main";
+import { formatSummary, paramsSchema, parseReportNames } from "./main";
 
 const summary: ExperimentSummary = {
   projectName: "Document processing",
@@ -53,6 +53,44 @@ describe("parseReportNames", () => {
   });
 });
 
+describe("runtime validation", () => {
+  const inputs = {
+    api_key: "test-key",
+    root: ".",
+    paths: "evals/run.rb",
+    use_proxy: "false",
+    terminate_on_failure: "false",
+    report_scores: "",
+    report_metrics: "",
+  };
+
+  it.each(["", "bundler"])(
+    "accepts Ruby with package manager %j",
+    package_manager => {
+      expect(
+        paramsSchema.safeParse({
+          ...inputs,
+          runtime: "ruby",
+          package_manager,
+        }).success,
+      ).toBe(true);
+    },
+  );
+
+  it.each(["npm", "pnpm", "pip", "uv", "go"])(
+    "rejects %s for Ruby",
+    package_manager => {
+      expect(
+        paramsSchema.safeParse({
+          ...inputs,
+          runtime: "ruby",
+          package_manager,
+        }).success,
+      ).toBe(false);
+    },
+  );
+});
+
 describe("formatSummary", () => {
   it("reports scores and metrics as sections in one table by default", () => {
     const result = formatSummary(summary);

eval/src/main.ts

@@ -8,6 +8,7 @@ import { z } from "zod";
 const nodeManagers = ["npm", "pnpm"];
 const pythonManagers = ["pip", "uv"];
 const goManagers = ["go"];
+const rubyManagers = ["bundler"];
 const booleanInput = z.stringbool({ truthy: ["true"], falsy: ["false"] });
 
 export function parseReportNames(value: string) {
@@ -25,14 +26,20 @@ function capitalize(value: string) {
   return value.charAt(0).toUpperCase() + value.slice(1);
 }
 
-const paramsSchema = z
+export const paramsSchema = z
   .strictObject({
     api_key: z.string(),
     root: z.string(),
     paths: z.string(),
-    runtime: z.enum(["node", "python", "go"]),
+    runtime: z.enum(["node", "python", "go", "ruby"]),
     package_manager: z
-      .enum(["", ...nodeManagers, ...pythonManagers, ...goManagers])
+      .enum([
+        "",
+        ...nodeManagers,
+        ...pythonManagers,
+        ...goManagers,
+        ...rubyManagers,
+      ])
       .describe("The preferred package manager for the runtime selected")
       .default(""),
     use_proxy: booleanInput,
@@ -54,6 +61,9 @@ const paramsSchema = z
       if (data.runtime === "go") {
         return goManagers.includes(data.package_manager as any);
       }
+      if (data.runtime === "ruby") {
+        return rubyManagers.includes(data.package_manager as any);
+      }
       return false;
     },
     {

examples/ruby/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gem "braintrust", "0.4.1"