Caleb Evans (calebevans-ab) · Oct 17, 2026 · Oct 17, 2026 · Oct 17, 2026 · Oct 17, 2026 · Oct 17, 2026
diff --git a/.github/pr-welcome-community.md b/.github/pr-welcome-community.md
@@ -25,7 +25,7 @@ As needed or by request, Airbyte Maintainers can execute the following slash com
 - `/uv-lock` - Updates uv.lock file
 - `/test-pr` - Runs tests with the updated PyAirbyte
 - `/prerelease` - Builds and publishes a prerelease version to PyPI
-- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite)
+- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed, on this PR and on its base branch, and posts the difference (`/evals full=true` compares the full suite). Use `/evals compare=main` (or a release, commit, or PR) to compare against a different target, or `/evals compare=false` to run only this PR. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`.
 
 </details>
 

diff --git a/.github/pr-welcome-internal.md b/.github/pr-welcome-internal.md
@@ -25,7 +25,7 @@ Airbyte Maintainers can execute the following slash commands on your PR:
 - `/uv-lock` - Updates uv.lock file
 - `/test-pr` - Runs tests with the updated PyAirbyte
 - `/prerelease` - Builds and publishes a prerelease version to PyPI
-- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite)
+- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed, on this PR and on its base branch, and posts the difference (`/evals full=true` compares the full suite). Use `/evals compare=main` (or a release, commit, or PR) to compare against a different target, or `/evals compare=false` to run only this PR. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`.
 
 </details>
 

diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml
@@ -16,6 +16,11 @@ on:
         type: string
         required: false
         default: 'false'
+      compare:
+        description: "Compare with the PR's base (default). Pass false to skip, or a release, PR, commit SHA or branch to compare against."
+        type: string
+        required: false
+        default: 'true'
 
 env:
   AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }}
@@ -37,6 +42,7 @@ jobs:
     env:
       PR: ${{ inputs.pr }}
       FULL: ${{ inputs.full }}
+      COMPARE: ${{ inputs.compare }}
       EVALS_REPO: airbytehq/cloud-mcp-evals
     steps:
     - name: Resolve workflow variables
@@ -51,13 +57,64 @@ jobs:
         set -euo pipefail
         [[ "$PR" =~ ^[0-9]+$ ]] || { echo "::error::Invalid PR number: $PR"; exit 1; }
         if [ "$FULL" = "true" ]; then
-          echo "changed-only=false" >> "$GITHUB_OUTPUT"
-          echo "label=full suite" >> "$GITHUB_OUTPUT"
+          changed_only=false
+          label="full suite"
         else
-          echo "changed-only=true" >> "$GITHUB_OUTPUT"
-          echo "label=changed tools only" >> "$GITHUB_OUTPUT"
+          changed_only=true
+          label="changed tools only"
+        fi
+        compare_to=""
+        if [ "$COMPARE" = "true" ]; then
+          compare=true
+          label="$label, compared with base"
+        elif [ "$COMPARE" = "false" ]; then
+          compare=false
+        elif [[ "$COMPARE" =~ ^#?[A-Za-z0-9][A-Za-z0-9_./-]*$ ]]; then
+          compare=true
+          compare_to="$COMPARE"
+          label="$label, compared with $COMPARE"
+        else
+          echo "::error::Invalid compare value: $COMPARE"
+          exit 1
         fi
-        echo "short-sha=${SHA:0:7}" >> "$GITHUB_OUTPUT"
+        {
+          echo "changed-only=$changed_only"
+          echo "label=$label"
+          echo "compare=$compare"
+          echo "compare_to=$compare_to"
+          echo "short-sha=${SHA:0:7}"
+        } >> "$GITHUB_OUTPUT"
+
+    - name: Read run arguments
+      id: args
+      if: inputs.comment-id != ''
+      env:
+        COMMENT_ID: ${{ inputs.comment-id }}
+        GH_TOKEN: ${{ github.token }}
+        BASE_LABEL: ${{ steps.mode.outputs.label }}
+      run: |
+        set -euo pipefail
+        COMMENT_BODY="$(gh api "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID" --jq .body)"
+        export COMMENT_BODY
+        RUN_ARGS="$(python3 - <<'PY'
+        import os
+        import re
+
+        first_line = os.environ["COMMENT_BODY"].split("\n", 1)[0].rstrip("\r")
+        first_line = re.sub(r"^\s*/evals(?:\s+|$)", "", first_line, count=1)
+        first_line = re.sub(r"(?<!\S)(?:full|ref|compare)=[^\s]+", "", first_line)
+        print(first_line.strip())
+        PY
+        )"
+        delimiter="$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
+        {
+          printf 'run-args<<%s\n' "$delimiter"
+          printf '%s\n' "$RUN_ARGS"
+          printf '%s\n' "$delimiter"
+        } >> "$GITHUB_OUTPUT"
+        label="$BASE_LABEL"
+        [[ -n "$RUN_ARGS" ]] && printf -v label "%s, \`%s\`" "$BASE_LABEL" "$RUN_ARGS"
+        printf 'label=%s\n' "$label" | tee -a "$GITHUB_OUTPUT"
 
     - name: Append comment with job run link
       id: first-comment-action
@@ -67,7 +124,7 @@ jobs:
         issue-number: ${{ inputs.pr }}
         body: |
 
-          > Cloud MCP evals queued for `${{ steps.mode.outputs.short-sha }}` (${{ steps.mode.outputs.label }}). [Check job output.](${{ steps.vars.outputs.run-url }})
+          > Cloud MCP evals queued for `${{ steps.mode.outputs.short-sha }}` (${{ steps.args.outputs.label || steps.mode.outputs.label }}). [Check job output.](${{ steps.vars.outputs.run-url }})
 
     # The token lasts one hour; a full evals run takes about 20 minutes.
     - name: Authenticate as GitHub App
@@ -85,11 +142,24 @@ jobs:
       env:
         GH_TOKEN: ${{ steps.get-app-token.outputs.token }}
         CHANGED_ONLY: ${{ steps.mode.outputs.changed-only }}
+        RUN_ARGS: ${{ steps.args.outputs.run-args }}
+        COMPARE: ${{ steps.mode.outputs.compare }}
+        COMPARE_TO: ${{ steps.mode.outputs.compare_to }}
       run: |
         set -euo pipefail
         since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+        extra=()
+        [[ -n "$RUN_ARGS" ]] && extra=(-f run_args="$RUN_ARGS")
+        compare_args=()
+        if [ "$COMPARE" = "true" ]; then
+          compare_args+=(-f compare=true)
+        fi
+        if [ -n "$COMPARE_TO" ]; then
+          compare_args+=(-f "compare_to=$COMPARE_TO")
+        fi
         gh workflow run evals.yml -R "$EVALS_REPO" --ref main \
-          -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY"
+          -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" \
+          "${extra[@]}" "${compare_args[@]}"
         title="PyAirbyte evals: #$PR"
         for _ in $(seq 18); do
           sleep 10
@@ -119,35 +189,67 @@ jobs:
         GH_TOKEN: ${{ steps.get-app-token.outputs.token }}
         RUN_ID: ${{ steps.dispatch.outputs.run-id }}
         RUN_URL: ${{ steps.dispatch.outputs.run-url }}
-        LABEL: ${{ steps.mode.outputs.label }}
+        LABEL: ${{ steps.args.outputs.label || steps.mode.outputs.label }}
         CHANGED_ONLY: ${{ steps.mode.outputs.changed-only }}
+        COMPARE: ${{ steps.mode.outputs.compare }}
       run: |
         set -euo pipefail
         # Eval failures are expected (model variance); they are reported, not fatal.
         gh run watch "$RUN_ID" -R "$EVALS_REPO" --interval 30 > /dev/null
         conclusion=$(gh run view "$RUN_ID" -R "$EVALS_REPO" --json conclusion --jq .conclusion)
+        body_file="$RUNNER_TEMP/evals-comment.md"
+        if [ "$COMPARE" != "true" ]; then
+          dir=$(mktemp -d)
+          if gh run download "$RUN_ID" -R "$EVALS_REPO" \
+            -n "comment-$RUN_ID" -D "$dir"; then
+            comment="$dir/comment.md"
+            if [ -f "$comment" ]; then
+              cp "$comment" "$body_file"
+              echo "body-path=$body_file" >> "$GITHUB_OUTPUT"
+              cat "$body_file"
+              exit 0
+            fi
+          fi
+        fi
         # A changed-tools run with nothing to run uploads no evidence; any other
         # listing or download failure fails the job.
         artifacts=$(gh api "repos/$EVALS_REPO/actions/runs/$RUN_ID/artifacts" --paginate \
           --jq '.artifacts[] | select(.name | startswith("all-evidence-")) | .name')
         results=""
-        if [ -n "$artifacts" ]; then
+        if [ "$COMPARE" != "true" ] && [ -n "$artifacts" ]; then
           dir=$(mktemp -d)
           gh run download "$RUN_ID" -R "$EVALS_REPO" -p 'all-evidence-*' -D "$dir"
           results=$(find "$dir" -name results.json | head -n 1)
           [ -n "$results" ] || { echo "::error::all-evidence artifact has no results.json."; exit 1; }
         fi
-        if [ -n "$results" ]; then
+        if [ "$COMPARE" = "true" ]; then
+          dir=$(mktemp -d)
+          if gh run download "$RUN_ID" -R "$EVALS_REPO" \
+            -n "comparison-$RUN_ID" -D "$dir"; then
+            comparison="$dir/comparison.md"
+            if [ -f "$comparison" ]; then
+              body="$(cat "$comparison")"
+              body="$body"$'\n\n'"[Full summary]($RUN_URL)"
+            else
+              body="Evals run ended ($conclusion) before producing a comparison."
+              body="$body"$'\n\n'"[Full summary]($RUN_URL)"
+            fi
+          else
+            body="Evals run ended ($conclusion) before producing a comparison."
+            body="$body"$'\n\n'"[Full summary]($RUN_URL)"
+          fi
+          printf '%s\n' "$body" > "$body_file"
+        elif [ -n "$results" ]; then
           run_json=$(dirname "$results")/run.json
           commit=$(jq -r '.mcp.airbyte.direct_url.vcs_info.commit_id // "unknown"' "$run_json")
           expected=$(jq -r '.expected_trials // empty' "$run_json")
-          body=$(jq -r --arg expected "$expected" --arg commit "${commit:0:7}" --arg label "$LABEL" '
+          body=$(jq -r --arg expected "$expected" --arg commit "${commit:0:7}" --arg report_label "$LABEL" '
             (if $expected == "" then length else ($expected | tonumber) end) as $total
             | (map(select(.status == "pass")) | length) as $passed
             | ((group_by(.status) | map(select(.[0].status != "pass")
                  | "\(length) \({fail: "failed", error: "errored"}[.[0].status] // .[0].status)"))
                + (if $total > length then ["\($total - length) missing"] else [] end)) as $other
-            | "Cloud MCP evals on `\($commit)` (\($label)): **\($passed) / \($total) passed**"
+            | "Cloud MCP evals on `\($commit)` (\($report_label)): **\($passed) / \($total) passed**"
               + (if ($other | length) > 0 then " (\($other | join(", ")))" else "" end)' "$results")
           body="$body · [summary]($RUN_URL)"
         elif [ "$conclusion" = "success" ] && [ "$CHANGED_ONLY" = "true" ]; then
@@ -158,15 +260,17 @@ jobs:
         else
           body="Evals run ended ($conclusion) before producing results. [summary]($RUN_URL)"
         fi
-        echo "$body"
-        echo "body=$body" >> "$GITHUB_OUTPUT"
+        if [ "$COMPARE" != "true" ]; then
+          printf '> %s\n' "$body" > "$body_file"
+        fi
+        echo "body-path=$body_file" >> "$GITHUB_OUTPUT"
+        cat "$body_file"
 
     - name: Append results comment
       uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
       with:
         comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
-        body: |
-          > ${{ steps.report.outputs.body }}
+        body-path: ${{ steps.report.outputs.body-path }}
 
     - name: Append failure comment
       if: failure()