From fbc94198fee4264d1b3c88eacc27a5c86bb45f8f Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 16 Oct 2026 17:01:58 -0700 Subject: [PATCH 1/7] ci: pass `/evals` CLI arguments through to Cloud MCP evals (#1401) (cherry picked from commit 30775e81f170fff43d9c90c742db4e3fea07ca14) --- .github/pr-welcome-community.md | 2 +- .github/pr-welcome-internal.md | 2 +- .github/workflows/evals-command.yml | 40 ++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/.github/pr-welcome-community.md b/.github/pr-welcome-community.md index 35ec2a61e..6cdfeaf71 100644 --- a/.github/pr-welcome-community.md +++ b/.github/pr-welcome-community.md @@ -25,7 +25,7 @@ As needed or by request, Airbyte Maintainers can execute the following slash com - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite) +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. diff --git a/.github/pr-welcome-internal.md b/.github/pr-welcome-internal.md index c7be57e3e..3c8cdaa66 100644 --- a/.github/pr-welcome-internal.md +++ b/.github/pr-welcome-internal.md @@ -25,7 +25,7 @@ Airbyte Maintainers can execute the following slash commands on your PR: - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite) +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index 9e3dcc34b..87a383db0 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -59,6 +59,37 @@ jobs: fi echo "short-sha=${SHA:0:7}" >> "$GITHUB_OUTPUT" + - name: Read run arguments + id: args + if: inputs.comment-id != '' + env: + COMMENT_ID: ${{ inputs.comment-id }} + GH_TOKEN: ${{ github.token }} + BASE_LABEL: ${{ steps.mode.outputs.label }} + run: | + set -euo pipefail + COMMENT_BODY="$(gh api "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID" --jq .body)" + export COMMENT_BODY + RUN_ARGS="$(python3 - <<'PY' + import os + import re + + first_line = os.environ["COMMENT_BODY"].split("\n", 1)[0].rstrip("\r") + first_line = re.sub(r"^\s*/evals(?:\s+|$)", "", first_line, count=1) + first_line = re.sub(r"(?> "$GITHUB_OUTPUT" + label="$BASE_LABEL" + [[ -n "$RUN_ARGS" ]] && printf -v label "%s, \`%s\`" "$BASE_LABEL" "$RUN_ARGS" + printf 'label=%s\n' "$label" | tee -a "$GITHUB_OUTPUT" + - name: Append comment with job run link id: first-comment-action uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 @@ -67,7 +98,7 @@ jobs: issue-number: ${{ inputs.pr }} body: | - > Cloud MCP evals queued for `${{ steps.mode.outputs.short-sha }}` (${{ steps.mode.outputs.label }}). [Check job output.](${{ steps.vars.outputs.run-url }}) + > Cloud MCP evals queued for `${{ steps.mode.outputs.short-sha }}` (${{ steps.args.outputs.label || steps.mode.outputs.label }}). [Check job output.](${{ steps.vars.outputs.run-url }}) # The token lasts one hour; a full evals run takes about 20 minutes. - name: Authenticate as GitHub App @@ -85,11 +116,14 @@ jobs: env: GH_TOKEN: ${{ steps.get-app-token.outputs.token }} CHANGED_ONLY: ${{ steps.mode.outputs.changed-only }} + RUN_ARGS: ${{ steps.args.outputs.run-args }} run: | set -euo pipefail since=$(date -u +%Y-%m-%dT%H:%M:%SZ) + extra=() + [[ -n "$RUN_ARGS" ]] && extra=(-f run_args="$RUN_ARGS") gh workflow run evals.yml -R "$EVALS_REPO" --ref main \ - -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" + -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" "${extra[@]}" title="PyAirbyte evals: #$PR" for _ in $(seq 18); do sleep 10 @@ -119,7 +153,7 @@ jobs: GH_TOKEN: ${{ steps.get-app-token.outputs.token }} RUN_ID: ${{ steps.dispatch.outputs.run-id }} RUN_URL: ${{ steps.dispatch.outputs.run-url }} - LABEL: ${{ steps.mode.outputs.label }} + LABEL: ${{ steps.args.outputs.label || steps.mode.outputs.label }} CHANGED_ONLY: ${{ steps.mode.outputs.changed-only }} run: | set -euo pipefail From 9bf7958bf852b6e70e139b5244349f8e71a284bf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:05:50 +0000 Subject: [PATCH 2/7] feat(ci): add eval comparison mode --- .github/pr-welcome-community.md | 2 +- .github/pr-welcome-internal.md | 2 +- .github/workflows/evals-command.yml | 71 +++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/.github/pr-welcome-community.md b/.github/pr-welcome-community.md index 6cdfeaf71..5037636d2 100644 --- a/.github/pr-welcome-community.md +++ b/.github/pr-welcome-community.md @@ -25,7 +25,7 @@ As needed or by request, Airbyte Maintainers can execute the following slash com - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. diff --git a/.github/pr-welcome-internal.md b/.github/pr-welcome-internal.md index 3c8cdaa66..e4a671a27 100644 --- a/.github/pr-welcome-internal.md +++ b/.github/pr-welcome-internal.md @@ -25,7 +25,7 @@ Airbyte Maintainers can execute the following slash commands on your PR: - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index 87a383db0..629389847 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -16,6 +16,11 @@ on: type: string required: false default: 'false' + compare: + description: "Also run the PR's base and post the comparison" + type: string + required: false + default: 'false' env: AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} @@ -37,6 +42,7 @@ jobs: env: PR: ${{ inputs.pr }} FULL: ${{ inputs.full }} + COMPARE: ${{ inputs.compare }} EVALS_REPO: airbytehq/cloud-mcp-evals steps: - name: Resolve workflow variables @@ -50,14 +56,24 @@ jobs: run: | set -euo pipefail [[ "$PR" =~ ^[0-9]+$ ]] || { echo "::error::Invalid PR number: $PR"; exit 1; } + [[ "$COMPARE" == "true" || "$COMPARE" == "false" ]] \ + || { echo "::error::Invalid compare value: $COMPARE"; exit 1; } if [ "$FULL" = "true" ]; then - echo "changed-only=false" >> "$GITHUB_OUTPUT" - echo "label=full suite" >> "$GITHUB_OUTPUT" + changed_only=false + label="full suite" else - echo "changed-only=true" >> "$GITHUB_OUTPUT" - echo "label=changed tools only" >> "$GITHUB_OUTPUT" + changed_only=true + label="changed tools only" fi - echo "short-sha=${SHA:0:7}" >> "$GITHUB_OUTPUT" + if [ "$COMPARE" = "true" ]; then + label="$label, compared with base" + fi + { + echo "changed-only=$changed_only" + echo "label=$label" + echo "compare=$COMPARE" + echo "short-sha=${SHA:0:7}" + } >> "$GITHUB_OUTPUT" - name: Read run arguments id: args @@ -76,7 +92,7 @@ jobs: first_line = os.environ["COMMENT_BODY"].split("\n", 1)[0].rstrip("\r") first_line = re.sub(r"^\s*/evals(?:\s+|$)", "", first_line, count=1) - first_line = re.sub(r"(? "$body_file" + elif [ -n "$results" ]; then run_json=$(dirname "$results")/run.json commit=$(jq -r '.mcp.airbyte.direct_url.vcs_info.commit_id // "unknown"' "$run_json") expected=$(jq -r '.expected_trials // empty' "$run_json") - body=$(jq -r --arg expected "$expected" --arg commit "${commit:0:7}" --arg label "$LABEL" ' + body=$(jq -r --arg expected "$expected" --arg commit "${commit:0:7}" --arg report_label "$LABEL" ' (if $expected == "" then length else ($expected | tonumber) end) as $total | (map(select(.status == "pass")) | length) as $passed | ((group_by(.status) | map(select(.[0].status != "pass") | "\(length) \({fail: "failed", error: "errored"}[.[0].status] // .[0].status)")) + (if $total > length then ["\($total - length) missing"] else [] end)) as $other - | "Cloud MCP evals on `\($commit)` (\($label)): **\($passed) / \($total) passed**" + | "Cloud MCP evals on `\($commit)` (\($report_label)): **\($passed) / \($total) passed**" + (if ($other | length) > 0 then " (\($other | join(", ")))" else "" end)' "$results") body="$body ยท [summary]($RUN_URL)" elif [ "$conclusion" = "success" ] && [ "$CHANGED_ONLY" = "true" ]; then @@ -192,15 +233,17 @@ jobs: else body="Evals run ended ($conclusion) before producing results. [summary]($RUN_URL)" fi - echo "$body" - echo "body=$body" >> "$GITHUB_OUTPUT" + if [ "$COMPARE" != "true" ]; then + printf '> %s\n' "$body" > "$body_file" + fi + echo "body-path=$body_file" >> "$GITHUB_OUTPUT" + cat "$body_file" - name: Append results comment uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 with: comment-id: ${{ steps.first-comment-action.outputs.comment-id }} - body: | - > ${{ steps.report.outputs.body }} + body-path: ${{ steps.report.outputs.body-path }} - name: Append failure comment if: failure() From 2fb9dd6e426a6d3a4abd07cd99e53f50fe52db56 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:06:51 +0000 Subject: [PATCH 3/7] ci: skip the evidence download in compare mode --- .github/workflows/evals-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index 629389847..6e0b0108c 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -188,7 +188,7 @@ jobs: artifacts=$(gh api "repos/$EVALS_REPO/actions/runs/$RUN_ID/artifacts" --paginate \ --jq '.artifacts[] | select(.name | startswith("all-evidence-")) | .name') results="" - if [ -n "$artifacts" ]; then + if [ "$COMPARE" != "true" ] && [ -n "$artifacts" ]; then dir=$(mktemp -d) gh run download "$RUN_ID" -R "$EVALS_REPO" -p 'all-evidence-*' -D "$dir" results=$(find "$dir" -name results.json | head -n 1) From 5d37ac9e867e967378b3d285d8250a629f7b653a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 06:44:43 +0000 Subject: [PATCH 4/7] docs(ci): clarify eval repeat argument syntax --- .github/pr-welcome-community.md | 2 +- .github/pr-welcome-internal.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/pr-welcome-community.md b/.github/pr-welcome-community.md index 5037636d2..43a8fefba 100644 --- a/.github/pr-welcome-community.md +++ b/.github/pr-welcome-community.md @@ -25,7 +25,7 @@ As needed or by request, Airbyte Maintainers can execute the following slash com - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. diff --git a/.github/pr-welcome-internal.md b/.github/pr-welcome-internal.md index e4a671a27..931a58638 100644 --- a/.github/pr-welcome-internal.md +++ b/.github/pr-welcome-internal.md @@ -25,7 +25,7 @@ Airbyte Maintainers can execute the following slash commands on your PR: - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. From e08ba7d1ad6e00fe01a529275e1166756cc754de Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 07:04:29 +0000 Subject: [PATCH 5/7] feat(ci): compare evals against arbitrary targets --- .github/pr-welcome-community.md | 2 +- .github/pr-welcome-internal.md | 2 +- .github/workflows/evals-command.yml | 22 ++++++++++++++++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/pr-welcome-community.md b/.github/pr-welcome-community.md index 43a8fefba..614075009 100644 --- a/.github/pr-welcome-community.md +++ b/.github/pr-welcome-community.md @@ -25,7 +25,7 @@ As needed or by request, Airbyte Maintainers can execute the following slash com - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Use `/evals compare=main` (or a release, commit, or PR) to compare against a target other than the base. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. diff --git a/.github/pr-welcome-internal.md b/.github/pr-welcome-internal.md index 931a58638..d4552d792 100644 --- a/.github/pr-welcome-internal.md +++ b/.github/pr-welcome-internal.md @@ -25,7 +25,7 @@ Airbyte Maintainers can execute the following slash commands on your PR: - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Use `/evals compare=main` (or a release, commit, or PR) to compare against a target other than the base. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index 6e0b0108c..1cbc8ad2e 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -17,7 +17,7 @@ on: required: false default: 'false' compare: - description: "Also run the PR's base and post the comparison" + description: "Compare with the PR's base, or pass a release, PR, commit SHA or branch." type: string required: false default: 'false' @@ -56,8 +56,6 @@ jobs: run: | set -euo pipefail [[ "$PR" =~ ^[0-9]+$ ]] || { echo "::error::Invalid PR number: $PR"; exit 1; } - [[ "$COMPARE" == "true" || "$COMPARE" == "false" ]] \ - || { echo "::error::Invalid compare value: $COMPARE"; exit 1; } if [ "$FULL" = "true" ]; then changed_only=false label="full suite" @@ -65,13 +63,25 @@ jobs: changed_only=true label="changed tools only" fi + compare_to="" if [ "$COMPARE" = "true" ]; then + compare=true label="$label, compared with base" + elif [ "$COMPARE" = "false" ]; then + compare=false + elif [[ "$COMPARE" =~ ^#?[A-Za-z0-9][A-Za-z0-9_./-]*$ ]]; then + compare=true + compare_to="$COMPARE" + label="$label, compared with $COMPARE" + else + echo "::error::Invalid compare value: $COMPARE" + exit 1 fi { echo "changed-only=$changed_only" echo "label=$label" - echo "compare=$COMPARE" + echo "compare=$compare" + echo "compare_to=$compare_to" echo "short-sha=${SHA:0:7}" } >> "$GITHUB_OUTPUT" @@ -134,6 +144,7 @@ jobs: CHANGED_ONLY: ${{ steps.mode.outputs.changed-only }} RUN_ARGS: ${{ steps.args.outputs.run-args }} COMPARE: ${{ steps.mode.outputs.compare }} + COMPARE_TO: ${{ steps.mode.outputs.compare_to }} run: | set -euo pipefail since=$(date -u +%Y-%m-%dT%H:%M:%SZ) @@ -143,6 +154,9 @@ jobs: if [ "$COMPARE" = "true" ]; then compare_args+=(-f compare=true) fi + if [ -n "$COMPARE_TO" ]; then + compare_args+=(-f "compare_to=$COMPARE_TO") + fi gh workflow run evals.yml -R "$EVALS_REPO" --ref main \ -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" \ "${extra[@]}" "${compare_args[@]}" From b4196eec6756488c152a3ed75c70b0ea4bcef437 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 21:37:47 +0000 Subject: [PATCH 6/7] feat(ci): paste the evals results comment in /evals --- .github/workflows/evals-command.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index 1cbc8ad2e..da44a067b 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -197,6 +197,20 @@ jobs: # Eval failures are expected (model variance); they are reported, not fatal. gh run watch "$RUN_ID" -R "$EVALS_REPO" --interval 30 > /dev/null conclusion=$(gh run view "$RUN_ID" -R "$EVALS_REPO" --json conclusion --jq .conclusion) + body_file="$RUNNER_TEMP/evals-comment.md" + if [ "$COMPARE" != "true" ]; then + dir=$(mktemp -d) + if gh run download "$RUN_ID" -R "$EVALS_REPO" \ + -n "comment-$RUN_ID" -D "$dir"; then + comment="$dir/comment.md" + if [ -f "$comment" ]; then + cp "$comment" "$body_file" + echo "body-path=$body_file" >> "$GITHUB_OUTPUT" + cat "$body_file" + exit 0 + fi + fi + fi # A changed-tools run with nothing to run uploads no evidence; any other # listing or download failure fails the job. artifacts=$(gh api "repos/$EVALS_REPO/actions/runs/$RUN_ID/artifacts" --paginate \ @@ -208,7 +222,6 @@ jobs: results=$(find "$dir" -name results.json | head -n 1) [ -n "$results" ] || { echo "::error::all-evidence artifact has no results.json."; exit 1; } fi - body_file="$RUNNER_TEMP/evals-comment.md" if [ "$COMPARE" = "true" ]; then dir=$(mktemp -d) if gh run download "$RUN_ID" -R "$EVALS_REPO" \ From cc84ee9b20b754c42d10c9d380b2c191700d8368 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 04:22:14 +0000 Subject: [PATCH 7/7] feat(ci): compare with the base by default in /evals --- .github/pr-welcome-community.md | 2 +- .github/pr-welcome-internal.md | 2 +- .github/workflows/evals-command.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/pr-welcome-community.md b/.github/pr-welcome-community.md index 614075009..adf33f3e0 100644 --- a/.github/pr-welcome-community.md +++ b/.github/pr-welcome-community.md @@ -25,7 +25,7 @@ As needed or by request, Airbyte Maintainers can execute the following slash com - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Use `/evals compare=main` (or a release, commit, or PR) to compare against a target other than the base. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed, on this PR and on its base branch, and posts the difference (`/evals full=true` compares the full suite). Use `/evals compare=main` (or a release, commit, or PR) to compare against a different target, or `/evals compare=false` to run only this PR. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. diff --git a/.github/pr-welcome-internal.md b/.github/pr-welcome-internal.md index d4552d792..676026d77 100644 --- a/.github/pr-welcome-internal.md +++ b/.github/pr-welcome-internal.md @@ -25,7 +25,7 @@ Airbyte Maintainers can execute the following slash commands on your PR: - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite; `/evals compare=true` runs the PR and its base and posts the difference). Use `/evals compare=main` (or a release, commit, or PR) to compare against a target other than the base. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed, on this PR and on its base branch, and posts the difference (`/evals full=true` compares the full suite). Use `/evals compare=main` (or a release, commit, or PR) to compare against a different target, or `/evals compare=false` to run only this PR. Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. Write options as `--repeats 3`, not `--repeats=3`. diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index da44a067b..adf653d58 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -17,10 +17,10 @@ on: required: false default: 'false' compare: - description: "Compare with the PR's base, or pass a release, PR, commit SHA or branch." + description: "Compare with the PR's base (default). Pass false to skip, or a release, PR, commit SHA or branch to compare against." type: string required: false - default: 'false' + default: 'true' env: AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }}