From fb0afd4b9dce9bca6897a51cad7b9485fd49da17 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:28:05 +0000 Subject: [PATCH 1/2] feat(pyairbyte): pass eval run args to workflow Co-Authored-By: alexandre@airbyte.io --- .github/pr-welcome-community.md | 2 +- .github/pr-welcome-internal.md | 2 +- .github/workflows/evals-command.yml | 38 ++++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/pr-welcome-community.md b/.github/pr-welcome-community.md index 35ec2a61e..6cdfeaf71 100644 --- a/.github/pr-welcome-community.md +++ b/.github/pr-welcome-community.md @@ -25,7 +25,7 @@ As needed or by request, Airbyte Maintainers can execute the following slash com - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite) +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. diff --git a/.github/pr-welcome-internal.md b/.github/pr-welcome-internal.md index c7be57e3e..3c8cdaa66 100644 --- a/.github/pr-welcome-internal.md +++ b/.github/pr-welcome-internal.md @@ -25,7 +25,7 @@ Airbyte Maintainers can execute the following slash commands on your PR: - `/uv-lock` - Updates uv.lock file - `/test-pr` - Runs tests with the updated PyAirbyte - `/prerelease` - Builds and publishes a prerelease version to PyPI -- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite) +- `/evals` - Runs the Cloud MCP evals that use the MCP tools this PR changed (`/evals full=true` runs the full suite). Add any `cloud_mcp_evals run` arguments after it, e.g. `/evals full=true --repeats 3 scenarios/regressions`. diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index 9e3dcc34b..811e6e190 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -59,6 +59,37 @@ jobs: fi echo "short-sha=${SHA:0:7}" >> "$GITHUB_OUTPUT" + - name: Read run arguments + id: args + if: inputs.comment-id != '' + env: + COMMENT_ID: ${{ inputs.comment-id }} + GH_TOKEN: ${{ github.token }} + BASE_LABEL: ${{ steps.mode.outputs.label }} + run: | + set -euo pipefail + COMMENT_BODY="$(gh api "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID" --jq .body)" + export COMMENT_BODY + RUN_ARGS="$(python3 - <<'PY' + import os + import re + + first_line = os.environ["COMMENT_BODY"].split("\n", 1)[0].rstrip("\r") + first_line = re.sub(r"^\s*/evals(?:\s+|$)", "", first_line, count=1) + first_line = re.sub(r"(?> "$GITHUB_OUTPUT" + label="$BASE_LABEL" + [[ -n "$RUN_ARGS" ]] && printf -v label "%s, \`%s\`" "$BASE_LABEL" "$RUN_ARGS" + printf 'label=%s\n' "$label" | tee -a "$GITHUB_OUTPUT" + - name: Append comment with job run link id: first-comment-action uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 @@ -67,7 +98,7 @@ jobs: issue-number: ${{ inputs.pr }} body: | - > Cloud MCP evals queued for `${{ steps.mode.outputs.short-sha }}` (${{ steps.mode.outputs.label }}). [Check job output.](${{ steps.vars.outputs.run-url }}) + > Cloud MCP evals queued for `${{ steps.mode.outputs.short-sha }}` (${{ steps.args.outputs.label || steps.mode.outputs.label }}). [Check job output.](${{ steps.vars.outputs.run-url }}) # The token lasts one hour; a full evals run takes about 20 minutes. - name: Authenticate as GitHub App @@ -85,11 +116,12 @@ jobs: env: GH_TOKEN: ${{ steps.get-app-token.outputs.token }} CHANGED_ONLY: ${{ steps.mode.outputs.changed-only }} + RUN_ARGS: ${{ steps.args.outputs.run-args }} run: | set -euo pipefail since=$(date -u +%Y-%m-%dT%H:%M:%SZ) gh workflow run evals.yml -R "$EVALS_REPO" --ref main \ - -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" + -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" -f run_args="$RUN_ARGS" title="PyAirbyte evals: #$PR" for _ in $(seq 18); do sleep 10 @@ -119,7 +151,7 @@ jobs: GH_TOKEN: ${{ steps.get-app-token.outputs.token }} RUN_ID: ${{ steps.dispatch.outputs.run-id }} RUN_URL: ${{ steps.dispatch.outputs.run-url }} - LABEL: ${{ steps.mode.outputs.label }} + LABEL: ${{ steps.args.outputs.label || steps.mode.outputs.label }} CHANGED_ONLY: ${{ steps.mode.outputs.changed-only }} run: | set -euo pipefail From 7a2df0b14259c982ba7cfbbb8e760e679b9fcf33 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:29:27 +0000 Subject: [PATCH 2/2] ci: only send run_args when the comment has some Co-Authored-By: alexandre@airbyte.io --- .github/workflows/evals-command.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/evals-command.yml b/.github/workflows/evals-command.yml index 811e6e190..87a383db0 100644 --- a/.github/workflows/evals-command.yml +++ b/.github/workflows/evals-command.yml @@ -120,8 +120,10 @@ jobs: run: | set -euo pipefail since=$(date -u +%Y-%m-%dT%H:%M:%SZ) + extra=() + [[ -n "$RUN_ARGS" ]] && extra=(-f run_args="$RUN_ARGS") gh workflow run evals.yml -R "$EVALS_REPO" --ref main \ - -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" -f run_args="$RUN_ARGS" + -f pyairbyte="#$PR" -f changed_tools_only="$CHANGED_ONLY" "${extra[@]}" title="PyAirbyte evals: #$PR" for _ in $(seq 18); do sleep 10