From caad4130f22e231c2095616cfb8368c5f16c0676 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Sun, 8 Nov 2026 17:47:26 -0800 Subject: [PATCH 1/2] Run Copilot Decisions integration test daily Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/cjs.yml | 37 --------------------- .github/workflows/decisions-integration.yml | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/decisions-integration.yml diff --git a/.github/workflows/cjs.yml b/.github/workflows/cjs.yml index 3936f5bf883..1eec460e346 100644 --- a/.github/workflows/cjs.yml +++ b/.github/workflows/cjs.yml @@ -352,42 +352,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: cd actions/setup/js && npm run test:js-integration-artifact - decisions-integration: - name: Copilot Decisions API Integration - needs: [checkout-cache] - # Fork PRs and Dependabot runs have read-only tokens, even with the permission below. - if: github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - copilot-requests: write - concurrency: - group: ci-${{ github.ref }}-decisions-integration - cancel-in-progress: true - steps: - - name: Restore checkout - id: restore-checkout - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: . - key: checkout-cjs-${{ github.run_id }}-${{ github.sha }} - - name: Checkout on cache miss - if: steps.restore-checkout.outputs.cache-hit != 'true' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 - with: - node-version: "24" - - name: Run live Decisions client integration test - env: - GITHUB_TOKEN: ${{ github.token }} - run: node --test actions/setup/js/scripts/test_decisions_client.cjs - nvx-musl-integration: name: NVX musl inference integration needs: [checkout-cache] @@ -435,7 +399,6 @@ jobs: - docker-root-owned-cleanup - lint-js - artifact-integration - - decisions-integration - nvx-musl-integration if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/decisions-integration.yml b/.github/workflows/decisions-integration.yml new file mode 100644 index 00000000000..fb6c28befcd --- /dev/null +++ b/.github/workflows/decisions-integration.yml @@ -0,0 +1,35 @@ +name: Copilot Decisions API Integration + +on: + schedule: + - cron: '17 6 * * *' # Daily at 06:17 UTC. + workflow_dispatch: + +permissions: {} + +jobs: + decisions-integration: + name: Copilot Decisions API Integration + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + copilot-requests: write + concurrency: + group: ci-${{ github.ref }}-decisions-integration + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 + with: + node-version: "24" + - name: Run live Decisions client integration test + env: + GITHUB_TOKEN: ${{ github.token }} + run: node --test actions/setup/js/scripts/test_decisions_client.cjs From c966bb34c31c60ecabaf2dbdeaa62df8d266082f Mon Sep 17 00:00:00 2001 From: pelikhan Date: Sun, 8 Nov 2026 17:50:25 -0800 Subject: [PATCH 2/2] Remove redundant Decisions API integration smoke test Copilot evals already provide live API coverage. Remove the standalone test and scheduled workflow, update its documentation, and drop the obsolete CJS inference permission exception. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/decisions-integration.yml | 35 ------------------- actions/setup/README.md | 12 ++----- .../js/scripts/test_decisions_client.cjs | 27 -------------- scripts/check-cgo-cjs-workflow-purity.sh | 4 --- scripts/check-cgo-cjs-workflow-purity_test.sh | 20 ++++++----- 5 files changed, 13 insertions(+), 85 deletions(-) delete mode 100644 .github/workflows/decisions-integration.yml delete mode 100644 actions/setup/js/scripts/test_decisions_client.cjs diff --git a/.github/workflows/decisions-integration.yml b/.github/workflows/decisions-integration.yml deleted file mode 100644 index fb6c28befcd..00000000000 --- a/.github/workflows/decisions-integration.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Copilot Decisions API Integration - -on: - schedule: - - cron: '17 6 * * *' # Daily at 06:17 UTC. - workflow_dispatch: - -permissions: {} - -jobs: - decisions-integration: - name: Copilot Decisions API Integration - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - copilot-requests: write - concurrency: - group: ci-${{ github.ref }}-decisions-integration - cancel-in-progress: true - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 - with: - node-version: "24" - - name: Run live Decisions client integration test - env: - GITHUB_TOKEN: ${{ github.token }} - run: node --test actions/setup/js/scripts/test_decisions_client.cjs diff --git a/actions/setup/README.md b/actions/setup/README.md index 6b20ce9c0d2..ac511c9510a 100644 --- a/actions/setup/README.md +++ b/actions/setup/README.md @@ -139,16 +139,8 @@ answer validation, so callers can collect costs even when the answer batch is invalid. Evals use it to save response token usage and catalog-derived AIC and USD costs to the accounting artifact. -The CJS workflow's **Copilot Decisions API Integration** job runs -`node --test actions/setup/js/scripts/test_decisions_client.cjs` with -`GITHUB_TOKEN` from a job granting `copilot-requests: write`. It makes one live -choice request using synthetic evidence and requires a named `basket_fruit` -answer of type `choice` with the exact value `apples` rather than `bananas`. -This single-request API, authentication, and protocol smoke check is not a -statistical model-quality evaluation; it uses no confidence threshold or retries. -Missing credentials, API errors, and incorrect or malformed answers fail the test. -The job excludes fork pull requests and Dependabot-triggered runs, whose tokens -are read-only, and does not use PATs or repository secrets. +Live Decisions API coverage is provided by Copilot evals rather than a separate +integration smoke test. ### Synthetic message previews diff --git a/actions/setup/js/scripts/test_decisions_client.cjs b/actions/setup/js/scripts/test_decisions_client.cjs deleted file mode 100644 index 9f5ce1b23c8..00000000000 --- a/actions/setup/js/scripts/test_decisions_client.cjs +++ /dev/null @@ -1,27 +0,0 @@ -// @ts-check -"use strict"; - -const assert = require("node:assert/strict"); -const { test } = require("node:test"); -const { invokeDecisions } = require("../decisions_client.cjs"); - -test("Copilot Decisions accepts the copilot-requests: write job token", { timeout: 45000 }, async () => { - const apiKey = process.env.GITHUB_TOKEN; - assert.ok(apiKey?.trim(), "GITHUB_TOKEN with copilot-requests: write is required"); - - const response = await invokeDecisions({ - baseURL: "https://api.githubcopilot.com/v1", - apiKey, - integrationId: "agentic-workflows", - model: "gpt-6-luna-decisions", - input: "The basket contains three apples and no other fruit.", - questions: [{ type: "choice", name: "basket_fruit", instructions: "Which fruit does the basket contain?", choices: [{ value: "apples" }, { value: "bananas" }] }], - timeoutMs: 30000, - }); - - assert.equal(response.answers.length, 1); - const answer = response.answers[0]; - assert.equal(answer.name, "basket_fruit"); - assert.equal(answer.type, "choice"); - assert.equal(answer.choice, "apples", "The Decisions API must recognize the apples in the supplied evidence"); -}); diff --git a/scripts/check-cgo-cjs-workflow-purity.sh b/scripts/check-cgo-cjs-workflow-purity.sh index f3fd81d7cbe..cfbc82dfb14 100644 --- a/scripts/check-cgo-cjs-workflow-purity.sh +++ b/scripts/check-cgo-cjs-workflow-purity.sh @@ -73,7 +73,6 @@ PY write_permissions_file="$tmp_dir/write-permissions.txt" if ! python3 - "$workflow" >"$write_permissions_file" <<'PY'; then -import pathlib import re import sys @@ -82,9 +81,6 @@ workflow = sys.argv[1] # `actions: write` is required by the cleanup jobs that delete the shared # checkout cache at the end of a run; it grants no access to repository content. allowed_write_scopes = {"actions"} -# CJS may exercise Copilot inference with its job token, not repository writes. -if pathlib.Path(workflow) == pathlib.Path(".github/workflows/cjs.yml"): - allowed_write_scopes.add("copilot-requests") def strip_comment(line): diff --git a/scripts/check-cgo-cjs-workflow-purity_test.sh b/scripts/check-cgo-cjs-workflow-purity_test.sh index 417b0271f0f..9142ab6434d 100644 --- a/scripts/check-cgo-cjs-workflow-purity_test.sh +++ b/scripts/check-cgo-cjs-workflow-purity_test.sh @@ -127,7 +127,7 @@ else fail "multiple input output was incorrect" "$(cat "$T5_OUT")" fi -echo "Test 6: CJS permits Copilot inference with only its built-in token..." +echo "Test 6: CJS rejects Copilot write permission even with its built-in token..." T6="$TMP_ROOT/t6" T6_CJS="$T6/.github/workflows/cjs.yml" write_workflow "$T6_CJS" " permissions: @@ -139,9 +139,11 @@ write_workflow "$T6_CJS" " permissions: GITHUB_TOKEN: \${{ github.token }}" T6_OUT="$TMP_ROOT/t6-output.txt" if (cd "$T6" && bash "$PURITY_SCRIPT" .github/workflows/cjs.yml >"$T6_OUT" 2>&1); then - pass "CJS permits Copilot inference with only its built-in token" + fail "CJS Copilot write permission should exit 1" "$(cat "$T6_OUT")" +elif grep -q "copilot-requests: write" "$T6_OUT"; then + pass "CJS rejects Copilot write permission even with its built-in token" else - fail "CJS Copilot inference permission should pass" "$(cat "$T6_OUT")" + fail "CJS write permission output was incorrect" "$(cat "$T6_OUT")" fi echo "Test 7: CGO still rejects Copilot write permission..." @@ -159,7 +161,7 @@ else fail "CGO write permission output was incorrect" "$(cat "$T7_OUT")" fi -echo "Test 8: Copilot inference does not permit CJS repository writes or PAT secrets..." +echo "Test 8: CJS rejects repository writes and PAT secrets..." T8="$TMP_ROOT/t8" T8_CJS="$T8/.github/workflows/cjs.yml" write_workflow "$T8_CJS" " permissions: @@ -171,12 +173,12 @@ T8_OUT="$TMP_ROOT/t8-output.txt" if (cd "$T8" && bash "$PURITY_SCRIPT" .github/workflows/cjs.yml >"$T8_OUT" 2>&1); then fail "CJS repository writes and PAT secrets should exit 1" "$(cat "$T8_OUT")" elif grep -q "contents: write" "$T8_OUT" && grep -q "secrets.COPILOT_GITHUB_TOKEN" "$T8_OUT"; then - pass "Copilot inference does not permit CJS repository writes or PAT secrets" + pass "CJS rejects repository writes and PAT secrets" else fail "CJS repository write and secret output was incorrect" "$(cat "$T8_OUT")" fi -echo "Test 9: same-basename CJS impostors reject Copilot write permission..." +echo "Test 9: other CJS paths reject Copilot write permission..." T9="$TMP_ROOT/t9" write_workflow "$T9/cjs.yml" " permissions: copilot-requests: write @@ -187,11 +189,11 @@ write_workflow "$T9/other/cjs.yml" " permissions: { contents: read, copilot-r - run: echo ok" T9_OUT="$TMP_ROOT/t9-output.txt" if (cd "$T9" && bash "$PURITY_SCRIPT" cjs.yml other/cjs.yml >"$T9_OUT" 2>&1); then - fail "CJS impostors should exit 1" "$(cat "$T9_OUT")" + fail "other CJS paths should exit 1" "$(cat "$T9_OUT")" elif grep -q "^cjs.yml:.*copilot-requests: write" "$T9_OUT" && grep -q "^other/cjs.yml:.*copilot-requests: write" "$T9_OUT"; then - pass "same-basename CJS impostors reject Copilot write permission" + pass "other CJS paths reject Copilot write permission" else - fail "CJS impostor permission output was incorrect" "$(cat "$T9_OUT")" + fail "other CJS path permission output was incorrect" "$(cat "$T9_OUT")" fi echo