diff --git a/.github/workflows/cjs.yml b/.github/workflows/cjs.yml index 3936f5bf883..1eec460e346 100644 --- a/.github/workflows/cjs.yml +++ b/.github/workflows/cjs.yml @@ -352,42 +352,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: cd actions/setup/js && npm run test:js-integration-artifact - decisions-integration: - name: Copilot Decisions API Integration - needs: [checkout-cache] - # Fork PRs and Dependabot runs have read-only tokens, even with the permission below. - if: github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - copilot-requests: write - concurrency: - group: ci-${{ github.ref }}-decisions-integration - cancel-in-progress: true - steps: - - name: Restore checkout - id: restore-checkout - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: . - key: checkout-cjs-${{ github.run_id }}-${{ github.sha }} - - name: Checkout on cache miss - if: steps.restore-checkout.outputs.cache-hit != 'true' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 - with: - node-version: "24" - - name: Run live Decisions client integration test - env: - GITHUB_TOKEN: ${{ github.token }} - run: node --test actions/setup/js/scripts/test_decisions_client.cjs - nvx-musl-integration: name: NVX musl inference integration needs: [checkout-cache] @@ -435,7 +399,6 @@ jobs: - docker-root-owned-cleanup - lint-js - artifact-integration - - decisions-integration - nvx-musl-integration if: always() runs-on: ubuntu-latest diff --git a/actions/setup/README.md b/actions/setup/README.md index 6b20ce9c0d2..ac511c9510a 100644 --- a/actions/setup/README.md +++ b/actions/setup/README.md @@ -139,16 +139,8 @@ answer validation, so callers can collect costs even when the answer batch is invalid. Evals use it to save response token usage and catalog-derived AIC and USD costs to the accounting artifact. -The CJS workflow's **Copilot Decisions API Integration** job runs -`node --test actions/setup/js/scripts/test_decisions_client.cjs` with -`GITHUB_TOKEN` from a job granting `copilot-requests: write`. It makes one live -choice request using synthetic evidence and requires a named `basket_fruit` -answer of type `choice` with the exact value `apples` rather than `bananas`. -This single-request API, authentication, and protocol smoke check is not a -statistical model-quality evaluation; it uses no confidence threshold or retries. -Missing credentials, API errors, and incorrect or malformed answers fail the test. -The job excludes fork pull requests and Dependabot-triggered runs, whose tokens -are read-only, and does not use PATs or repository secrets. +Live Decisions API coverage is provided by Copilot evals rather than a separate +integration smoke test. ### Synthetic message previews diff --git a/actions/setup/js/scripts/test_decisions_client.cjs b/actions/setup/js/scripts/test_decisions_client.cjs deleted file mode 100644 index 9f5ce1b23c8..00000000000 --- a/actions/setup/js/scripts/test_decisions_client.cjs +++ /dev/null @@ -1,27 +0,0 @@ -// @ts-check -"use strict"; - -const assert = require("node:assert/strict"); -const { test } = require("node:test"); -const { invokeDecisions } = require("../decisions_client.cjs"); - -test("Copilot Decisions accepts the copilot-requests: write job token", { timeout: 45000 }, async () => { - const apiKey = process.env.GITHUB_TOKEN; - assert.ok(apiKey?.trim(), "GITHUB_TOKEN with copilot-requests: write is required"); - - const response = await invokeDecisions({ - baseURL: "https://api.githubcopilot.com/v1", - apiKey, - integrationId: "agentic-workflows", - model: "gpt-6-luna-decisions", - input: "The basket contains three apples and no other fruit.", - questions: [{ type: "choice", name: "basket_fruit", instructions: "Which fruit does the basket contain?", choices: [{ value: "apples" }, { value: "bananas" }] }], - timeoutMs: 30000, - }); - - assert.equal(response.answers.length, 1); - const answer = response.answers[0]; - assert.equal(answer.name, "basket_fruit"); - assert.equal(answer.type, "choice"); - assert.equal(answer.choice, "apples", "The Decisions API must recognize the apples in the supplied evidence"); -}); diff --git a/scripts/check-cgo-cjs-workflow-purity.sh b/scripts/check-cgo-cjs-workflow-purity.sh index f3fd81d7cbe..cfbc82dfb14 100644 --- a/scripts/check-cgo-cjs-workflow-purity.sh +++ b/scripts/check-cgo-cjs-workflow-purity.sh @@ -73,7 +73,6 @@ PY write_permissions_file="$tmp_dir/write-permissions.txt" if ! python3 - "$workflow" >"$write_permissions_file" <<'PY'; then -import pathlib import re import sys @@ -82,9 +81,6 @@ workflow = sys.argv[1] # `actions: write` is required by the cleanup jobs that delete the shared # checkout cache at the end of a run; it grants no access to repository content. allowed_write_scopes = {"actions"} -# CJS may exercise Copilot inference with its job token, not repository writes. -if pathlib.Path(workflow) == pathlib.Path(".github/workflows/cjs.yml"): - allowed_write_scopes.add("copilot-requests") def strip_comment(line): diff --git a/scripts/check-cgo-cjs-workflow-purity_test.sh b/scripts/check-cgo-cjs-workflow-purity_test.sh index 417b0271f0f..9142ab6434d 100644 --- a/scripts/check-cgo-cjs-workflow-purity_test.sh +++ b/scripts/check-cgo-cjs-workflow-purity_test.sh @@ -127,7 +127,7 @@ else fail "multiple input output was incorrect" "$(cat "$T5_OUT")" fi -echo "Test 6: CJS permits Copilot inference with only its built-in token..." +echo "Test 6: CJS rejects Copilot write permission even with its built-in token..." T6="$TMP_ROOT/t6" T6_CJS="$T6/.github/workflows/cjs.yml" write_workflow "$T6_CJS" " permissions: @@ -139,9 +139,11 @@ write_workflow "$T6_CJS" " permissions: GITHUB_TOKEN: \${{ github.token }}" T6_OUT="$TMP_ROOT/t6-output.txt" if (cd "$T6" && bash "$PURITY_SCRIPT" .github/workflows/cjs.yml >"$T6_OUT" 2>&1); then - pass "CJS permits Copilot inference with only its built-in token" + fail "CJS Copilot write permission should exit 1" "$(cat "$T6_OUT")" +elif grep -q "copilot-requests: write" "$T6_OUT"; then + pass "CJS rejects Copilot write permission even with its built-in token" else - fail "CJS Copilot inference permission should pass" "$(cat "$T6_OUT")" + fail "CJS write permission output was incorrect" "$(cat "$T6_OUT")" fi echo "Test 7: CGO still rejects Copilot write permission..." @@ -159,7 +161,7 @@ else fail "CGO write permission output was incorrect" "$(cat "$T7_OUT")" fi -echo "Test 8: Copilot inference does not permit CJS repository writes or PAT secrets..." +echo "Test 8: CJS rejects repository writes and PAT secrets..." T8="$TMP_ROOT/t8" T8_CJS="$T8/.github/workflows/cjs.yml" write_workflow "$T8_CJS" " permissions: @@ -171,12 +173,12 @@ T8_OUT="$TMP_ROOT/t8-output.txt" if (cd "$T8" && bash "$PURITY_SCRIPT" .github/workflows/cjs.yml >"$T8_OUT" 2>&1); then fail "CJS repository writes and PAT secrets should exit 1" "$(cat "$T8_OUT")" elif grep -q "contents: write" "$T8_OUT" && grep -q "secrets.COPILOT_GITHUB_TOKEN" "$T8_OUT"; then - pass "Copilot inference does not permit CJS repository writes or PAT secrets" + pass "CJS rejects repository writes and PAT secrets" else fail "CJS repository write and secret output was incorrect" "$(cat "$T8_OUT")" fi -echo "Test 9: same-basename CJS impostors reject Copilot write permission..." +echo "Test 9: other CJS paths reject Copilot write permission..." T9="$TMP_ROOT/t9" write_workflow "$T9/cjs.yml" " permissions: copilot-requests: write @@ -187,11 +189,11 @@ write_workflow "$T9/other/cjs.yml" " permissions: { contents: read, copilot-r - run: echo ok" T9_OUT="$TMP_ROOT/t9-output.txt" if (cd "$T9" && bash "$PURITY_SCRIPT" cjs.yml other/cjs.yml >"$T9_OUT" 2>&1); then - fail "CJS impostors should exit 1" "$(cat "$T9_OUT")" + fail "other CJS paths should exit 1" "$(cat "$T9_OUT")" elif grep -q "^cjs.yml:.*copilot-requests: write" "$T9_OUT" && grep -q "^other/cjs.yml:.*copilot-requests: write" "$T9_OUT"; then - pass "same-basename CJS impostors reject Copilot write permission" + pass "other CJS paths reject Copilot write permission" else - fail "CJS impostor permission output was incorrect" "$(cat "$T9_OUT")" + fail "other CJS path permission output was incorrect" "$(cat "$T9_OUT")" fi echo