From 094ccbf95b53690e24ee0bbd355541bf3a36c85d Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 21 Oct 2026 09:37:04 -0700 Subject: [PATCH 1/3] Fix Vally 0.19 schema loading outside TypeSpec benchmarks --- .../azsdk-common-apiview-feedback-resolution/evals/eval.yaml | 2 +- .../skills/azsdk-common-generate-sdk-locally/evals/eval.yaml | 2 +- .../skills/azsdk-common-prepare-release-plan/evals/eval.yaml | 2 +- .github/skills/azsdk-common-sdk-release/evals/eval.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml index 22daf2988b0a..b2473a5341b5 100644 --- a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-apiview-feedback-resolution type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol diff --git a/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml b/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml index 3ce10aed89b6..5856f2c07a64 100644 --- a/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml +++ b/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-generate-sdk-locally type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml index 548a435dd598..615883a4d2e1 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-prepare-release-plan type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol diff --git a/.github/skills/azsdk-common-sdk-release/evals/eval.yaml b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml index e4604b9e2b05..0e94c3019d34 100644 --- a/.github/skills/azsdk-common-sdk-release/evals/eval.yaml +++ b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-sdk-release type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol From d337545eca77347dce31f882775bfa335370981a Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 21 Oct 2026 10:14:03 -0700 Subject: [PATCH 2/3] Fix hermetic eval fixtures and mock prerequisites --- .../evals/trigger.eval.yaml | 9 +++++++++ .../evals/schedule-risk.eval.yaml | 4 +++- .../evals/trigger.eval.yaml | 3 +++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml b/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml index 2ebf85d89760..b33524cda990 100644 --- a/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml @@ -20,6 +20,10 @@ scoring: stimuli: - name: trigger-generate-python-sdk-locally prompt: "Generate the Python SDK locally from my TypeSpec project at specification/compute/resource-manager/Microsoft.Compute/ComputeRP/tspconfig.yaml." + environment: + skills: + - .. + - ../../azsdk-common-sdk-breaking-change graders: - type: skill-invocation config: @@ -28,7 +32,12 @@ stimuli: # Should NOT trigger - name: anti-trigger-mitigate-breaking-change prompt: "Fix the SDK breaking changes" + environment: + skills: + - .. + - ../../azsdk-common-sdk-breaking-change graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-breaking-change"] disallowed: ["azsdk-common-generate-sdk-locally"] diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml index bc5e8a9bca30..a87fdcf1f099 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml @@ -58,6 +58,7 @@ stimuli: config: substring: "https://github.com/Azure/azure-rest-api-specs/pull/12345" + # The new target stays future-proof; the mock's older plan supplies the warning. - name: create-relays-past-due-guidance environment: skills: @@ -65,7 +66,8 @@ stimuli: prompt: | Create a public preview release plan for TypeSpec project specification/contosowidgetmanager/Contoso.WidgetManager with target - release month June 2026, spec PR + release month next calendar month (resolve to an explicit month and year + using the current date), spec PR https://github.com/Azure/azure-rest-api-specs/pull/38387, service tree ID a7f2b8e4-9c1d-4a3e-b6f9-2d8e5a7c3b1f, and product tree ID f1a8c5d2-6e4b-4f7a-9c2d-8b5e1f3a6c9e. Use the Azure SDK MCP tools. diff --git a/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml b/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml index 933bdcfb40f5..6a0ba6107815 100644 --- a/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml @@ -20,6 +20,9 @@ scoring: stimuli: - name: trigger-detect-sdk-breaking-changes-for-service prompt: "Detect and mitigate SDK breaking changes for the service webpubsub" + environment: + skills: + - .. graders: - type: skill-invocation config: From cfa44a54c92a430877f366634628072e62d0be2c Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 21 Oct 2026 11:41:22 -0700 Subject: [PATCH 3/3] Trim eval upgrade to minimal configuration fixes --- .github/skills/azsdk-common-sdk-release/evals/eval.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/skills/azsdk-common-sdk-release/evals/eval.yaml b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml index 0e94c3019d34..b73345200b0d 100644 --- a/.github/skills/azsdk-common-sdk-release/evals/eval.yaml +++ b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml @@ -60,12 +60,16 @@ stimuli: - name: basic-trigger prompt: "I need to release a new version of my Azure SDK package. Walk me through the process." + environment: + skills: + - .. graders: - type: output-contains config: substring: "release" - constraints: - expect_skills: ["azsdk-common-sdk-release"] + - type: skill-invocation + config: + required: ["azsdk-common-sdk-release"] - name: anti-trigger prompt: "How do I write unit tests for my Azure SDK client?"