feat(ci): add prompt-agent e2e tests for copilot harness toolbox and skill samples

.azure-pipelines/prompt-agent-tests/cli/04-foundry-toolbox/test-spec.yml

@@ -0,0 +1,22 @@
+version: 1
+sample:
+  owner: zhihuan
+  experiences: [azd]
+tests:
+  - name: toolbox-web-search-cites-official-python-release
+    turns:
+      - input: >-
+          Search the web for the Python 3.13.0 final release date on python.org.
+          State the release year and include the official python.org release URL.
+        assertions:
+          - source: assistant_text
+            type: contains
+            value: '2024'
+          - source: assistant_text
+            type: regex
+            value: 'https?://(?:www\.)?python\.org/downloads/release/python-3130/?'
+          - source: assistant_text
+            type: regex
+            value: '(?i)traceback|unhandled exception'
+            min_matches: 0
+            max_matches: 0

.azure-pipelines/prompt-agent-tests/cli/05-skills/test-spec.yml

@@ -0,0 +1,37 @@
+version: 1
+sample:
+  owner: zhihuan
+  experiences: [azd]
+tests:
+  - name: lisbon-travel-guide-follows-skill-sections
+    turns:
+      - input: >-
+          Make me a 3-day travel guide for Lisbon focused on food and viewpoints.
+          Use the attached travel-guide skill and include all six required sections.
+        assertions:
+          - source: assistant_text
+            type: contains
+            value: Lisbon
+          - source: assistant_text
+            type: regex
+            value: '(?i)overview'
+          - source: assistant_text
+            type: regex
+            value: '(?i)day[- ]by[- ]day itinerary'
+          - source: assistant_text
+            type: regex
+            value: '(?i)neighborhoods'
+          - source: assistant_text
+            type: regex
+            value: '(?i)food\s*(?:&|and)\s*drink'
+          - source: assistant_text
+            type: regex
+            value: '(?i)practical tips'
+          - source: assistant_text
+            type: regex
+            value: '(?i)photo[- ]worthy stops'
+          - source: assistant_text
+            type: regex
+            value: '(?i)traceback|unhandled exception'
+            min_matches: 0
+            max_matches: 0

.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml

@@ -0,0 +1,36 @@
+version: 1
+sample:
+  owner: zhihuan
+  experiences: [azd]
+tests:
+  - name: official-source-python-release-research
+    turns:
+      - input: >-
+          What is the latest stable Python release? Use official Python sources.
+          Return these labeled fields: VERSION, RELEASE_DATE, CHANGE_1, CHANGE_2,
+          RELEASE_URL, and WHATS_NEW_URL. Cite the official Python release page
+          and What's New page.
+        assertions:
+          - source: assistant_text
+            type: regex
+            value: '(?i)\bVERSION(?:`|\*\*)?[ \t]*:[ \t]*[^\r\n]*?\b\d+\.\d+\.\d+\b'
+          - source: assistant_text
+            type: regex
+            value: '(?i)\bRELEASE_DATE(?:`|\*\*)?[ \t]*:[ \t]*\S+'
+          - source: assistant_text
+            type: regex
+            value: '(?i)\bCHANGE_1(?:`|\*\*)?[ \t]*:[ \t]*\S+'
+          - source: assistant_text
+            type: regex
+            value: '(?i)\bCHANGE_2(?:`|\*\*)?[ \t]*:[ \t]*\S+'
+          - source: assistant_text
+            type: regex
+            value: '(?i)\bRELEASE_URL(?:`|\*\*)?[ \t]*:[ \t]*[^\r\n]*?https?://(?:www\.)?python\.org/downloads/release/python-\d+'
+          - source: assistant_text
+            type: regex
+            value: '(?i)\bWHATS_NEW_URL(?:`|\*\*)?[ \t]*:[ \t]*[^\r\n]*?https?://docs\.python\.org/[^\s)>\]]*?/whatsnew/\d+\.\d+\.html'
+          - source: assistant_text
+            type: regex
+            value: '(?i)traceback|unhandled exception'
+            min_matches: 0
+            max_matches: 0

.azure-pipelines/prompt-agents-samples-ci.yml

@@ -150,7 +150,7 @@ stages:
         jobs:
           - job: Validate
             timeoutInMinutes: 210
-            cancelTimeoutInMinutes: 65
+            cancelTimeoutInMinutes: 75
             strategy:
               matrix: $[ stageDependencies.Discover.Discover.outputs['matrix.matrix_${{ profile }}'] ]
               maxParallel: 1

.azure-pipelines/scripts/prompt-agents/README.md

@@ -92,8 +92,9 @@ SDK invocation. Bash owns command sequencing, timeouts, and EXIT-trap cleanup.
 evidence, and exact-target absence parsing; the shared Python evaluator is reused.
 The runner supports string inputs and assistant-text assertions for plain prompt
 agents and managed-harness prompt agents (any valid `harness.type` identifier). Managed manifests must declare
-one project, one agent, and one toolbox containing only built-in
-`web_search`. Skills, connections, additional dependencies, and lifecycle hooks
+one project, one agent, and either one toolbox containing only built-in
+`web_search` or one file-backed `azure.ai.skill` service under `./skills/<skill-name>`.
+Connections, additional dependencies, and lifecycle hooks
 are rejected before deployment. This is capability validation, not a sample-name
 allowlist.
 
@@ -142,9 +143,10 @@ bash .azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh \
 The adapter verifies the endpoint against the ARM project, adopts a temporary
 manifest with a unique agent name, replaces the literal model with the selected
 existing deployment, and removes model provisioning declarations. Managed runs
-also give the toolbox a unique name and update both `uses` and `toolbox.name`.
-The runner deploys the owned toolbox before the agent and verifies that the
-agent's MCP attachment matches its versioned endpoint. It never calls
+also give the owned toolbox or skill a unique name and update `uses`,
+`toolboxes`/`toolbox`, `skills`, MCP `server_label`, and `./skills/<name>` archive
+references. The runner deploys the owned toolbox or skill before the agent and verifies that the
+agent's attachment matches its versioned resource. It never calls
 `provision`, `up`, or `down`. The original sample is unchanged. The artifacts
 directory must be new and outside the sample.
 
@@ -172,9 +174,9 @@ HTTP 404 counts as absent; auth/network/configuration errors fail verification.
 This check parses azd's error text and must be revalidated when CLI formatting
 changes. Cleanup failures fail the run even if behavior assertions pass.
 Logging failures do not prevent an agent deletion attempt.
-For managed runs, toolbox deletion is attempted after agent cleanup even if
-agent cleanup fails. A successful project-scoped `azd ai toolbox list` must omit
-the owned name before or after its deployment; malformed/failed list responses
+For managed runs, owned toolbox, skill, and toolbox-MCP connection deletion are attempted after agent cleanup even if
+agent cleanup fails. Successful project-scoped `azd ai toolbox list`, `azd ai skill list`, and `azd ai connection list` calls must omit
+the owned names before or after deployment; malformed/failed list responses
 cannot confirm absence. This relies on the CLI's list pagination and visibility.
 Only resources recorded as deployment-attempted are deleted. The in-process
 cleanup uses short per-command timeouts to leave cancellation time for the

.azure-pipelines/scripts/prompt-agents/blacklist.json

@@ -1,11 +1,8 @@
 {
   "samples/cli/prompt-agents/02-tools": "The azd text-only E2E runner cannot assert emitted function-call names and arguments.",
-  "samples/cli/prompt-agents/04-foundry-toolbox": "Requires azure.ai.agents plural toolboxes connection support (Azure/azure-dev#10631).",
-  "samples/cli/prompt-agents/05-skills": "The E2E runner does not manage skill resource ownership and cleanup.",
   "samples/cli/prompt-agents/06-azure-search-rag": "Requires a seeded CI Search index, runtime RBAC, and connection lifecycle support.",
   "samples/cli/prompt-agents/microsoft-copilot-harness/adventureworks-sql-analyst": "Requires Azure SQL Server, connector namespace infrastructure, skills, and postprovision hooks.",
   "samples/cli/prompt-agents/microsoft-copilot-harness/browser-automation-blog-research": "Requires Playwright workspace Bicep infrastructure and project connections.",
   "samples/cli/prompt-agents/microsoft-copilot-harness/csv-chart-analyst": "Requires postdeploy RBAC hooks and the custom file-upload Python client.",
-  "samples/cli/prompt-agents/microsoft-copilot-harness/quickstart-web-research": "Requires skill resource lifecycle management and custom smoke-test orchestration.",
   "samples/cli/prompt-agents/microsoft-copilot-harness/web-research-email-digest": "Requires Work IQ M365 connections, user consent, and routine lifecycle support."
 }

.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh

@@ -102,6 +102,8 @@ if ! jq -e -s --arg id "$project_id" --arg endpoint "$project_endpoint" --slurpf
     ($c.project_endpoint | rtrimstr("/")) == $endpoint and
     ($o.deployment_attempted | type) == "boolean" and
     ($o.toolbox_deployment_attempted | type) == "boolean" and
+    (($o.skill_deployment_attempted // false) | type) == "boolean" and
+    (($o.connection_deployment_attempted // false) | type) == "boolean" and
     ($o.working_directory | type) == "string" and
     ($o.working_directory | test("^(?:/[^/]|[A-Za-z]:[/\\\\])")) and
     ($o.working_directory | test("[\\x00-\\x1f\\x7f]") | not) and
@@ -111,26 +113,53 @@ if ! jq -e -s --arg id "$project_id" --arg endpoint "$project_endpoint" --slurpf
     $c.service_env_key == ($c.agent_service | gsub("[- ]"; "_") | ascii_upcase) and
     ($c.service_env_key | test("^[A-Z0-9_]+$")) and
     $o.toolbox_name == $c.toolbox_name and
-    (if $c.execution_profile == "managed" then
-       $c.toolbox_name == ("ci-e2e-tb-" + ($c.agent_name | ltrimstr("ci-prompt-")))
-     else $c.execution_profile == "prompt" and $c.toolbox_name == "" and
-       $o.toolbox_deployment_attempted == false end) and
+    ($o.toolbox_endpoint // "") == ($c.toolbox_endpoint // "") and
+    ($o.skill_name // "") == ($c.skill_name // "") and
+    ($o.connection_name // "") == ($c.connection_name // "") and
+    (($c.agent_name | ltrimstr("ci-prompt-")) as $nonce |
+      if $c.execution_profile == "managed" then
+        (
+          ($c.toolbox_name == ("ci-e2e-tb-" + $nonce) and ($c.skill_name // "") == "" and ($o.skill_deployment_attempted // false) == false and
+            (if ($o.connection_deployment_attempted // false) or (($c.connection_name // "") != "") or (($c.toolbox_endpoint // "") != "") then
+              (($c.connection_name // "") | type) == "string" and (($c.connection_name // "") != "") and
+              (($c.toolbox_endpoint // "") | type) == "string" and (($c.toolbox_endpoint // "") != "") and
+              (($c.connection_name // "") | test("^ci-e2e-tb-" + ($nonce[:10]) + "-[0-9a-f]{8}-mcp$"))
+            else (($c.connection_name // "") == "" and ($c.toolbox_endpoint // "") == "") end))
+          or
+          (($c.skill_name // "") == ("ci-e2e-sk-" + $nonce) and $c.toolbox_name == "" and $o.toolbox_deployment_attempted == false and
+            ($c.toolbox_endpoint // "") == "" and ($c.connection_name // "") == "" and ($o.connection_deployment_attempted // false) == false)
+        )
+      else $c.execution_profile == "prompt" and $c.toolbox_name == "" and $o.toolbox_deployment_attempted == false and
+        ($c.toolbox_endpoint // "") == "" and ($c.skill_name // "") == "" and ($o.skill_deployment_attempted // false) == false and
+        ($c.connection_name // "") == "" and ($o.connection_deployment_attempted // false) == false end) and
     ($o.cleanup | type) == "object" and
     ($o.cleanup | to_entries | all(
       ((.key == $o.agent_name and $o.deployment_attempted) or
-       (.key == $o.toolbox_name and $o.toolbox_deployment_attempted)) and
+       (.key == $o.toolbox_name and $o.toolbox_deployment_attempted) or
+       (.key == ($o.skill_name // "") and ($o.skill_deployment_attempted // false)) or
+       (.key == ($o.connection_name // "") and ($o.connection_deployment_attempted // false))) and
       (.value | type) == "string" and (.value | length) > 0)))
 ' "$ownership" > "$recovery/validation-stdout.txt" 2> "$recovery/validation-stderr.txt"; then
   fail "Invalid ownership/context or trusted project mismatch; refusing recovery commands."
 fi
+if [ -n "$(jq -r '.connection_name // ""' "$context")" ] || [ -n "$(jq -r '.toolbox_endpoint // ""' "$context")" ]; then
+  if ! timeout --kill-after=10s 60s "$python" "$helper" verify-connection --context "$context" \
+    >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt"; then
+    fail "Invalid ownership/context or trusted project mismatch; refusing recovery commands."
+  fi
+fi
 validated=true
 name=$(jq -r .agent_name "$context")
 service=$(jq -r .agent_service "$context")
 env_key=$(jq -r .service_env_key "$context")
 toolbox=$(jq -r .toolbox_name "$context")
+skill=$(jq -r '.skill_name // ""' "$context")
+connection=$(jq -r '.connection_name // ""' "$context")
 agent_pending=$(jq -r '.deployment_attempted and .cleanup[.agent_name] != "absent"' "$ownership")
 toolbox_pending=$(jq -r '.toolbox_deployment_attempted and .cleanup[.toolbox_name] != "absent"' "$ownership")
-if [ "$agent_pending" = false ] && [ "$toolbox_pending" = false ]; then exit 0; fi
+skill_pending=$(jq -r '(.skill_deployment_attempted // false) and .cleanup[.skill_name] != "absent"' "$ownership")
+connection_pending=$(jq -r '(.connection_deployment_attempted // false) and (.connection_name // "") != "" and .cleanup[.connection_name] != "absent"' "$ownership")
+if [ "$agent_pending" = false ] && [ "$toolbox_pending" = false ] && [ "$skill_pending" = false ] && [ "$connection_pending" = false ]; then exit 0; fi
 
 work=$(jq -r .working_directory "$ownership")
 case "${OSTYPE:-}" in msys*|cygwin*) work=$(cygpath -u "$work") ;; esac
@@ -177,8 +206,24 @@ toolbox_absent() {
     --metadata "$(native_path "$last_stdout")" --context "$context" \
     >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt"
 }
+skill_absent() {
+  timeout --kill-after=10s 60s "$python" "$helper" skill-absent \
+    --metadata "$(native_path "$last_stdout")" --context "$context" \
+    >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt"
+}
+connection_absent() {
+  timeout --kill-after=10s 60s "$python" "$helper" connection-absent \
+    --metadata "$(native_path "$last_stdout")" --context "$context" \
+    >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt"
+}
+verify_connection() {
+  timeout --kill-after=10s 60s "$python" "$helper" verify-connection \
+    --metadata "$(native_path "$last_stdout")" --context "$context" \
+    >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt"
+}
 
 # Pin both process variables and the retained azd environment; do not trust defaults.
+export AZD_ALPHA_ENABLE_AGENTS_MANAGED_HARNESS=true
 export FOUNDRY_PROJECT_ENDPOINT="$project_endpoint" AZURE_AI_PROJECT_ENDPOINT="$project_endpoint"
 export AZURE_AI_PROJECT_ID="$project_id" AZURE_ENV_NAME="$name" "AGENT_${env_key}_NAME=$name"
 azd_project=(azd -C "$(native_path "$project")")
@@ -225,7 +270,49 @@ cleanup_toolbox() {
   done
   return 1
 }
+cleanup_skill() {
+  local attempt
+  save_status "$skill" unverified || return 1
+  run_command "${azd_project[@]}" ai skill list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1
+  if skill_absent; then save_status "$skill" absent; return $?; fi
+  jq -e -s --arg name "$skill" '
+    length == 1 and (.[0] | type) == "array" and
+    (.[0] | all(type == "object" and (.name | type) == "string" and (.name | test("\\S")))) and
+    (.[0] | any((.name | ascii_downcase) == $name))
+  ' "$(native_path "$last_stdout")" >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" || return 1
+  run_command "${azd_project[@]}" ai skill delete "$skill" --project-endpoint "$project_endpoint" \
+    --force --no-prompt --output json || cleanup_failed=true
+  for attempt in 1 2 3; do
+    run_command "${azd_project[@]}" ai skill list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1
+    if skill_absent; then save_status "$skill" absent; return $?; fi
+    [ "$attempt" -eq 3 ] || sleep 5
+  done
+  return 1
+}
+cleanup_connection() {
+  local attempt
+  save_status "$connection" unverified || return 1
+  run_command "${azd_project[@]}" ai connection list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1
+  if connection_absent; then save_status "$connection" absent; return $?; fi
+  jq -e -s --arg name "$connection" '
+    length == 1 and (.[0] | type) == "array" and
+    (.[0] | all(type == "object" and (.name | type) == "string" and (.name | test("\\S")))) and
+    (.[0] | any((.name | ascii_downcase) == $name))
+  ' "$(native_path "$last_stdout")" >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" || return 1
+  run_command "${azd_project[@]}" ai connection show "$connection" --project-endpoint "$project_endpoint" --output json --no-prompt || return 1
+  verify_connection || return 1
+  run_command "${azd_project[@]}" ai connection delete "$connection" --project-endpoint "$project_endpoint" \
+    --force --no-prompt || cleanup_failed=true
+  for attempt in 1 2 3; do
+    run_command "${azd_project[@]}" ai connection list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1
+    if connection_absent; then save_status "$connection" absent; return $?; fi
+    [ "$attempt" -eq 3 ] || sleep 5
+  done
+  return 1
+}
 if [ "$agent_pending" = true ]; then cleanup_agent || cleanup_failed=true; fi
-# Agent errors must not strand an independently owned toolbox.
+# Agent errors must not strand independently owned resources.
+if [ "$connection_pending" = true ]; then cleanup_connection || cleanup_failed=true; fi
 if [ "$toolbox_pending" = true ]; then cleanup_toolbox || cleanup_failed=true; fi
+if [ "$skill_pending" = true ]; then cleanup_skill || cleanup_failed=true; fi
 if [ "$cleanup_failed" = true ]; then exit 1; fi