From 22f7d998547b34f4e9b8364d0a5e7f6da0a86276 Mon Sep 17 00:00:00 2001 From: hund030 Date: Mon, 2 Nov 2026 11:36:50 +0800 Subject: [PATCH 1/3] feat(ci): add prompt-agent e2e tests for copilot harness toolbox and skill samples - extend the prompt-agent CI runner and recovery script to manage plural `toolboxes:`, endpoint-scoped toolbox MCP connections, and `azure.ai.skill` resources - add `test-spec.yml` contracts and enable `04-foundry-toolbox`, `05-skills`, and `microsoft-copilot-harness/quickstart-web-research` in the managed E2E matrix --- .../cli/04-foundry-toolbox/test-spec.yml | 22 ++ .../cli/05-skills/test-spec.yml | 37 ++ .../quickstart-web-research/test-spec.yml | 36 ++ .azure-pipelines/prompt-agents-samples-ci.yml | 2 +- .../scripts/prompt-agents/README.md | 18 +- .../scripts/prompt-agents/blacklist.json | 3 - .../prompt-agents/cleanup-prompt-agent.sh | 86 +++- .../prompt-agents/prompt_agent_helpers.py | 369 +++++++++++++++--- .../prompt-agents/provision-prompt-agent.sh | 19 +- .../scripts/prompt-agents/run-prompt-agent.sh | 74 +++- .../tests/test_prompt_agent_helpers.py | 43 +- .../tests/test_prompt_agent_lifecycle.py | 201 ++++++++-- .../tests/test_prompt_agent_pipeline.py | 2 +- .../tests/test_prompt_agent_recovery.py | 55 ++- samples/cli/prompt-agents/AGENTS.md | 2 +- .../quickstart-web-research/azure.yaml | 1 + 16 files changed, 857 insertions(+), 113 deletions(-) create mode 100644 .azure-pipelines/prompt-agent-tests/cli/04-foundry-toolbox/test-spec.yml create mode 100644 .azure-pipelines/prompt-agent-tests/cli/05-skills/test-spec.yml create mode 100644 .azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml diff --git a/.azure-pipelines/prompt-agent-tests/cli/04-foundry-toolbox/test-spec.yml b/.azure-pipelines/prompt-agent-tests/cli/04-foundry-toolbox/test-spec.yml new file mode 100644 index 000000000..5db7f9b24 --- /dev/null +++ b/.azure-pipelines/prompt-agent-tests/cli/04-foundry-toolbox/test-spec.yml @@ -0,0 +1,22 @@ +version: 1 +sample: + owner: zhihuan + experiences: [azd] +tests: + - name: toolbox-web-search-cites-official-python-release + turns: + - input: >- + Search the web for the Python 3.13.0 final release date on python.org. + State the release year and include the official python.org release URL. + assertions: + - source: assistant_text + type: contains + value: '2024' + - source: assistant_text + type: regex + value: 'https?://(?:www\.)?python\.org/downloads/release/python-3130/?' + - source: assistant_text + type: regex + value: '(?i)traceback|unhandled exception' + min_matches: 0 + max_matches: 0 diff --git a/.azure-pipelines/prompt-agent-tests/cli/05-skills/test-spec.yml b/.azure-pipelines/prompt-agent-tests/cli/05-skills/test-spec.yml new file mode 100644 index 000000000..73dd3fcd5 --- /dev/null +++ b/.azure-pipelines/prompt-agent-tests/cli/05-skills/test-spec.yml @@ -0,0 +1,37 @@ +version: 1 +sample: + owner: zhihuan + experiences: [azd] +tests: + - name: lisbon-travel-guide-follows-skill-sections + turns: + - input: >- + Make me a 3-day travel guide for Lisbon focused on food and viewpoints. + Use the attached travel-guide skill and include all six required sections. + assertions: + - source: assistant_text + type: contains + value: Lisbon + - source: assistant_text + type: regex + value: '(?i)overview' + - source: assistant_text + type: regex + value: '(?i)day[- ]by[- ]day itinerary' + - source: assistant_text + type: regex + value: '(?i)neighborhoods' + - source: assistant_text + type: regex + value: '(?i)food\s*(?:&|and)\s*drink' + - source: assistant_text + type: regex + value: '(?i)practical tips' + - source: assistant_text + type: regex + value: '(?i)photo[- ]worthy stops' + - source: assistant_text + type: regex + value: '(?i)traceback|unhandled exception' + min_matches: 0 + max_matches: 0 diff --git a/.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml b/.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml new file mode 100644 index 000000000..583c3cca3 --- /dev/null +++ b/.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml @@ -0,0 +1,36 @@ +version: 1 +sample: + owner: zhihuan + experiences: [azd] +tests: + - name: official-source-python-release-research + turns: + - input: >- + What is the latest stable Python release? Use official Python sources. + Return these labeled fields: VERSION, RELEASE_DATE, CHANGE_1, CHANGE_2, + RELEASE_URL, and WHATS_NEW_URL. Cite the official Python release page + and What's New page. + assertions: + - source: assistant_text + type: regex + value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?VERSION(?:`|\*\*)?[ \t]*:[ \t]*.*?\b\d+\.\d+\.\d+\b' + - source: assistant_text + type: regex + value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?RELEASE_DATE(?:`|\*\*)?[ \t]*:[ \t]*\S+' + - source: assistant_text + type: regex + value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?CHANGE_1(?:`|\*\*)?[ \t]*:[ \t]*\S+' + - source: assistant_text + type: regex + value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?CHANGE_2(?:`|\*\*)?[ \t]*:[ \t]*\S+' + - source: assistant_text + type: regex + value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?RELEASE_URL(?:`|\*\*)?[ \t]*:[ \t]*.*?https?://(?:www\.)?python\.org/downloads/release/python-\d+' + - source: assistant_text + type: regex + value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?WHATS_NEW_URL(?:`|\*\*)?[ \t]*:[ \t]*.*?https?://docs\.python\.org/[^\s)>\]]*?/whatsnew/\d+\.\d+\.html' + - source: assistant_text + type: regex + value: '(?i)traceback|unhandled exception' + min_matches: 0 + max_matches: 0 diff --git a/.azure-pipelines/prompt-agents-samples-ci.yml b/.azure-pipelines/prompt-agents-samples-ci.yml index 7ac9f2a4d..21f1c2fe3 100644 --- a/.azure-pipelines/prompt-agents-samples-ci.yml +++ b/.azure-pipelines/prompt-agents-samples-ci.yml @@ -150,7 +150,7 @@ stages: jobs: - job: Validate timeoutInMinutes: 210 - cancelTimeoutInMinutes: 65 + cancelTimeoutInMinutes: 75 strategy: matrix: $[ stageDependencies.Discover.Discover.outputs['matrix.matrix_${{ profile }}'] ] maxParallel: 1 diff --git a/.azure-pipelines/scripts/prompt-agents/README.md b/.azure-pipelines/scripts/prompt-agents/README.md index 4159deddf..b71b5b3ed 100644 --- a/.azure-pipelines/scripts/prompt-agents/README.md +++ b/.azure-pipelines/scripts/prompt-agents/README.md @@ -92,8 +92,9 @@ SDK invocation. Bash owns command sequencing, timeouts, and EXIT-trap cleanup. evidence, and exact-target absence parsing; the shared Python evaluator is reused. The runner supports string inputs and assistant-text assertions for plain prompt agents and managed-harness prompt agents (any valid `harness.type` identifier). Managed manifests must declare -one project, one agent, and one toolbox containing only built-in -`web_search`. Skills, connections, additional dependencies, and lifecycle hooks +one project, one agent, and either one toolbox containing only built-in +`web_search` or one file-backed `azure.ai.skill` service under `./skills/`. +Connections, additional dependencies, and lifecycle hooks are rejected before deployment. This is capability validation, not a sample-name allowlist. @@ -142,9 +143,10 @@ bash .azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh \ The adapter verifies the endpoint against the ARM project, adopts a temporary manifest with a unique agent name, replaces the literal model with the selected existing deployment, and removes model provisioning declarations. Managed runs -also give the toolbox a unique name and update both `uses` and `toolbox.name`. -The runner deploys the owned toolbox before the agent and verifies that the -agent's MCP attachment matches its versioned endpoint. It never calls +also give the owned toolbox or skill a unique name and update `uses`, +`toolboxes`/`toolbox`, `skills`, MCP `server_label`, and `./skills/` archive +references. The runner deploys the owned toolbox or skill before the agent and verifies that the +agent's attachment matches its versioned resource. It never calls `provision`, `up`, or `down`. The original sample is unchanged. The artifacts directory must be new and outside the sample. @@ -172,9 +174,9 @@ HTTP 404 counts as absent; auth/network/configuration errors fail verification. This check parses azd's error text and must be revalidated when CLI formatting changes. Cleanup failures fail the run even if behavior assertions pass. Logging failures do not prevent an agent deletion attempt. -For managed runs, toolbox deletion is attempted after agent cleanup even if -agent cleanup fails. A successful project-scoped `azd ai toolbox list` must omit -the owned name before or after its deployment; malformed/failed list responses +For managed runs, owned toolbox, skill, and toolbox-MCP connection deletion are attempted after agent cleanup even if +agent cleanup fails. Successful project-scoped `azd ai toolbox list`, `azd ai skill list`, and `azd ai connection list` calls must omit +the owned names before or after deployment; malformed/failed list responses cannot confirm absence. This relies on the CLI's list pagination and visibility. Only resources recorded as deployment-attempted are deleted. The in-process cleanup uses short per-command timeouts to leave cancellation time for the diff --git a/.azure-pipelines/scripts/prompt-agents/blacklist.json b/.azure-pipelines/scripts/prompt-agents/blacklist.json index b9ffa36f8..3004a3bde 100644 --- a/.azure-pipelines/scripts/prompt-agents/blacklist.json +++ b/.azure-pipelines/scripts/prompt-agents/blacklist.json @@ -1,11 +1,8 @@ { "samples/cli/prompt-agents/02-tools": "The azd text-only E2E runner cannot assert emitted function-call names and arguments.", - "samples/cli/prompt-agents/04-foundry-toolbox": "Requires azure.ai.agents plural toolboxes connection support (Azure/azure-dev#10631).", - "samples/cli/prompt-agents/05-skills": "The E2E runner does not manage skill resource ownership and cleanup.", "samples/cli/prompt-agents/06-azure-search-rag": "Requires a seeded CI Search index, runtime RBAC, and connection lifecycle support.", "samples/cli/prompt-agents/microsoft-copilot-harness/adventureworks-sql-analyst": "Requires Azure SQL Server, connector namespace infrastructure, skills, and postprovision hooks.", "samples/cli/prompt-agents/microsoft-copilot-harness/browser-automation-blog-research": "Requires Playwright workspace Bicep infrastructure and project connections.", "samples/cli/prompt-agents/microsoft-copilot-harness/csv-chart-analyst": "Requires postdeploy RBAC hooks and the custom file-upload Python client.", - "samples/cli/prompt-agents/microsoft-copilot-harness/quickstart-web-research": "Requires skill resource lifecycle management and custom smoke-test orchestration.", "samples/cli/prompt-agents/microsoft-copilot-harness/web-research-email-digest": "Requires Work IQ M365 connections, user consent, and routine lifecycle support." } diff --git a/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh b/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh index 7bb877734..a1ceed53c 100644 --- a/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh +++ b/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh @@ -102,6 +102,8 @@ if ! jq -e -s --arg id "$project_id" --arg endpoint "$project_endpoint" --slurpf ($c.project_endpoint | rtrimstr("/")) == $endpoint and ($o.deployment_attempted | type) == "boolean" and ($o.toolbox_deployment_attempted | type) == "boolean" and + (($o.skill_deployment_attempted // false) | type) == "boolean" and + (($o.connection_deployment_attempted // false) | type) == "boolean" and ($o.working_directory | type) == "string" and ($o.working_directory | test("^(?:/[^/]|[A-Za-z]:[/\\\\])")) and ($o.working_directory | test("[\\x00-\\x1f\\x7f]") | not) and @@ -111,14 +113,29 @@ if ! jq -e -s --arg id "$project_id" --arg endpoint "$project_endpoint" --slurpf $c.service_env_key == ($c.agent_service | gsub("[- ]"; "_") | ascii_upcase) and ($c.service_env_key | test("^[A-Z0-9_]+$")) and $o.toolbox_name == $c.toolbox_name and - (if $c.execution_profile == "managed" then - $c.toolbox_name == ("ci-e2e-tb-" + ($c.agent_name | ltrimstr("ci-prompt-"))) - else $c.execution_profile == "prompt" and $c.toolbox_name == "" and - $o.toolbox_deployment_attempted == false end) and + ($o.skill_name // "") == ($c.skill_name // "") and + ($o.connection_name // "") == ($c.connection_name // "") and + (($c.agent_name | ltrimstr("ci-prompt-")) as $nonce | + if $c.execution_profile == "managed" then + ( + ($c.toolbox_name == ("ci-e2e-tb-" + $nonce) and ($c.skill_name // "") == "" and ($o.skill_deployment_attempted // false) == false and + (if ($o.connection_deployment_attempted // false) then + (($c.connection_name // "") | type) == "string" and + (($c.connection_name // "") | test("^ci-e2e-tb-" + ($nonce[:10]) + "-[0-9a-f]{8}-mcp$")) + else (($c.connection_name // "") == "" or (($c.connection_name // "") | test("^ci-e2e-tb-" + ($nonce[:10]) + "-[0-9a-f]{8}-mcp$"))) end)) + or + (($c.skill_name // "") == ("ci-e2e-sk-" + $nonce) and $c.toolbox_name == "" and $o.toolbox_deployment_attempted == false and + ($c.connection_name // "") == "" and ($o.connection_deployment_attempted // false) == false) + ) + else $c.execution_profile == "prompt" and $c.toolbox_name == "" and $o.toolbox_deployment_attempted == false and + ($c.skill_name // "") == "" and ($o.skill_deployment_attempted // false) == false and + ($c.connection_name // "") == "" and ($o.connection_deployment_attempted // false) == false end) and ($o.cleanup | type) == "object" and ($o.cleanup | to_entries | all( ((.key == $o.agent_name and $o.deployment_attempted) or - (.key == $o.toolbox_name and $o.toolbox_deployment_attempted)) and + (.key == $o.toolbox_name and $o.toolbox_deployment_attempted) or + (.key == ($o.skill_name // "") and ($o.skill_deployment_attempted // false)) or + (.key == ($o.connection_name // "") and ($o.connection_deployment_attempted // false))) and (.value | type) == "string" and (.value | length) > 0))) ' "$ownership" > "$recovery/validation-stdout.txt" 2> "$recovery/validation-stderr.txt"; then fail "Invalid ownership/context or trusted project mismatch; refusing recovery commands." @@ -128,9 +145,13 @@ name=$(jq -r .agent_name "$context") service=$(jq -r .agent_service "$context") env_key=$(jq -r .service_env_key "$context") toolbox=$(jq -r .toolbox_name "$context") +skill=$(jq -r '.skill_name // ""' "$context") +connection=$(jq -r '.connection_name // ""' "$context") agent_pending=$(jq -r '.deployment_attempted and .cleanup[.agent_name] != "absent"' "$ownership") toolbox_pending=$(jq -r '.toolbox_deployment_attempted and .cleanup[.toolbox_name] != "absent"' "$ownership") -if [ "$agent_pending" = false ] && [ "$toolbox_pending" = false ]; then exit 0; fi +skill_pending=$(jq -r '(.skill_deployment_attempted // false) and .cleanup[.skill_name] != "absent"' "$ownership") +connection_pending=$(jq -r '(.connection_deployment_attempted // false) and (.connection_name // "") != "" and .cleanup[.connection_name] != "absent"' "$ownership") +if [ "$agent_pending" = false ] && [ "$toolbox_pending" = false ] && [ "$skill_pending" = false ] && [ "$connection_pending" = false ]; then exit 0; fi work=$(jq -r .working_directory "$ownership") case "${OSTYPE:-}" in msys*|cygwin*) work=$(cygpath -u "$work") ;; esac @@ -177,8 +198,19 @@ toolbox_absent() { --metadata "$(native_path "$last_stdout")" --context "$context" \ >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" } +skill_absent() { + timeout --kill-after=10s 60s "$python" "$helper" skill-absent \ + --metadata "$(native_path "$last_stdout")" --context "$context" \ + >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" +} +connection_absent() { + timeout --kill-after=10s 60s "$python" "$helper" connection-absent \ + --metadata "$(native_path "$last_stdout")" --context "$context" \ + >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" +} # Pin both process variables and the retained azd environment; do not trust defaults. +export AZD_ALPHA_ENABLE_AGENTS_MANAGED_HARNESS=true export FOUNDRY_PROJECT_ENDPOINT="$project_endpoint" AZURE_AI_PROJECT_ENDPOINT="$project_endpoint" export AZURE_AI_PROJECT_ID="$project_id" AZURE_ENV_NAME="$name" "AGENT_${env_key}_NAME=$name" azd_project=(azd -C "$(native_path "$project")") @@ -225,7 +257,47 @@ cleanup_toolbox() { done return 1 } +cleanup_skill() { + local attempt + save_status "$skill" unverified || return 1 + run_command "${azd_project[@]}" ai skill list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1 + if skill_absent; then save_status "$skill" absent; return $?; fi + jq -e -s --arg name "$skill" ' + length == 1 and (.[0] | type) == "array" and + (.[0] | all(type == "object" and (.name | type) == "string" and (.name | test("\\S")))) and + (.[0] | any((.name | ascii_downcase) == $name)) + ' "$(native_path "$last_stdout")" >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" || return 1 + run_command "${azd_project[@]}" ai skill delete "$skill" --project-endpoint "$project_endpoint" \ + --force --no-prompt --output json || cleanup_failed=true + for attempt in 1 2 3; do + run_command "${azd_project[@]}" ai skill list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1 + if skill_absent; then save_status "$skill" absent; return $?; fi + [ "$attempt" -eq 3 ] || sleep 5 + done + return 1 +} +cleanup_connection() { + local attempt + save_status "$connection" unverified || return 1 + run_command "${azd_project[@]}" ai connection list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1 + if connection_absent; then save_status "$connection" absent; return $?; fi + jq -e -s --arg name "$connection" ' + length == 1 and (.[0] | type) == "array" and + (.[0] | all(type == "object" and (.name | type) == "string" and (.name | test("\\S")))) and + (.[0] | any((.name | ascii_downcase) == $name)) + ' "$(native_path "$last_stdout")" >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" || return 1 + run_command "${azd_project[@]}" ai connection delete "$connection" --project-endpoint "$project_endpoint" \ + --force --no-prompt || cleanup_failed=true + for attempt in 1 2 3; do + run_command "${azd_project[@]}" ai connection list --project-endpoint "$project_endpoint" --output json --no-prompt || return 1 + if connection_absent; then save_status "$connection" absent; return $?; fi + [ "$attempt" -eq 3 ] || sleep 5 + done + return 1 +} if [ "$agent_pending" = true ]; then cleanup_agent || cleanup_failed=true; fi -# Agent errors must not strand an independently owned toolbox. +# Agent errors must not strand independently owned resources. +if [ "$connection_pending" = true ]; then cleanup_connection || cleanup_failed=true; fi if [ "$toolbox_pending" = true ]; then cleanup_toolbox || cleanup_failed=true; fi +if [ "$skill_pending" = true ]; then cleanup_skill || cleanup_failed=true; fi if [ "$cleanup_failed" = true ]; then exit 1; fi diff --git a/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py b/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py index 3e4dd11bb..2a4b014a8 100644 --- a/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py +++ b/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py @@ -5,6 +5,7 @@ import argparse from datetime import datetime, timezone +import hashlib import json from pathlib import Path import re @@ -52,6 +53,82 @@ def owned_toolbox_name(context: dict) -> str: return name +def owned_skill_name(context: dict) -> str: + name = context.get("skill_name") + agent_name = context.get("agent_name") + if (context.get("execution_profile") != "managed" + or not isinstance(agent_name, str) or not re.fullmatch(r"ci-prompt-[A-Za-z0-9-]+", agent_name) + or name != "ci-e2e-sk-" + agent_name.removeprefix("ci-prompt-")): + raise LifecycleError("context must identify the managed agent's owned skill") + return name + + +def toolbox_connection_name(toolbox_name: str, version: str, endpoint: str) -> str: + digest = hashlib.sha256(endpoint.encode("utf-8")).hexdigest()[:8] + prefix = re.sub(r"[^A-Za-z0-9_-]", "-", f"{toolbox_name}-v{version}").lstrip("-_")[:20] + return f"{prefix}-{digest}-mcp" + + +def _extract_skill_frontmatter(content: str) -> tuple[str, str]: + trimmed = content.lstrip("\ufeff \t\r\n") + if not trimmed.startswith("---"): + raise LifecycleError("SKILL.md must begin with a YAML front matter block") + rest = trimmed[3:].lstrip("\r\n") + before, sep, after = rest.partition("\n---") + if not sep: + raise LifecycleError("SKILL.md front matter is missing its closing delimiter") + nl = after.find("\n") + body = after[nl + 1:] if nl >= 0 else "" + return before, body + + +def _validate_skill_archive(archive: str) -> str: + if not isinstance(archive, str) or not archive.strip(): + raise LifecycleError("skill service must declare a nonempty archive path") + normalized = archive.strip().replace("\\", "/") + parts = [part for part in normalized.split("/") if part not in ("", ".")] + if len(parts) != 2 or parts[0] != "skills" or ".." in parts or not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", parts[1]): + raise LifecycleError("skill archive must be ./skills/") + return parts[1] + + +def _validate_managed_tools(tools: object, toolbox_key: str) -> None: + if not isinstance(tools, list) or any(not isinstance(tool, dict) for tool in tools): + raise LifecycleError("managed agent tools must be an array of mappings") + mcp_count = 0 + for tool in tools: + tool_type = tool.get("type") + if tool_type == "web_search": + if set(tool) != {"type"}: + raise LifecycleError("managed web_search tool cannot have extra properties") + elif tool_type in ("microsoft_copilot_toolset_preview", "github_copilot_toolset_preview"): + if set(tool) - {"type", "defaultConfig"}: + raise LifecycleError("managed copilot toolset supports only defaultConfig") + default_config = tool.get("defaultConfig", {}) + if (not isinstance(default_config, dict) or set(default_config) - {"enabled"} + or not isinstance(default_config.get("enabled", False), bool)): + raise LifecycleError("managed copilot toolset defaultConfig.enabled must be a boolean") + elif tool_type == "mcp" and toolbox_key: + mcp_count += 1 + if mcp_count > 1 or tool != { + "type": "mcp", "server_label": toolbox_key, "require_approval": "never", + }: + raise LifecycleError("managed toolbox MCP tool must target only the owned toolbox with require_approval: never") + else: + raise LifecycleError("managed agents support only web_search, copilot toolset, and owned toolbox MCP tools") + + +def _validate_agent_toolboxes(agent: dict, toolbox_key: str) -> None: + has_legacy = "toolbox" in agent + has_plural = "toolboxes" in agent + if has_legacy == has_plural: + raise LifecycleError("managed toolbox agent must declare either toolboxes or legacy toolbox") + if has_legacy and agent.get("toolbox") != {"name": toolbox_key}: + raise LifecycleError("managed agent legacy toolbox must match the owned toolbox") + if has_plural and agent.get("toolboxes") not in ([toolbox_key], [{"name": toolbox_key}]): + raise LifecycleError("managed agent toolboxes must reference only the owned toolbox") + + def validate_manifest(document: dict, service_key: str) -> str: if not isinstance(document, dict): raise LifecycleError("manifest must be a mapping") @@ -60,6 +137,10 @@ def validate_manifest(document: dict, service_key: str) -> str: raise LifecycleError("services must be a mapping of service mappings") if "hooks" in document or any("hooks" in service for service in services.values()): raise LifecycleError("lifecycle hooks are not supported by the prompt CI adapter") + if any(service.get("host") == "azure.ai.connection" for service in services.values()) or any( + "connections" in item for item in [document, *services.values()] + ): + raise LifecycleError("connections are not supported by the prompt CI adapter") projects = [key for key, service in services.items() if service.get("host") == "azure.ai.project"] agent = services.get(service_key, {}) if agent.get("host") != "azure.ai.agent" or agent.get("kind") != "prompt": @@ -67,27 +148,45 @@ def validate_manifest(document: dict, service_key: str) -> str: profile = execution_profile(agent) if profile == "managed": toolboxes = [key for key, service in services.items() if service.get("host") == "azure.ai.toolbox"] - if len(projects) != 1 or len(toolboxes) != 1 or len(services) != 3: - raise LifecycleError("managed agents require exactly one borrowed project, one toolbox, and one agent") - if any(key in item for item in [document, *services.values()] for key in ("skills", "connections")): - raise LifecycleError("managed skills and connections are not supported") - toolbox_key = toolboxes[0] - toolbox = services[toolbox_key] - if (set(toolbox) - {"host", "uses", "description", "tools"} - or toolbox.get("uses") != projects or toolbox.get("tools") != [{"type": "web_search"}]): - raise LifecycleError("managed toolbox must use only the borrowed project and contain only web_search") - if (agent.get("uses") not in (projects + toolboxes, toolboxes + projects) - or agent.get("project", ".") != "." - or agent.get("toolbox") != {"name": toolbox_key}): - raise LifecycleError("managed agent must reference the project and toolbox once, with only toolbox.name") - if agent.get("tools", []) != [] or any(key in agent for key in ("codeConfiguration", "image")): - raise LifecycleError("managed agents cannot have direct tools or other dependencies") + skills = [key for key, service in services.items() if service.get("host") == "azure.ai.skill"] + if len(projects) != 1 or len(toolboxes) + len(skills) != 1 or len(services) != 3: + raise LifecycleError("managed agents require exactly one borrowed project and either one toolbox or one skill") + if any(key in agent for key in ("codeConfiguration", "image", "memory", "policies")): + raise LifecycleError("managed agents cannot have code, image, memory, or policy dependencies") + if agent.get("project", ".") != ".": + raise LifecycleError("managed agent must use project: .") + if toolboxes: + if "skills" in document or any("skills" in service for service in services.values()): + raise LifecycleError("managed toolbox agents cannot also declare skills") + toolbox_key = toolboxes[0] + toolbox = services[toolbox_key] + if (set(toolbox) - {"host", "uses", "description", "tools"} + or toolbox.get("uses") != projects or toolbox.get("tools") != [{"type": "web_search"}]): + raise LifecycleError("managed toolbox must use only the borrowed project and contain only web_search") + if agent.get("uses") not in (projects + toolboxes, toolboxes + projects): + raise LifecycleError("managed agent must reference the project and toolbox once") + _validate_agent_toolboxes(agent, toolbox_key) + _validate_managed_tools(agent.get("tools", []), toolbox_key) + return projects[0] + if any(key in agent for key in ("toolbox", "toolboxes")): + raise LifecycleError("managed skill agents cannot declare toolboxes") + skill_key = skills[0] + skill = services[skill_key] + if set(skill) != {"host", "uses", "archive"} or skill.get("uses") != projects: + raise LifecycleError("managed skill must use only the borrowed project and declare archive") + if _validate_skill_archive(skill["archive"]) != skill_key: + raise LifecycleError("managed skill archive directory must match the skill service key") + if agent.get("uses") not in (projects + skills, skills + projects): + raise LifecycleError("managed agent must reference the project and skill once") + if "skills" in agent and agent["skills"] not in ([skill_key], [{"name": skill_key}]): + raise LifecycleError("managed agent skills must reference only the owned skill") + _validate_managed_tools(agent.get("tools", []), "") return projects[0] if len(projects) != 1 or len(services) != 2: raise LifecycleError("only an agent and one borrowed project are supported; skills, toolboxes and connections are not supported by this adapter") if agent.get("uses") != projects or agent.get("project", ".") != ".": raise LifecycleError("agent must consume only the borrowed project and use project: .") - if any(key in agent for key in ("skills", "toolbox", "codeConfiguration", "image")): + if any(key in agent for key in ("skills", "toolbox", "toolboxes", "codeConfiguration", "image")): raise LifecycleError("agent has dependencies outside the prompt CLI adapter") tools = agent.get("tools", []) if not isinstance(tools, list): @@ -103,6 +202,34 @@ def validate_manifest(document: dict, service_key: str) -> str: return projects[0] +def _rewrite_skill_bundle(source_dir: Path, original_skill: str, owned_skill: str) -> None: + skills_dir = source_dir / "skills" + original_dir = skills_dir / original_skill + owned_dir = skills_dir / owned_skill + if not original_dir.is_dir() or owned_dir.exists(): + raise LifecycleError("skill archive directory is missing or conflicts with owned skill directory") + entries = [entry for entry in skills_dir.iterdir() if not entry.name.startswith(".")] + if entries != [original_dir]: + raise LifecycleError("skills directory must contain only the declared skill bundle") + skill_md = original_dir / "SKILL.md" + if not skill_md.is_file() or skill_md.is_symlink(): + raise LifecycleError("skill archive directory must contain a regular SKILL.md file") + front, body = _extract_skill_frontmatter(skill_md.read_text(encoding="utf-8")) + if not body.strip(): + raise LifecycleError("SKILL.md must contain nonempty instructions") + metadata = yaml.load(front, Loader=ManifestLoader) + if not isinstance(metadata, dict) or not str(metadata.get("description", "")).strip(): + raise LifecycleError("SKILL.md front matter must declare a nonempty description") + if metadata.get("name", original_skill) != original_skill: + raise LifecycleError("SKILL.md front matter name must match the skill service key") + metadata["name"] = owned_skill + skill_md.write_text( + "---\n" + yaml.safe_dump(metadata, sort_keys=False).strip() + "\n---\n" + body, + encoding="utf-8", newline="\n", + ) + original_dir.rename(owned_dir) + + def prepare_manifest(repo: Path, sample: Path, model: str, name: str) -> tuple[dict, str]: record = sample_record(repo, sample / "azure.yaml") document = yaml.load((sample / "azure.yaml").read_text(encoding="utf-8"), Loader=ManifestLoader) @@ -113,20 +240,53 @@ def prepare_manifest(repo: Path, sample: Path, model: str, name: str) -> tuple[d agent["description"] = f"Temporary prompt-agent CI resource: {name}" agent["model"] = model if execution_profile(agent) == "managed": - original_toolbox = agent["toolbox"]["name"] - toolbox_name = owned_toolbox_name({ - "execution_profile": "managed", "agent_name": name, - "toolbox_name": "ci-e2e-tb-" + name.removeprefix("ci-prompt-"), - }) - if any(key.casefold() == toolbox_name.casefold() for key in document["services"] if key != original_toolbox): - raise LifecycleError("owned toolbox name conflicts with an existing service") - document["services"] = { - toolbox_name if key == original_toolbox else key: service - for key, service in document["services"].items() - } - agent["uses"] = [toolbox_name if key == original_toolbox else key for key in agent["uses"]] - agent["toolbox"]["name"] = toolbox_name + toolboxes = [key for key, service in document["services"].items() if service.get("host") == "azure.ai.toolbox"] + skills = [key for key, service in document["services"].items() if service.get("host") == "azure.ai.skill"] + if toolboxes: + original_toolbox = toolboxes[0] + toolbox_name = owned_toolbox_name({ + "execution_profile": "managed", "agent_name": name, + "toolbox_name": "ci-e2e-tb-" + name.removeprefix("ci-prompt-"), + }) + if any(key.casefold() == toolbox_name.casefold() for key in document["services"] if key != original_toolbox): + raise LifecycleError("owned toolbox name conflicts with an existing service") + document["services"] = { + toolbox_name if key == original_toolbox else key: service + for key, service in document["services"].items() + } + agent["uses"] = [toolbox_name if key == original_toolbox else key for key in agent["uses"]] + if "toolbox" in agent: + agent["toolbox"] = {"name": toolbox_name} + else: + agent["toolboxes"] = [ + {"name": toolbox_name} if isinstance(item, dict) else toolbox_name + for item in agent["toolboxes"] + ] + for tool in agent.get("tools", []): + if tool.get("type") == "mcp" and tool.get("server_label") == original_toolbox: + tool["server_label"] = toolbox_name + else: + original_skill = skills[0] + skill_name = owned_skill_name({ + "execution_profile": "managed", "agent_name": name, + "skill_name": "ci-e2e-sk-" + name.removeprefix("ci-prompt-"), + }) + if any(key.casefold() == skill_name.casefold() for key in document["services"] if key != original_skill): + raise LifecycleError("owned skill name conflicts with an existing service") + document["services"] = { + skill_name if key == original_skill else key: service + for key, service in document["services"].items() + } + document["services"][skill_name]["archive"] = f"./skills/{skill_name}" + agent["uses"] = [skill_name if key == original_skill else key for key in agent["uses"]] + if "skills" in agent: + agent["skills"] = [ + {"name": skill_name} if isinstance(item, dict) else skill_name + for item in agent["skills"] + ] document["services"][project_key].pop("deployments", None) + if isinstance(document.get("infra"), dict) and set(document["infra"]) == {"provider", "path"} and document["infra"].get("provider") == "microsoft.foundry": + document["infra"] = {"provider": "microsoft.foundry"} validate_manifest(document, service_key) return document, service_key @@ -162,11 +322,19 @@ def prepare(args) -> None: sample != repo / root and sample.is_relative_to(repo / root) for root in SAMPLE_ROOTS ): raise LifecycleError("sample must be a repository-relative directory below a configured prompt sample root") + original_doc = yaml.load((sample / "azure.yaml").read_text(encoding="utf-8"), Loader=ManifestLoader) + original_skills = [ + key for key, svc in (original_doc.get("services") or {}).items() + if isinstance(svc, dict) and svc.get("host") == "azure.ai.skill" + ] if isinstance(original_doc, dict) else [] name = "ci-prompt-" + uuid.uuid4().hex document, service_key = prepare_manifest(repo, sample, args.model_deployment, name) agent = document["services"][service_key] profile = execution_profile(agent) - toolbox_name = agent["toolbox"]["name"] if profile == "managed" else "" + toolboxes = [key for key, svc in document["services"].items() if svc.get("host") == "azure.ai.toolbox"] + skills = [key for key, svc in document["services"].items() if svc.get("host") == "azure.ai.skill"] + toolbox_name = toolboxes[0] if profile == "managed" and toolboxes else "" + skill_name = skills[0] if profile == "managed" and skills else "" spec = args.spec or repo / fixture_dir_for_sample(sample.relative_to(repo).as_posix()) / "test-spec.yml" plan = plan_contract(spec, profile) artifacts = args.artifacts.resolve() @@ -180,6 +348,12 @@ def prepare(args) -> None: source = work / "input" if source.exists() or artifacts.is_relative_to(source): raise LifecycleError("work/input must be new and separate from artifacts") + shutil.copytree(sample, source, ignore=shutil.ignore_patterns( + ".azure", ".env", ".env.*", ".git", ".venv", "__pycache__", + )) + if skill_name: + _rewrite_skill_bundle(source, original_skills[0], skill_name) + (source / "azure.yaml").write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8", newline="\n") artifacts.mkdir(parents=True, exist_ok=False) evidence = artifacts / "evidence" evidence.mkdir() @@ -191,21 +365,21 @@ def prepare(args) -> None: "account_name": args.project_id.split("/")[-3], "execution_profile": profile, "toolbox_name": toolbox_name, + "skill_name": skill_name, + "connection_name": "", "project_endpoint": args.project_endpoint, }) write_json(artifacts / "ownership.json", { "agent_name": name, "agent_service": service_key, "toolbox_name": toolbox_name, "toolbox_deployment_attempted": False, + "skill_name": skill_name, "skill_deployment_attempted": False, + "connection_name": "", "connection_deployment_attempted": False, "project_id": args.project_id, "project_endpoint": args.project_endpoint, "created_at": datetime.now(timezone.utc).isoformat(), "working_directory": str(work), "deployment_attempted": False, "cleanup": {}, "response_retention": "not verified: azd does not expose prompt response deletion", }) - shutil.copytree(sample, source, ignore=shutil.ignore_patterns( - ".azure", ".env", ".env.*", ".git", ".venv", "__pycache__", - )) - (source / "azure.yaml").write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8", newline="\n") turns = [] for test in plan["tests"]: for index, turn in enumerate(test["turns"]): @@ -225,13 +399,32 @@ def verify_adopted(manifest: Path, context: dict, model_deployment: str) -> None profile = execution_profile(agent) if profile != context.get("execution_profile", "prompt"): raise LifecycleError("init changed the requested execution profile") - if profile == "managed" and agent["toolbox"] != {"name": owned_toolbox_name(context)}: - raise LifecycleError("init changed the owned toolbox reference") + if profile == "managed": + if context.get("toolbox_name"): + expected_tb = owned_toolbox_name(context) + if "toolbox" in agent and agent["toolbox"] != {"name": expected_tb}: + raise LifecycleError("init changed the owned toolbox reference") + if "toolboxes" in agent and agent["toolboxes"] not in ([expected_tb], [{"name": expected_tb}]): + raise LifecycleError("init changed the owned toolboxes reference") + elif context.get("skill_name"): + expected_sk = owned_skill_name(context) + if expected_sk not in document["services"] or document["services"][expected_sk].get("archive") != f"./skills/{expected_sk}": + raise LifecycleError("init changed the owned skill service") + if "skills" in agent and agent["skills"] not in ([expected_sk], [{"name": expected_sk}]): + raise LifecycleError("init changed the owned skills reference") + else: + raise LifecycleError("managed context must identify an owned toolbox or skill") if agent.get("name") != context["agent_name"] or agent.get("model") != model_deployment: raise LifecycleError("init changed the requested agent identity, model, or kind") -def verify_deployed(metadata: Path, context: dict, model_deployment: str, toolbox_metadata: Path | None = None) -> None: +def verify_deployed( + metadata: Path, + context: dict, + model_deployment: str, + toolbox_metadata: Path | None = None, + skill_metadata: Path | None = None, +) -> None: document = json.loads(metadata.read_text(encoding="utf-8")) if not isinstance(document, dict): raise LifecycleError("azd show metadata must be an object") @@ -246,18 +439,35 @@ def verify_deployed(metadata: Path, context: dict, model_deployment: str, toolbo if profile != context.get("execution_profile", "prompt"): raise LifecycleError("azd show returned an unexpected execution profile") if profile == "managed": - name = owned_toolbox_name(context) - tools = definition.get("tools") - if (not isinstance(tools, list) or len(tools) != 1 or not isinstance(tools[0], dict) - or tools[0].get("type") != "mcp" or tools[0].get("server_label") != name - or not isinstance(tools[0].get("server_url"), str)): - raise LifecycleError("managed agent must attach exactly the owned toolbox MCP server") - if toolbox_metadata is None: - raise LifecycleError("managed agent verification requires toolbox metadata") - verify_toolbox(toolbox_metadata, context, tools[0]["server_url"]) - - -def verify_toolbox(metadata: Path, context: dict, endpoint: str) -> None: + if context.get("toolbox_name"): + name = owned_toolbox_name(context) + tools = definition.get("tools") + mcp_tools = [tool for tool in (tools or []) if isinstance(tool, dict) and tool.get("type") == "mcp"] + if (not isinstance(tools, list) or len(mcp_tools) != 1 + or mcp_tools[0].get("server_label") != name + or not isinstance(mcp_tools[0].get("server_url"), str)): + raise LifecycleError("managed agent must attach exactly the owned toolbox MCP server") + if toolbox_metadata is None: + raise LifecycleError("managed agent verification requires toolbox metadata") + version = verify_toolbox(toolbox_metadata, context, mcp_tools[0]["server_url"]) + connection_id = mcp_tools[0].get("project_connection_id") + if connection_id is not None and connection_id != toolbox_connection_name(name, version, mcp_tools[0]["server_url"]): + raise LifecycleError("managed agent toolbox connection ID does not match the owned toolbox endpoint") + elif context.get("skill_name"): + name = owned_skill_name(context) + skills = definition.get("skills") + if skill_metadata is None: + raise LifecycleError("managed skill agent verification requires skill metadata") + version = verify_skill(skill_metadata, context) + if (not isinstance(skills, list) or len(skills) != 1 or not isinstance(skills[0], dict) + or skills[0].get("name") != name or skills[0].get("version") != version + or skills[0].get("type", "skill_reference") != "skill_reference"): + raise LifecycleError("managed agent must attach the owned skill and version") + else: + raise LifecycleError("managed context must identify an owned toolbox or skill") + + +def verify_toolbox(metadata: Path, context: dict, endpoint: str) -> str: name = owned_toolbox_name(context) document = json.loads(metadata.read_text(encoding="utf-8")) if not isinstance(document, dict): @@ -281,6 +491,22 @@ def verify_toolbox(metadata: Path, context: dict, endpoint: str) -> None: + "/versions/" + quote(version["version"], safe="") + "/mcp?api-version=v1") if not isinstance(endpoint, str) or endpoint != expected or document.get("endpoint") != expected: raise LifecycleError("toolbox MCP endpoint must exactly match the owned project, toolbox, and version") + return version["version"] + + +def verify_skill(metadata: Path, context: dict, version: str | None = None) -> str: + name = owned_skill_name(context) + document = json.loads(metadata.read_text(encoding="utf-8")) + if not isinstance(document, dict): + raise LifecycleError("skill show metadata must be an object") + default_version = document.get("default_version") + latest_version = document.get("latest_version") + if (document.get("name") != name or not isinstance(default_version, str) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", default_version) + or (latest_version is not None and latest_version != default_version) + or (version is not None and default_version != version)): + raise LifecycleError("skill show returned unexpected owned name or version") + return default_version def toolbox_absent(metadata: Path, context: dict) -> bool: @@ -295,6 +521,30 @@ def toolbox_absent(metadata: Path, context: dict) -> bool: return all(toolbox["name"].casefold() != name.casefold() for toolbox in toolboxes) +def skill_absent(metadata: Path, context: dict) -> bool: + name = owned_skill_name(context) + skills = json.loads(metadata.read_text(encoding="utf-8")) + if not isinstance(skills, list) or any( + not isinstance(skill, dict) or not isinstance(skill.get("name"), str) or not skill["name"].strip() + for skill in skills + ): + raise LifecycleError("skill list metadata must contain an array of nonempty skill names") + return all(skill["name"].casefold() != name.casefold() for skill in skills) + + +def connection_absent(metadata: Path, context: dict) -> bool: + name = context.get("connection_name") + if not isinstance(name, str) or not re.fullmatch(r"ci-e2e-tb-[0-9a-f]{10}-[0-9a-f]{8}-mcp", name): + raise LifecycleError("context must identify the owned toolbox connection") + connections = json.loads(metadata.read_text(encoding="utf-8")) + if not isinstance(connections, list) or any( + not isinstance(conn, dict) or not isinstance(conn.get("name"), str) or not conn["name"].strip() + for conn in connections + ): + raise LifecycleError("connection list metadata must contain an array of nonempty connection names") + return all(conn["name"].casefold() != name.casefold() for conn in connections) + + def agent_absent(stderr: str, exit_code: int, name: str, endpoint: str) -> bool: """Fail closed unless azd reports an exact-target prompt GET with HTTP 404.""" if exit_code in (0, 124, 137): @@ -348,13 +598,18 @@ def main(argv=None) -> int: deployed = commands.add_parser("verify-deployed") deployed.add_argument("--metadata", type=Path, required=True) deployed.add_argument("--toolbox-metadata", type=Path) + deployed.add_argument("--skill-metadata", type=Path) for verification in (adopted, deployed): verification.add_argument("--context", type=Path, required=True) verification.add_argument("--model-deployment", required=True) toolbox = commands.add_parser("verify-toolbox") toolbox.add_argument("--endpoint", required=True) toolbox_absence = commands.add_parser("toolbox-absent") - for verification in (toolbox, toolbox_absence): + skill = commands.add_parser("verify-skill") + skill.add_argument("--version", required=True) + skill_absence = commands.add_parser("skill-absent") + connection_absence = commands.add_parser("connection-absent") + for verification in (toolbox, toolbox_absence, skill, skill_absence, connection_absence): verification.add_argument("--metadata", type=Path, required=True) verification.add_argument("--context", type=Path, required=True) absence = commands.add_parser("absent") @@ -379,11 +634,21 @@ def main(argv=None) -> int: if args.command == "verify-adopted": verify_adopted(args.manifest, context, args.model_deployment) elif args.command == "verify-deployed": - verify_deployed(args.metadata, context, args.model_deployment, args.toolbox_metadata) + verify_deployed( + args.metadata, context, args.model_deployment, + args.toolbox_metadata, args.skill_metadata, + ) elif args.command == "verify-toolbox": - verify_toolbox(args.metadata, context, args.endpoint) + version = verify_toolbox(args.metadata, context, args.endpoint) + print(toolbox_connection_name(owned_toolbox_name(context), version, args.endpoint)) elif args.command == "toolbox-absent": return 0 if toolbox_absent(args.metadata, context) else 1 + elif args.command == "verify-skill": + verify_skill(args.metadata, context, args.version) + elif args.command == "skill-absent": + return 0 if skill_absent(args.metadata, context) else 1 + elif args.command == "connection-absent": + return 0 if connection_absent(args.metadata, context) else 1 else: streams = [args.stderr] + ([args.stdout] if args.stdout else []) return 0 if any(agent_absent( diff --git a/.azure-pipelines/scripts/prompt-agents/provision-prompt-agent.sh b/.azure-pipelines/scripts/prompt-agents/provision-prompt-agent.sh index c4da4dc1d..dc341aace 100644 --- a/.azure-pipelines/scripts/prompt-agents/provision-prompt-agent.sh +++ b/.azure-pipelines/scripts/prompt-agents/provision-prompt-agent.sh @@ -48,12 +48,21 @@ from pathlib import Path import shutil import yaml -sample = Path(os.environ['SAMPLE']) +samples = Path(os.environ['SAMPLE']) work = Path(os.environ['WORK']) -doc = yaml.safe_load((sample / 'azure.yaml').read_text(encoding='utf-8')) +doc = yaml.safe_load((samples / 'azure.yaml').read_text(encoding='utf-8')) services = doc.get('services') if isinstance(doc, dict) else None -if not isinstance(doc, dict) or doc.get('infra') != {'provider': 'microsoft.foundry'}: +infra = doc.get('infra') if isinstance(doc, dict) else None +if not isinstance(doc, dict) or not isinstance(infra, dict) or infra.get('provider') != 'microsoft.foundry' or set(infra) - {'provider', 'path'}: raise SystemExit('Disposable provisioning requires only the microsoft.foundry provider') +if 'path' in infra: + infra_path = str(infra['path']).replace('\\', '/').strip('/') + if not infra_path or any(part in ('', '.', '..') for part in infra_path.split('/')): + raise SystemExit('Invalid infra.path') + infra_dir = samples / infra_path + if infra_dir.exists() and any(p.suffix.lower() in ('.bicep', '.bicepparam', '.tf', '.tfvars', '.json') for p in infra_dir.rglob('*') if p.is_file()): + raise SystemExit('Custom IaC files under infra.path are forbidden for disposable provisioning') +doc['infra'] = {'provider': 'microsoft.foundry'} projects = [s for s in services.values() if isinstance(s, dict) and s.get('host') == 'azure.ai.project'] if isinstance(services, dict) else [] if len(projects) != 1 or len(projects[0].get('deployments', [])) != 1: raise SystemExit('Exactly one project with one declared model deployment is required') @@ -65,10 +74,12 @@ if 'hooks' in doc or any('hooks' in s for s in services.values()): deployment = projects[0]['deployments'][0] if not isinstance(deployment, dict) or not isinstance(deployment.get('name'), str) or not deployment['name']: raise SystemExit('Model deployment must have a name') -shutil.copytree(sample, work / 'input', ignore=shutil.ignore_patterns('.azure', '.env', '.env.*', '.git', '.venv', '__pycache__')) +shutil.copytree(samples, work / 'input', ignore=shutil.ignore_patterns('.azure', '.env', '.env.*', '.git', '.venv', '__pycache__')) +(work / 'input' / 'azure.yaml').write_text(yaml.safe_dump(doc, sort_keys=False), encoding='utf-8') PY cd "$work" +export AZD_ALPHA_ENABLE_AGENTS_MANAGED_HARNESS=true AZURE_SUBSCRIPTION_ID="$subscription" AZURE_LOCATION="$location" \ azd ai agent init -m "$(native_path "$work/input/azure.yaml")" --src project \ --no-prompt --environment "$environment" diff --git a/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh b/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh index d3ef6398e..4f0771715 100644 --- a/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh +++ b/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh @@ -55,6 +55,7 @@ prepare=(prepare --repo "$(native_path "$repo")" --sample "$sample" --project-id --project-endpoint "$project_endpoint" --model-deployment "$model" --artifacts "$(native_path "$artifacts")" --work "$(native_path "$work")") [ -z "$spec" ] || prepare+=(--spec "$(native_path "$spec")") +export AZD_ALPHA_ENABLE_AGENTS_MANAGED_HARNESS=true "$python" "$helper" "${prepare[@]}" context=$(native_path "$artifacts/context.json") name=$(jq -r .agent_name "$context") @@ -62,10 +63,16 @@ service=$(jq -r .agent_service "$context") env_key=$(jq -r .service_env_key "$context") account=$(jq -r .account_name "$context") toolbox=$(jq -r .toolbox_name "$context") +skill=$(jq -r .skill_name "$context") +connection= deployed=false toolbox_deployed=false +skill_deployed=false +connection_deployed=false cleanup_status=not_needed toolbox_cleanup_status=not_needed +skill_cleanup_status=not_needed +connection_cleanup_status=not_needed cleanup_failed=false command_index=0 last_code=0 @@ -73,11 +80,18 @@ last_stdout= last_stderr= save_ownership() { jq --argjson attempted "$deployed" --arg status "$cleanup_status" --arg name "$name" \ - --argjson toolbox_attempted "$toolbox_deployed" --arg toolbox "$toolbox" --arg toolbox_status "$toolbox_cleanup_status" ' + --argjson toolbox_attempted "$toolbox_deployed" --arg toolbox "$toolbox" --arg toolbox_status "$toolbox_cleanup_status" \ + --argjson skill_attempted "$skill_deployed" --arg skill "$skill" --arg skill_status "$skill_cleanup_status" \ + --argjson connection_attempted "$connection_deployed" --arg connection "$connection" --arg connection_status "$connection_cleanup_status" ' .deployment_attempted = $attempted | .toolbox_deployment_attempted = $toolbox_attempted | + .skill_deployment_attempted = $skill_attempted | + .connection_name = $connection | + .connection_deployment_attempted = $connection_attempted | .cleanup = ((if $attempted then {($name): $status} else {} end) + - (if $toolbox_attempted then {($toolbox): $toolbox_status} else {} end)) + (if $toolbox_attempted then {($toolbox): $toolbox_status} else {} end) + + (if $skill_attempted then {($skill): $skill_status} else {} end) + + (if $connection_attempted and ($connection | length) > 0 then {($connection): $connection_status} else {} end)) ' "$(native_path "$artifacts/ownership.json")" > "$work/ownership.json" && cp "$work/ownership.json" "$artifacts/ownership.json" } @@ -131,7 +145,7 @@ finish() { trap - EXIT INT TERM set +e # Leave time for the separately authenticated recovery and infra teardown tasks. - local cleanup_timeout=10 + local cleanup_timeout=20 if [ "$deployed" = true ]; then cleanup_status=unverified if run_command "$cleanup_timeout" cleanup azd env set "AGENT_${env_key}_NAME" "$name" && [ "$last_code" -eq 0 ]; then @@ -149,6 +163,22 @@ finish() { cleanup_failed=true fi [ "$cleanup_status" = absent ] || cleanup_failed=true + if [ "$connection_deployed" = true ] && [ -n "$connection" ]; then + connection_cleanup_status=unverified + run_command "$cleanup_timeout" cleanup azd ai connection delete "$connection" \ + --project-endpoint "$project_endpoint" --force --no-prompt + [ "$last_code" -eq 0 ] || cleanup_failed=true + for attempt in 1 2 3; do + if ! run_command "$cleanup_timeout" cleanup azd ai connection list \ + --project-endpoint "$project_endpoint" --output json --no-prompt; then break; fi + [ "$last_code" -eq 0 ] || break + if "$python" "$helper" connection-absent --metadata "$(native_path "$last_stdout")" --context "$context"; then + connection_cleanup_status=absent; break + fi + [ "$attempt" -eq 3 ] || sleep 5 + done + [ "$connection_cleanup_status" = absent ] || cleanup_failed=true + fi fi if [ "$toolbox_deployed" = true ]; then toolbox_cleanup_status=unverified @@ -166,6 +196,22 @@ finish() { done [ "$toolbox_cleanup_status" = absent ] || cleanup_failed=true fi + if [ "$skill_deployed" = true ]; then + skill_cleanup_status=unverified + run_command "$cleanup_timeout" cleanup azd ai skill delete "$skill" \ + --project-endpoint "$project_endpoint" --force --no-prompt --output json + [ "$last_code" -eq 0 ] || cleanup_failed=true + for attempt in 1 2 3; do + if ! run_command "$cleanup_timeout" cleanup azd ai skill list \ + --project-endpoint "$project_endpoint" --output json --no-prompt; then break; fi + [ "$last_code" -eq 0 ] || break + if "$python" "$helper" skill-absent --metadata "$(native_path "$last_stdout")" --context "$context"; then + skill_cleanup_status=absent; break + fi + [ "$attempt" -eq 3 ] || sleep 5 + done + [ "$skill_cleanup_status" = absent ] || cleanup_failed=true + fi save_ownership || cleanup_failed=true "$python" "$evaluator" evaluate --plan "$(native_path "$artifacts/plan.json")" \ --evidence-dir "$(native_path "$artifacts/evidence")" \ @@ -223,9 +269,29 @@ if [ -n "$toolbox" ]; then toolbox_endpoint=$(tr -d '\r\n' < "$last_stdout") run_command "$command_timeout" checked azd ai toolbox show "$toolbox" --project-endpoint "$project_endpoint" --output json --no-prompt cp "$last_stdout" "$artifacts/toolbox.json" - "$python" "$helper" verify-toolbox --metadata "$(native_path "$artifacts/toolbox.json")" --context "$context" --endpoint "$toolbox_endpoint" + connection=$("$python" "$helper" verify-toolbox --metadata "$(native_path "$artifacts/toolbox.json")" --context "$context" --endpoint "$toolbox_endpoint" | tr -d '\r\n') + jq --arg conn "$connection" '.connection_name = $conn' "$context" > "$work/context.json" && + cp "$work/context.json" "$artifacts/context.json" verify_options=(--toolbox-metadata "$(native_path "$artifacts/toolbox.json")") fi +if [ -n "$skill" ]; then + run_command "$command_timeout" checked azd ai skill list --project-endpoint "$project_endpoint" --output json --no-prompt + "$python" "$helper" skill-absent --metadata "$(native_path "$last_stdout")" --context "$context" || { + echo "cannot confirm generated skill name is unused; refusing deployment" >&2; exit 1; + } + skill_deployed=true + save_ownership + run_command "$command_timeout" checked azd deploy "$skill" --no-prompt --timeout "$command_timeout" + skill_env_key=${skill//-/_} + skill_env_key=${skill_env_key^^} + run_command "$command_timeout" checked azd env get-value "SKILL_${skill_env_key}_VERSION" + skill_version=$(tr -d '\r\n' < "$last_stdout") + run_command "$command_timeout" checked azd ai skill show "$skill" --project-endpoint "$project_endpoint" --output json --no-prompt + cp "$last_stdout" "$artifacts/skill.json" + "$python" "$helper" verify-skill --metadata "$(native_path "$artifacts/skill.json")" --context "$context" --version "$skill_version" + verify_options=(--skill-metadata "$(native_path "$artifacts/skill.json")") +fi +if [ -n "$connection" ]; then connection_deployed=true; fi deployed=true save_ownership run_command "$command_timeout" checked azd deploy "$service" --no-prompt --timeout "$command_timeout" diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py index 9b86aca0a..7a82bda3a 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py @@ -35,6 +35,15 @@ def test_prompt_and_managed_manifest_restrictions(self): for harness_type in ("github_copilot_preview", "microsoft_copilot_preview"): document["services"]["agent"]["harness"] = {"type": harness_type} self.assertEqual(helpers.validate_manifest(document, "agent"), "project") + document["services"]["agent"].pop("toolbox") + document["services"]["agent"].update( + toolboxes=["toolbox"], + tools=[ + {"type": "microsoft_copilot_toolset_preview", "defaultConfig": {"enabled": False}}, + {"type": "mcp", "server_label": "toolbox", "require_approval": "never"}, + ], + ) + self.assertEqual(helpers.validate_manifest(document, "agent"), "project") for bad_harness in ("", {"type": ""}, {"type": "bad name"}, {"type": "ok", "extra": True}): document["services"]["agent"]["harness"] = bad_harness with self.assertRaisesRegex(helpers.LifecycleError, "managed harness.type must be a nonempty identifier"): @@ -43,6 +52,19 @@ def test_prompt_and_managed_manifest_restrictions(self): document["services"]["toolbox"]["tools"] = [{"type": "mcp"}] with self.assertRaisesRegex(helpers.LifecycleError, "only web_search"): helpers.validate_manifest(document, "agent") + skill_doc = {"services": { + "project": {"host": "azure.ai.project"}, + "travel-guide": {"host": "azure.ai.skill", "uses": ["project"], "archive": "./skills/travel-guide"}, + "agent": { + "host": "azure.ai.agent", "kind": "prompt", "project": ".", + "uses": ["project", "travel-guide"], "harness": {"type": "microsoft_copilot_preview"}, + "skills": ["travel-guide"], "tools": [{"type": "web_search"}], + }, + }} + self.assertEqual(helpers.validate_manifest(skill_doc, "agent"), "project") + skill_doc["services"]["travel-guide"]["archive"] = "../outside" + with self.assertRaisesRegex(helpers.LifecycleError, "skill archive"): + helpers.validate_manifest(skill_doc, "agent") def test_absence_requires_exact_owned_target(self): name = "ci-prompt-owned" @@ -56,12 +78,15 @@ def test_absence_requires_exact_owned_target(self): self.assertFalse(helpers.agent_absent(message, 1, "ci-prompt-other", endpoint)) self.assertFalse(helpers.agent_absent(message.replace("404: 404", "403: 403"), 1, name, endpoint)) - def test_toolbox_endpoint_and_absence_are_pinned(self): + def test_toolbox_skill_and_connection_absence_are_pinned(self): + nonce = "a" * 32 context = { - "execution_profile": "managed", "agent_name": "ci-prompt-owned", - "toolbox_name": "ci-e2e-tb-owned", "project_endpoint": "https://project.invalid/api/projects/project", + "execution_profile": "managed", "agent_name": "ci-prompt-" + nonce, + "toolbox_name": "ci-e2e-tb-" + nonce, "skill_name": "ci-e2e-sk-" + nonce, + "project_endpoint": "https://project.invalid/api/projects/project", } - endpoint = context["project_endpoint"] + "/toolboxes/ci-e2e-tb-owned/versions/3/mcp?api-version=v1" + endpoint = context["project_endpoint"] + f"/toolboxes/{context['toolbox_name']}/versions/3/mcp?api-version=v1" + context["connection_name"] = helpers.toolbox_connection_name(context["toolbox_name"], "3", endpoint) metadata = { "toolbox": {"name": context["toolbox_name"]}, "version": {"name": context["toolbox_name"], "version": "3", "tools": [{"type": "web_search"}]}, @@ -70,7 +95,7 @@ def test_toolbox_endpoint_and_absence_are_pinned(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "metadata.json" path.write_text(json.dumps(metadata), encoding="utf-8") - helpers.verify_toolbox(path, context, endpoint) + self.assertEqual(helpers.verify_toolbox(path, context, endpoint), "3") with self.assertRaisesRegex(helpers.LifecycleError, "MCP endpoint"): helpers.verify_toolbox(path, context, endpoint.replace("versions/3", "versions/4")) path.write_text(json.dumps({"toolboxes": [{"name": context["toolbox_name"].upper()}]}), encoding="utf-8") @@ -78,6 +103,14 @@ def test_toolbox_endpoint_and_absence_are_pinned(self): path.write_text('{"toolboxes": [{}]}', encoding="utf-8") with self.assertRaisesRegex(helpers.LifecycleError, "nonempty toolbox names"): helpers.toolbox_absent(path, context) + path.write_text(json.dumps({"name": context["skill_name"], "default_version": "1", "latest_version": "1"}), encoding="utf-8") + self.assertEqual(helpers.verify_skill(path, context, "1"), "1") + with self.assertRaisesRegex(helpers.LifecycleError, "skill show returned"): + helpers.verify_skill(path, context, "2") + path.write_text(json.dumps([{"name": context["skill_name"].upper()}]), encoding="utf-8") + self.assertFalse(helpers.skill_absent(path, context)) + path.write_text(json.dumps([{"name": context["connection_name"].upper()}]), encoding="utf-8") + self.assertFalse(helpers.connection_absent(path, context)) if __name__ == "__main__": diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py index 336f586fe..757f7d434 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py @@ -24,9 +24,11 @@ # State is outside the disposable azd workspace so cleanup remains observable. FAKE_CLI = r''' +import hashlib import json import os from pathlib import Path +import re import shutil import sys import time @@ -47,11 +49,23 @@ def absent(): return (f'failed to get prompt agent "{name}": GET {config["endpoint"]}/agents/{name}?api-version=v1\n' 'RESPONSE 404: 404 Not Found\n') +def conn_name(tb_name, version, endpoint): + digest = hashlib.sha256(endpoint.encode("utf-8")).hexdigest()[:8] + prefix = re.sub(r"[^A-Za-z0-9_-]", "-", f"{tb_name}-v{version}").lstrip("-_")[:20] + return f"{prefix}-{digest}-mcp" + save() stdout, stderr, code = "", "", 0 is_toolbox = argv[1:3] == ["ai", "toolbox"] +is_skill = argv[1:3] == ["ai", "skill"] +is_connection = argv[1:3] == ["ai", "connection"] operation = argv[3] if argv[1:2] == ["ai"] else argv[1] -failure = "toolbox_" + operation if is_toolbox else operation +failure = ( + "toolbox_" + operation if is_toolbox + else "skill_" + operation if is_skill + else "connection_" + operation if is_connection + else operation +) if argv[0] == "az": assert argv == ["az", "resource", "show", "--ids", config["project_id"], "--query", 'properties.endpoints."AI Foundry API"', "-o", "tsv"], argv @@ -63,6 +77,7 @@ def absent(): assert argv == ["azd", "extension", "list", "--installed", "--output", "json"], argv stdout = '[{"id":"microsoft.foundry"}]\n' elif operation == "init": + assert os.environ.get("AZD_ALPHA_ENABLE_AGENTS_MANAGED_HARNESS") == "true" source = Path(argv[argv.index("-m") + 1]) assert source == Path.cwd().parent / "input/azure.yaml", argv assert not list(Path.cwd().iterdir()) @@ -71,13 +86,20 @@ def absent(): "--project-id", config["project_id"], "--model-deployment", config["model"], "--agent-name", state["name"], "--src", ".", "--no-prompt", "--environment", state["name"]], argv - shutil.copyfile(source, "azure.yaml") + shutil.copytree(source.parent, ".", dirs_exist_ok=True) document = yaml.safe_load(Path("azure.yaml").read_text(encoding="utf-8")) - if config["managed"]: - state["toolbox"] = document["services"][config["service"]]["toolbox"]["name"] + if config["managed"] == "toolbox": + agent_svc = document["services"][config["service"]] + tb_ref = agent_svc["toolboxes"][0] if "toolboxes" in agent_svc else agent_svc["toolbox"] + state["toolbox"] = tb_ref["name"] if isinstance(tb_ref, dict) else tb_ref assert state["toolbox"] == "ci-e2e-tb-" + state["name"].removeprefix("ci-prompt-") state["toolbox_endpoint"] = (config["endpoint"] + "/toolboxes/" + state["toolbox"] + "/versions/3/mcp?api-version=v1") + state["connection"] = conn_name(state["toolbox"], "3", state["toolbox_endpoint"]) + elif config["managed"] == "skill": + skills = [k for k, v in document["services"].items() if v.get("host") == "azure.ai.skill"] + state["skill"] = skills[0] + assert state["skill"] == "ci-e2e-sk-" + state["name"].removeprefix("ci-prompt-") environment = Path(".azure") / state["name"] environment.mkdir(parents=True) (environment / ".env").write_text("", encoding="utf-8") @@ -86,9 +108,16 @@ def absent(): Path("azure.yaml").write_text(yaml.safe_dump(document), encoding="utf-8") elif operation == "env": if argv[2] == "get-value": - assert argv == ["azd", "env", "get-value", "TOOLBOX_" + state["toolbox"].replace("-", "_").upper() + "_MCP_ENDPOINT"] - assert state["toolbox_deployed"] and not state["deployed"] - stdout = config.get("toolbox_endpoint", state["toolbox_endpoint"]) + "\n" + if config["managed"] == "toolbox": + assert argv == ["azd", "env", "get-value", "TOOLBOX_" + state["toolbox"].replace("-", "_").upper() + "_MCP_ENDPOINT"] + assert state["toolbox_deployed"] and not state["deployed"] + stdout = config.get("toolbox_endpoint", state["toolbox_endpoint"]) + "\n" + elif config["managed"] == "skill": + assert argv == ["azd", "env", "get-value", "SKILL_" + state["skill"].replace("-", "_").upper() + "_VERSION"] + assert state["skill_deployed"] and not state["deployed"] + stdout = config.get("skill_version", "1") + "\n" + else: + raise AssertionError(argv) else: assert argv[1:3] == ["env", "set"] and len(argv) == 5, argv state["environment"][argv[3]] = argv[4] @@ -119,26 +148,80 @@ def absent(): assert state["toolbox_deployed"] assert not state["deployed"] or state["deleted"] state["toolbox_deleted"] = True +elif is_skill: + expected = ["azd", "ai", "skill", operation] + if operation != "list": + expected.append(state["skill"]) + expected += ["--project-endpoint", config["endpoint"]] + expected += (["--force", "--no-prompt", "--output", "json"] if operation == "delete" + else ["--output", "json", "--no-prompt"]) + assert argv == expected, argv + if operation == "list": + mode = config.get("skill_cleanup" if state["skill_deleted"] else "skill_preflight", "absent") + names = ["unowned-skill"] + if mode == "collision" or state["skill_deployed"] and not state["skill_deleted"]: + names.append(state["skill"].upper()) + stdout = json.dumps([{"name": name} for name in names]) + if mode == "auth": + code, stderr = 1, "403 Forbidden listing skills\n" + elif mode == "malformed": + stdout = '[{}]' + elif operation == "show": + assert state["skill_deployed"] and not state["deployed"] + stdout = json.dumps({"name": state["skill"], "default_version": "1", "latest_version": "1"}) + elif operation == "delete": + assert state["skill_deployed"] + assert not state["deployed"] or state["deleted"] + state["skill_deleted"] = True +elif is_connection: + if operation == "list": + assert argv == ["azd", "ai", "connection", "list", "--project-endpoint", config["endpoint"], "--output", "json", "--no-prompt"], argv + names = ["unowned-conn"] + if state["deployed"] and not state["connection_deleted"]: + names.append(state["connection"].upper()) + stdout = json.dumps([{"name": name} for name in names]) + elif operation == "delete": + assert argv == ["azd", "ai", "connection", "delete", state["connection"], "--project-endpoint", config["endpoint"], "--force", "--no-prompt"], argv + assert state["deployed"] and state["deleted"] + state["connection_deleted"] = True + else: + raise AssertionError(argv) elif operation == "deploy": assert state["environment"]["AGENT_CHAT_AGENT_NAME"] == state["name"] - target = state["toolbox"] if config["managed"] and argv[2] == state["toolbox"] else config["service"] + if config["managed"] == "toolbox" and argv[2] == state["toolbox"]: + target = state["toolbox"] + elif config["managed"] == "skill" and argv[2] == state["skill"]: + target = state["skill"] + else: + target = config["service"] assert argv == ["azd", "deploy", target, "--no-prompt", "--timeout", "20"], argv if target == config["service"]: - assert not config["managed"] or state["toolbox_deployed"] + assert not config["managed"] or state["toolbox_deployed"] or state["skill_deployed"] state["deployed"] = True - else: + elif config["managed"] == "toolbox": state["toolbox_deployed"] = True failure = "toolbox_deploy" + else: + state["skill_deployed"] = True + failure = "skill_deploy" elif operation == "show": assert argv == ["azd", "ai", "agent", "show", config["service"], "--output", "json", "--no-prompt"], argv mode = config.get("cleanup" if state["deleted"] else "preflight", "absent") if state["deployed"] and not state["deleted"] or mode == "collision": definition = {"kind": "prompt", "model": config["model"]} - if config["managed"]: - definition.update(harness={"type": "github_copilot_preview"}, tools=[{ - "type": "mcp", "server_label": state["toolbox"], - "server_url": state["toolbox_endpoint"], "require_approval": "never"}]) - definition["tools"][0].update(config.get("mcp_changes", {})) + if config["managed"] == "toolbox": + definition.update(harness={"type": "microsoft_copilot_preview"}, tools=[ + {"type": "microsoft_copilot_toolset_preview", "default_config": {"enabled": False}}, + {"type": "mcp", "server_label": state["toolbox"], + "server_url": state["toolbox_endpoint"], "project_connection_id": state["connection"], + "require_approval": "never"}, + ]) + definition["tools"][1].update(config.get("mcp_changes", {})) + elif config["managed"] == "skill": + definition.update(harness={"type": "microsoft_copilot_preview"}, skills=[{ + "type": "skill_reference", "name": state["skill"], "version": "1", + }], tools=[{"type": "web_search"}]) + definition["skills"][0].update(config.get("skill_changes", {})) definition.update(config.get("definition_changes", {})) metadata = {"name": state["name"], "versions": {"latest": {"version": "7", "definition": definition}}} metadata.update(config.get("metadata_changes", {})) @@ -234,18 +317,41 @@ def configure(self, *, managed=False): agent = {"host": "azure.ai.agent", "kind": "prompt", "name": "source-agent", "model": "source-model", "project": ".", "uses": ["project"]} services = {"project": {"host": "azure.ai.project", "deployments": [{"name": "source-model"}]}} - if managed: - agent.update(harness={"type": "github_copilot_preview"}, uses=["project", "sample-toolbox"], - toolbox={"name": "sample-toolbox"}) + mode = "toolbox" if managed is True else managed + if mode == "toolbox": + agent.update( + harness={"type": "microsoft_copilot_preview"}, + uses=["project", "sample-toolbox"], + toolboxes=["sample-toolbox"], + tools=[ + {"type": "microsoft_copilot_toolset_preview", "defaultConfig": {"enabled": False}}, + {"type": "mcp", "server_label": "sample-toolbox", "require_approval": "never"}, + ], + ) services["sample-toolbox"] = {"host": "azure.ai.toolbox", "uses": ["project"], "tools": [{"type": "web_search"}]} + elif mode == "skill": + skill_dir = self.sample / "skills/sample-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "---\nname: sample-skill\ndescription: Sample skill\n---\n# Instructions\nFollow steps.\n", + encoding="utf-8", + ) + agent.update( + harness={"type": "microsoft_copilot_preview"}, + uses=["project", "sample-skill"], + skills=["sample-skill"], + tools=[{"type": "web_search"}], + ) + services["sample-skill"] = {"host": "azure.ai.skill", "uses": ["project"], + "archive": "./skills/sample-skill"} services[SERVICE] = agent self.document = {"name": "basic", "services": services} self.contract = {"version": 1, "sample": {"owner": "tester", "experiences": ["azd"]}, "tests": [{"name": "answer", "turns": [{"input": "Say hello", "assertions": [ {"source": "assistant_text", "type": "equals", "value": "Hello"}]}]}]} self.config = {"endpoint": ENDPOINT, "project_id": PROJECT_ID, "model": MODEL, - "service": SERVICE, "managed": managed} + "service": SERVICE, "managed": mode} def read_json(self, name): return json.loads((self.artifacts / name).read_text(encoding="utf-8")) @@ -258,7 +364,9 @@ def run_agent(self, *, passed=True, prepared=True): self.fixture.write_text(yaml.safe_dump(self.contract), encoding="utf-8") self.state_path.write_text(json.dumps({"config": self.config, "commands": [], "workspaces": [], "environment": {}, "inputs": [], "deployed": False, "deleted": False, - "toolbox_deployed": False, "toolbox_deleted": False}), encoding="utf-8") + "toolbox_deployed": False, "toolbox_deleted": False, + "skill_deployed": False, "skill_deleted": False, + "connection_deleted": False}), encoding="utf-8") before = {p.relative_to(self.sample): p.read_bytes() for p in self.sample.rglob("*") if p.is_file()} env = {**os.environ, "PATH": str(self.bin) + os.pathsep + self.tool_path, "PYTHON": Path(sys.executable).as_posix(), "PYTHONUTF8": "1", @@ -298,12 +406,23 @@ def run_agent(self, *, passed=True, prepared=True): def assert_cleaned(self): ownership = self.read_json("ownership.json") expected = {self.state["name"]: "absent"} - if self.config["managed"]: + if self.config["managed"] == "toolbox": expected[self.state["toolbox"]] = "absent" + expected[self.state["connection"]] = "absent" self.assertTrue(self.state["toolbox_deleted"]) + self.assertTrue(self.state["connection_deleted"]) self.assertEqual(len(self.operations("delete", "toolbox")), 1) + self.assertEqual(len(self.operations("delete", "connection")), 1) self.assertLess(self.commands.index(self.operations("delete")[0]), + self.commands.index(self.operations("delete", "connection")[0])) + self.assertLess(self.commands.index(self.operations("delete", "connection")[0]), self.commands.index(self.operations("delete", "toolbox")[0])) + elif self.config["managed"] == "skill": + expected[self.state["skill"]] = "absent" + self.assertTrue(self.state["skill_deleted"]) + self.assertEqual(len(self.operations("delete", "skill")), 1) + self.assertLess(self.commands.index(self.operations("delete")[0]), + self.commands.index(self.operations("delete", "skill")[0])) self.assertEqual(ownership["cleanup"], expected) self.assertEqual(self.result["cleanup"], expected) self.assertEqual(len(self.operations("delete")), 1) @@ -313,8 +432,16 @@ def assert_not_owned(self): ownership = self.read_json("ownership.json") self.assertFalse(ownership["deployment_attempted"]) self.assertFalse(ownership["toolbox_deployment_attempted"]) + self.assertFalse(ownership["skill_deployment_attempted"]) + self.assertFalse(ownership["connection_deployment_attempted"]) self.assertEqual(ownership["cleanup"], {}) - self.assertEqual(self.operations("delete") + self.operations("delete", "toolbox"), []) + self.assertEqual( + self.operations("delete") + + self.operations("delete", "toolbox") + + self.operations("delete", "skill") + + self.operations("delete", "connection"), + [], + ) def test_success_adopts_manifest_records_evidence_and_cleans(self): self.document["services"][SERVICE]["tools"] = [{"type": "mcp", "server_label": "public", @@ -453,6 +580,36 @@ def test_managed_wrong_attachment_cleans_without_invoking(self): self.assertEqual(self.operations("invoke"), []) self.assert_cleaned() + def test_managed_skill_success_orders_and_cleans_both_resources(self): + self.configure(managed="skill") + self.run_agent() + self.assert_cleaned() + skill = self.state["skill"] + adopted = yaml.safe_load((self.artifacts / "azure.yaml").read_text(encoding="utf-8")) + self.assertEqual(list(adopted["services"]), ["project", skill, SERVICE]) + self.assertEqual(adopted["services"][SERVICE]["uses"], ["project", skill]) + self.assertEqual(adopted["services"][skill]["archive"], f"./skills/{skill}") + self.assertEqual(self.read_json("skill.json")["default_version"], "1") + self.assertEqual(self.read_json("context.json")["execution_profile"], "managed") + self.assertLess(self.commands.index(["azd", "deploy", skill, "--no-prompt", "--timeout", "20"]), + self.commands.index(["azd", "deploy", SERVICE, "--no-prompt", "--timeout", "20"])) + + def test_managed_skill_deploy_failure_cleans_only_skill(self): + self.configure(managed="skill") + self.config["fail_operation"] = "skill_deploy" + self.run_agent(passed=False) + self.assertFalse(self.state["deployed"]) + self.assertEqual(self.operations("delete"), []) + self.assertTrue(self.state["skill_deleted"]) + self.assertEqual(self.result["cleanup"], {self.state["skill"]: "absent"}) + + def test_managed_wrong_skill_attachment_cleans_without_invoking(self): + self.configure(managed="skill") + self.config["skill_changes"] = {"version": "99"} + self.assertIn("owned skill and version", self.run_agent(passed=False).stderr) + self.assertEqual(self.operations("invoke"), []) + self.assert_cleaned() + if __name__ == "__main__": unittest.main() diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_pipeline.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_pipeline.py index 0c09322a6..c796cc77a 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_pipeline.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_pipeline.py @@ -124,7 +124,7 @@ def test_recovery_is_authenticated_always_task_after_runner(self): cleanup_timeout = int(re.search(r"local cleanup_timeout=(\d+)", runner)[1]) self.assertGreater( self.job["cancelTimeoutInMinutes"] * 60, - 600 + 10 + 9 * (cleanup_timeout + 10) + 20 + + 600 + 10 + 14 * (cleanup_timeout + 10) + 25 + (self.recovery["timeoutInMinutes"] + 30) * 60, ) diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py index 4b7cb5344..5ef6d031c 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py @@ -19,6 +19,8 @@ MODEL = "borrowed-model" NAME = "ci-prompt-" + "a" * 32 TOOLBOX = "ci-e2e-tb-" + "a" * 32 +SKILL = "ci-e2e-sk-" + "a" * 32 +CONN = "ci-e2e-tb-aaaaaaaaaa-12345678-mcp" SERVICE = "chat-agent" FAKE_CLI = r''' @@ -35,6 +37,7 @@ assert argv[:2] == ["azd", "-C"], argv assert Path(argv[2]) == Path(state["project"]), argv assert (Path(argv[2]) / ".azure" / state["name"] / ".env").is_file() +assert os.environ.get("AZD_ALPHA_ENABLE_AGENTS_MANAGED_HARNESS") == "true" expected = {"FOUNDRY_PROJECT_ENDPOINT": state["endpoint"], "AZURE_AI_PROJECT_ENDPOINT": state["endpoint"], "AZURE_AI_PROJECT_ID": state["project_id"], @@ -68,6 +71,22 @@ assert args == ["ai", "toolbox", "delete", state["toolbox"], "--project-endpoint", state["endpoint"], "--force", "--no-prompt", "--output", "json"], args state["toolbox_deleted"] = True + elif args[:3] == ["ai", "skill", "list"]: + assert args == ["ai", "skill", "list", "--project-endpoint", state["endpoint"], "--output", "json", "--no-prompt"], args + names = ["unowned-skill"] + ([] if state["skill_deleted"] else [state["skill"]]) + print(json.dumps([{"name": name} for name in names])) + elif args[:3] == ["ai", "skill", "delete"]: + assert args == ["ai", "skill", "delete", state["skill"], "--project-endpoint", state["endpoint"], + "--force", "--no-prompt", "--output", "json"], args + state["skill_deleted"] = True + elif args[:3] == ["ai", "connection", "list"]: + assert args == ["ai", "connection", "list", "--project-endpoint", state["endpoint"], "--output", "json", "--no-prompt"], args + names = ["unowned-conn"] + ([] if state["connection_deleted"] else [state["connection"]]) + print(json.dumps([{"name": name} for name in names])) + elif args[:3] == ["ai", "connection", "delete"]: + assert args == ["ai", "connection", "delete", state["connection"], "--project-endpoint", state["endpoint"], + "--force", "--no-prompt"], args + state["connection_deleted"] = True else: raise AssertionError(args) path.write_text(json.dumps(state), encoding="utf-8") @@ -138,8 +157,10 @@ def run_recovery(self, *, code=0, ledger=True, auth_failure=False): (self.artifacts / "ownership.json").write_text(json.dumps(self.ownership), encoding="utf-8") (self.artifacts / "context.json").write_text(json.dumps(self.context), encoding="utf-8") self.state_path.write_text(json.dumps({"project": str(self.project), "project_id": PROJECT_ID, - "endpoint": ENDPOINT, "name": NAME, "toolbox": TOOLBOX, "commands": [], "pinned": {}, - "agent_deleted": False, "toolbox_deleted": False, "auth_failure": auth_failure}), encoding="utf-8") + "endpoint": ENDPOINT, "name": NAME, "toolbox": TOOLBOX, "skill": SKILL, "connection": CONN, + "commands": [], "pinned": {}, + "agent_deleted": False, "toolbox_deleted": False, "skill_deleted": False, + "connection_deleted": False, "auth_failure": auth_failure}), encoding="utf-8") env = {**os.environ, "PATH": str(self.bin) + os.pathsep + self.tool_path, "PYTHON": Path(sys.executable).as_posix(), "PYTHONUTF8": "1", "PYTHONDONTWRITEBYTECODE": "1", "RECOVERY_TEST_STATE": self.state_path.as_posix(), "TEST_BIN": self.bin.as_posix(), @@ -173,18 +194,42 @@ def test_already_absent_needs_no_workspace(self): self.assertEqual(self.read_json("recovery-result.json")["cleanup_status"], "complete") def test_pending_resources_clean_in_order_without_upgrading_result(self): + self.ownership.update(connection_name=CONN, connection_deployment_attempted=True) + self.context["connection_name"] = CONN (self.artifacts / "result.json").write_text('{"status":"failed","evaluation_exit_code":1}', encoding="utf-8") before = (self.project / "azure.yaml").read_bytes() self.run_recovery() operations = [cmd[3:6] for cmd in self.state["commands"] if cmd[3] == "ai"] self.assertEqual(operations, [["ai", "agent", "show"], ["ai", "agent", "delete"], - ["ai", "agent", "show"], ["ai", "toolbox", "list"], - ["ai", "toolbox", "delete"], ["ai", "toolbox", "list"]]) - self.assertEqual(self.read_json("ownership.json")["cleanup"], {NAME: "absent", TOOLBOX: "absent"}) + ["ai", "agent", "show"], ["ai", "connection", "list"], + ["ai", "connection", "delete"], ["ai", "connection", "list"], + ["ai", "toolbox", "list"], ["ai", "toolbox", "delete"], + ["ai", "toolbox", "list"]]) + self.assertEqual(self.read_json("ownership.json")["cleanup"], {NAME: "absent", CONN: "absent", TOOLBOX: "absent"}) self.assertEqual(self.read_json("result.json")["status"], "failed") self.assertEqual(self.read_json("result.json")["evaluation_exit_code"], 1) self.assertEqual((self.project / "azure.yaml").read_bytes(), before) + def test_pending_skill_cleans_in_order(self): + manifest = {"name": "recovery-skill", "services": { + "project": {"host": "azure.ai.project"}, + SKILL: {"host": "azure.ai.skill", "uses": ["project"], "archive": f"./skills/{SKILL}"}, + SERVICE: {"host": "azure.ai.agent", "kind": "prompt", "project": ".", + "name": NAME, "model": MODEL, "uses": ["project", SKILL], + "harness": {"type": "microsoft_copilot_preview"}, "skills": [SKILL]}, + }} + for parent in (self.project, self.artifacts): + (parent / "azure.yaml").write_text(yaml.safe_dump(manifest), encoding="utf-8", newline="\n") + self.ownership.update(toolbox_name="", toolbox_deployment_attempted=False, + skill_name=SKILL, skill_deployment_attempted=True) + self.context.update(toolbox_name="", skill_name=SKILL) + self.run_recovery() + operations = [cmd[3:6] for cmd in self.state["commands"] if cmd[3] == "ai"] + self.assertEqual(operations, [["ai", "agent", "show"], ["ai", "agent", "delete"], + ["ai", "agent", "show"], ["ai", "skill", "list"], + ["ai", "skill", "delete"], ["ai", "skill", "list"]]) + self.assertEqual(self.read_json("ownership.json")["cleanup"], {NAME: "absent", SKILL: "absent"}) + def test_agent_auth_error_still_cleans_toolbox_and_fails_result(self): (self.artifacts / "result.json").write_text('{"status":"passed"}', encoding="utf-8") self.run_recovery(code=1, auth_failure=True) diff --git a/samples/cli/prompt-agents/AGENTS.md b/samples/cli/prompt-agents/AGENTS.md index ad64fb456..55b960564 100644 --- a/samples/cli/prompt-agents/AGENTS.md +++ b/samples/cli/prompt-agents/AGENTS.md @@ -26,7 +26,7 @@ Every non-excluded sample under `samples/cli/prompt-agents/` must either: exposes only terminal assistant text, and contract validation rejects unsupported sources even on filtered-out tests. 2. Or, if the sample requires capabilities outside the prompt CI adapter (such as - client-executed function tools, file-based skills, connections, extra Azure + client-executed function tools, connections, extra Azure resources like Azure AI Search, or lifecycle hooks), add an entry with a non-empty reason in [`.azure-pipelines/scripts/prompt-agents/blacklist.json`](../../../.azure-pipelines/scripts/prompt-agents/blacklist.json). diff --git a/samples/cli/prompt-agents/microsoft-copilot-harness/quickstart-web-research/azure.yaml b/samples/cli/prompt-agents/microsoft-copilot-harness/quickstart-web-research/azure.yaml index 7cc665e6e..a2c14b906 100644 --- a/samples/cli/prompt-agents/microsoft-copilot-harness/quickstart-web-research/azure.yaml +++ b/samples/cli/prompt-agents/microsoft-copilot-harness/quickstart-web-research/azure.yaml @@ -5,6 +5,7 @@ requiredVersions: extensions: azure.ai.agents: '>=1.0.0-beta.21' azure.ai.projects: '>=1.0.0-beta.14' + azure.ai.skills: '>=1.0.0-beta.8' name: quickstart-web-research From 2ce1d5fbe425ab2c0aa1118470425b5ab9e42ce9 Mon Sep 17 00:00:00 2001 From: hund030 Date: Mon, 9 Nov 2026 16:34:14 +0800 Subject: [PATCH 2/3] fix(ci): harden prompt-agent toolbox connection and skill ownership checks - verify preflight absence and post-deploy metadata for endpoint-scoped toolbox MCP connections, and accept `null` from empty `azd ai connection list` - enforce mutually exclusive toolbox vs. skill topology in `verify_adopted()` and reject symlinks in source skill bundles before workspace copy --- .../prompt-agents/cleanup-prompt-agent.sh | 25 ++- .../prompt-agents/prompt_agent_helpers.py | 176 ++++++++++++++---- .../scripts/prompt-agents/run-prompt-agent.sh | 24 ++- .../tests/test_prompt_agent_helpers.py | 123 ++++++++++++ .../tests/test_prompt_agent_lifecycle.py | 39 +++- .../tests/test_prompt_agent_recovery.py | 63 ++++++- 6 files changed, 387 insertions(+), 63 deletions(-) diff --git a/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh b/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh index a1ceed53c..0ef18a1c2 100644 --- a/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh +++ b/.azure-pipelines/scripts/prompt-agents/cleanup-prompt-agent.sh @@ -113,22 +113,24 @@ if ! jq -e -s --arg id "$project_id" --arg endpoint "$project_endpoint" --slurpf $c.service_env_key == ($c.agent_service | gsub("[- ]"; "_") | ascii_upcase) and ($c.service_env_key | test("^[A-Z0-9_]+$")) and $o.toolbox_name == $c.toolbox_name and + ($o.toolbox_endpoint // "") == ($c.toolbox_endpoint // "") and ($o.skill_name // "") == ($c.skill_name // "") and ($o.connection_name // "") == ($c.connection_name // "") and (($c.agent_name | ltrimstr("ci-prompt-")) as $nonce | if $c.execution_profile == "managed" then ( ($c.toolbox_name == ("ci-e2e-tb-" + $nonce) and ($c.skill_name // "") == "" and ($o.skill_deployment_attempted // false) == false and - (if ($o.connection_deployment_attempted // false) then - (($c.connection_name // "") | type) == "string" and + (if ($o.connection_deployment_attempted // false) or (($c.connection_name // "") != "") or (($c.toolbox_endpoint // "") != "") then + (($c.connection_name // "") | type) == "string" and (($c.connection_name // "") != "") and + (($c.toolbox_endpoint // "") | type) == "string" and (($c.toolbox_endpoint // "") != "") and (($c.connection_name // "") | test("^ci-e2e-tb-" + ($nonce[:10]) + "-[0-9a-f]{8}-mcp$")) - else (($c.connection_name // "") == "" or (($c.connection_name // "") | test("^ci-e2e-tb-" + ($nonce[:10]) + "-[0-9a-f]{8}-mcp$"))) end)) + else (($c.connection_name // "") == "" and ($c.toolbox_endpoint // "") == "") end)) or (($c.skill_name // "") == ("ci-e2e-sk-" + $nonce) and $c.toolbox_name == "" and $o.toolbox_deployment_attempted == false and - ($c.connection_name // "") == "" and ($o.connection_deployment_attempted // false) == false) + ($c.toolbox_endpoint // "") == "" and ($c.connection_name // "") == "" and ($o.connection_deployment_attempted // false) == false) ) else $c.execution_profile == "prompt" and $c.toolbox_name == "" and $o.toolbox_deployment_attempted == false and - ($c.skill_name // "") == "" and ($o.skill_deployment_attempted // false) == false and + ($c.toolbox_endpoint // "") == "" and ($c.skill_name // "") == "" and ($o.skill_deployment_attempted // false) == false and ($c.connection_name // "") == "" and ($o.connection_deployment_attempted // false) == false end) and ($o.cleanup | type) == "object" and ($o.cleanup | to_entries | all( @@ -140,6 +142,12 @@ if ! jq -e -s --arg id "$project_id" --arg endpoint "$project_endpoint" --slurpf ' "$ownership" > "$recovery/validation-stdout.txt" 2> "$recovery/validation-stderr.txt"; then fail "Invalid ownership/context or trusted project mismatch; refusing recovery commands." fi +if [ -n "$(jq -r '.connection_name // ""' "$context")" ] || [ -n "$(jq -r '.toolbox_endpoint // ""' "$context")" ]; then + if ! timeout --kill-after=10s 60s "$python" "$helper" verify-connection --context "$context" \ + >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt"; then + fail "Invalid ownership/context or trusted project mismatch; refusing recovery commands." + fi +fi validated=true name=$(jq -r .agent_name "$context") service=$(jq -r .agent_service "$context") @@ -208,6 +216,11 @@ connection_absent() { --metadata "$(native_path "$last_stdout")" --context "$context" \ >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" } +verify_connection() { + timeout --kill-after=10s 60s "$python" "$helper" verify-connection \ + --metadata "$(native_path "$last_stdout")" --context "$context" \ + >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" +} # Pin both process variables and the retained azd environment; do not trust defaults. export AZD_ALPHA_ENABLE_AGENTS_MANAGED_HARNESS=true @@ -286,6 +299,8 @@ cleanup_connection() { (.[0] | all(type == "object" and (.name | type) == "string" and (.name | test("\\S")))) and (.[0] | any((.name | ascii_downcase) == $name)) ' "$(native_path "$last_stdout")" >> "$recovery/validation-stdout.txt" 2>> "$recovery/validation-stderr.txt" || return 1 + run_command "${azd_project[@]}" ai connection show "$connection" --project-endpoint "$project_endpoint" --output json --no-prompt || return 1 + verify_connection || return 1 run_command "${azd_project[@]}" ai connection delete "$connection" --project-endpoint "$project_endpoint" \ --force --no-prompt || cleanup_failed=true for attempt in 1 2 3; do diff --git a/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py b/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py index 2a4b014a8..de0192786 100644 --- a/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py +++ b/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py @@ -7,6 +7,7 @@ from datetime import datetime, timezone import hashlib import json +import os from pathlib import Path import re import shutil @@ -69,6 +70,36 @@ def toolbox_connection_name(toolbox_name: str, version: str, endpoint: str) -> s return f"{prefix}-{digest}-mcp" +def _validate_project_endpoint(context: dict) -> str: + project_endpoint = context.get("project_endpoint") + if not isinstance(project_endpoint, str): + raise LifecycleError("context must contain the project endpoint") + project = urlsplit(project_endpoint) + if (project.scheme != "https" or not project.hostname or project.query or project.fragment + or project.username is not None or any(c.isspace() for c in project_endpoint)): + raise LifecycleError("context project endpoint must be an HTTPS URL without credentials/query/fragment") + return project_endpoint.rstrip("/") + + +def _expected_connection_name(context: dict) -> tuple[str, str]: + name = owned_toolbox_name(context) + project_endpoint = _validate_project_endpoint(context) + toolbox_endpoint = context.get("toolbox_endpoint") + connection_name = context.get("connection_name") + prefix = project_endpoint + "/toolboxes/" + quote(name, safe="") + "/versions/" + suffix = "/mcp?api-version=v1" + if (not isinstance(toolbox_endpoint, str) or not toolbox_endpoint.startswith(prefix) + or not toolbox_endpoint.endswith(suffix)): + raise LifecycleError("context must contain the owned toolbox MCP endpoint") + version = toolbox_endpoint[len(prefix):-len(suffix)] + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", version): + raise LifecycleError("context toolbox endpoint must contain a valid toolbox version") + expected = toolbox_connection_name(name, version, toolbox_endpoint) + if not isinstance(connection_name, str) or connection_name != expected: + raise LifecycleError("context must identify the owned toolbox connection") + return expected, toolbox_endpoint + + def _extract_skill_frontmatter(content: str) -> tuple[str, str]: trimmed = content.lstrip("\ufeff \t\r\n") if not trimmed.startswith("---"): @@ -202,17 +233,27 @@ def validate_manifest(document: dict, service_key: str) -> str: return projects[0] -def _rewrite_skill_bundle(source_dir: Path, original_skill: str, owned_skill: str) -> None: - skills_dir = source_dir / "skills" +def _validate_source_skill_bundle(sample_dir: Path, original_skill: str) -> tuple[dict, str]: + if sample_dir.is_symlink() or not sample_dir.is_dir(): + raise LifecycleError("sample directory must be a real directory") + skills_dir = sample_dir / "skills" + if skills_dir.is_symlink() or not skills_dir.is_dir(): + raise LifecycleError("skill archive directory is missing or is a symlink") + skills_root = skills_dir.resolve() + for root, dirs, files in os.walk(skills_dir, followlinks=False): + root_path = Path(root) + for item in (*dirs, *files): + entry = root_path / item + if entry.is_symlink() or not (entry.is_file() or entry.is_dir()): + raise LifecycleError("skill bundle cannot contain symlinks or special files") + if not entry.resolve().is_relative_to(skills_root): + raise LifecycleError("skill bundle entry resolves outside skills directory") original_dir = skills_dir / original_skill - owned_dir = skills_dir / owned_skill - if not original_dir.is_dir() or owned_dir.exists(): - raise LifecycleError("skill archive directory is missing or conflicts with owned skill directory") entries = [entry for entry in skills_dir.iterdir() if not entry.name.startswith(".")] - if entries != [original_dir]: + if entries != [original_dir] or not original_dir.is_dir() or original_dir.is_symlink(): raise LifecycleError("skills directory must contain only the declared skill bundle") skill_md = original_dir / "SKILL.md" - if not skill_md.is_file() or skill_md.is_symlink(): + if skill_md.is_symlink() or not skill_md.is_file(): raise LifecycleError("skill archive directory must contain a regular SKILL.md file") front, body = _extract_skill_frontmatter(skill_md.read_text(encoding="utf-8")) if not body.strip(): @@ -222,6 +263,17 @@ def _rewrite_skill_bundle(source_dir: Path, original_skill: str, owned_skill: st raise LifecycleError("SKILL.md front matter must declare a nonempty description") if metadata.get("name", original_skill) != original_skill: raise LifecycleError("SKILL.md front matter name must match the skill service key") + return metadata, body + + +def _rewrite_skill_bundle(source_dir: Path, original_skill: str, owned_skill: str) -> None: + metadata, body = _validate_source_skill_bundle(source_dir, original_skill) + skills_dir = source_dir / "skills" + original_dir = skills_dir / original_skill + owned_dir = skills_dir / owned_skill + if owned_dir.exists() or owned_dir.is_symlink(): + raise LifecycleError("skill archive directory is missing or conflicts with owned skill directory") + skill_md = original_dir / "SKILL.md" metadata["name"] = owned_skill skill_md.write_text( "---\n" + yaml.safe_dump(metadata, sort_keys=False).strip() + "\n---\n" + body, @@ -267,6 +319,7 @@ def prepare_manifest(repo: Path, sample: Path, model: str, name: str) -> tuple[d tool["server_label"] = toolbox_name else: original_skill = skills[0] + _validate_source_skill_bundle(sample, original_skill) skill_name = owned_skill_name({ "execution_profile": "managed", "agent_name": name, "skill_name": "ci-e2e-sk-" + name.removeprefix("ci-prompt-"), @@ -348,7 +401,7 @@ def prepare(args) -> None: source = work / "input" if source.exists() or artifacts.is_relative_to(source): raise LifecycleError("work/input must be new and separate from artifacts") - shutil.copytree(sample, source, ignore=shutil.ignore_patterns( + shutil.copytree(sample, source, symlinks=True, ignore=shutil.ignore_patterns( ".azure", ".env", ".env.*", ".git", ".venv", "__pycache__", )) if skill_name: @@ -365,13 +418,14 @@ def prepare(args) -> None: "account_name": args.project_id.split("/")[-3], "execution_profile": profile, "toolbox_name": toolbox_name, + "toolbox_endpoint": "", "skill_name": skill_name, "connection_name": "", "project_endpoint": args.project_endpoint, }) write_json(artifacts / "ownership.json", { "agent_name": name, "agent_service": service_key, - "toolbox_name": toolbox_name, "toolbox_deployment_attempted": False, + "toolbox_name": toolbox_name, "toolbox_endpoint": "", "toolbox_deployment_attempted": False, "skill_name": skill_name, "skill_deployment_attempted": False, "connection_name": "", "connection_deployment_attempted": False, "project_id": args.project_id, "project_endpoint": args.project_endpoint, @@ -395,35 +449,78 @@ def verify_adopted(manifest: Path, context: dict, model_deployment: str) -> None if not isinstance(document, dict): raise LifecycleError("adopted manifest must be a mapping") validate_manifest(document, context["agent_service"]) - agent = document["services"][context["agent_service"]] + services = document["services"] + agent = services[context["agent_service"]] profile = execution_profile(agent) if profile != context.get("execution_profile", "prompt"): raise LifecycleError("init changed the requested execution profile") + has_toolbox = bool(context.get("toolbox_name")) + has_skill = bool(context.get("skill_name")) if profile == "managed": - if context.get("toolbox_name"): + if has_toolbox == has_skill: + raise LifecycleError("managed context must identify an owned toolbox or skill") + if has_toolbox: + if context.get("skill_name", "") != "": + raise LifecycleError("managed toolbox context cannot declare a skill") expected_tb = owned_toolbox_name(context) - if "toolbox" in agent and agent["toolbox"] != {"name": expected_tb}: - raise LifecycleError("init changed the owned toolbox reference") - if "toolboxes" in agent and agent["toolboxes"] not in ([expected_tb], [{"name": expected_tb}]): - raise LifecycleError("init changed the owned toolboxes reference") - elif context.get("skill_name"): + if (expected_tb not in services + or services[expected_tb].get("host") != "azure.ai.toolbox" + or "skills" in agent + or any(svc.get("host") == "azure.ai.skill" for svc in services.values())): + raise LifecycleError("init changed the owned toolbox service") + try: + _validate_agent_toolboxes(agent, expected_tb) + except LifecycleError as exc: + raise LifecycleError("init changed the owned toolbox reference") from exc + if context.get("connection_name") or context.get("toolbox_endpoint"): + _expected_connection_name(context) + else: + if (context.get("toolbox_name", "") != "" + or context.get("connection_name", "") != "" + or context.get("toolbox_endpoint", "") != ""): + raise LifecycleError("managed skill context cannot declare a toolbox or connection") expected_sk = owned_skill_name(context) - if expected_sk not in document["services"] or document["services"][expected_sk].get("archive") != f"./skills/{expected_sk}": + if (expected_sk not in services + or services[expected_sk].get("host") != "azure.ai.skill" + or services[expected_sk].get("archive") != f"./skills/{expected_sk}" + or "toolbox" in agent or "toolboxes" in agent + or any(svc.get("host") == "azure.ai.toolbox" for svc in services.values())): raise LifecycleError("init changed the owned skill service") if "skills" in agent and agent["skills"] not in ([expected_sk], [{"name": expected_sk}]): raise LifecycleError("init changed the owned skills reference") - else: - raise LifecycleError("managed context must identify an owned toolbox or skill") + elif (context.get("toolbox_name", "") != "" + or context.get("skill_name", "") != "" + or context.get("connection_name", "") != "" + or context.get("toolbox_endpoint", "") != ""): + raise LifecycleError("prompt context cannot declare managed resources") if agent.get("name") != context["agent_name"] or agent.get("model") != model_deployment: raise LifecycleError("init changed the requested agent identity, model, or kind") +def verify_connection(metadata: Path | None, context: dict) -> None: + expected_name, expected_target = _expected_connection_name(context) + if metadata is None: + return + document = json.loads(metadata.read_text(encoding="utf-8")) + if not isinstance(document, dict): + raise LifecycleError("connection show metadata must be an object") + kind = document.get("kind") + auth_type = document.get("authType") + if (document.get("name") != expected_name + or document.get("target") != expected_target + or not isinstance(kind, str) or kind.casefold() != "remotetool" + or not isinstance(auth_type, str) + or auth_type.replace("-", "").replace("_", "").casefold() != "userentratoken"): + raise LifecycleError("connection show returned unexpected owned name, target, kind, or authType") + + def verify_deployed( metadata: Path, context: dict, model_deployment: str, toolbox_metadata: Path | None = None, skill_metadata: Path | None = None, + connection_metadata: Path | None = None, ) -> None: document = json.loads(metadata.read_text(encoding="utf-8")) if not isinstance(document, dict): @@ -439,6 +536,8 @@ def verify_deployed( if profile != context.get("execution_profile", "prompt"): raise LifecycleError("azd show returned an unexpected execution profile") if profile == "managed": + if bool(context.get("toolbox_name")) == bool(context.get("skill_name")): + raise LifecycleError("managed context must identify an owned toolbox or skill") if context.get("toolbox_name"): name = owned_toolbox_name(context) tools = definition.get("tools") @@ -447,13 +546,18 @@ def verify_deployed( or mcp_tools[0].get("server_label") != name or not isinstance(mcp_tools[0].get("server_url"), str)): raise LifecycleError("managed agent must attach exactly the owned toolbox MCP server") - if toolbox_metadata is None: - raise LifecycleError("managed agent verification requires toolbox metadata") + if toolbox_metadata is None or connection_metadata is None: + raise LifecycleError("managed agent verification requires toolbox and connection metadata") version = verify_toolbox(toolbox_metadata, context, mcp_tools[0]["server_url"]) + expected_conn = toolbox_connection_name(name, version, mcp_tools[0]["server_url"]) connection_id = mcp_tools[0].get("project_connection_id") - if connection_id is not None and connection_id != toolbox_connection_name(name, version, mcp_tools[0]["server_url"]): + if (not isinstance(connection_id, str) + or connection_id != context.get("connection_name") + or connection_id != expected_conn + or mcp_tools[0]["server_url"] != context.get("toolbox_endpoint")): raise LifecycleError("managed agent toolbox connection ID does not match the owned toolbox endpoint") - elif context.get("skill_name"): + verify_connection(connection_metadata, context) + else: name = owned_skill_name(context) skills = definition.get("skills") if skill_metadata is None: @@ -463,8 +567,6 @@ def verify_deployed( or skills[0].get("name") != name or skills[0].get("version") != version or skills[0].get("type", "skill_reference") != "skill_reference"): raise LifecycleError("managed agent must attach the owned skill and version") - else: - raise LifecycleError("managed context must identify an owned toolbox or skill") def verify_toolbox(metadata: Path, context: dict, endpoint: str) -> str: @@ -480,14 +582,8 @@ def verify_toolbox(metadata: Path, context: dict, endpoint: str) -> str: or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", version["version"]) or version.get("tools") != [{"type": "web_search"}]): raise LifecycleError("toolbox show returned unexpected owned name, version, or tools") - project_endpoint = context.get("project_endpoint") - if not isinstance(project_endpoint, str): - raise LifecycleError("context must contain the project endpoint") - project = urlsplit(project_endpoint) - if (project.scheme != "https" or not project.hostname or project.query or project.fragment - or project.username is not None or any(c.isspace() for c in project_endpoint)): - raise LifecycleError("context project endpoint must be an HTTPS URL without credentials/query/fragment") - expected = (project_endpoint.rstrip("/") + "/toolboxes/" + quote(name, safe="") + project_endpoint = _validate_project_endpoint(context) + expected = (project_endpoint + "/toolboxes/" + quote(name, safe="") + "/versions/" + quote(version["version"], safe="") + "/mcp?api-version=v1") if not isinstance(endpoint, str) or endpoint != expected or document.get("endpoint") != expected: raise LifecycleError("toolbox MCP endpoint must exactly match the owned project, toolbox, and version") @@ -533,10 +629,10 @@ def skill_absent(metadata: Path, context: dict) -> bool: def connection_absent(metadata: Path, context: dict) -> bool: - name = context.get("connection_name") - if not isinstance(name, str) or not re.fullmatch(r"ci-e2e-tb-[0-9a-f]{10}-[0-9a-f]{8}-mcp", name): - raise LifecycleError("context must identify the owned toolbox connection") + name, _ = _expected_connection_name(context) connections = json.loads(metadata.read_text(encoding="utf-8")) + if connections is None: + return True if not isinstance(connections, list) or any( not isinstance(conn, dict) or not isinstance(conn.get("name"), str) or not conn["name"].strip() for conn in connections @@ -599,6 +695,7 @@ def main(argv=None) -> int: deployed.add_argument("--metadata", type=Path, required=True) deployed.add_argument("--toolbox-metadata", type=Path) deployed.add_argument("--skill-metadata", type=Path) + deployed.add_argument("--connection-metadata", type=Path) for verification in (adopted, deployed): verification.add_argument("--context", type=Path, required=True) verification.add_argument("--model-deployment", required=True) @@ -608,6 +705,9 @@ def main(argv=None) -> int: skill = commands.add_parser("verify-skill") skill.add_argument("--version", required=True) skill_absence = commands.add_parser("skill-absent") + connection = commands.add_parser("verify-connection") + connection.add_argument("--metadata", type=Path) + connection.add_argument("--context", type=Path, required=True) connection_absence = commands.add_parser("connection-absent") for verification in (toolbox, toolbox_absence, skill, skill_absence, connection_absence): verification.add_argument("--metadata", type=Path, required=True) @@ -636,7 +736,7 @@ def main(argv=None) -> int: elif args.command == "verify-deployed": verify_deployed( args.metadata, context, args.model_deployment, - args.toolbox_metadata, args.skill_metadata, + args.toolbox_metadata, args.skill_metadata, args.connection_metadata, ) elif args.command == "verify-toolbox": version = verify_toolbox(args.metadata, context, args.endpoint) @@ -647,6 +747,8 @@ def main(argv=None) -> int: verify_skill(args.metadata, context, args.version) elif args.command == "skill-absent": return 0 if skill_absent(args.metadata, context) else 1 + elif args.command == "verify-connection": + verify_connection(args.metadata, context) elif args.command == "connection-absent": return 0 if connection_absent(args.metadata, context) else 1 else: diff --git a/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh b/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh index 4f0771715..aa2765d6f 100644 --- a/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh +++ b/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh @@ -64,6 +64,7 @@ env_key=$(jq -r .service_env_key "$context") account=$(jq -r .account_name "$context") toolbox=$(jq -r .toolbox_name "$context") skill=$(jq -r .skill_name "$context") +toolbox_endpoint= connection= deployed=false toolbox_deployed=false @@ -80,11 +81,12 @@ last_stdout= last_stderr= save_ownership() { jq --argjson attempted "$deployed" --arg status "$cleanup_status" --arg name "$name" \ - --argjson toolbox_attempted "$toolbox_deployed" --arg toolbox "$toolbox" --arg toolbox_status "$toolbox_cleanup_status" \ + --argjson toolbox_attempted "$toolbox_deployed" --arg toolbox "$toolbox" --arg toolbox_endpoint "$toolbox_endpoint" --arg toolbox_status "$toolbox_cleanup_status" \ --argjson skill_attempted "$skill_deployed" --arg skill "$skill" --arg skill_status "$skill_cleanup_status" \ --argjson connection_attempted "$connection_deployed" --arg connection "$connection" --arg connection_status "$connection_cleanup_status" ' .deployment_attempted = $attempted | .toolbox_deployment_attempted = $toolbox_attempted | + .toolbox_endpoint = $toolbox_endpoint | .skill_deployment_attempted = $skill_attempted | .connection_name = $connection | .connection_deployment_attempted = $connection_attempted | @@ -266,13 +268,19 @@ if [ -n "$toolbox" ]; then toolbox_env_key=${toolbox//-/_} toolbox_env_key=${toolbox_env_key^^} run_command "$command_timeout" checked azd env get-value "TOOLBOX_${toolbox_env_key}_MCP_ENDPOINT" - toolbox_endpoint=$(tr -d '\r\n' < "$last_stdout") + candidate_endpoint=$(tr -d '\r\n' < "$last_stdout") run_command "$command_timeout" checked azd ai toolbox show "$toolbox" --project-endpoint "$project_endpoint" --output json --no-prompt cp "$last_stdout" "$artifacts/toolbox.json" - connection=$("$python" "$helper" verify-toolbox --metadata "$(native_path "$artifacts/toolbox.json")" --context "$context" --endpoint "$toolbox_endpoint" | tr -d '\r\n') - jq --arg conn "$connection" '.connection_name = $conn' "$context" > "$work/context.json" && - cp "$work/context.json" "$artifacts/context.json" - verify_options=(--toolbox-metadata "$(native_path "$artifacts/toolbox.json")") + candidate_connection=$("$python" "$helper" verify-toolbox --metadata "$(native_path "$artifacts/toolbox.json")" --context "$context" --endpoint "$candidate_endpoint" | tr -d '\r\n') + jq --arg conn "$candidate_connection" --arg ep "$candidate_endpoint" '.connection_name = $conn | .toolbox_endpoint = $ep' "$context" > "$work/context.json" + run_command "$command_timeout" checked azd ai connection list --project-endpoint "$project_endpoint" --output json --no-prompt + "$python" "$helper" connection-absent --metadata "$(native_path "$last_stdout")" --context "$(native_path "$work/context.json")" || { + echo "cannot confirm generated connection name is unused; refusing deployment" >&2; exit 1; + } + cp "$work/context.json" "$artifacts/context.json" + toolbox_endpoint=$candidate_endpoint + connection=$candidate_connection + verify_options=(--toolbox-metadata "$(native_path "$artifacts/toolbox.json")" --connection-metadata "$(native_path "$artifacts/connection.json")") fi if [ -n "$skill" ]; then run_command "$command_timeout" checked azd ai skill list --project-endpoint "$project_endpoint" --output json --no-prompt @@ -295,6 +303,10 @@ if [ -n "$connection" ]; then connection_deployed=true; fi deployed=true save_ownership run_command "$command_timeout" checked azd deploy "$service" --no-prompt --timeout "$command_timeout" +if [ -n "$connection" ]; then + run_command "$command_timeout" checked azd ai connection show "$connection" --project-endpoint "$project_endpoint" --output json --no-prompt + cp "$last_stdout" "$artifacts/connection.json" +fi run_command "$command_timeout" checked azd ai agent show "$service" --output json --no-prompt cp "$last_stdout" "$artifacts/agent.json" "$python" "$helper" verify-deployed --metadata "$(native_path "$artifacts/agent.json")" --context "$context" --model-deployment "$model" "${verify_options[@]}" diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py index 7a82bda3a..a28cbb23c 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py @@ -2,11 +2,13 @@ import importlib.util import json +import os from pathlib import Path import sys import tempfile import unittest from unittest import mock +import yaml SCRIPT = Path(__file__).resolve().parents[1] / "prompt_agent_helpers.py" @@ -86,6 +88,7 @@ def test_toolbox_skill_and_connection_absence_are_pinned(self): "project_endpoint": "https://project.invalid/api/projects/project", } endpoint = context["project_endpoint"] + f"/toolboxes/{context['toolbox_name']}/versions/3/mcp?api-version=v1" + context["toolbox_endpoint"] = endpoint context["connection_name"] = helpers.toolbox_connection_name(context["toolbox_name"], "3", endpoint) metadata = { "toolbox": {"name": context["toolbox_name"]}, @@ -109,8 +112,128 @@ def test_toolbox_skill_and_connection_absence_are_pinned(self): helpers.verify_skill(path, context, "2") path.write_text(json.dumps([{"name": context["skill_name"].upper()}]), encoding="utf-8") self.assertFalse(helpers.skill_absent(path, context)) + conn_doc = { + "name": context["connection_name"], "target": endpoint, + "kind": "RemoteTool", "authType": "UserEntraToken", + } + path.write_text(json.dumps(conn_doc), encoding="utf-8") + helpers.verify_connection(path, context) + for field, bad_val in (("name", "wrong"), ("target", "https://other.invalid"), ("kind", "ApiKey"), ("authType", "ApiKey")): + path.write_text(json.dumps({**conn_doc, field: bad_val}), encoding="utf-8") + with self.assertRaisesRegex(helpers.LifecycleError, "connection show returned"): + helpers.verify_connection(path, context) + tb_path = Path(directory) / "toolbox.json" + tb_path.write_text(json.dumps(metadata), encoding="utf-8") + conn_path = Path(directory) / "connection.json" + conn_path.write_text(json.dumps(conn_doc), encoding="utf-8") + agent_path = Path(directory) / "agent.json" + agent_doc = { + "name": context["agent_name"], + "versions": {"latest": {"version": "1", "definition": { + "kind": "prompt", "model": "gpt-4o", + "harness": {"type": "microsoft_copilot_preview"}, + "tools": [{ + "type": "mcp", "server_label": context["toolbox_name"], + "server_url": endpoint, "project_connection_id": context["connection_name"], + }], + }}}, + } + agent_path.write_text(json.dumps(agent_doc), encoding="utf-8") + tb_only_ctx = {**context, "skill_name": ""} + helpers.verify_deployed(agent_path, tb_only_ctx, "gpt-4o", tb_path, None, conn_path) + with self.assertRaisesRegex(helpers.LifecycleError, "toolbox and connection metadata"): + helpers.verify_deployed(agent_path, tb_only_ctx, "gpt-4o", tb_path, None, None) + no_conn_agent = json.loads(json.dumps(agent_doc)) + no_conn_agent["versions"]["latest"]["definition"]["tools"][0].pop("project_connection_id") + agent_path.write_text(json.dumps(no_conn_agent), encoding="utf-8") + with self.assertRaisesRegex(helpers.LifecycleError, "connection ID"): + helpers.verify_deployed(agent_path, tb_only_ctx, "gpt-4o", tb_path, None, conn_path) path.write_text(json.dumps([{"name": context["connection_name"].upper()}]), encoding="utf-8") self.assertFalse(helpers.connection_absent(path, context)) + path.write_text("null", encoding="utf-8") + self.assertTrue(helpers.connection_absent(path, context)) + with self.assertRaisesRegex(helpers.LifecycleError, "owned toolbox connection"): + helpers.connection_absent(path, {**context, "connection_name": "ci-e2e-tb-aaaaaaaaaa-00000000-mcp"}) + + def test_verify_adopted_enforces_exclusive_toolbox_and_skill_topology(self): + nonce = "b" * 32 + agent_name = "ci-prompt-" + nonce + tb_name = "ci-e2e-tb-" + nonce + sk_name = "ci-e2e-sk-" + nonce + tb_context = { + "execution_profile": "managed", "agent_name": agent_name, "agent_service": "agent", + "toolbox_name": tb_name, "skill_name": "", "connection_name": "", "toolbox_endpoint": "", + "project_endpoint": "https://project.invalid/api/projects/project", + } + sk_context = {**tb_context, "toolbox_name": "", "skill_name": sk_name} + tb_doc = {"services": { + "project": {"host": "azure.ai.project"}, + tb_name: {"host": "azure.ai.toolbox", "uses": ["project"], "tools": [{"type": "web_search"}]}, + "agent": { + "host": "azure.ai.agent", "kind": "prompt", "project": ".", + "name": agent_name, "model": "gpt-4o", "uses": ["project", tb_name], + "harness": {"type": "microsoft_copilot_preview"}, "toolboxes": [tb_name], + }, + }} + sk_doc = {"services": { + "project": {"host": "azure.ai.project"}, + sk_name: {"host": "azure.ai.skill", "uses": ["project"], "archive": f"./skills/{sk_name}"}, + "agent": { + "host": "azure.ai.agent", "kind": "prompt", "project": ".", + "name": agent_name, "model": "gpt-4o", "uses": ["project", sk_name], + "harness": {"type": "microsoft_copilot_preview"}, "skills": [sk_name], + }, + }} + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory) / "azure.yaml" + manifest.write_text(yaml.safe_dump(tb_doc), encoding="utf-8") + helpers.verify_adopted(manifest, tb_context, "gpt-4o") + with self.assertRaisesRegex(helpers.LifecycleError, "owned skill service"): + helpers.verify_adopted(manifest, sk_context, "gpt-4o") + with self.assertRaisesRegex(helpers.LifecycleError, "owned toolbox or skill"): + helpers.verify_adopted(manifest, {**tb_context, "skill_name": sk_name}, "gpt-4o") + no_tb = yaml.safe_load(manifest.read_text(encoding="utf-8")) + no_tb["services"]["agent"].pop("toolboxes") + manifest.write_text(yaml.safe_dump(no_tb), encoding="utf-8") + with self.assertRaisesRegex(helpers.LifecycleError, "toolboxes or legacy toolbox"): + helpers.verify_adopted(manifest, tb_context, "gpt-4o") + manifest.write_text(yaml.safe_dump(sk_doc), encoding="utf-8") + helpers.verify_adopted(manifest, sk_context, "gpt-4o") + with self.assertRaisesRegex(helpers.LifecycleError, "owned toolbox service"): + helpers.verify_adopted(manifest, tb_context, "gpt-4o") + + def test_source_skill_bundle_rejects_symlinks_before_copy(self): + with tempfile.TemporaryDirectory() as directory: + sample = Path(directory) / "sample" + skill_dir = sample / "skills/travel-guide" + skill_dir.mkdir(parents=True) + skill_md = skill_dir / "SKILL.md" + skill_md.write_text( + "---\nname: travel-guide\ndescription: Guide\n---\nFollow instructions.\n", + encoding="utf-8", + ) + aux = skill_dir / "notes.txt" + aux.write_text("helper notes\n", encoding="utf-8") + helpers._validate_source_skill_bundle(sample, "travel-guide") + for target in (sample / "skills", skill_dir, skill_md, aux): + with self.subTest(target=target.name): + orig_is_symlink = Path.is_symlink + with mock.patch.object( + Path, "is_symlink", autospec=True, + side_effect=lambda p, t=target: True if p == t else orig_is_symlink(p), + ): + with self.assertRaisesRegex(helpers.LifecycleError, "symlink"): + helpers._validate_source_skill_bundle(sample, "travel-guide") + outside = Path(directory) / "outside.txt" + outside.write_text("outside\n", encoding="utf-8") + link = skill_dir / "linked.txt" + try: + os.symlink(outside, link) + except OSError: + pass + else: + with self.assertRaisesRegex(helpers.LifecycleError, "symlink"): + helpers._validate_source_skill_bundle(sample, "travel-guide") if __name__ == "__main__": diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py index 757f7d434..d804276c5 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py @@ -176,10 +176,20 @@ def conn_name(tb_name, version, endpoint): elif is_connection: if operation == "list": assert argv == ["azd", "ai", "connection", "list", "--project-endpoint", config["endpoint"], "--output", "json", "--no-prompt"], argv + mode = config.get("connection_cleanup" if state["connection_deleted"] else "connection_preflight", "absent") names = ["unowned-conn"] - if state["deployed"] and not state["connection_deleted"]: + if mode == "collision" or state["deployed"] and not state["connection_deleted"]: names.append(state["connection"].upper()) - stdout = json.dumps([{"name": name} for name in names]) + stdout = "null\n" if mode == "null" else json.dumps([{"name": name} for name in names]) + elif operation == "show": + assert argv == ["azd", "ai", "connection", "show", state["connection"], "--project-endpoint", config["endpoint"], "--output", "json", "--no-prompt"], argv + assert state["deployed"] + conn_doc = { + "name": state["connection"], "target": state["toolbox_endpoint"], + "kind": "RemoteTool", "authType": "UserEntraToken", + } + conn_doc.update(config.get("connection_changes", {})) + stdout = json.dumps(conn_doc) elif operation == "delete": assert argv == ["azd", "ai", "connection", "delete", state["connection"], "--project-endpoint", config["endpoint"], "--force", "--no-prompt"], argv assert state["deployed"] and state["deleted"] @@ -566,17 +576,36 @@ def test_managed_preflight_never_deletes_unowned_resources(self): def test_managed_toolbox_deploy_failure_cleans_only_toolbox(self): self.configure(managed=True) - self.config["fail_operation"] = "toolbox_deploy" + self.config["fail_operation"] = "toolbox_show" self.run_agent(passed=False) self.assertFalse(self.state["deployed"]) self.assertEqual(self.operations("delete"), []) self.assertTrue(self.state["toolbox_deleted"]) self.assertEqual(self.result["cleanup"], {self.state["toolbox"]: "absent"}) + for doc in (self.read_json("ownership.json"), self.read_json("context.json")): + self.assertEqual(doc["toolbox_endpoint"], "") + self.assertEqual(doc["connection_name"], "") + self.assertFalse(self.read_json("ownership.json")["connection_deployment_attempted"]) + + def test_managed_connection_preflight_collision_cleans_only_toolbox(self): + self.configure(managed=True) + self.config["connection_preflight"] = "collision" + self.assertIn("connection name is unused", self.run_agent(passed=False).stderr) + self.assertFalse(self.state["deployed"]) + self.assertEqual(self.operations("delete"), []) + self.assertEqual(self.operations("delete", "connection"), []) + self.assertTrue(self.state["toolbox_deleted"]) + self.assertEqual(self.result["cleanup"], {self.state["toolbox"]: "absent"}) + for doc in (self.read_json("ownership.json"), self.read_json("context.json")): + self.assertEqual(doc["toolbox_endpoint"], "") + self.assertEqual(doc["connection_name"], "") + self.assertFalse(self.read_json("ownership.json")["connection_deployment_attempted"]) def test_managed_wrong_attachment_cleans_without_invoking(self): self.configure(managed=True) - self.config["mcp_changes"] = {"server_url": "https://other.invalid/mcp"} - self.assertIn("MCP endpoint", self.run_agent(passed=False).stderr) + self.config["connection_changes"] = {"target": "https://other.invalid/mcp"} + self.config["connection_cleanup"] = "null" + self.assertIn("connection show returned", self.run_agent(passed=False).stderr) self.assertEqual(self.operations("invoke"), []) self.assert_cleaned() diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py index 5ef6d031c..f19276f38 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py @@ -1,5 +1,6 @@ """Exercise the real recovery shell with isolated azd/git stubs, never Azure.""" +import hashlib import json import os from pathlib import Path @@ -19,8 +20,9 @@ MODEL = "borrowed-model" NAME = "ci-prompt-" + "a" * 32 TOOLBOX = "ci-e2e-tb-" + "a" * 32 +TOOLBOX_ENDPOINT = f"{ENDPOINT}/toolboxes/{TOOLBOX}/versions/3/mcp?api-version=v1" SKILL = "ci-e2e-sk-" + "a" * 32 -CONN = "ci-e2e-tb-aaaaaaaaaa-12345678-mcp" +CONN = f"ci-e2e-tb-aaaaaaaaaa-{hashlib.sha256(TOOLBOX_ENDPOINT.encode('utf-8')).hexdigest()[:8]}-mcp" SERVICE = "chat-agent" FAKE_CLI = r''' @@ -81,8 +83,19 @@ state["skill_deleted"] = True elif args[:3] == ["ai", "connection", "list"]: assert args == ["ai", "connection", "list", "--project-endpoint", state["endpoint"], "--output", "json", "--no-prompt"], args - names = ["unowned-conn"] + ([] if state["connection_deleted"] else [state["connection"]]) - print(json.dumps([{"name": name} for name in names])) + if state["connection_deleted"] and state.get("connection_null"): + print("null") + else: + names = ["unowned-conn"] + ([] if state["connection_deleted"] else [state["connection"]]) + print(json.dumps([{"name": name} for name in names])) + elif args[:3] == ["ai", "connection", "show"]: + assert args == ["ai", "connection", "show", state["connection"], "--project-endpoint", state["endpoint"], + "--output", "json", "--no-prompt"], args + print(json.dumps({ + "name": state["connection"], "target": state["toolbox_endpoint"], + "kind": "RemoteTool", "authType": "UserEntraToken", + **state.get("connection_changes", {}), + })) elif args[:3] == ["ai", "connection", "delete"]: assert args == ["ai", "connection", "delete", state["connection"], "--project-endpoint", state["endpoint"], "--force", "--no-prompt"], args @@ -152,15 +165,17 @@ def setUp(self): def read_json(self, name): return json.loads((self.artifacts / name).read_text(encoding="utf-8")) - def run_recovery(self, *, code=0, ledger=True, auth_failure=False): + def run_recovery(self, *, code=0, ledger=True, auth_failure=False, connection_null=False, connection_changes=None): if ledger: (self.artifacts / "ownership.json").write_text(json.dumps(self.ownership), encoding="utf-8") (self.artifacts / "context.json").write_text(json.dumps(self.context), encoding="utf-8") self.state_path.write_text(json.dumps({"project": str(self.project), "project_id": PROJECT_ID, - "endpoint": ENDPOINT, "name": NAME, "toolbox": TOOLBOX, "skill": SKILL, "connection": CONN, + "endpoint": ENDPOINT, "name": NAME, "toolbox": TOOLBOX, "toolbox_endpoint": TOOLBOX_ENDPOINT, + "skill": SKILL, "connection": CONN, "commands": [], "pinned": {}, "agent_deleted": False, "toolbox_deleted": False, "skill_deleted": False, - "connection_deleted": False, "auth_failure": auth_failure}), encoding="utf-8") + "connection_deleted": False, "auth_failure": auth_failure, + "connection_null": connection_null, "connection_changes": connection_changes or {}}), encoding="utf-8") env = {**os.environ, "PATH": str(self.bin) + os.pathsep + self.tool_path, "PYTHON": Path(sys.executable).as_posix(), "PYTHONUTF8": "1", "PYTHONDONTWRITEBYTECODE": "1", "RECOVERY_TEST_STATE": self.state_path.as_posix(), "TEST_BIN": self.bin.as_posix(), @@ -194,15 +209,16 @@ def test_already_absent_needs_no_workspace(self): self.assertEqual(self.read_json("recovery-result.json")["cleanup_status"], "complete") def test_pending_resources_clean_in_order_without_upgrading_result(self): - self.ownership.update(connection_name=CONN, connection_deployment_attempted=True) - self.context["connection_name"] = CONN + self.ownership.update(toolbox_endpoint=TOOLBOX_ENDPOINT, connection_name=CONN, connection_deployment_attempted=True) + self.context.update(toolbox_endpoint=TOOLBOX_ENDPOINT, connection_name=CONN) (self.artifacts / "result.json").write_text('{"status":"failed","evaluation_exit_code":1}', encoding="utf-8") before = (self.project / "azure.yaml").read_bytes() - self.run_recovery() + self.run_recovery(connection_null=True) operations = [cmd[3:6] for cmd in self.state["commands"] if cmd[3] == "ai"] self.assertEqual(operations, [["ai", "agent", "show"], ["ai", "agent", "delete"], ["ai", "agent", "show"], ["ai", "connection", "list"], - ["ai", "connection", "delete"], ["ai", "connection", "list"], + ["ai", "connection", "show"], ["ai", "connection", "delete"], + ["ai", "connection", "list"], ["ai", "toolbox", "list"], ["ai", "toolbox", "delete"], ["ai", "toolbox", "list"]]) self.assertEqual(self.read_json("ownership.json")["cleanup"], {NAME: "absent", CONN: "absent", TOOLBOX: "absent"}) @@ -210,6 +226,33 @@ def test_pending_resources_clean_in_order_without_upgrading_result(self): self.assertEqual(self.read_json("result.json")["evaluation_exit_code"], 1) self.assertEqual((self.project / "azure.yaml").read_bytes(), before) + def test_unowned_connection_digest_or_target_refuses_deletion(self): + self.ownership.update( + deployment_attempted=False, toolbox_endpoint="", connection_name="", + connection_deployment_attempted=False, cleanup={TOOLBOX: "unverified"}, + ) + self.context.update(toolbox_endpoint="", connection_name="") + self.run_recovery() + self.assertEqual( + [cmd[3:6] for cmd in self.state["commands"] if cmd[3] == "ai"], + [["ai", "toolbox", "list"], ["ai", "toolbox", "delete"], ["ai", "toolbox", "list"]], + ) + self.assertEqual(self.read_json("ownership.json")["cleanup"], {TOOLBOX: "absent"}) + bad_conn = "ci-e2e-tb-aaaaaaaaaa-00000000-mcp" + self.ownership.update( + deployment_attempted=True, toolbox_endpoint=TOOLBOX_ENDPOINT, + connection_name=bad_conn, connection_deployment_attempted=True, cleanup={}, + ) + self.context.update(toolbox_endpoint=TOOLBOX_ENDPOINT, connection_name=bad_conn) + self.run_recovery(code=1) + self.assertEqual(self.state["commands"], []) + self.assertFalse(self.read_json("recovery-result.json")["ownership_validated"]) + self.ownership.update(connection_name=CONN) + self.context.update(connection_name=CONN) + self.run_recovery(code=1, connection_changes={"target": "https://other.invalid/mcp"}) + self.assertFalse(self.state["connection_deleted"]) + self.assertEqual(self.read_json("ownership.json")["cleanup"], {NAME: "absent", CONN: "unverified", TOOLBOX: "absent"}) + def test_pending_skill_cleans_in_order(self): manifest = {"name": "recovery-skill", "services": { "project": {"host": "azure.ai.project"}, From ffcb978c1febec6448f9400c181fd0e9cd5300bd Mon Sep 17 00:00:00 2001 From: hund030 Date: Tue, 10 Nov 2026 10:34:51 +0800 Subject: [PATCH 3/3] fix(ci): rebind toolbox connection auth for sp invoke and relax preamble regex - rebind the verified toolbox MCP connection to `project-managed-identity` before CI invocation so app-only service principal callers do not fail on `UserEntraToken` OBO exchange - use word-boundary anchors in `quickstart-web-research/test-spec.yml` so labeled fields match when streamed immediately after a pre-tool preamble --- .../quickstart-web-research/test-spec.yml | 12 +++---- .../prompt-agents/prompt_agent_helpers.py | 34 ++++++++++++++----- .../scripts/prompt-agents/run-prompt-agent.sh | 11 ++++++ .../tests/test_prompt_agent_helpers.py | 34 ++++++++++++++++++- .../tests/test_prompt_agent_lifecycle.py | 18 +++++++++- .../tests/test_prompt_agent_recovery.py | 2 +- 6 files changed, 94 insertions(+), 17 deletions(-) diff --git a/.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml b/.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml index 583c3cca3..da4c78e4a 100644 --- a/.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml +++ b/.azure-pipelines/prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml @@ -13,22 +13,22 @@ tests: assertions: - source: assistant_text type: regex - value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?VERSION(?:`|\*\*)?[ \t]*:[ \t]*.*?\b\d+\.\d+\.\d+\b' + value: '(?i)\bVERSION(?:`|\*\*)?[ \t]*:[ \t]*[^\r\n]*?\b\d+\.\d+\.\d+\b' - source: assistant_text type: regex - value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?RELEASE_DATE(?:`|\*\*)?[ \t]*:[ \t]*\S+' + value: '(?i)\bRELEASE_DATE(?:`|\*\*)?[ \t]*:[ \t]*\S+' - source: assistant_text type: regex - value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?CHANGE_1(?:`|\*\*)?[ \t]*:[ \t]*\S+' + value: '(?i)\bCHANGE_1(?:`|\*\*)?[ \t]*:[ \t]*\S+' - source: assistant_text type: regex - value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?CHANGE_2(?:`|\*\*)?[ \t]*:[ \t]*\S+' + value: '(?i)\bCHANGE_2(?:`|\*\*)?[ \t]*:[ \t]*\S+' - source: assistant_text type: regex - value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?RELEASE_URL(?:`|\*\*)?[ \t]*:[ \t]*.*?https?://(?:www\.)?python\.org/downloads/release/python-\d+' + value: '(?i)\bRELEASE_URL(?:`|\*\*)?[ \t]*:[ \t]*[^\r\n]*?https?://(?:www\.)?python\.org/downloads/release/python-\d+' - source: assistant_text type: regex - value: '(?im)^[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?(?:`|\*\*)?WHATS_NEW_URL(?:`|\*\*)?[ \t]*:[ \t]*.*?https?://docs\.python\.org/[^\s)>\]]*?/whatsnew/\d+\.\d+\.html' + value: '(?i)\bWHATS_NEW_URL(?:`|\*\*)?[ \t]*:[ \t]*[^\r\n]*?https?://docs\.python\.org/[^\s)>\]]*?/whatsnew/\d+\.\d+\.html' - source: assistant_text type: regex value: '(?i)traceback|unhandled exception' diff --git a/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py b/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py index de0192786..71dcdbaec 100644 --- a/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py +++ b/.azure-pipelines/scripts/prompt-agents/prompt_agent_helpers.py @@ -81,7 +81,7 @@ def _validate_project_endpoint(context: dict) -> str: return project_endpoint.rstrip("/") -def _expected_connection_name(context: dict) -> tuple[str, str]: +def _expected_connection_name(context: dict) -> tuple[str, str, str]: name = owned_toolbox_name(context) project_endpoint = _validate_project_endpoint(context) toolbox_endpoint = context.get("toolbox_endpoint") @@ -97,7 +97,7 @@ def _expected_connection_name(context: dict) -> tuple[str, str]: expected = toolbox_connection_name(name, version, toolbox_endpoint) if not isinstance(connection_name, str) or connection_name != expected: raise LifecycleError("context must identify the owned toolbox connection") - return expected, toolbox_endpoint + return expected, toolbox_endpoint, version def _extract_skill_frontmatter(content: str) -> tuple[str, str]: @@ -497,8 +497,13 @@ def verify_adopted(manifest: Path, context: dict, model_deployment: str) -> None raise LifecycleError("init changed the requested agent identity, model, or kind") -def verify_connection(metadata: Path | None, context: dict) -> None: - expected_name, expected_target = _expected_connection_name(context) +def verify_connection( + metadata: Path | None, + context: dict, + *, + allowed_auth_types: tuple[str, ...] = ("userentratoken", "projectmanagedidentity"), +) -> None: + expected_name, expected_target, expected_version = _expected_connection_name(context) if metadata is None: return document = json.loads(metadata.read_text(encoding="utf-8")) @@ -506,11 +511,22 @@ def verify_connection(metadata: Path | None, context: dict) -> None: raise LifecycleError("connection show metadata must be an object") kind = document.get("kind") auth_type = document.get("authType") + conn_meta = document.get("metadata") + normalized_allowed = { + item.replace("-", "").replace("_", "").casefold() + for item in allowed_auth_types + } if (document.get("name") != expected_name or document.get("target") != expected_target or not isinstance(kind, str) or kind.casefold() != "remotetool" or not isinstance(auth_type, str) - or auth_type.replace("-", "").replace("_", "").casefold() != "userentratoken"): + or auth_type.replace("-", "").replace("_", "").casefold() not in normalized_allowed + or (conn_meta is not None and ( + not isinstance(conn_meta, dict) + or conn_meta.get("type") != "toolbox_MCP" + or conn_meta.get("toolboxName") != owned_toolbox_name(context) + or str(conn_meta.get("toolboxVersion", "")) != expected_version + ))): raise LifecycleError("connection show returned unexpected owned name, target, kind, or authType") @@ -556,7 +572,7 @@ def verify_deployed( or connection_id != expected_conn or mcp_tools[0]["server_url"] != context.get("toolbox_endpoint")): raise LifecycleError("managed agent toolbox connection ID does not match the owned toolbox endpoint") - verify_connection(connection_metadata, context) + verify_connection(connection_metadata, context, allowed_auth_types=("userentratoken",)) else: name = owned_skill_name(context) skills = definition.get("skills") @@ -629,7 +645,7 @@ def skill_absent(metadata: Path, context: dict) -> bool: def connection_absent(metadata: Path, context: dict) -> bool: - name, _ = _expected_connection_name(context) + name, _, _ = _expected_connection_name(context) connections = json.loads(metadata.read_text(encoding="utf-8")) if connections is None: return True @@ -708,6 +724,7 @@ def main(argv=None) -> int: connection = commands.add_parser("verify-connection") connection.add_argument("--metadata", type=Path) connection.add_argument("--context", type=Path, required=True) + connection.add_argument("--auth-type") connection_absence = commands.add_parser("connection-absent") for verification in (toolbox, toolbox_absence, skill, skill_absence, connection_absence): verification.add_argument("--metadata", type=Path, required=True) @@ -748,7 +765,8 @@ def main(argv=None) -> int: elif args.command == "skill-absent": return 0 if skill_absent(args.metadata, context) else 1 elif args.command == "verify-connection": - verify_connection(args.metadata, context) + allowed = (args.auth_type,) if args.auth_type else ("userentratoken", "projectmanagedidentity") + verify_connection(args.metadata, context, allowed_auth_types=allowed) elif args.command == "connection-absent": return 0 if connection_absent(args.metadata, context) else 1 else: diff --git a/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh b/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh index aa2765d6f..6e0381703 100644 --- a/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh +++ b/.azure-pipelines/scripts/prompt-agents/run-prompt-agent.sh @@ -310,6 +310,17 @@ fi run_command "$command_timeout" checked azd ai agent show "$service" --output json --no-prompt cp "$last_stdout" "$artifacts/agent.json" "$python" "$helper" verify-deployed --metadata "$(native_path "$artifacts/agent.json")" --context "$context" --model-deployment "$model" "${verify_options[@]}" +if [ -n "$connection" ]; then + toolbox_version=$(jq -r .version.version "$(native_path "$artifacts/toolbox.json")") + run_command "$command_timeout" checked azd ai connection create "$connection" \ + --kind remote-tool --target "$toolbox_endpoint" --auth-type project-managed-identity \ + --audience https://ai.azure.com --metadata "type=toolbox_MCP" \ + --metadata "toolboxName=$toolbox" --metadata "toolboxVersion=$toolbox_version" \ + --force --project-endpoint "$project_endpoint" --no-prompt + run_command "$command_timeout" checked azd ai connection show "$connection" --project-endpoint "$project_endpoint" --output json --no-prompt + cp "$last_stdout" "$artifacts/connection.json" + "$python" "$helper" verify-connection --metadata "$(native_path "$artifacts/connection.json")" --context "$context" --auth-type projectmanagedidentity +fi jq -c '.[]' "$(native_path "$artifacts/turns.json")" > "$work/turns.jsonl" while IFS= read -r turn; do number=$(jq -r .global_turn <<< "$turn") diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py index a28cbb23c..7751185e0 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_helpers.py @@ -115,10 +115,22 @@ def test_toolbox_skill_and_connection_absence_are_pinned(self): conn_doc = { "name": context["connection_name"], "target": endpoint, "kind": "RemoteTool", "authType": "UserEntraToken", + "metadata": {"type": "toolbox_MCP", "toolboxName": context["toolbox_name"], "toolboxVersion": "3"}, } path.write_text(json.dumps(conn_doc), encoding="utf-8") helpers.verify_connection(path, context) - for field, bad_val in (("name", "wrong"), ("target", "https://other.invalid"), ("kind", "ApiKey"), ("authType", "ApiKey")): + helpers.verify_connection(path, context, allowed_auth_types=("userentratoken",)) + with self.assertRaisesRegex(helpers.LifecycleError, "connection show returned"): + helpers.verify_connection(path, context, allowed_auth_types=("projectmanagedidentity",)) + path.write_text(json.dumps({**conn_doc, "authType": "ProjectManagedIdentity"}), encoding="utf-8") + helpers.verify_connection(path, context) + helpers.verify_connection(path, context, allowed_auth_types=("projectmanagedidentity",)) + with self.assertRaisesRegex(helpers.LifecycleError, "connection show returned"): + helpers.verify_connection(path, context, allowed_auth_types=("userentratoken",)) + for field, bad_val in ( + ("name", "wrong"), ("target", "https://other.invalid"), ("kind", "ApiKey"), + ("authType", "ApiKey"), ("metadata", {"type": "toolbox_MCP", "toolboxName": context["toolbox_name"], "toolboxVersion": "1"}), + ): path.write_text(json.dumps({**conn_doc, field: bad_val}), encoding="utf-8") with self.assertRaisesRegex(helpers.LifecycleError, "connection show returned"): helpers.verify_connection(path, context) @@ -235,6 +247,26 @@ def test_source_skill_bundle_rejects_symlinks_before_copy(self): with self.assertRaisesRegex(helpers.LifecycleError, "symlink"): helpers._validate_source_skill_bundle(sample, "travel-guide") + def test_quickstart_web_research_spec_accepts_streamed_preamble(self): + spec = SCRIPT.parents[2] / "prompt-agent-tests/cli/microsoft-copilot-harness/quickstart-web-research/test-spec.yml" + plan = helpers.plan_contract(spec, "managed") + streamed = ( + 'I\'m checking the official Python release and "What\'s New" pages directly, ' + 'then I\'ll return the requested fields with citations.' + 'VERSION: Python 3.15.2 - [official Python release page](https://www.python.org/downloads/release/python-3152/)\n' + 'RELEASE_DATE: October 6, 2026\n' + 'CHANGE_1: Improved JIT compiler performance\n' + 'CHANGE_2: Enhanced REPL diagnostics\n' + 'RELEASE_URL: https://www.python.org/downloads/release/python-3152/\n' + 'WHATS_NEW_URL: https://docs.python.org/3/whatsnew/3.15.html\n' + ) + for assertion in plan["tests"][0]["turns"][0]["assertions"]: + matches = list(helpers.re.finditer(assertion["value"], streamed)) + if assertion.get("max_matches") == 0: + self.assertEqual(matches, []) + else: + self.assertGreaterEqual(len(matches), 1, assertion["value"]) + if __name__ == "__main__": unittest.main() diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py index d804276c5..026a39f8d 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_lifecycle.py @@ -181,12 +181,24 @@ def conn_name(tb_name, version, endpoint): if mode == "collision" or state["deployed"] and not state["connection_deleted"]: names.append(state["connection"].upper()) stdout = "null\n" if mode == "null" else json.dumps([{"name": name} for name in names]) + elif operation == "create": + assert argv == [ + "azd", "ai", "connection", "create", state["connection"], + "--kind", "remote-tool", "--target", state["toolbox_endpoint"], + "--auth-type", "project-managed-identity", "--audience", "https://ai.azure.com", + "--metadata", "type=toolbox_MCP", "--metadata", f"toolboxName={state['toolbox']}", + "--metadata", "toolboxVersion=3", "--force", + "--project-endpoint", config["endpoint"], "--no-prompt", + ], argv + assert state["deployed"] + state["connection_auth_type"] = "ProjectManagedIdentity" elif operation == "show": assert argv == ["azd", "ai", "connection", "show", state["connection"], "--project-endpoint", config["endpoint"], "--output", "json", "--no-prompt"], argv assert state["deployed"] conn_doc = { "name": state["connection"], "target": state["toolbox_endpoint"], - "kind": "RemoteTool", "authType": "UserEntraToken", + "kind": "RemoteTool", "authType": state.get("connection_auth_type", "UserEntraToken"), + "metadata": {"type": "toolbox_MCP", "toolboxName": state["toolbox"], "toolboxVersion": "3"}, } conn_doc.update(config.get("connection_changes", {})) stdout = json.dumps(conn_doc) @@ -249,6 +261,8 @@ def conn_name(tb_name, version, endpoint): stderr = text code = 1 elif operation == "invoke": + if config["managed"] == "toolbox": + assert state.get("connection_auth_type") == "ProjectManagedIdentity" filename = argv[argv.index("--input-file") + 1] expected = ["azd", "ai", "agent", "invoke", config["service"], "--input-file", filename, "--protocol", "responses", "--no-prompt"] @@ -561,6 +575,8 @@ def test_managed_success_orders_and_cleans_both_resources(self): self.assertEqual(list(adopted["services"]), ["project", toolbox, SERVICE]) self.assertEqual(adopted["services"][SERVICE]["uses"], ["project", toolbox]) self.assertEqual(self.read_json("toolbox.json")["endpoint"], self.state["toolbox_endpoint"]) + self.assertEqual(self.read_json("connection.json")["authType"], "ProjectManagedIdentity") + self.assertEqual(len(self.operations("create", "connection")), 1) self.assertEqual(self.read_json("context.json")["execution_profile"], "managed") self.assertLess(self.commands.index(["azd", "deploy", toolbox, "--no-prompt", "--timeout", "20"]), self.commands.index(["azd", "deploy", SERVICE, "--no-prompt", "--timeout", "20"])) diff --git a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py index f19276f38..c2fc65707 100644 --- a/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py +++ b/.azure-pipelines/scripts/prompt-agents/tests/test_prompt_agent_recovery.py @@ -213,7 +213,7 @@ def test_pending_resources_clean_in_order_without_upgrading_result(self): self.context.update(toolbox_endpoint=TOOLBOX_ENDPOINT, connection_name=CONN) (self.artifacts / "result.json").write_text('{"status":"failed","evaluation_exit_code":1}', encoding="utf-8") before = (self.project / "azure.yaml").read_bytes() - self.run_recovery(connection_null=True) + self.run_recovery(connection_null=True, connection_changes={"authType": "ProjectManagedIdentity"}) operations = [cmd[3:6] for cmd in self.state["commands"] if cmd[3] == "ai"] self.assertEqual(operations, [["ai", "agent", "show"], ["ai", "agent", "delete"], ["ai", "agent", "show"], ["ai", "connection", "list"],