From 7e417c8f8b3cf37bf598ef07aa526e92d9623409 Mon Sep 17 00:00:00 2001 From: JiahuiGu Date: Wed, 4 Nov 2026 20:34:41 +0800 Subject: [PATCH 1/6] fix(samples): use latest task in LangGraph HITL conversations --- .../07-human-in-the-loop/test-spec.yml | 32 +++++++++++++++++++ .../responses/07-human-in-the-loop/README.md | 16 +++++++--- .../main.py | 20 ++++++++++-- 3 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 .azure-pipelines/hosted-agent-tests/python/langgraph/responses/07-human-in-the-loop/test-spec.yml diff --git a/.azure-pipelines/hosted-agent-tests/python/langgraph/responses/07-human-in-the-loop/test-spec.yml b/.azure-pipelines/hosted-agent-tests/python/langgraph/responses/07-human-in-the-loop/test-spec.yml new file mode 100644 index 000000000..277c07a27 --- /dev/null +++ b/.azure-pipelines/hosted-agent-tests/python/langgraph/responses/07-human-in-the-loop/test-spec.yml @@ -0,0 +1,32 @@ +version: 1 + +sample: + owner: jiahuigu + experiences: [azd, vscode] + +tests: + - name: approved-tasks-use-latest-user-message + turns: + - input: Draft a one-sentence launch email containing the exact marker SEATTLE-LAUNCH-071. Do not include any other city or event marker. + approvals: + mcp: + - server_label: langgraph + name: __hosted_agent_adapter_interrupt__ + assertions: + - source: assistant_text + type: contains + value: SEATTLE-LAUNCH-071 + - input: This is a new task. Draft a one-sentence invitation containing the exact marker PARIS-EVENT-072. Do not mention the previous task or its marker. + approvals: + mcp: + - server_label: langgraph + name: __hosted_agent_adapter_interrupt__ + assertions: + - source: assistant_text + type: contains + value: PARIS-EVENT-072 + - source: assistant_text + type: contains + value: SEATTLE-LAUNCH-071 + min_matches: 0 + max_matches: 0 diff --git a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md index 6aa494564..d4ecfa74e 100644 --- a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md +++ b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md @@ -7,11 +7,11 @@ A [LangGraph](https://langchain-ai.github.io/langgraph/) **human-in-the-loop** a ### Graph shape ``` -START → draft → await_approval ──approve──► END (emit final draft) - │ - └──revise(feedback)──► draft (loop) - │ - └──reject──► host returns response.failed (interrupt_rejected) +START → start_task → draft → await_approval ──approve──► END (emit final draft) + │ + └──revise(feedback)──► draft (loop) + │ + └──reject──► host returns response.failed (interrupt_rejected) ``` The graph state declares three channels: @@ -20,6 +20,12 @@ The graph state declares three channels: - **`draft`** — the current proposal text under review. - **`revision_history`** — list of `{draft, feedback}` pairs from prior revision rounds; each round seeds the next `draft` call with the rejected draft and the reviewer's feedback. +After completing a task, a new user message in the same conversation starts a new +task. `start_task` clears the previous draft and revision history, while `draft` +selects the latest user message rather than the first conversation message. +Revision feedback resumes the existing task directly and retains its revision +history; it does not pass through `start_task`. + Routing out of `await_approval` is done by returning a [`Command(goto=...)`](https://langchain-ai.github.io/langgraph/how-tos/command/) instead of static edges, so the same node can finalize, loop, or stay paused depending on the resume value. State is persisted by `FoundryCheckpointSaver` and keyed by the `conversation.id` from the Responses request, so follow-up requests continue the paused run even after a process restart. The saver uses Foundry State Store when hosted and a local file-backed store during local development. See [main.py](src/langgraph-human-in-the-loop-responses/main.py) for the full implementation. diff --git a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/langgraph-human-in-the-loop-responses/main.py b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/langgraph-human-in-the-loop-responses/main.py index a7391bd15..22fb88690 100644 --- a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/langgraph-human-in-the-loop-responses/main.py +++ b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/langgraph-human-in-the-loop-responses/main.py @@ -92,11 +92,23 @@ class State(TypedDict): def _build_graph(model: ChatOpenAI): + def start_task(state: State) -> dict: + return {"draft": "", "revision_history": []} + async def draft(state: State) -> dict: - task = state["messages"][0].content + task = next( + ( + message + for message in reversed(state["messages"]) + if isinstance(message, HumanMessage) + ), + None, + ) + if task is None: + raise ValueError("A user message is required to draft a proposal") msgs: list[BaseMessage] = [ SystemMessage(content=_DRAFT_PROMPT), - HumanMessage(content=f"Task: {task}"), + HumanMessage(content=task.content), ] for rev in state.get("revision_history", []): msgs.append(AIMessage(content=rev["draft"])) @@ -125,9 +137,11 @@ def await_approval(state: State) -> Command: ) builder = StateGraph(State) + builder.add_node("start_task", start_task) builder.add_node("draft", draft) builder.add_node("await_approval", await_approval) - builder.add_edge(START, "draft") + builder.add_edge(START, "start_task") + builder.add_edge("start_task", "draft") builder.add_edge("draft", "await_approval") checkpointer = FoundryCheckpointSaver(user_isolation=True) return builder.compile(checkpointer=checkpointer) From 5b80ac603fe17d591d8b0eef16d5dc1301e83f30 Mon Sep 17 00:00:00 2001 From: JiahuiGu Date: Thu, 5 Nov 2026 00:29:19 +0800 Subject: [PATCH 2/6] test(samples): cover HITL revision lifecycle over Responses --- .azure-pipelines/hosted-agent-tests/README.md | 20 ++ .../tests/test_langgraph_sample_lifecycle.py | 198 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 .azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py diff --git a/.azure-pipelines/hosted-agent-tests/README.md b/.azure-pipelines/hosted-agent-tests/README.md index e4d5aa100..cca8a6523 100644 --- a/.azure-pipelines/hosted-agent-tests/README.md +++ b/.azure-pipelines/hosted-agent-tests/README.md @@ -147,6 +147,26 @@ as rejecting MCP approval policies for Invocations. The PR policy applies both checks to new sample contracts. Cloud runners must validate and plan the contracts they consume before executing their turns. +### Local sample lifecycle regression checks + +The shared tests directory also contains deterministic lifecycle checks against +the actual sample graph and local Responses HTTP host. Run the HITL check from +the repository root using the sample runtime dependencies: + +```bash +uv run --frozen --project samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/langgraph-human-in-the-loop-responses python .azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py +``` + +This standard-library `unittest` check sends revision feedback through the rich +`function_call_output` resume channel, approves the revised draft, then submits +a new task in the same conversation. It asserts the exact model inputs: the +revision includes its draft and feedback, while the new task includes neither. +Mutation checks ensure that removing history reset or feedback application is +detected at the corresponding assertion. The model is scripted and the graph +checkpointer is in-memory; no Azure credentials or model calls are required. +These are local regression checks, not cloud or UI acceptance. They require the +sample runtime environment and are separate from the policy-only test suite. + ## Complete document shape ```yaml diff --git a/.azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py b/.azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py new file mode 100644 index 000000000..c20b3e66b --- /dev/null +++ b/.azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py @@ -0,0 +1,198 @@ +"""Local sample lifecycle regressions using runtime dependencies, not Azure access. + +Run from the repository root in the HITL sample's Python environment: + python .azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest.mock import AsyncMock, patch + +from langchain_azure_ai.agents.hosting import ResponsesHostServer +from langchain_core.messages import AIMessage, HumanMessage +from langgraph.checkpoint.memory import InMemorySaver +from starlette.testclient import TestClient + +ROOT = Path(__file__).resolve().parents[4] +SAMPLE = ROOT / ( + "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/" + "src/langgraph-human-in-the-loop-responses/main.py" +) + + +class LangGraphSampleLifecycleTests(unittest.TestCase): + def exercise_lifecycle(self, source: str) -> None: + state_root = self.enterContext(tempfile.TemporaryDirectory()) + source_path = Path(state_root) / "main.py" + source_path.write_text(source, encoding="utf-8") + spec = importlib.util.spec_from_file_location( + "hitl_sample_lifecycle", source_path + ) + self.assertIsNotNone(spec) + self.assertIsNotNone(spec.loader) + module = importlib.util.module_from_spec(spec) + model = types.SimpleNamespace( + ainvoke=AsyncMock( + side_effect=[ + AIMessage(content="SEATTLE-DRAFT-ORIGINAL"), + AIMessage(content="SEATTLE-DRAFT-REVISED"), + AIMessage(content="PARIS-DRAFT"), + ] + ) + ) + with ( + patch.dict(sys.modules, {module.__name__: module}), + patch.dict( + os.environ, + {"AGENTSERVER_STATE_ROOT": state_root, "OTEL_SDK_DISABLED": "true"}, + ), + patch("dotenv.load_dotenv"), + ): + spec.loader.exec_module(module) + with patch.object( + module, "FoundryCheckpointSaver", return_value=InMemorySaver() + ): + graph = module._build_graph(model) + with TestClient(ResponsesHostServer(graph).app) as client: + conversation = {"id": "hitl-lifecycle"} + + def post(input_items: object) -> dict: + response = client.post( + "/responses", + json={"conversation": conversation, "input": input_items}, + ) + self.assertEqual(response.status_code, 200, response.text) + payload = response.json() + self.assertEqual(payload["status"], "completed", payload) + return payload + + def pending(payload: dict) -> dict: + requests = [ + item + for item in payload["output"] + if item.get("type") == "mcp_approval_request" + ] + self.assertEqual(len(requests), 1, payload) + return requests[0] + + def approve(payload: dict) -> dict: + return post( + [ + { + "type": "mcp_approval_response", + "approval_request_id": pending(payload)["id"], + "approve": True, + } + ] + ) + + def assistant_text(payload: dict) -> str: + return "".join( + part.get("text", "") + for item in payload["output"] + if item.get("type") == "message" + for part in item.get("content", []) + ) + + initial = post("Draft a Seattle launch email.") + pending(initial) + original_call = model.ainvoke.await_args_list[0] + self.assertEqual( + original_call.args[0][1].content, "Draft a Seattle launch email." + ) + sentinels = [ + item + for item in initial["output"] + if item.get("type") == "function_call" + and item.get("name") == "__hosted_agent_adapter_interrupt__" + ] + self.assertEqual(len(sentinels), 1, initial) + feedback = "Keep the Seattle launch date, but shorten the email." + revised = post( + [ + { + "type": "function_call_output", + "call_id": sentinels[0]["call_id"], + "output": json.dumps({"resume": {"feedback": feedback}}), + } + ] + ) + self.assertEqual( + json.loads(pending(revised)["arguments"])["value"]["draft"], + "SEATTLE-DRAFT-REVISED", + ) + self.assertEqual(model.ainvoke.await_count, 2) + revision_messages = model.ainvoke.await_args_list[1].args[0] + self.assertEqual( + [message.content for message in revision_messages[1:]], + [ + "Draft a Seattle launch email.", + "SEATTLE-DRAFT-ORIGINAL", + f"Revision feedback: {feedback}", + ], + "revision must apply the current draft and feedback", + ) + self.assertIsInstance(revision_messages[2], AIMessage) + self.assertIsInstance(revision_messages[3], HumanMessage) + approved = approve(revised) + self.assertFalse( + any( + item.get("type") == "mcp_approval_request" + for item in approved["output"] + ), + approved, + ) + self.assertEqual(model.ainvoke.await_count, 2) + self.assertEqual(assistant_text(approved), "SEATTLE-DRAFT-REVISED") + + following = post("Draft a Paris event invitation.") + self.assertEqual(model.ainvoke.await_count, 3) + following_messages = model.ainvoke.await_args_list[2].args[0] + self.assertEqual( + [message.content for message in following_messages[1:]], + ["Draft a Paris event invitation."], + "new task must exclude the previous draft and revision feedback", + ) + self.assertEqual( + json.loads(pending(following)["arguments"])["value"]["draft"], + "PARIS-DRAFT", + ) + final = approve(following) + self.assertEqual(model.ainvoke.await_count, 3) + self.assertEqual(assistant_text(final), "PARIS-DRAFT") + self.assertFalse( + any( + item.get("type") == "mcp_approval_request" + for item in final["output"] + ), + final, + ) + + def test_revision_then_new_task_isolates_model_context(self) -> None: + self.exercise_lifecycle(SAMPLE.read_text(encoding="utf-8")) + + def test_detects_missing_history_reset(self) -> None: + source = SAMPLE.read_text(encoding="utf-8") + original = 'return {"draft": "", "revision_history": []}' + self.assertIn(original, source) + with self.assertRaisesRegex(AssertionError, "new task must exclude"): + self.exercise_lifecycle(source.replace(original, 'return {"draft": ""}', 1)) + + def test_detects_missing_feedback_application(self) -> None: + source = SAMPLE.read_text(encoding="utf-8") + original = 'for rev in state.get("revision_history", []):' + self.assertIn(original, source) + with self.assertRaisesRegex(AssertionError, "revision must apply"): + self.exercise_lifecycle(source.replace(original, "for rev in []:", 1)) + + +if __name__ == "__main__": + unittest.main() From 5e17122a1c8502b5fe4456e9235a026e6df133c2 Mon Sep 17 00:00:00 2001 From: JiahuiGu Date: Thu, 5 Nov 2026 01:03:11 +0800 Subject: [PATCH 3/6] fix(ci): isolate and run frozen HITL runtime regressions --- .azure-pipelines/hosted-agent-tests/README.md | 24 ++-- .azure-pipelines/hosted-agents-samples-ci.yml | 6 + .../hosted-agents/run_runtime_tests.py | 42 ++++++ .../test_langgraph_sample_lifecycle.py | 29 +--- .../tests/test_runtime_test_entrypoint.py | 134 ++++++++++++++++++ .github/workflows/scripts-selftest.yml | 16 +++ 6 files changed, 217 insertions(+), 34 deletions(-) create mode 100644 .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py rename .azure-pipelines/scripts/hosted-agents/{tests => runtime-tests}/test_langgraph_sample_lifecycle.py (84%) create mode 100644 .azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py diff --git a/.azure-pipelines/hosted-agent-tests/README.md b/.azure-pipelines/hosted-agent-tests/README.md index cca8a6523..b8135b137 100644 --- a/.azure-pipelines/hosted-agent-tests/README.md +++ b/.azure-pipelines/hosted-agent-tests/README.md @@ -149,23 +149,29 @@ they consume before executing their turns. ### Local sample lifecycle regression checks -The shared tests directory also contains deterministic lifecycle checks against -the actual sample graph and local Responses HTTP host. Run the HITL check from -the repository root using the sample runtime dependencies: +Runtime lifecycle checks live in the separate `runtime-tests` directory, outside +the lightweight `tests/test_*.py` policy discovery path. They exercise the actual +sample graph and local Responses HTTP host. Run them from the repository root: ```bash -uv run --frozen --project samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/langgraph-human-in-the-loop-responses python .azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py +python -m pip install "uv==0.11.7" +python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py ``` This standard-library `unittest` check sends revision feedback through the rich `function_call_output` resume channel, approves the revised draft, then submits a new task in the same conversation. It asserts the exact model inputs: the revision includes its draft and feedback, while the new task includes neither. -Mutation checks ensure that removing history reset or feedback application is -detected at the corresponding assertion. The model is scripted and the graph -checkpointer is in-memory; no Azure credentials or model calls are required. -These are local regression checks, not cloud or UI acceptance. They require the -sample runtime environment and are separate from the policy-only test suite. +The model is scripted and the graph checkpointer is in-memory; no Azure +credentials or model calls are required. No test rewrites or matches the sample's +source implementation. Mutation checks are development evidence only. + +The entrypoint first runs `uv sync --frozen --python 3.13` against the sample's +committed runtime project, then executes runtime test discovery in that isolated +environment. Dependency or test failures propagate; missing dependencies are not +skipped. GitHub scripts self-test runs it in a dedicated credential-free job, +and ADO runs it in a separate step after lightweight policy tests. These are +local runtime regressions, not cloud or UI acceptance. ## Complete document shape diff --git a/.azure-pipelines/hosted-agents-samples-ci.yml b/.azure-pipelines/hosted-agents-samples-ci.yml index 1b302a497..7a7fe291d 100644 --- a/.azure-pipelines/hosted-agents-samples-ci.yml +++ b/.azure-pipelines/hosted-agents-samples-ci.yml @@ -186,6 +186,12 @@ stages: fi displayName: Test and validate hosted-agent test specs + - bash: | + set -euo pipefail + python3 -m pip install --disable-pip-version-check "uv==0.11.7" + python3 .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py + displayName: Run frozen hosted-agent runtime regressions + - bash: .azure-pipelines/scripts/hosted-agents/tests/test-hosted-agent-ci-toolboxes.sh displayName: Test hosted-agent CI toolbox lifecycle scripts diff --git a/.azure-pipelines/scripts/hosted-agents/run_runtime_tests.py b/.azure-pipelines/scripts/hosted-agents/run_runtime_tests.py new file mode 100644 index 000000000..1d9243950 --- /dev/null +++ b/.azure-pipelines/scripts/hosted-agents/run_runtime_tests.py @@ -0,0 +1,42 @@ +"""Run hosted-agent runtime regressions outside the lightweight policy suite.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +def main() -> None: + root = Path(__file__).resolve().parents[3] + project = root / ( + "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/" + "src/langgraph-human-in-the-loop-responses" + ) + subprocess.run( + ["uv", "sync", "--frozen", "--python", "3.13", "--project", str(project)], + cwd=root, + check=True, + ) + subprocess.run( + [ + "uv", + "run", + "--no-sync", + "--project", + str(project), + "python", + "-m", + "unittest", + "discover", + "-s", + ".azure-pipelines/scripts/hosted-agents/runtime-tests", + "-p", + "test_*.py", + ], + cwd=root, + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/.azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py b/.azure-pipelines/scripts/hosted-agents/runtime-tests/test_langgraph_sample_lifecycle.py similarity index 84% rename from .azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py rename to .azure-pipelines/scripts/hosted-agents/runtime-tests/test_langgraph_sample_lifecycle.py index c20b3e66b..2c27a1bbe 100644 --- a/.azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py +++ b/.azure-pipelines/scripts/hosted-agents/runtime-tests/test_langgraph_sample_lifecycle.py @@ -1,7 +1,7 @@ """Local sample lifecycle regressions using runtime dependencies, not Azure access. -Run from the repository root in the HITL sample's Python environment: - python .azure-pipelines/scripts/hosted-agents/tests/test_langgraph_sample_lifecycle.py +Run with the sample's frozen runtime dependencies: + python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py """ from __future__ import annotations @@ -29,13 +29,9 @@ class LangGraphSampleLifecycleTests(unittest.TestCase): - def exercise_lifecycle(self, source: str) -> None: + def test_revision_then_new_task_isolates_model_context(self) -> None: state_root = self.enterContext(tempfile.TemporaryDirectory()) - source_path = Path(state_root) / "main.py" - source_path.write_text(source, encoding="utf-8") - spec = importlib.util.spec_from_file_location( - "hitl_sample_lifecycle", source_path - ) + spec = importlib.util.spec_from_file_location("hitl_sample_lifecycle", SAMPLE) self.assertIsNotNone(spec) self.assertIsNotNone(spec.loader) module = importlib.util.module_from_spec(spec) @@ -176,23 +172,6 @@ def assistant_text(payload: dict) -> str: final, ) - def test_revision_then_new_task_isolates_model_context(self) -> None: - self.exercise_lifecycle(SAMPLE.read_text(encoding="utf-8")) - - def test_detects_missing_history_reset(self) -> None: - source = SAMPLE.read_text(encoding="utf-8") - original = 'return {"draft": "", "revision_history": []}' - self.assertIn(original, source) - with self.assertRaisesRegex(AssertionError, "new task must exclude"): - self.exercise_lifecycle(source.replace(original, 'return {"draft": ""}', 1)) - - def test_detects_missing_feedback_application(self) -> None: - source = SAMPLE.read_text(encoding="utf-8") - original = 'for rev in state.get("revision_history", []):' - self.assertIn(original, source) - with self.assertRaisesRegex(AssertionError, "revision must apply"): - self.exercise_lifecycle(source.replace(original, "for rev in []:", 1)) - if __name__ == "__main__": unittest.main() diff --git a/.azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py b/.azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py new file mode 100644 index 000000000..cdd383848 --- /dev/null +++ b/.azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py @@ -0,0 +1,134 @@ +"""Keep runtime regression execution isolated, frozen, and wired into both CIs.""" + +from __future__ import annotations + +import importlib.util +import subprocess +import unittest +from pathlib import Path +from unittest.mock import patch + +import yaml + +ROOT = Path(__file__).resolve().parents[4] +SCRIPTS = ROOT / ".azure-pipelines/scripts/hosted-agents" +SPEC = importlib.util.spec_from_file_location( + "hosted_agent_runtime_tests", SCRIPTS / "run_runtime_tests.py" +) +assert SPEC and SPEC.loader +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +class RuntimeTestEntrypointTests(unittest.TestCase): + def test_syncs_frozen_sample_before_running_separate_discovery(self): + project = str( + ROOT + / "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop" + / "src/langgraph-human-in-the-loop-responses" + ) + with patch.object(runner.subprocess, "run") as run: + runner.main() + self.assertEqual(run.call_count, 2) + self.assertEqual( + run.call_args_list[0].args[0], + ["uv", "sync", "--frozen", "--python", "3.13", "--project", project], + ) + self.assertEqual( + run.call_args_list[1].args[0], + [ + "uv", + "run", + "--no-sync", + "--project", + project, + "python", + "-m", + "unittest", + "discover", + "-s", + ".azure-pipelines/scripts/hosted-agents/runtime-tests", + "-p", + "test_*.py", + ], + ) + for call in run.call_args_list: + self.assertEqual(call.kwargs, {"cwd": ROOT, "check": True}) + self.assertFalse( + (SCRIPTS / "tests/test_langgraph_sample_lifecycle.py").exists() + ) + self.assertTrue( + (SCRIPTS / "runtime-tests/test_langgraph_sample_lifecycle.py").is_file() + ) + + def test_sync_failure_prevents_test_execution(self): + with ( + patch.object( + runner.subprocess, + "run", + side_effect=subprocess.CalledProcessError(1, "uv"), + ) as run, + self.assertRaises(subprocess.CalledProcessError), + ): + runner.main() + self.assertEqual(run.call_count, 1) + + def test_runtime_failure_propagates(self): + with ( + patch.object( + runner.subprocess, + "run", + side_effect=[None, subprocess.CalledProcessError(1, "unittest")], + ), + self.assertRaises(subprocess.CalledProcessError), + ): + runner.main() + + def test_github_and_ado_run_the_entrypoint_without_skipping(self): + workflow = yaml.load( + (ROOT / ".github/workflows/scripts-selftest.yml").read_text( + encoding="utf-8" + ), + Loader=yaml.BaseLoader, + ) + job = workflow["jobs"]["hosted-agent-runtime-regressions"] + self.assertEqual(workflow["permissions"], {"contents": "read"}) + self.assertIn( + "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/**", + workflow["on"]["pull_request"]["paths"], + ) + self.assertEqual(job["steps"][0]["with"]["persist-credentials"], "false") + for step in job["steps"]: + self.assertNotIn("if", step) + self.assertNotIn("continue-on-error", step) + self.assertIn('"uv==0.11.7"', job["steps"][-2]["run"]) + self.assertEqual( + job["steps"][-1]["run"], + "python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py", + ) + pipeline = yaml.safe_load( + (ROOT / ".azure-pipelines/hosted-agents-samples-ci.yml").read_text( + encoding="utf-8" + ) + ) + discover = next( + stage for stage in pipeline["stages"] if stage.get("stage") == "Discover" + ) + steps = discover["jobs"][0]["steps"] + runtime = next( + step + for step in steps + if step.get("displayName") == "Run frozen hosted-agent runtime regressions" + ) + self.assertIn('"uv==0.11.7"', runtime["bash"]) + self.assertIn("set -euo pipefail", runtime["bash"]) + self.assertIn( + "python3 .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py", + runtime["bash"], + ) + self.assertNotIn("condition", runtime) + self.assertNotIn("continueOnError", runtime) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 1c76e6468..580154b68 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -27,11 +27,27 @@ on: - '.github/workflows/validation-pilot.yml' - '.github/scripts/discover-validation-samples.py' - 'samples/*/prompt-agents/**' + - 'samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/**' permissions: contents: read jobs: + hosted-agent-runtime-regressions: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.13' + - name: Install runtime test resolver + run: python -m pip install --disable-pip-version-check "uv==0.11.7" + - name: Run frozen hosted-agent runtime regressions + run: python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py + ado-pipelines-harness: runs-on: ubuntu-latest timeout-minutes: 15 From 58042aea57a25d54d8af7084248c590ee09cfb81 Mon Sep 17 00:00:00 2001 From: JiahuiGu Date: Tue, 10 Nov 2026 16:14:13 +0800 Subject: [PATCH 4/6] test(samples): scope HITL regressions without changing cloud CI --- .azure-pipelines/hosted-agent-tests/README.md | 26 ----------- .azure-pipelines/hosted-agents-samples-ci.yml | 6 --- .../scripts/run_hosted_agent_runtime_tests.py | 4 +- .../test_langgraph_sample_lifecycle.py | 4 +- .../test_hosted_agent_runtime_entrypoint.py | 44 ++++++------------- .github/workflows/scripts-selftest.yml | 3 +- .../responses/07-human-in-the-loop/README.md | 20 +++++++++ 7 files changed, 40 insertions(+), 67 deletions(-) rename .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py => .github/scripts/run_hosted_agent_runtime_tests.py (88%) rename {.azure-pipelines/scripts/hosted-agents => .github/scripts}/runtime-tests/test_langgraph_sample_lifecycle.py (98%) rename .azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py => .github/scripts/test/test_hosted_agent_runtime_entrypoint.py (70%) diff --git a/.azure-pipelines/hosted-agent-tests/README.md b/.azure-pipelines/hosted-agent-tests/README.md index 94e558fa4..5d96c1e5f 100644 --- a/.azure-pipelines/hosted-agent-tests/README.md +++ b/.azure-pipelines/hosted-agent-tests/README.md @@ -147,32 +147,6 @@ as rejecting MCP approval policies for Invocations. The PR policy applies both checks to new sample contracts. Cloud runners must validate and plan the contracts they consume before executing their turns. -### Local sample lifecycle regression checks - -Runtime lifecycle checks live in the separate `runtime-tests` directory, outside -the lightweight `tests/test_*.py` policy discovery path. They exercise the actual -sample graph and local Responses HTTP host. Run them from the repository root: - -```bash -python -m pip install "uv==0.11.7" -python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py -``` - -This standard-library `unittest` check sends revision feedback through the rich -`function_call_output` resume channel, approves the revised draft, then submits -a new task in the same conversation. It asserts the exact model inputs: the -revision includes its draft and feedback, while the new task includes neither. -The model is scripted and the graph checkpointer is in-memory; no Azure -credentials or model calls are required. No test rewrites or matches the sample's -source implementation. Mutation checks are development evidence only. - -The entrypoint first runs `uv sync --frozen --python 3.13` against the sample's -committed runtime project, then executes runtime test discovery in that isolated -environment. Dependency or test failures propagate; missing dependencies are not -skipped. GitHub scripts self-test runs it in a dedicated credential-free job, -and ADO runs it in a separate step after lightweight policy tests. These are -local runtime regressions, not cloud or UI acceptance. - ## Complete document shape ```yaml diff --git a/.azure-pipelines/hosted-agents-samples-ci.yml b/.azure-pipelines/hosted-agents-samples-ci.yml index 7a7fe291d..1b302a497 100644 --- a/.azure-pipelines/hosted-agents-samples-ci.yml +++ b/.azure-pipelines/hosted-agents-samples-ci.yml @@ -186,12 +186,6 @@ stages: fi displayName: Test and validate hosted-agent test specs - - bash: | - set -euo pipefail - python3 -m pip install --disable-pip-version-check "uv==0.11.7" - python3 .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py - displayName: Run frozen hosted-agent runtime regressions - - bash: .azure-pipelines/scripts/hosted-agents/tests/test-hosted-agent-ci-toolboxes.sh displayName: Test hosted-agent CI toolbox lifecycle scripts diff --git a/.azure-pipelines/scripts/hosted-agents/run_runtime_tests.py b/.github/scripts/run_hosted_agent_runtime_tests.py similarity index 88% rename from .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py rename to .github/scripts/run_hosted_agent_runtime_tests.py index 1d9243950..10449078c 100644 --- a/.azure-pipelines/scripts/hosted-agents/run_runtime_tests.py +++ b/.github/scripts/run_hosted_agent_runtime_tests.py @@ -7,7 +7,7 @@ def main() -> None: - root = Path(__file__).resolve().parents[3] + root = Path(__file__).resolve().parents[2] project = root / ( "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/" "src/langgraph-human-in-the-loop-responses" @@ -29,7 +29,7 @@ def main() -> None: "unittest", "discover", "-s", - ".azure-pipelines/scripts/hosted-agents/runtime-tests", + ".github/scripts/runtime-tests", "-p", "test_*.py", ], diff --git a/.azure-pipelines/scripts/hosted-agents/runtime-tests/test_langgraph_sample_lifecycle.py b/.github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py similarity index 98% rename from .azure-pipelines/scripts/hosted-agents/runtime-tests/test_langgraph_sample_lifecycle.py rename to .github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py index 2c27a1bbe..7a5d9280d 100644 --- a/.azure-pipelines/scripts/hosted-agents/runtime-tests/test_langgraph_sample_lifecycle.py +++ b/.github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py @@ -1,7 +1,7 @@ """Local sample lifecycle regressions using runtime dependencies, not Azure access. Run with the sample's frozen runtime dependencies: - python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py + python .github/scripts/run_hosted_agent_runtime_tests.py """ from __future__ import annotations @@ -21,7 +21,7 @@ from langgraph.checkpoint.memory import InMemorySaver from starlette.testclient import TestClient -ROOT = Path(__file__).resolve().parents[4] +ROOT = Path(__file__).resolve().parents[3] SAMPLE = ROOT / ( "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/" "src/langgraph-human-in-the-loop-responses/main.py" diff --git a/.azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py b/.github/scripts/test/test_hosted_agent_runtime_entrypoint.py similarity index 70% rename from .azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py rename to .github/scripts/test/test_hosted_agent_runtime_entrypoint.py index cdd383848..73b10373c 100644 --- a/.azure-pipelines/scripts/hosted-agents/tests/test_runtime_test_entrypoint.py +++ b/.github/scripts/test/test_hosted_agent_runtime_entrypoint.py @@ -1,4 +1,4 @@ -"""Keep runtime regression execution isolated, frozen, and wired into both CIs.""" +"""Keep runtime regression execution isolated, frozen, and wired into GitHub CI.""" from __future__ import annotations @@ -10,10 +10,10 @@ import yaml -ROOT = Path(__file__).resolve().parents[4] -SCRIPTS = ROOT / ".azure-pipelines/scripts/hosted-agents" +ROOT = Path(__file__).resolve().parents[3] +SCRIPTS = ROOT / ".github/scripts" SPEC = importlib.util.spec_from_file_location( - "hosted_agent_runtime_tests", SCRIPTS / "run_runtime_tests.py" + "hosted_agent_runtime_tests", SCRIPTS / "run_hosted_agent_runtime_tests.py" ) assert SPEC and SPEC.loader runner = importlib.util.module_from_spec(SPEC) @@ -47,16 +47,14 @@ def test_syncs_frozen_sample_before_running_separate_discovery(self): "unittest", "discover", "-s", - ".azure-pipelines/scripts/hosted-agents/runtime-tests", + ".github/scripts/runtime-tests", "-p", "test_*.py", ], ) for call in run.call_args_list: self.assertEqual(call.kwargs, {"cwd": ROOT, "check": True}) - self.assertFalse( - (SCRIPTS / "tests/test_langgraph_sample_lifecycle.py").exists() - ) + self.assertFalse((SCRIPTS / "test/test_langgraph_sample_lifecycle.py").exists()) self.assertTrue( (SCRIPTS / "runtime-tests/test_langgraph_sample_lifecycle.py").is_file() ) @@ -84,7 +82,7 @@ def test_runtime_failure_propagates(self): ): runner.main() - def test_github_and_ado_run_the_entrypoint_without_skipping(self): + def test_github_runs_the_entrypoint_without_skipping(self): workflow = yaml.load( (ROOT / ".github/workflows/scripts-selftest.yml").read_text( encoding="utf-8" @@ -104,30 +102,16 @@ def test_github_and_ado_run_the_entrypoint_without_skipping(self): self.assertIn('"uv==0.11.7"', job["steps"][-2]["run"]) self.assertEqual( job["steps"][-1]["run"], - "python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py", - ) - pipeline = yaml.safe_load( - (ROOT / ".azure-pipelines/hosted-agents-samples-ci.yml").read_text( - encoding="utf-8" - ) + "python .github/scripts/run_hosted_agent_runtime_tests.py", ) - discover = next( - stage for stage in pipeline["stages"] if stage.get("stage") == "Discover" - ) - steps = discover["jobs"][0]["steps"] - runtime = next( - step - for step in steps - if step.get("displayName") == "Run frozen hosted-agent runtime regressions" - ) - self.assertIn('"uv==0.11.7"', runtime["bash"]) - self.assertIn("set -euo pipefail", runtime["bash"]) self.assertIn( - "python3 .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py", - runtime["bash"], + "python .github/scripts/test/test_hosted_agent_runtime_entrypoint.py", + next( + step["run"] + for step in workflow["jobs"]["ado-pipelines-harness"]["steps"] + if step.get("name") == "Test hosted-agent helpers and fixture contracts" + ), ) - self.assertNotIn("condition", runtime) - self.assertNotIn("continueOnError", runtime) if __name__ == "__main__": diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 580154b68..1ec22b43b 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -46,7 +46,7 @@ jobs: - name: Install runtime test resolver run: python -m pip install --disable-pip-version-check "uv==0.11.7" - name: Run frozen hosted-agent runtime regressions - run: python .azure-pipelines/scripts/hosted-agents/run_runtime_tests.py + run: python .github/scripts/run_hosted_agent_runtime_tests.py ado-pipelines-harness: runs-on: ubuntu-latest @@ -66,6 +66,7 @@ jobs: - name: Test hosted-agent helpers and fixture contracts run: | python -m unittest discover -s .azure-pipelines/scripts/hosted-agents/tests -p 'test_*.py' + python .github/scripts/test/test_hosted_agent_runtime_entrypoint.py bash .azure-pipelines/scripts/hosted-agents/tests/test-hosted-agent-ci-toolboxes.sh bash .azure-pipelines/scripts/hosted-agents/tests/test-hosted-agent-session-quota.sh python .github/scripts/test/test_ado_pipeline_fixtures.py diff --git a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md index d4ecfa74e..f2463d218 100644 --- a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md +++ b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md @@ -47,6 +47,26 @@ The Responses host emits two paired output items for each pause, both keyed by t The graph factory is registered in `langgraph.json` and loaded by `langchain_azure_ai.agents.hosting.run`, which exposes the OpenAI-compatible Responses endpoint at `/responses` and handles conversation history, interrupt serialization, and streaming lifecycle events automatically. +### Lifecycle regression check + +From the repository root, run the shared runtime regression entrypoint: + +```bash +python -m pip install "uv==0.11.7" +python .github/scripts/run_hosted_agent_runtime_tests.py +``` + +It installs this sample's committed dependencies with `uv sync --frozen` before +running the actual graph through a local Responses HTTP host. A scripted model +lets the test assert that rich `function_call_output` revision feedback reaches +the model, and that a new task after approval receives neither the prior draft +nor its revision feedback. No Azure credentials or model calls are needed. + +GitHub scripts self-test runs this in a dedicated runtime job, separate from +lightweight policy tests. Missing dependencies and test failures are not skipped. +The cloud contract separately tests latest-task selection in Code and Container +deployments; this local regression is not cloud or UI acceptance. + ## Option 1: Azure Developer CLI (`azd`) ### Prerequisites From efb4ec2aefb9ffc5bc5cedc7f251e2be22bf8e79 Mon Sep 17 00:00:00 2001 From: JiahuiGu Date: Fri, 13 Nov 2026 14:53:25 +0800 Subject: [PATCH 5/6] docs(samples): remove internal regression details from HITL README --- .../responses/07-human-in-the-loop/README.md | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md index f2463d218..d4ecfa74e 100644 --- a/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md +++ b/samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/README.md @@ -47,26 +47,6 @@ The Responses host emits two paired output items for each pause, both keyed by t The graph factory is registered in `langgraph.json` and loaded by `langchain_azure_ai.agents.hosting.run`, which exposes the OpenAI-compatible Responses endpoint at `/responses` and handles conversation history, interrupt serialization, and streaming lifecycle events automatically. -### Lifecycle regression check - -From the repository root, run the shared runtime regression entrypoint: - -```bash -python -m pip install "uv==0.11.7" -python .github/scripts/run_hosted_agent_runtime_tests.py -``` - -It installs this sample's committed dependencies with `uv sync --frozen` before -running the actual graph through a local Responses HTTP host. A scripted model -lets the test assert that rich `function_call_output` revision feedback reaches -the model, and that a new task after approval receives neither the prior draft -nor its revision feedback. No Azure credentials or model calls are needed. - -GitHub scripts self-test runs this in a dedicated runtime job, separate from -lightweight policy tests. Missing dependencies and test failures are not skipped. -The cloud contract separately tests latest-task selection in Code and Container -deployments; this local regression is not cloud or UI acceptance. - ## Option 1: Azure Developer CLI (`azd`) ### Prerequisites From 06f2a9c5f7d6c1f41520e7ce3e27d8c698b8ef35 Mon Sep 17 00:00:00 2001 From: JiahuiGu Date: Fri, 13 Nov 2026 15:12:18 +0800 Subject: [PATCH 6/6] test(samples): simplify and isolate HITL lifecycle CI --- .../scripts/run_hosted_agent_runtime_tests.py | 42 ------- .../test_langgraph_sample_lifecycle.py | 4 +- .../test_hosted_agent_runtime_entrypoint.py | 118 ------------------ .../workflows/hosted-agent-runtime-tests.yml | 32 +++++ .github/workflows/scripts-selftest.yml | 17 --- 5 files changed, 34 insertions(+), 179 deletions(-) delete mode 100644 .github/scripts/run_hosted_agent_runtime_tests.py delete mode 100644 .github/scripts/test/test_hosted_agent_runtime_entrypoint.py create mode 100644 .github/workflows/hosted-agent-runtime-tests.yml diff --git a/.github/scripts/run_hosted_agent_runtime_tests.py b/.github/scripts/run_hosted_agent_runtime_tests.py deleted file mode 100644 index 10449078c..000000000 --- a/.github/scripts/run_hosted_agent_runtime_tests.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Run hosted-agent runtime regressions outside the lightweight policy suite.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - - -def main() -> None: - root = Path(__file__).resolve().parents[2] - project = root / ( - "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/" - "src/langgraph-human-in-the-loop-responses" - ) - subprocess.run( - ["uv", "sync", "--frozen", "--python", "3.13", "--project", str(project)], - cwd=root, - check=True, - ) - subprocess.run( - [ - "uv", - "run", - "--no-sync", - "--project", - str(project), - "python", - "-m", - "unittest", - "discover", - "-s", - ".github/scripts/runtime-tests", - "-p", - "test_*.py", - ], - cwd=root, - check=True, - ) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py b/.github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py index 7a5d9280d..6884f7745 100644 --- a/.github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py +++ b/.github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py @@ -1,7 +1,7 @@ """Local sample lifecycle regressions using runtime dependencies, not Azure access. -Run with the sample's frozen runtime dependencies: - python .github/scripts/run_hosted_agent_runtime_tests.py +Run from the repository root in the sample's frozen runtime environment: + python .github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py """ from __future__ import annotations diff --git a/.github/scripts/test/test_hosted_agent_runtime_entrypoint.py b/.github/scripts/test/test_hosted_agent_runtime_entrypoint.py deleted file mode 100644 index 73b10373c..000000000 --- a/.github/scripts/test/test_hosted_agent_runtime_entrypoint.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Keep runtime regression execution isolated, frozen, and wired into GitHub CI.""" - -from __future__ import annotations - -import importlib.util -import subprocess -import unittest -from pathlib import Path -from unittest.mock import patch - -import yaml - -ROOT = Path(__file__).resolve().parents[3] -SCRIPTS = ROOT / ".github/scripts" -SPEC = importlib.util.spec_from_file_location( - "hosted_agent_runtime_tests", SCRIPTS / "run_hosted_agent_runtime_tests.py" -) -assert SPEC and SPEC.loader -runner = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(runner) - - -class RuntimeTestEntrypointTests(unittest.TestCase): - def test_syncs_frozen_sample_before_running_separate_discovery(self): - project = str( - ROOT - / "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop" - / "src/langgraph-human-in-the-loop-responses" - ) - with patch.object(runner.subprocess, "run") as run: - runner.main() - self.assertEqual(run.call_count, 2) - self.assertEqual( - run.call_args_list[0].args[0], - ["uv", "sync", "--frozen", "--python", "3.13", "--project", project], - ) - self.assertEqual( - run.call_args_list[1].args[0], - [ - "uv", - "run", - "--no-sync", - "--project", - project, - "python", - "-m", - "unittest", - "discover", - "-s", - ".github/scripts/runtime-tests", - "-p", - "test_*.py", - ], - ) - for call in run.call_args_list: - self.assertEqual(call.kwargs, {"cwd": ROOT, "check": True}) - self.assertFalse((SCRIPTS / "test/test_langgraph_sample_lifecycle.py").exists()) - self.assertTrue( - (SCRIPTS / "runtime-tests/test_langgraph_sample_lifecycle.py").is_file() - ) - - def test_sync_failure_prevents_test_execution(self): - with ( - patch.object( - runner.subprocess, - "run", - side_effect=subprocess.CalledProcessError(1, "uv"), - ) as run, - self.assertRaises(subprocess.CalledProcessError), - ): - runner.main() - self.assertEqual(run.call_count, 1) - - def test_runtime_failure_propagates(self): - with ( - patch.object( - runner.subprocess, - "run", - side_effect=[None, subprocess.CalledProcessError(1, "unittest")], - ), - self.assertRaises(subprocess.CalledProcessError), - ): - runner.main() - - def test_github_runs_the_entrypoint_without_skipping(self): - workflow = yaml.load( - (ROOT / ".github/workflows/scripts-selftest.yml").read_text( - encoding="utf-8" - ), - Loader=yaml.BaseLoader, - ) - job = workflow["jobs"]["hosted-agent-runtime-regressions"] - self.assertEqual(workflow["permissions"], {"contents": "read"}) - self.assertIn( - "samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/**", - workflow["on"]["pull_request"]["paths"], - ) - self.assertEqual(job["steps"][0]["with"]["persist-credentials"], "false") - for step in job["steps"]: - self.assertNotIn("if", step) - self.assertNotIn("continue-on-error", step) - self.assertIn('"uv==0.11.7"', job["steps"][-2]["run"]) - self.assertEqual( - job["steps"][-1]["run"], - "python .github/scripts/run_hosted_agent_runtime_tests.py", - ) - self.assertIn( - "python .github/scripts/test/test_hosted_agent_runtime_entrypoint.py", - next( - step["run"] - for step in workflow["jobs"]["ado-pipelines-harness"]["steps"] - if step.get("name") == "Test hosted-agent helpers and fixture contracts" - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/hosted-agent-runtime-tests.yml b/.github/workflows/hosted-agent-runtime-tests.yml new file mode 100644 index 000000000..9199abb12 --- /dev/null +++ b/.github/workflows/hosted-agent-runtime-tests.yml @@ -0,0 +1,32 @@ +name: Hosted-agent runtime regressions + +on: + workflow_dispatch: + pull_request: + paths: + - 'samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/**' + - '.github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py' + - '.github/workflows/hosted-agent-runtime-tests.yml' + +permissions: + contents: read + +jobs: + hitl-lifecycle: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + RUNTIME_PROJECT: samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/src/langgraph-human-in-the-loop-responses + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.13' + - name: Install runtime test resolver + run: python -m pip install --disable-pip-version-check "uv==0.11.7" + - name: Install frozen sample dependencies + run: uv sync --frozen --python 3.13 --project "$RUNTIME_PROJECT" + - name: Verify HITL revision lifecycle + run: uv run --no-sync --project "$RUNTIME_PROJECT" python .github/scripts/runtime-tests/test_langgraph_sample_lifecycle.py diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 1ec22b43b..1c76e6468 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -27,27 +27,11 @@ on: - '.github/workflows/validation-pilot.yml' - '.github/scripts/discover-validation-samples.py' - 'samples/*/prompt-agents/**' - - 'samples/python/hosted-agents/langgraph/responses/07-human-in-the-loop/**' permissions: contents: read jobs: - hosted-agent-runtime-regressions: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.13' - - name: Install runtime test resolver - run: python -m pip install --disable-pip-version-check "uv==0.11.7" - - name: Run frozen hosted-agent runtime regressions - run: python .github/scripts/run_hosted_agent_runtime_tests.py - ado-pipelines-harness: runs-on: ubuntu-latest timeout-minutes: 15 @@ -66,7 +50,6 @@ jobs: - name: Test hosted-agent helpers and fixture contracts run: | python -m unittest discover -s .azure-pipelines/scripts/hosted-agents/tests -p 'test_*.py' - python .github/scripts/test/test_hosted_agent_runtime_entrypoint.py bash .azure-pipelines/scripts/hosted-agents/tests/test-hosted-agent-ci-toolboxes.sh bash .azure-pipelines/scripts/hosted-agents/tests/test-hosted-agent-session-quota.sh python .github/scripts/test/test_ado_pipeline_fixtures.py