diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 626631a48..0b68f8a6a 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -158,11 +158,11 @@ jobs: retention-days: 14 integration-local-gateway-shard: - name: integration-local-gateway (${{ matrix.shard }}/2) + name: integration-local-gateway (${{ matrix.shard }}/3) strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] runs-on: ubuntu-latest timeout-minutes: 40 steps: @@ -183,7 +183,7 @@ jobs: timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' - name: Slow integration tier on the local defaults env: - AGENTENV_TEST_SHARD: ${{ matrix.shard }}/2 + AGENTENV_TEST_SHARD: ${{ matrix.shard }}/3 run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ tst/integration/env/gateway/gateway_test.py diff --git a/AGENTS.md b/AGENTS.md index 5219364fa..a5f285c4a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -283,7 +283,7 @@ pass, but a failing one does not block the merge. `.github/workflows/local-backends.yml` runs `unit`, `ui`, `integration-local`, `integration-local-slow`, `integration-local-gateway` (the slow tier's `gateway_test.py`, run beside the rest of it) and `installer`. The slow tier runs on eight runners (`integration-local-slow (1/8)` to `(8/8)`) and the -gateway tests on two, and `integration-local-slow` and `integration-local-gateway` each pass when all +gateway tests on three, and `integration-local-slow` and `integration-local-gateway` each pass when all of their shards do. The Python jobs install from `uv.lock` (public PyPI only), with the uv that `.github/ci-requirements.txt` pins, and use no secrets and no external services (a `registry:2` container for the integration jobs). The `unit` job also fails if `uv.lock` resolves anything from a diff --git a/Makefile b/Makefile index 42932baef..189b9d793 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ unit-test: ## Run the unit test suite in parallel (no external services) # every module's fixtures with worker-unique tags. CI splits the slow tier # across runners instead (tst/util/shards.py). To run one runner's share: # AGENTENV_TEST_SHARD=2/8 make int-test-slow PYTEST_ARGS=--ignore=tst/integration/env/gateway/gateway_test.py -# AGENTENV_TEST_SHARD=1/2 .venv/bin/python -m pytest -m int_test_slow tst/integration/env/gateway/gateway_test.py +# AGENTENV_TEST_SHARD=1/3 .venv/bin/python -m pytest -m int_test_slow tst/integration/env/gateway/gateway_test.py INT_PYTEST_ARGS_FAST ?= --dist=loadscope --tb=short --durations=20 -p no:cacheprovider INT_PYTEST_ARGS_SLOW ?= --tb=short --durations=20 -p no:cacheprovider INT_FAST_WORKERS ?= auto diff --git a/tst/integration/env/envs/server_env_local_test.py b/tst/integration/env/envs/server_env_local_test.py index 702ad5012..b725729d0 100644 --- a/tst/integration/env/envs/server_env_local_test.py +++ b/tst/integration/env/envs/server_env_local_test.py @@ -38,7 +38,7 @@ from agent_env.task_step.task_steps.teardown_sandboxes import TORN_DOWN_KEY, TeardownSandboxesTaskStep from tst.util.a2a_test_agent import put_test_agent -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] _REPO = Path(__file__).resolve().parents[4] diff --git a/tst/integration/openenv/build_app_test.py b/tst/integration/openenv/build_app_test.py index 6f0b1083e..00027646c 100644 --- a/tst/integration/openenv/build_app_test.py +++ b/tst/integration/openenv/build_app_test.py @@ -29,7 +29,7 @@ from tst.integration.env.envs.server_env_local_test import VERIFY_CRATE, _docker, _free_port, _put_items_image from tst.util.a2a_test_agent import put_test_agent -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] RAISING_VERIFY = "async def verify(mcp_url):\n raise RuntimeError('the verifier broke')\n" @@ -230,7 +230,6 @@ async def test_every_failure_that_ends_an_episode_is_an_error_observation_and_a_ @pytest.mark.asyncio -@pytest.mark.int_test_slow async def test_each_session_holds_its_own_deployment_and_every_way_it_ends_tears_it_down(items): root = items.bundle("dock", {"crate": items()}) diff --git a/tst/integration/slow_durations.json b/tst/integration/slow_durations.json index fedd287df..ef5f65932 100644 --- a/tst/integration/slow_durations.json +++ b/tst/integration/slow_durations.json @@ -8,38 +8,39 @@ "tst/integration/agent/a2a_agent/a2a_agent_test.py": 0.0, "tst/integration/agent/a2a_agent/agent_triggers_e2e_test.py": 0.0, "tst/integration/agent/proxy_modal_vm_test.py": 0.0, - "tst/integration/artifact/artifacts/docker_image_artifact_test.py": 24.3, - "tst/integration/cli/bundle_init_images_test.py": 25.0, - "tst/integration/cli/cli_test.py": 59.8, - "tst/integration/cli/run_bundle_built_agents_test.py": 47.9, - "tst/integration/cli/run_bundle_built_envs_test.py::test_a_multi_whose_children_name_one_container_is_refused_before_anything_is_built_or_written": 0.2, - "tst/integration/cli/run_bundle_built_envs_test.py::test_a_multi_whose_servers_share_a_tool_name_is_refused_at_deploy_naming_the_tool_and_both_servers": 25.8, - "tst/integration/cli/run_bundle_built_envs_test.py::test_a_script_on_an_envs_host_reaches_it_and_its_mcp_endpoint_at_the_addresses_its_input_gives[gateway]": 25.1, - "tst/integration/cli/run_bundle_built_envs_test.py::test_a_script_on_an_envs_host_reaches_it_and_its_mcp_endpoint_at_the_addresses_its_input_gives[server]": 2.3, - "tst/integration/cli/run_bundle_built_envs_test.py::test_a_server_that_crashes_on_start_fails_its_deploy_naming_it_and_ending_with_what_it_printed": 19.5, - "tst/integration/cli/run_bundle_built_envs_test.py::test_a_stored_mcp_task_imports_as_a_bundle_that_runs_through_the_gateway": 51.6, - "tst/integration/cli/run_bundle_built_envs_test.py::test_a_universe_composed_every_way_seeds_each_server_of_a_multi_and_a_shared_one_seeds_its_server": 31.9, - "tst/integration/cli/run_bundle_built_envs_test.py::test_an_env_folder_is_built_named_by_its_card_deployed_and_rebuilt_only_when_it_changes": 19.1, - "tst/integration/cli/run_bundle_built_envs_test.py::test_an_output_a_bundles_task_wrote_is_written_under_the_folder_that_imports_it": 27.3, - "tst/integration/cli/run_bundle_built_envs_test.py::test_each_kind_of_bundle_env_deploys_through_the_gateway_and_a_child_edit_rewrites_its_multi": 250.5, - "tst/integration/cli/run_bundle_built_envs_test.py::test_environments_and_a_universe_written_from_a_bundle_load_into_an_env_through_the_gateway": 54.3, - "tst/integration/cli/run_bundle_built_envs_test.py::test_the_shipped_echo_mcp_example_runs_by_name_and_a_rerun_writes_nothing": 62.5, - "tst/integration/cli/run_bundle_built_envs_test.py::test_the_shipped_mail_mcp_example_runs_and_a_task_outside_a_bundle_deploys_its_env_by_its_alias": 50.0, - "tst/integration/cli/run_bundle_claude_code_install_test.py": 168.8, - "tst/integration/cli/run_bundle_claude_code_test.py": 166.6, - "tst/integration/cli/run_bundle_cli_folder_test.py": 84.5, - "tst/integration/cli/run_bundle_image_folder_test.py": 59.7, - "tst/integration/cli/run_bundle_image_ref_test.py": 20.2, - "tst/integration/cli/run_bundle_installed_agent_test.py": 22.0, - "tst/integration/cli/run_bundle_openhands_test.py": 249.1, - "tst/integration/cli/run_bundle_skills_test.py": 26.2, - "tst/integration/cli/run_role_models_test.py": 27.1, - "tst/integration/computer/base_computer_test.py": 84.3, - "tst/integration/computer/recording_chaos_test.py": 173.6, - "tst/integration/computer/scaffold_test.py": 51.8, - "tst/integration/env/env_test.py": 15.4, - "tst/integration/env/envs/build_context_local_test.py": 216.8, - "tst/integration/env/envs/multi_env_local_test.py": 99.5, + "tst/integration/artifact/artifacts/docker_image_artifact_test.py": 18.0, + "tst/integration/cli/bundle_init_images_test.py": 20.5, + "tst/integration/cli/cli_test.py": 59.5, + "tst/integration/cli/run_bundle_built_agents_test.py": 48.5, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_multi_whose_children_name_one_container_is_refused_before_anything_is_built_or_written": 0.1, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_multi_whose_servers_share_a_tool_name_is_refused_at_deploy_naming_the_tool_and_both_servers": 41.4, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_script_on_an_envs_host_reaches_it_and_its_mcp_endpoint_at_the_addresses_its_input_gives[gateway]": 40.0, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_script_on_an_envs_host_reaches_it_and_its_mcp_endpoint_at_the_addresses_its_input_gives[server]": 2.0, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_server_that_crashes_on_start_fails_its_deploy_naming_it_and_ending_with_what_it_printed": 27.6, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_stored_mcp_task_imports_as_a_bundle_that_runs_through_the_gateway": 63.0, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_universe_composed_every_way_seeds_each_server_of_a_multi_and_a_shared_one_seeds_its_server": 29.2, + "tst/integration/cli/run_bundle_built_envs_test.py::test_an_env_folder_is_built_named_by_its_card_deployed_and_rebuilt_only_when_it_changes": 18.0, + "tst/integration/cli/run_bundle_built_envs_test.py::test_an_output_a_bundles_task_wrote_is_written_under_the_folder_that_imports_it": 35.3, + "tst/integration/cli/run_bundle_built_envs_test.py::test_each_kind_of_bundle_env_deploys_through_the_gateway_and_a_child_edit_rewrites_its_multi": 150.4, + "tst/integration/cli/run_bundle_built_envs_test.py::test_environments_and_a_universe_written_from_a_bundle_load_into_an_env_through_the_gateway": 63.3, + "tst/integration/cli/run_bundle_built_envs_test.py::test_the_shipped_echo_mcp_example_runs_by_name_and_a_rerun_writes_nothing": 63.9, + "tst/integration/cli/run_bundle_built_envs_test.py::test_the_shipped_mail_mcp_example_runs_and_a_task_outside_a_bundle_deploys_its_env_by_its_alias": 54.2, + "tst/integration/cli/run_bundle_claude_code_install_test.py": 127.1, + "tst/integration/cli/run_bundle_claude_code_test.py": 146.5, + "tst/integration/cli/run_bundle_cli_folder_test.py": 80.6, + "tst/integration/cli/run_bundle_image_folder_test.py": 34.9, + "tst/integration/cli/run_bundle_image_ref_test.py": 14.4, + "tst/integration/cli/run_bundle_installed_agent_test.py": 8.6, + "tst/integration/cli/run_bundle_openhands_test.py": 258.5, + "tst/integration/cli/run_bundle_skills_test.py": 23.2, + "tst/integration/cli/run_role_models_test.py": 36.8, + "tst/integration/computer/base_computer_test.py": 75.1, + "tst/integration/computer/recording_chaos_test.py": 167.6, + "tst/integration/computer/scaffold_test.py": 109.6, + "tst/integration/env/env_test.py": 11.3, + "tst/integration/env/envs/build_context_local_test.py": 182.4, + "tst/integration/env/envs/multi_env_local_test.py": 111.4, + "tst/integration/env/envs/server_env_local_test.py": 72.0, "tst/integration/env/envs/server_env_modal_test.py": 0.0, "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[daytona]": 0.0, @@ -105,7 +106,7 @@ "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_cardless_environment_resolves_name_from_environment_name": 38.6, + "tst/integration/env/gateway/gateway_test.py::test_cardless_environment_resolves_name_from_environment_name": 36.5, "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[modal]": 0.0, @@ -200,7 +201,7 @@ "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[scale]": 29.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[scale]": 29.3, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[boxd]": 0.0, @@ -208,17 +209,17 @@ "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[scale]": 28.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[scale]": 28.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_request_behind_a_write_through_the_rest_proxy": 26.3, + "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_request_behind_a_write_through_the_rest_proxy": 26.1, "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_role_behind_each_write": 26.3, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[scale]": 36.5, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[scale]": 36.6, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[boxd]": 0.0, @@ -226,16 +227,16 @@ "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[scale]": 51.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[scale]": 51.2, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_consistent_mode": 41.6, + "tst/integration/env/gateway/gateway_test.py::test_gateway_consistent_mode": 41.1, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[scale]": 30.9, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[scale]": 30.1, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[boxd]": 0.0, @@ -251,7 +252,7 @@ "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[scale]": 30.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[scale]": 29.7, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[boxd]": 0.0, @@ -259,7 +260,7 @@ "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[scale]": 26.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[scale]": 26.4, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[boxd]": 0.0, @@ -267,24 +268,24 @@ "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[scale]": 37.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[scale]": 37.3, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_nl_executor[modal_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_website_trigger_fires_on_a_request_through_the_website_proxy": 40.3, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_agentenv_environment": 39.2, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_env_including_website": 70.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_website_trigger_fires_on_a_request_through_the_website_proxy": 58.4, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_agentenv_environment": 28.3, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_env_including_website": 72.6, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[scale]": 28.4, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[scale]": 207.5, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_mcp_server": 208.5, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_website_env": 227.8, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_website_env_v1": 40.4, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_mcp_server": 186.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_website_env": 227.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_website_env_v1": 41.5, "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[modal]": 0.0, @@ -381,7 +382,7 @@ "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_load_from_snapshot": 41.4, + "tst/integration/env/gateway/gateway_test.py::test_load_from_snapshot": 42.9, "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[modal]": 0.0, @@ -470,7 +471,7 @@ "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_renamed_registration_serves_its_registered_name": 29.5, + "tst/integration/env/gateway/gateway_test.py::test_renamed_registration_serves_its_registered_name": 37.7, "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[modal]": 0.0, @@ -519,8 +520,8 @@ "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_snapshot_agent_state_v1_and_fallback": 28.2, - "tst/integration/env/gateway/gateway_test.py::test_snapshot_cli": 55.3, + "tst/integration/env/gateway/gateway_test.py::test_snapshot_agent_state_v1_and_fallback": 48.4, + "tst/integration/env/gateway/gateway_test.py::test_snapshot_cli": 37.5, "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[modal]": 0.0, @@ -566,7 +567,7 @@ "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[scale]": 27.4, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[scale]": 27.1, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[boxd]": 0.0, @@ -609,8 +610,8 @@ "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_validate_universe_compatibility": 42.0, - "tst/integration/env/gateway/gateway_test.py::test_verify_universe_roundtrip_export_all_v1_and_fallback": 27.6, + "tst/integration/env/gateway/gateway_test.py::test_validate_universe_compatibility": 27.1, + "tst/integration/env/gateway/gateway_test.py::test_verify_universe_roundtrip_export_all_v1_and_fallback": 27.2, "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[modal]": 0.0, @@ -659,9 +660,9 @@ "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/task_step/task_steps_test.py": 167.8, + "tst/integration/env/task_step/task_steps_test.py": 180.1, "tst/integration/eval/eval_test.py": 1.4, - "tst/integration/openenv/build_app_test.py": 22.6, + "tst/integration/openenv/build_app_test.py": 78.8, "tst/integration/providers/sandbox_providers/boxd_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/daytona_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/e2b_sandbox_smoke_test.py": 0.0, @@ -669,13 +670,13 @@ "tst/integration/providers/sandbox_providers/tenki_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/tensorlake_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py": 0.0, - "tst/integration/providers/sandbox_providers/vm_sandbox_build_test.py": 2.3, - "tst/integration/store/local_parallel_deploy_test.py": 54.5, - "tst/integration/task_step/test_agent_on_local_sandbox.py": 9.2, + "tst/integration/providers/sandbox_providers/vm_sandbox_build_test.py": 1.3, + "tst/integration/store/local_parallel_deploy_test.py": 88.0, + "tst/integration/task_step/test_agent_on_local_sandbox.py": 6.5, "tst/integration/task_step/test_enable_docker_e2e.py": 0.0, "tst/integration/task_step/test_multiturn_verifier_delivery.py": 0.0, - "tst/integration/task_step/test_next_env_event_hook_local.py": 108.1, - "tst/integration/task_step/test_register_env_triggers_local.py": 273.2, + "tst/integration/task_step/test_next_env_event_hook_local.py": 147.5, + "tst/integration/task_step/test_register_env_triggers_local.py": 92.0, "tst/integration/utils/docker/prune_test.py": 8.2 } } diff --git a/tst/integration/store/local_parallel_deploy_test.py b/tst/integration/store/local_parallel_deploy_test.py index e31738d32..4964e80c4 100644 --- a/tst/integration/store/local_parallel_deploy_test.py +++ b/tst/integration/store/local_parallel_deploy_test.py @@ -8,6 +8,7 @@ prior step", which would serialise the branches and prove nothing. """ +import contextlib import json import socket import subprocess @@ -57,6 +58,14 @@ def _wait_ready(url: str, codes: tuple[int, ...], timeout: int = 60) -> bool: @pytest.fixture def local_stack(monkeypatch, tmp_path): + with local_stack_at(monkeypatch, tmp_path) as host: + yield host + + +@contextlib.contextmanager +def local_stack_at(monkeypatch: pytest.MonkeyPatch, path: Path): + """A throwaway ``registry:2`` and the local stores in ``path`` for as long as the block runs: ``local_stack`` for one + test, or under a module's own MonkeyPatch for a stack its tests share. Yields the registry's host.""" if _docker("info").returncode != 0: pytest.skip("docker daemon not available") @@ -70,7 +79,7 @@ def local_stack(monkeypatch, tmp_path): _docker("rm", "-f", reg_name) pytest.skip("local registry did not become ready") - monkeypatch.chdir(tmp_path) + monkeypatch.chdir(path) for var in ("DOCUMENT", "OBJECT", "IMAGE"): monkeypatch.setenv(f"AGENT_ENV_{var}_STORE", "local") configure() diff --git a/tst/integration/task_step/test_register_env_triggers_local.py b/tst/integration/task_step/test_register_env_triggers_local.py index dfd738f70..0a695d011 100644 --- a/tst/integration/task_step/test_register_env_triggers_local.py +++ b/tst/integration/task_step/test_register_env_triggers_local.py @@ -1,7 +1,8 @@ """Named nl-trigger executors on a local gateway: each nl action reaches only the executor it names, that executor acts in the env as its own role, personas react to each other through `when.roles`, replies wait out their delay and working hours in virtual time, deadlines fire on silence, and the trajectory names who made each call. Requires a docker daemon; -spins up a throwaway ``registry:2``.""" +spins up a throwaway ``registry:2``. The tests share one deployed env, which ``items_env`` resets before each. +""" import asyncio import functools @@ -14,29 +15,48 @@ import httpx import pytest +import pytest_asyncio from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER -from agent_env.env.gateway.constants import TRIGGER_IN_FLIGHT_STATUSES +from agent_env.env.gateway.constants import METHOD_RESET, RPC_PATH, TRIGGER_IN_FLIGHT_STATUSES from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep from tst.integration.env.envs.server_env_local_test import _put_items_image -from tst.integration.store.local_parallel_deploy_test import _seed_bootstrap_envs, local_stack as local_stack +from tst.integration.store.local_parallel_deploy_test import _seed_bootstrap_envs, local_stack_at pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] +@pytest_asyncio.fixture(scope="module") +async def suite(tmp_path_factory): + """One items env for the module, on a stack of its own: deploying one is most of what a test here costs.""" + with pytest.MonkeyPatch.context() as monkeypatch, local_stack_at(monkeypatch, tmp_path_factory.mktemp("stack")): + _seed_bootstrap_envs() + env, deployed, host = await _deploy_items("trig") + try: + yield env, deployed, host + finally: + await env.close() + + +@pytest_asyncio.fixture +async def items_env(suite): + """The module's env as a fresh deploy leaves it: no triggers, executors or events, no clock, and no items.""" + _, deployed, _ = suite + async with httpx.AsyncClient(base_url=deployed.gateway_url, timeout=60) as client: + for path in ("/triggers/clear", "/clock/clear"): + (await client.post(path)).raise_for_status() + reset = (await client.post(RPC_PATH, json={"jsonrpc": "2.0", "id": "reset", "method": METHOD_RESET, + "params": {}})).raise_for_status().json() + assert "error" not in reset, reset + return suite + + @pytest.mark.asyncio -async def test_each_nl_action_reaches_the_executor_it_names(local_stack): - _seed_bootstrap_envs() - uid = uuid.uuid4().hex[:8] - items = MCPServerEnv.put(id=f"trig-items-{uid}", environment_name="items", - docker_image_artifact=_put_items_image(f"trig-items-{uid}")) - env = MultiEnv.put(id=f"trig-{uid}", mcp_server_envs=[items], name="suite") - deployed = await env.deploy(sandbox_type="local", ttl_seconds=900) - # host.docker.internal: loopback on Docker Desktop, the bridge gateway on Linux. - host = env._sandbox.host_ips[-1] +async def test_each_nl_action_reaches_the_executor_it_names(items_env): + env, deployed, host = items_env started: list[_Executor] = [] triggers = [ {"id": "to-alice", "when": {"type": "action", "tool": "list_items"}, @@ -60,7 +80,6 @@ async def test_each_nl_action_reaches_the_executor_it_names(local_stack): finally: for executor in started: executor.stop() - await env.close() statuses = {t["id"]: t["status"] for t in state["triggers"]} assert statuses == {"to-alice": "fired", "to-bob": "fired", "on-add": "armed"}, state["events"] @@ -75,14 +94,8 @@ async def test_each_nl_action_reaches_the_executor_it_names(local_stack): @pytest.mark.asyncio -async def test_personas_react_to_each_other_until_one_stays_quiet(local_stack): - _seed_bootstrap_envs() - uid = uuid.uuid4().hex[:8] - items = MCPServerEnv.put(id=f"chain-items-{uid}", environment_name="items", - docker_image_artifact=_put_items_image(f"chain-items-{uid}")) - env = MultiEnv.put(id=f"chain-{uid}", mcp_server_envs=[items], name="suite") - deployed = await env.deploy(sandbox_type="local", ttl_seconds=900) - host = env._sandbox.host_ips[-1] +async def test_personas_react_to_each_other_until_one_stays_quiet(items_env): + env, deployed, host = items_env started: list[_Executor] = [] triggers = [ {"id": "denise-hears-solver", "when": {"type": "action", "tool": "list_items"}, @@ -109,7 +122,6 @@ async def test_personas_react_to_each_other_until_one_stays_quiet(local_stack): finally: for executor in started: executor.stop() - await env.close() assert (denise.instructions, bob.instructions) == (["DENISE-TO-SOLVER", "DENISE-TO-BOB"], ["BOB-TO-DENISE", "BOB-TO-DENISE"]) @@ -122,9 +134,8 @@ async def test_personas_react_to_each_other_until_one_stays_quiet(local_stack): @pytest.mark.asyncio -async def test_a_reply_waits_its_delay_and_hours_and_its_writes_name_the_fire(local_stack): - _seed_bootstrap_envs() - env, deployed, host = await _deploy_items("pace") +async def test_a_reply_waits_its_delay_and_hours_and_its_writes_name_the_fire(items_env): + env, deployed, host = items_env started: list[_Executor] = [] triggers = [{"id": "bob-hears-solver", "when": {"type": "action", "tool": "list_items"}, "actions": [{"type": "nl", "executor": "bob", "instruction": "BOB-ACTS", "delay": "PT2M"}]}] @@ -132,6 +143,8 @@ async def test_a_reply_waits_its_delay_and_hours_and_its_writes_name_the_fire(lo bob = _Executor("bob", "bob@example.com", deployed.gateway_url, host, started) context = TaskStepContext(deployed_envs=[deployed], deployed_agents=[bob.agent], metadata={}) async with httpx.AsyncClient(base_url=deployed.gateway_url, timeout=60) as client: + # The env is shared, so its trajectory holds the earlier tests' calls: read this test's, after them. + earlier = len((await client.get("/trajectory")).raise_for_status().text.splitlines()) await RegisterEnvTriggersStep(id="reg", version=None, env_id=env.id, triggers=triggers, watch_roles=["default"], executor_timeout_seconds=60, executor_hours={"bob": {"start": "09:00", "end": "17:00"}}).execute(context) @@ -139,11 +152,11 @@ async def test_a_reply_waits_its_delay_and_hours_and_its_writes_name_the_fire(lo await _arm_clock(client, "2026-10-14T08:50:00Z", 60) (await client.post("/step", json={"action": "call_tool", "tool_name": "list_items", "arguments": {}})).raise_for_status() state = await _quiet(client, {"bob-hears-solver": 1}) - trajectory = [json.loads(line) for line in (await client.get("/trajectory")).raise_for_status().text.splitlines()] + lines = (await client.get("/trajectory")).raise_for_status().text.splitlines() + trajectory = [json.loads(line) for line in lines[earlier:]] finally: for executor in started: executor.stop() - await env.close() paced = [e for e in state["events"] if e["kind"] in ("delayed", "deferred", "action_ok")] assert [e["kind"] for e in paced] == ["delayed", "deferred", "action_ok"] @@ -161,9 +174,8 @@ async def test_a_reply_waits_its_delay_and_hours_and_its_writes_name_the_fire(lo @pytest.mark.asyncio -async def test_a_deadline_fires_on_silence_and_a_reply_cancels_it(local_stack): - _seed_bootstrap_envs() - env, deployed, host = await _deploy_items("deadline") +async def test_a_deadline_fires_on_silence_and_a_reply_cancels_it(items_env): + env, deployed, host = items_env started: list[_Executor] = [] triggers = [] for n in (1, 2): @@ -186,7 +198,6 @@ async def test_a_deadline_fires_on_silence_and_a_reply_cancels_it(local_stack): finally: for executor in started: executor.stop() - await env.close() statuses = {t["id"]: t["status"] for t in state["triggers"]} assert (statuses["deadline-1"], statuses["deadline-2"]) == ("fired", "cancelled"), state["events"][-8:] @@ -196,9 +207,8 @@ async def test_a_deadline_fires_on_silence_and_a_reply_cancels_it(local_stack): @pytest.mark.asyncio -async def test_the_step_can_wait_for_start_of_run_time_triggers(local_stack): - _seed_bootstrap_envs() - env, deployed, host = await _deploy_items("await") +async def test_the_step_can_wait_for_start_of_run_time_triggers(items_env): + env, deployed, host = items_env triggers = [{"id": "wf-1", "when": {"type": "time", "at": "2026-10-14T10:02:00Z"}, "actions": [{"type": "tool", "tool": "items_add_item", "args": {"item": "WORLD-1"}}]}, {"id": "wf-2", "when": {"type": "time", "at": "2026-10-14T10:03:00Z"}, @@ -206,20 +216,17 @@ async def test_the_step_can_wait_for_start_of_run_time_triggers(local_stack): {"id": "later", "when": {"type": "time", "at": "2026-10-14T18:00:00Z"}, "actions": [{"type": "tool", "tool": "items_add_item", "args": {"item": "LATER"}}]}] context = TaskStepContext(deployed_envs=[deployed], deployed_agents=[], metadata={}) - try: - async with httpx.AsyncClient(base_url=deployed.gateway_url, timeout=60) as client: - await _arm_clock(client, "2026-10-14T10:00:00Z", 60) - started = time.monotonic() - await RegisterEnvTriggersStep(id="reg", version=None, env_id=env.id, triggers=triggers, watch_roles=["default"], - await_fired=["wf-1", "wf-2"], await_timeout_seconds=60).execute(context) - waited = time.monotonic() - started - state = (await client.get("/triggers/state")).raise_for_status().json() - with pytest.raises(RuntimeError, match=r"did not all fire within 3s; statuses \{'later': 'armed'\}"): - await RegisterEnvTriggersStep(id="reg-later", version=None, env_id=env.id, triggers=triggers[2:], - watch_roles=["default"], await_fired=["later"], - await_timeout_seconds=3).execute(context) - finally: - await env.close() + async with httpx.AsyncClient(base_url=deployed.gateway_url, timeout=60) as client: + await _arm_clock(client, "2026-10-14T10:00:00Z", 60) + started = time.monotonic() + await RegisterEnvTriggersStep(id="reg", version=None, env_id=env.id, triggers=triggers, watch_roles=["default"], + await_fired=["wf-1", "wf-2"], await_timeout_seconds=60).execute(context) + waited = time.monotonic() - started + state = (await client.get("/triggers/state")).raise_for_status().json() + with pytest.raises(RuntimeError, match=r"did not all fire within 3s; statuses \{'later': 'armed'\}"): + await RegisterEnvTriggersStep(id="reg-later", version=None, env_id=env.id, triggers=triggers[2:], + watch_roles=["default"], await_fired=["later"], + await_timeout_seconds=3).execute(context) awaited_first, awaited_later = (e["awaited"] for e in context.metadata["env_trigger_registrations"]) assert awaited_first == {"wf-1": "fired", "wf-2": "fired"}