diff --git a/.github/scripts/slow_test_durations.py b/.github/scripts/slow_test_durations.py index 294f59c2e..13184b0f3 100644 --- a/.github/scripts/slow_test_durations.py +++ b/.github/scripts/slow_test_durations.py @@ -1,9 +1,10 @@ """Rewrite tst/integration/slow_durations.json from the slow tier's JUnit reports, so the CI shards stay balanced. - python .github/scripts/slow_test_durations.py junit-integration-slow-*.xml + gh run download --pattern 'junit-integration-local-*' + python .github/scripts/slow_test_durations.py junit-integration-local-*/junit-integration-slow.xml -Download the reports from a green ``main`` run (the ``junit-integration-local-*`` artifacts of every -``integration-local-slow`` and ``integration-local-gateway`` shard). Each test's time counts toward its file, +Use a green ``main`` run: it has a ``junit-integration-local-*`` artifact for every ``integration-local-slow`` and +``integration-local-gateway`` shard, which ``gh run download`` puts in a folder each. Each test's time counts toward its file, or toward the test itself for a file the table names in ``split_by_test``; that list is kept as it is. """ diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 3239618f9..626631a48 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -113,11 +113,11 @@ jobs: # (tst/util/shards.py). The gateway tests are a check of their own, split the same way. Each check passes # when all of its shards do (the integration-local-slow and integration-local-gateway jobs below). integration-local-slow-shard: - name: integration-local-slow (${{ matrix.shard }}/4) + name: integration-local-slow (${{ matrix.shard }}/8) strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4] + shard: [1, 2, 3, 4, 5, 6, 7, 8] runs-on: ubuntu-latest timeout-minutes: 40 steps: @@ -140,7 +140,7 @@ jobs: timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' - name: Slow integration tier on the local defaults env: - AGENTENV_TEST_SHARD: ${{ matrix.shard }}/4 + AGENTENV_TEST_SHARD: ${{ matrix.shard }}/8 run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ tst/integration --ignore=tst/integration/env/gateway/gateway_test.py diff --git a/AGENTS.md b/AGENTS.md index 5eca2721e..1564fc2ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -254,7 +254,8 @@ orchestrators call it the same way. What callers rely on: backends. Minutes-long tests (real image builds, sandbox VMs, gateways, agents) carry `@pytest.mark.int_test_slow` (or module-level `pytestmark`) and run serially. CI splits them across runners instead: `AGENTENV_TEST_SHARD=i/N` (`tst/util/shards.py`) runs the i-th of N groups, keeping a - file's tests together and balancing the groups by `tst/integration/slow_durations.json`. When the + file's tests together unless the table's `split_by_test` lists the file, and balancing the groups by + `tst/integration/slow_durations.json`. When the shards' times drift apart, rewrite the table from a `main` run's `junit-integration-local-*` artifacts with `.github/scripts/slow_test_durations.py`. - `tst/manual/` and `tst/benchmarks/` run only by hand, as each module's docstring says; no CI job @@ -281,7 +282,7 @@ resolved, and the checks the `main` ruleset requires: `unit`, `integration-local pass, but a failing one does not block the merge. `.github/workflows/local-backends.yml` runs `unit`, `ui`, `integration-local`, `integration-local-slow`, `integration-local-gateway` (the slow tier's `gateway_test.py`, run beside the rest of it) and -`installer`. The slow tier runs on four runners (`integration-local-slow (1/4)` to `(4/4)`) and the +`installer`. The slow tier runs on eight runners (`integration-local-slow (1/8)` to `(8/8)`) and the gateway tests on two, and `integration-local-slow` and `integration-local-gateway` each pass when all of their shards do. The Python jobs install from `uv.lock` (public PyPI only), with the uv that `.github/ci-requirements.txt` pins, and use no secrets and no external services (a `registry:2` diff --git a/Makefile b/Makefile index bf2ecb416..42932baef 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ unit-test: ## Run the unit test suite in parallel (no external services) # clock but is the only way to keep slow runs reliable without rewriting # every module's fixtures with worker-unique tags. CI splits the slow tier # across runners instead (tst/util/shards.py). To run one runner's share: -# AGENTENV_TEST_SHARD=2/4 make int-test-slow PYTEST_ARGS=--ignore=tst/integration/env/gateway/gateway_test.py +# AGENTENV_TEST_SHARD=2/8 make int-test-slow PYTEST_ARGS=--ignore=tst/integration/env/gateway/gateway_test.py # AGENTENV_TEST_SHARD=1/2 .venv/bin/python -m pytest -m int_test_slow tst/integration/env/gateway/gateway_test.py INT_PYTEST_ARGS_FAST ?= --dist=loadscope --tb=short --durations=20 -p no:cacheprovider INT_PYTEST_ARGS_SLOW ?= --tb=short --durations=20 -p no:cacheprovider diff --git a/tst/integration/slow_durations.json b/tst/integration/slow_durations.json index e8374021f..fedd287df 100644 --- a/tst/integration/slow_durations.json +++ b/tst/integration/slow_durations.json @@ -1,31 +1,45 @@ { "_comment": "Seconds each slow-tier unit took in CI, which AGENTENV_TEST_SHARD balances the CI shards by (tst/util/shards.py). A unit is a file, or a test of a file in split_by_test. Refresh with .github/scripts/slow_test_durations.py from the shards' JUnit reports.", "split_by_test": [ + "tst/integration/cli/run_bundle_built_envs_test.py", "tst/integration/env/gateway/gateway_test.py" ], "seconds": { "tst/integration/agent/a2a_agent/a2a_agent_test.py": 0.0, "tst/integration/agent/a2a_agent/agent_triggers_e2e_test.py": 0.0, - "tst/integration/agent/proxy_modal_vm_test.py": 13.6, - "tst/integration/artifact/artifacts/docker_image_artifact_test.py": 7.2, - "tst/integration/cli/bundle_init_images_test.py": 23.4, - "tst/integration/cli/cli_test.py": 58.3, - "tst/integration/cli/run_bundle_built_agents_test.py": 36.9, - "tst/integration/cli/run_bundle_built_envs_test.py": 534.3, - "tst/integration/cli/run_bundle_claude_code_install_test.py": 156.8, - "tst/integration/cli/run_bundle_claude_code_test.py": 90.6, - "tst/integration/cli/run_bundle_cli_folder_test.py": 74.7, - "tst/integration/cli/run_bundle_image_folder_test.py": 25.9, - "tst/integration/cli/run_bundle_image_ref_test.py": 14.1, - "tst/integration/cli/run_bundle_installed_agent_test.py": 8.2, - "tst/integration/cli/run_bundle_openhands_test.py": 228.5, - "tst/integration/cli/run_bundle_skills_test.py": 12.8, - "tst/integration/cli/run_role_models_test.py": 78.8, - "tst/integration/computer/recording_chaos_test.py": 74.6, - "tst/integration/computer/scaffold_test.py": 51.7, - "tst/integration/env/env_test.py": 4.1, - "tst/integration/env/envs/build_context_local_test.py": 170.2, - "tst/integration/env/envs/multi_env_local_test.py": 52.6, + "tst/integration/agent/proxy_modal_vm_test.py": 0.0, + "tst/integration/artifact/artifacts/docker_image_artifact_test.py": 24.3, + "tst/integration/cli/bundle_init_images_test.py": 25.0, + "tst/integration/cli/cli_test.py": 59.8, + "tst/integration/cli/run_bundle_built_agents_test.py": 47.9, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_multi_whose_children_name_one_container_is_refused_before_anything_is_built_or_written": 0.2, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_multi_whose_servers_share_a_tool_name_is_refused_at_deploy_naming_the_tool_and_both_servers": 25.8, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_script_on_an_envs_host_reaches_it_and_its_mcp_endpoint_at_the_addresses_its_input_gives[gateway]": 25.1, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_script_on_an_envs_host_reaches_it_and_its_mcp_endpoint_at_the_addresses_its_input_gives[server]": 2.3, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_server_that_crashes_on_start_fails_its_deploy_naming_it_and_ending_with_what_it_printed": 19.5, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_stored_mcp_task_imports_as_a_bundle_that_runs_through_the_gateway": 51.6, + "tst/integration/cli/run_bundle_built_envs_test.py::test_a_universe_composed_every_way_seeds_each_server_of_a_multi_and_a_shared_one_seeds_its_server": 31.9, + "tst/integration/cli/run_bundle_built_envs_test.py::test_an_env_folder_is_built_named_by_its_card_deployed_and_rebuilt_only_when_it_changes": 19.1, + "tst/integration/cli/run_bundle_built_envs_test.py::test_an_output_a_bundles_task_wrote_is_written_under_the_folder_that_imports_it": 27.3, + "tst/integration/cli/run_bundle_built_envs_test.py::test_each_kind_of_bundle_env_deploys_through_the_gateway_and_a_child_edit_rewrites_its_multi": 250.5, + "tst/integration/cli/run_bundle_built_envs_test.py::test_environments_and_a_universe_written_from_a_bundle_load_into_an_env_through_the_gateway": 54.3, + "tst/integration/cli/run_bundle_built_envs_test.py::test_the_shipped_echo_mcp_example_runs_by_name_and_a_rerun_writes_nothing": 62.5, + "tst/integration/cli/run_bundle_built_envs_test.py::test_the_shipped_mail_mcp_example_runs_and_a_task_outside_a_bundle_deploys_its_env_by_its_alias": 50.0, + "tst/integration/cli/run_bundle_claude_code_install_test.py": 168.8, + "tst/integration/cli/run_bundle_claude_code_test.py": 166.6, + "tst/integration/cli/run_bundle_cli_folder_test.py": 84.5, + "tst/integration/cli/run_bundle_image_folder_test.py": 59.7, + "tst/integration/cli/run_bundle_image_ref_test.py": 20.2, + "tst/integration/cli/run_bundle_installed_agent_test.py": 22.0, + "tst/integration/cli/run_bundle_openhands_test.py": 249.1, + "tst/integration/cli/run_bundle_skills_test.py": 26.2, + "tst/integration/cli/run_role_models_test.py": 27.1, + "tst/integration/computer/base_computer_test.py": 84.3, + "tst/integration/computer/recording_chaos_test.py": 173.6, + "tst/integration/computer/scaffold_test.py": 51.8, + "tst/integration/env/env_test.py": 15.4, + "tst/integration/env/envs/build_context_local_test.py": 216.8, + "tst/integration/env/envs/multi_env_local_test.py": 99.5, "tst/integration/env/envs/server_env_modal_test.py": 0.0, "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[daytona]": 0.0, @@ -91,7 +105,7 @@ "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_cardless_environment_resolves_name_from_environment_name": 38.8, + "tst/integration/env/gateway/gateway_test.py::test_cardless_environment_resolves_name_from_environment_name": 38.6, "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[modal]": 0.0, @@ -182,50 +196,50 @@ "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[scale]": 29.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[scale]": 29.1, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[scale]": 28.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[scale]": 28.8, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_request_behind_a_write_through_the_rest_proxy": 26.4, - "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_role_behind_each_write": 26.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_request_behind_a_write_through_the_rest_proxy": 26.3, + "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_role_behind_each_write": 26.3, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[scale]": 36.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[scale]": 36.5, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[scale]": 51.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[scale]": 51.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_consistent_mode": 41.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_consistent_mode": 41.6, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[scale]": 30.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[scale]": 30.9, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[sail_vm]": 0.0, @@ -233,44 +247,44 @@ "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[scale]": 30.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[scale]": 30.1, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[scale]": 26.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[scale]": 26.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[scale]": 37.5, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[scale]": 37.8, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_nl_executor[modal_vm]": 0.1, - "tst/integration/env/gateway/gateway_test.py::test_gateway_website_trigger_fires_on_a_request_through_the_website_proxy": 39.4, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_agentenv_environment": 28.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_env_including_website": 47.9, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_nl_executor[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_website_trigger_fires_on_a_request_through_the_website_proxy": 40.3, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_agentenv_environment": 39.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_env_including_website": 70.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[scale]": 28.3, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[scale]": 28.4, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_mcp_server": 189.2, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_website_env": 64.0, - "tst/integration/env/gateway/gateway_test.py::test_gateway_with_website_env_v1": 40.6, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_mcp_server": 208.5, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_website_env": 227.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_website_env_v1": 40.4, "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[modal]": 0.0, @@ -367,7 +381,7 @@ "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_load_from_snapshot": 41.8, + "tst/integration/env/gateway/gateway_test.py::test_load_from_snapshot": 41.4, "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[modal]": 0.0, @@ -456,7 +470,7 @@ "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_renamed_registration_serves_its_registered_name": 29.2, + "tst/integration/env/gateway/gateway_test.py::test_renamed_registration_serves_its_registered_name": 29.5, "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[modal]": 0.0, @@ -505,8 +519,8 @@ "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_snapshot_agent_state_v1_and_fallback": 27.4, - "tst/integration/env/gateway/gateway_test.py::test_snapshot_cli": 36.3, + "tst/integration/env/gateway/gateway_test.py::test_snapshot_agent_state_v1_and_fallback": 28.2, + "tst/integration/env/gateway/gateway_test.py::test_snapshot_cli": 55.3, "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[modal]": 0.0, @@ -552,7 +566,7 @@ "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[sail_vm]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[scale]": 26.6, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[scale]": 27.4, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[boxd]": 0.0, @@ -595,8 +609,8 @@ "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_validate_universe_compatibility": 43.4, - "tst/integration/env/gateway/gateway_test.py::test_verify_universe_roundtrip_export_all_v1_and_fallback": 27.3, + "tst/integration/env/gateway/gateway_test.py::test_validate_universe_compatibility": 42.0, + "tst/integration/env/gateway/gateway_test.py::test_verify_universe_roundtrip_export_all_v1_and_fallback": 27.6, "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[boxd]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[modal]": 0.0, @@ -614,7 +628,7 @@ "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[tensorlake]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[boxd]": 0.0, - "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[daytona]": 30.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[daytona]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[modal]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[modal_vm]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[sail_vm]": 0.0, @@ -645,23 +659,23 @@ "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[scale]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[tenki]": 0.0, "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[tensorlake]": 0.0, - "tst/integration/env/task_step/task_steps_test.py": 111.6, - "tst/integration/eval/eval_test.py": 0.2, - "tst/integration/openenv/build_app_test.py": 42.4, + "tst/integration/env/task_step/task_steps_test.py": 167.8, + "tst/integration/eval/eval_test.py": 1.4, + "tst/integration/openenv/build_app_test.py": 22.6, "tst/integration/providers/sandbox_providers/boxd_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/daytona_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/e2b_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/sail_vm_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/tenki_sandbox_smoke_test.py": 0.0, "tst/integration/providers/sandbox_providers/tensorlake_sandbox_smoke_test.py": 0.0, - "tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py": 0.1, - "tst/integration/providers/sandbox_providers/vm_sandbox_build_test.py": 2.8, - "tst/integration/store/local_parallel_deploy_test.py": 51.8, - "tst/integration/task_step/test_agent_on_local_sandbox.py": 8.4, + "tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py": 0.0, + "tst/integration/providers/sandbox_providers/vm_sandbox_build_test.py": 2.3, + "tst/integration/store/local_parallel_deploy_test.py": 54.5, + "tst/integration/task_step/test_agent_on_local_sandbox.py": 9.2, "tst/integration/task_step/test_enable_docker_e2e.py": 0.0, - "tst/integration/task_step/test_multiturn_verifier_delivery.py": 0.2, - "tst/integration/task_step/test_next_env_event_hook_local.py": 111.6, - "tst/integration/task_step/test_register_env_triggers_local.py": 285.8, - "tst/integration/utils/docker/prune_test.py": 2.6 + "tst/integration/task_step/test_multiturn_verifier_delivery.py": 0.0, + "tst/integration/task_step/test_next_env_event_hook_local.py": 108.1, + "tst/integration/task_step/test_register_env_triggers_local.py": 273.2, + "tst/integration/utils/docker/prune_test.py": 8.2 } } diff --git a/tst/unit/test_shards.py b/tst/unit/test_shards.py index bcbf55061..230d352cd 100644 --- a/tst/unit/test_shards.py +++ b/tst/unit/test_shards.py @@ -93,7 +93,8 @@ def test_the_same_table_and_tests_always_give_the_same_shards(): def test_the_committed_table_names_a_file_for_every_slow_tier_unit_it_splits(): table = shards.load_table() - assert table["split_by_test"] == [SPLIT] + assert SPLIT in table["split_by_test"] + assert all((ROOT / path).is_file() for path in table["split_by_test"]), "split_by_test names a test file that is gone" assert all((ROOT / unit.split("::")[0]).is_file() for unit in table["seconds"]), "the table names a test file that is gone"