From 1034e3c92f84e77d5f34a0bb023519700e577560 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 18:00:45 -0700 Subject: [PATCH 1/3] ci(integration): split the slow tier and the gateway tests across runners The slow tier runs its tests serially, since they share image tags and artifact ids, and it has grown to its 40-minute timeout. Each runner has its own Docker daemon, so split it across runners instead: AGENTENV_TEST_SHARD=i/N keeps the i-th of N groups, keeping a file's tests together and balancing the groups by the seconds in tst/integration/slow_durations.json. The slow tier runs on four runners and gateway_test.py on two; integration-local-slow and integration-local-gateway are now jobs that pass when all of their shards do, so the required check keeps its name. Each shard uploads its JUnit report, from which .github/scripts/slow_test_durations.py rewrites the table. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/scripts/slow_test_durations.py | 62 ++ .github/workflows/local-backends.yml | 88 ++- AGENTS.md | 10 +- Makefile | 4 +- tst/conftest.py | 20 + tst/integration/slow_durations.json | 667 ++++++++++++++++++++ tst/unit/test_shards.py | 103 +++ tst/unit/test_slow_test_durations_script.py | 69 ++ tst/util/shards.py | 58 ++ 9 files changed, 1069 insertions(+), 12 deletions(-) create mode 100644 .github/scripts/slow_test_durations.py create mode 100644 tst/integration/slow_durations.json create mode 100644 tst/unit/test_shards.py create mode 100644 tst/unit/test_slow_test_durations_script.py create mode 100644 tst/util/shards.py diff --git a/.github/scripts/slow_test_durations.py b/.github/scripts/slow_test_durations.py new file mode 100644 index 000000000..294f59c2e --- /dev/null +++ b/.github/scripts/slow_test_durations.py @@ -0,0 +1,62 @@ +"""Rewrite tst/integration/slow_durations.json from the slow tier's JUnit reports, so the CI shards stay balanced. + + python .github/scripts/slow_test_durations.py junit-integration-slow-*.xml + +Download the reports from a green ``main`` run (the ``junit-integration-local-*`` artifacts of every +``integration-local-slow`` and ``integration-local-gateway`` shard). Each test's time counts toward its file, +or toward the test itself for a file the table names in ``split_by_test``; that list is kept as it is. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import xml.etree.ElementTree as ET +from collections import defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TABLE = ROOT / "tst" / "integration" / "slow_durations.json" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("reports", nargs="+", help="JUnit XML reports written by pytest --junitxml") + parser.add_argument("--table", type=Path, default=TABLE, help="the durations table to rewrite") + parser.add_argument("--root", type=Path, default=ROOT, help="the checkout the reports' test files are in") + args = parser.parse_args() + + table = json.loads(args.table.read_text()) + split = set(table.get("split_by_test", [])) + seconds: dict[str, float] = defaultdict(float) + for report in args.reports: + for case in ET.parse(report).getroot().iter("testcase"): + nodeid = _nodeid(case.get("classname", ""), case.get("name", ""), args.root) + if nodeid is None: + print(f"slow test durations: no test file for {case.get('classname')!r}; left out") + continue + path = nodeid.split("::", 1)[0] + seconds[nodeid if path in split else path] += float(case.get("time") or 0) + if not seconds: + print("slow test durations: the reports have no test cases; the table is unchanged") + return 1 + table["seconds"] = {unit: round(value, 1) for unit, value in sorted(seconds.items())} + args.table.write_text(json.dumps(table, indent=1) + "\n") + print(f"slow test durations: {len(seconds)} units, {sum(seconds.values()) / 60:.1f} min, written to {args.table}") + return 0 + + +def _nodeid(classname: str, name: str, root: Path) -> str | None: + """The pytest node id of a JUnit test case: the longest dotted prefix of its classname that is a test + file, then any classes, then the test's name.""" + parts = classname.split(".") + for cut in range(len(parts), 0, -1): + path = "/".join(parts[:cut]) + ".py" + if (root / path).is_file(): + return "::".join([path, *parts[cut:], name]) + return None + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index b28a6da97..3239618f9 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -108,17 +108,16 @@ jobs: # The slow tier: builds the env images and the echo A2A agent (tst/data/a2a_agent) and deploys # them on the runner's Docker. The tests that need a real model, a remote sandbox, a registered # default A2A agent or MCP server sources from outside this repository skip with the capability reason. - # The gateway tests take as long as the rest of the tier, so they run beside it as a check of their own. - integration-local-slow: - name: ${{ matrix.check }} + # Its tests run serially, since they share image tags and artifact ids, so it is split across runners + # instead: AGENTENV_TEST_SHARD picks each runner's share, balanced by tst/integration/slow_durations.json + # (tst/util/shards.py). The gateway tests are a check of their own, split the same way. Each check passes + # when all of its shards do (the integration-local-slow and integration-local-gateway jobs below). + integration-local-slow-shard: + name: integration-local-slow (${{ matrix.shard }}/4) strategy: fail-fast: false matrix: - include: - - check: integration-local-slow - tests: tst/integration --ignore=tst/integration/env/gateway/gateway_test.py - - check: integration-local-gateway - tests: tst/integration/env/gateway/gateway_test.py + shard: [1, 2, 3, 4] runs-on: ubuntu-latest timeout-minutes: 40 steps: @@ -140,14 +139,85 @@ jobs: docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' - name: Slow integration tier on the local defaults + env: + AGENTENV_TEST_SHARD: ${{ matrix.shard }}/4 + run: | + .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ + tst/integration --ignore=tst/integration/env/gateway/gateway_test.py + - name: Skips must be declared capability gaps + if: always() + run: | + .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \ + --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources + # The tests' times, for .github/scripts/slow_test_durations.py to rebalance the shards with. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: junit-integration-local-slow-${{ matrix.shard }} + path: junit-integration-slow.xml + retention-days: 14 + + integration-local-gateway-shard: + name: integration-local-gateway (${{ matrix.shard }}/2) + strategy: + fail-fast: false + matrix: + shard: [1, 2] + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Install from uv.lock + run: | + python -m pip install --quiet -r .github/ci-requirements.txt + uv sync --locked --extra dev --python "$(command -v python)" + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + - name: Pull the base images from mirror.gcr.io + run: python3 .github/scripts/mirror_base_images.py src tst + - name: Start the local registry + run: | + docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 + timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' + - name: Slow integration tier on the local defaults + env: + AGENTENV_TEST_SHARD: ${{ matrix.shard }}/2 run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ - ${{ matrix.tests }} + tst/integration/env/gateway/gateway_test.py - name: Skips must be declared capability gaps if: always() run: | .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \ --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: junit-integration-local-gateway-${{ matrix.shard }} + path: junit-integration-slow.xml + retention-days: 14 + + # One check per tier under its own name (the main ruleset requires integration-local-slow): each passes + # when every shard of its tier passed. + integration-local-slow: + if: always() + needs: integration-local-slow-shard + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Every shard passed + run: test "${{ needs.integration-local-slow-shard.result }}" = success + + integration-local-gateway: + if: always() + needs: integration-local-gateway-shard + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Every shard passed + run: test "${{ needs.integration-local-gateway-shard.result }}" = success # Whether this change touches the code the installer tier covers; the tier costs minutes, so # it runs only then. A skipped job reports as passed to a required check. diff --git a/AGENTS.md b/AGENTS.md index 19b082bef..5eca2721e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -252,7 +252,11 @@ orchestrators call it the same way. What callers rely on: `_disable_network` fixture in a closer conftest when a loopback server is the point. - `tst/integration/` needs Docker and the local registry on `:5000` and runs on the local default backends. Minutes-long tests (real image builds, sandbox VMs, gateways, agents) carry - `@pytest.mark.int_test_slow` (or module-level `pytestmark`) and run serially. + `@pytest.mark.int_test_slow` (or module-level `pytestmark`) and run serially. CI splits them across + runners instead: `AGENTENV_TEST_SHARD=i/N` (`tst/util/shards.py`) runs the i-th of N groups, keeping a + file's tests together and balancing the groups by `tst/integration/slow_durations.json`. When the + shards' times drift apart, rewrite the table from a `main` run's `junit-integration-local-*` + artifacts with `.github/scripts/slow_test_durations.py`. - `tst/manual/` and `tst/benchmarks/` run only by hand, as each module's docstring says; no CI job collects them. - Skips are declared capability gaps, decided at collection time: the gates in @@ -277,7 +281,9 @@ resolved, and the checks the `main` ruleset requires: `unit`, `integration-local pass, but a failing one does not block the merge. `.github/workflows/local-backends.yml` runs `unit`, `ui`, `integration-local`, `integration-local-slow`, `integration-local-gateway` (the slow tier's `gateway_test.py`, run beside the rest of it) and -`installer`. The Python jobs install from `uv.lock` (public PyPI only), with the uv that +`installer`. The slow tier runs on four runners (`integration-local-slow (1/4)` to `(4/4)`) and the +gateway tests on two, and `integration-local-slow` and `integration-local-gateway` each pass when all +of their shards do. The Python jobs install from `uv.lock` (public PyPI only), with the uv that `.github/ci-requirements.txt` pins, and use no secrets and no external services (a `registry:2` container for the integration jobs). The `unit` job also fails if `uv.lock` resolves anything from a registry other than PyPI, if `agentenv-framework-protocol` is not the editable workspace member, if diff --git a/Makefile b/Makefile index 17cf0ef0f..e220a4306 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,9 @@ unit-test: ## Run the unit test suite in parallel (no external services) # "task-step-test-deploy-multi"), producing ERROR-at-setup cascades that # look like real bugs but are pure concurrency. Running serial costs wall # clock but is the only way to keep slow runs reliable without rewriting -# every module's fixtures with worker-unique tags. +# every module's fixtures with worker-unique tags. CI splits the slow tier +# across runners instead; AGENTENV_TEST_SHARD=2/4 make int-test-slow runs +# one runner's share (tst/util/shards.py). INT_PYTEST_ARGS_FAST ?= --dist=loadscope --tb=short --durations=20 -p no:cacheprovider INT_PYTEST_ARGS_SLOW ?= --tb=short --durations=20 -p no:cacheprovider INT_FAST_WORKERS ?= auto diff --git a/tst/conftest.py b/tst/conftest.py index af537f562..77692694a 100644 --- a/tst/conftest.py +++ b/tst/conftest.py @@ -1,5 +1,6 @@ """Shared pytest fixtures for all tests.""" +import os import shutil import signal import sys @@ -10,6 +11,7 @@ from agent_env.config import reset_config from agent_env.store.document_store import document_store as document_store_module from agent_env.store.routing import disable_namespace_routing +from tst.util import shards def _reset_store_singletons() -> None: @@ -40,6 +42,24 @@ def pytest_unconfigure(config): shutil.rmtree(config.stash[_STATE_DIR], ignore_errors=True) +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config, items): + """With ``AGENTENV_TEST_SHARD=i/N``, run only the i-th of N groups of the selected tests (``tst/util/shards.py``). + Last, so it splits what ``-m`` and the other filters left.""" + value = os.environ.get(shards.ENV_VAR) + if not value: + return + try: + index, count = shards.parse(value) + except ValueError as e: + raise pytest.UsageError(str(e)) from None + keep = set(shards.select([item.nodeid for item in items], index, count, shards.load_table())) + deselected = [item for item in items if item.nodeid not in keep] + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = [item for item in items if item.nodeid in keep] + + @pytest.fixture def sigint_handled(): """SIGINT handled as Python handles it by default, for a test that sends it: a CI job started in the background diff --git a/tst/integration/slow_durations.json b/tst/integration/slow_durations.json new file mode 100644 index 000000000..e8374021f --- /dev/null +++ b/tst/integration/slow_durations.json @@ -0,0 +1,667 @@ +{ + "_comment": "Seconds each slow-tier unit took in CI, which AGENTENV_TEST_SHARD balances the CI shards by (tst/util/shards.py). A unit is a file, or a test of a file in split_by_test. Refresh with .github/scripts/slow_test_durations.py from the shards' JUnit reports.", + "split_by_test": [ + "tst/integration/env/gateway/gateway_test.py" + ], + "seconds": { + "tst/integration/agent/a2a_agent/a2a_agent_test.py": 0.0, + "tst/integration/agent/a2a_agent/agent_triggers_e2e_test.py": 0.0, + "tst/integration/agent/proxy_modal_vm_test.py": 13.6, + "tst/integration/artifact/artifacts/docker_image_artifact_test.py": 7.2, + "tst/integration/cli/bundle_init_images_test.py": 23.4, + "tst/integration/cli/cli_test.py": 58.3, + "tst/integration/cli/run_bundle_built_agents_test.py": 36.9, + "tst/integration/cli/run_bundle_built_envs_test.py": 534.3, + "tst/integration/cli/run_bundle_claude_code_install_test.py": 156.8, + "tst/integration/cli/run_bundle_claude_code_test.py": 90.6, + "tst/integration/cli/run_bundle_cli_folder_test.py": 74.7, + "tst/integration/cli/run_bundle_image_folder_test.py": 25.9, + "tst/integration/cli/run_bundle_image_ref_test.py": 14.1, + "tst/integration/cli/run_bundle_installed_agent_test.py": 8.2, + "tst/integration/cli/run_bundle_openhands_test.py": 228.5, + "tst/integration/cli/run_bundle_skills_test.py": 12.8, + "tst/integration/cli/run_role_models_test.py": 78.8, + "tst/integration/computer/recording_chaos_test.py": 74.6, + "tst/integration/computer/scaffold_test.py": 51.7, + "tst/integration/env/env_test.py": 4.1, + "tst/integration/env/envs/build_context_local_test.py": 170.2, + "tst/integration/env/envs/multi_env_local_test.py": 52.6, + "tst/integration/env/envs/server_env_modal_test.py": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_airtable_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_amazon_fresh_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_apple_health_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_blackline_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_buildium_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cab_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_calendar_mcp_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_cardless_environment_resolves_name_from_environment_name": 38.8, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_clickup_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_confluence_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_crm_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_eight_sleep_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_email_mcp_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_env_validator_task": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_erp_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_figma_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fintrack_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fitbit_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_fresh_direct_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_garmin_health_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[scale]": 29.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_autonomous_time_trigger[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[scale]": 28.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_barrier_orders_a_templated_mirror[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_request_behind_a_write_through_the_rest_proxy": 26.4, + "tst/integration/env/gateway/gateway_test.py::test_gateway_changes_name_the_role_behind_each_write": 26.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[scale]": 36.8, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_extension[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[scale]": 51.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_clock_rate_accuracy[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_consistent_mode": 41.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[scale]": 30.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_queued_trigger_is_not_reported_settled[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[scale]": 28.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_time_trigger_extension[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[scale]": 30.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_rearms_after_a_failed_mirror[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[scale]": 26.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_trigger_registration_is_fail_loud[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[daytona]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[scale]": 37.5, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_extension[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_triggers_nl_executor[modal_vm]": 0.1, + "tst/integration/env/gateway/gateway_test.py::test_gateway_website_trigger_fires_on_a_request_through_the_website_proxy": 39.4, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_agentenv_environment": 28.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_env_including_website": 47.9, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[scale]": 28.3, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_multi_mcp_server[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_mcp_server": 189.2, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_single_website_env": 64.0, + "tst/integration/env/gateway/gateway_test.py::test_gateway_with_website_env_v1": 40.6, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_github_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gmail_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_gofundme_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_calendar_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_docs_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_drive_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_sheets_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_google_slides_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_hotel_management_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_instacart_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_jira_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_linear_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_load_from_snapshot": 41.8, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_messaging_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_mortgage_los_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_myfitnesspal_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_notion_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_obsidian_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oms_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_oracle_gl_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_polymarket_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_quickbooks_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_records_vault_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_reminder_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renamed_registration_serves_its_registered_name": 29.2, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_renpho_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ring_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sap_subledger_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_shopping_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_slack_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_smart_lock_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snapshot_agent_state_v1_and_fallback": 27.4, + "tst/integration/env/gateway/gateway_test.py::test_snapshot_cli": 36.3, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_async_virtual_clock[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_snowflake_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stayncook_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_strava_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_stripe_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[scale]": 26.6, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_sync_env_clock_step[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_system_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_target_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_ticketmaster_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_tms_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_trello_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_validate_universe_compatibility": 43.4, + "tst/integration/env/gateway/gateway_test.py::test_verify_universe_roundtrip_export_all_v1_and_fallback": 27.3, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_walmart_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_weather_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[daytona]": 30.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_whoop_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_wms_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zendesk_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[boxd]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[daytona]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[modal]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[modal_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[sail_vm]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[scale]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[tenki]": 0.0, + "tst/integration/env/gateway/gateway_test.py::test_zillow_clock_consumer[tensorlake]": 0.0, + "tst/integration/env/task_step/task_steps_test.py": 111.6, + "tst/integration/eval/eval_test.py": 0.2, + "tst/integration/openenv/build_app_test.py": 42.4, + "tst/integration/providers/sandbox_providers/boxd_sandbox_smoke_test.py": 0.0, + "tst/integration/providers/sandbox_providers/daytona_sandbox_smoke_test.py": 0.0, + "tst/integration/providers/sandbox_providers/e2b_sandbox_smoke_test.py": 0.0, + "tst/integration/providers/sandbox_providers/sail_vm_sandbox_smoke_test.py": 0.0, + "tst/integration/providers/sandbox_providers/tenki_sandbox_smoke_test.py": 0.0, + "tst/integration/providers/sandbox_providers/tensorlake_sandbox_smoke_test.py": 0.0, + "tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py": 0.1, + "tst/integration/providers/sandbox_providers/vm_sandbox_build_test.py": 2.8, + "tst/integration/store/local_parallel_deploy_test.py": 51.8, + "tst/integration/task_step/test_agent_on_local_sandbox.py": 8.4, + "tst/integration/task_step/test_enable_docker_e2e.py": 0.0, + "tst/integration/task_step/test_multiturn_verifier_delivery.py": 0.2, + "tst/integration/task_step/test_next_env_event_hook_local.py": 111.6, + "tst/integration/task_step/test_register_env_triggers_local.py": 285.8, + "tst/integration/utils/docker/prune_test.py": 2.6 + } +} diff --git a/tst/unit/test_shards.py b/tst/unit/test_shards.py new file mode 100644 index 000000000..9c854a0f0 --- /dev/null +++ b/tst/unit/test_shards.py @@ -0,0 +1,103 @@ +"""AGENTENV_TEST_SHARD (``tst/util/shards.py`` and its hook in ``tst/conftest.py``): the CI slow-tier shards.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from tst.util import shards + +ROOT = Path(__file__).resolve().parents[2] +SPLIT = "tst/integration/env/gateway/gateway_test.py" + + +def _table(seconds: dict[str, float]) -> dict: + return {"split_by_test": [SPLIT], "seconds": seconds} + + +def _groups(nodeids: list[str], count: int, table: dict) -> list[list[str]]: + return [shards.select(nodeids, index, count, table) for index in range(1, count + 1)] + + +@pytest.mark.parametrize("value,expected", [("1/4", (1, 4)), ("4/4", (4, 4)), ("1/1", (1, 1))]) +def test_parse_reads_the_shard_and_the_count(value, expected): + assert shards.parse(value) == expected + + +@pytest.mark.parametrize("value", ["", "2", "0/4", "5/4", "a/4", "1/4/2"]) +def test_parse_refuses_anything_but_i_of_n(value): + with pytest.raises(ValueError, match=shards.ENV_VAR): + shards.parse(value) + + +def test_every_test_runs_in_exactly_one_shard_in_its_collected_order(): + nodeids = [f"tst/integration/f{n}_test.py::test_{t}" for n in range(7) for t in range(3)] + groups = _groups(nodeids, 3, _table({f"tst/integration/f{n}_test.py": 10 * n for n in range(7)})) + assert sorted(sum(groups, [])) == sorted(nodeids) + for group in groups: + assert group == [nodeid for nodeid in nodeids if nodeid in group] + + +def test_a_files_tests_stay_together_and_the_heaviest_files_go_to_different_shards(): + seconds = {"tst/integration/big_test.py": 600, "tst/integration/mid_test.py": 500, "tst/integration/small_test.py": 100} + nodeids = [f"{path}::test_{t}" for path in seconds for t in range(4)] + groups = _groups(nodeids, 2, _table(seconds)) + assert {nodeid.split("::")[0] for nodeid in groups[0]} == {"tst/integration/big_test.py"} + assert {nodeid.split("::")[0] for nodeid in groups[1]} == {"tst/integration/mid_test.py", "tst/integration/small_test.py"} + + +def test_a_file_named_in_split_by_test_is_split_by_test(): + nodeids = [f"{SPLIT}::test_{t}" for t in range(4)] + groups = _groups(nodeids, 2, _table({nodeid: 60 for nodeid in nodeids})) + assert [len(group) for group in groups] == [2, 2] + + +def test_a_unit_the_table_does_not_know_weighs_the_median_of_the_known_ones(): + seconds = {"tst/integration/a_test.py": 10, "tst/integration/b_test.py": 20, "tst/integration/c_test.py": 900, + "tst/integration/skipped_test.py": 0} + nodeids = [f"{path}::test" for path in [*seconds, "tst/integration/new_test.py"]] + groups = _groups(nodeids, 2, _table(seconds)) + # c (900) alone on one shard; the rest, the new file weighing the median of 10, 20 and 900, on the other + assert groups[0] == ["tst/integration/c_test.py::test"] + + +def test_the_same_table_and_tests_always_give_the_same_shards(): + nodeids = [f"tst/integration/f{n}_test.py::test" for n in range(12)] + table = _table({f"tst/integration/f{n}_test.py": 30 for n in range(12)}) + assert _groups(nodeids, 4, table) == _groups(list(nodeids), 4, table) + + +def test_the_committed_table_names_a_file_for_every_slow_tier_unit_it_splits(): + table = shards.load_table() + assert table["split_by_test"] == [SPLIT] + assert all((ROOT / unit.split("::")[0]).is_file() for unit in table["seconds"]), "the table names a test file that is gone" + + +def _collect(*paths: str, shard: str | None) -> subprocess.CompletedProcess: + env = {key: value for key, value in os.environ.items() if key != shards.ENV_VAR} + if shard is not None: + env[shards.ENV_VAR] = shard + return subprocess.run([sys.executable, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider", *paths], + cwd=ROOT, env=env, capture_output=True, text=True) + + +def _collected(result: subprocess.CompletedProcess) -> set[str]: + return {line for line in result.stdout.splitlines() if "::" in line} + + +def test_the_hook_splits_a_collection_into_shards_that_together_are_all_of_it(): + paths = ("tst/unit/test_check_skip_policy.py", "tst/unit/test_tls_verification.py") + everything = _collected(_collect(*paths, shard=None)) + first, second = (_collected(_collect(*paths, shard=f"{index}/2")) for index in (1, 2)) + assert first and second and not first & second + assert first | second == everything + + +def test_the_hook_refuses_a_malformed_shard(): + result = _collect("tst/unit/test_tls_verification.py", shard="3/2") + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert shards.ENV_VAR in result.stderr diff --git a/tst/unit/test_slow_test_durations_script.py b/tst/unit/test_slow_test_durations_script.py new file mode 100644 index 000000000..80033f28e --- /dev/null +++ b/tst/unit/test_slow_test_durations_script.py @@ -0,0 +1,69 @@ +"""The slow-tier durations refresh in ``.github/scripts/slow_test_durations.py``.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "slow_test_durations.py" +SPLIT = "tst/integration/env/gateway/gateway_test.py" + + +def _checkout(tmp_path: Path) -> Path: + for path in ("tst/integration/cli/cli_test.py", "tst/integration/env/task_step/task_steps_test.py", SPLIT): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text("") + table = tmp_path / "slow_durations.json" + table.write_text(json.dumps({"_comment": "kept", "split_by_test": [SPLIT], "seconds": {"tst/integration/gone_test.py": 9}})) + return table + + +def _report(tmp_path: Path, name: str, cases: list[tuple[str, str, float]]) -> Path: + body = "".join(f'' for classname, test, time in cases) + report = tmp_path / name + report.write_text(f'{body}') + return report + + +def _refresh(table: Path, root: Path, *reports: Path) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(SCRIPT), *map(str, reports), "--table", str(table), "--root", str(root)], + capture_output=True, text=True) + + +def test_sums_each_files_tests_and_keeps_the_split_files_tests_apart(tmp_path): + table = _checkout(tmp_path) + one = _report(tmp_path, "one.xml", [("tst.integration.cli.cli_test.TestCliIntegration", "test_deploy", 30.0), + ("tst.integration.cli.cli_test.TestCliIntegration", "test_put", 12.5)]) + two = _report(tmp_path, "two.xml", [("tst.integration.env.gateway.gateway_test", "test_mode[scale]", 41.0), + ("tst.integration.env.gateway.gateway_test", "test_other", 7.0), + ("tst.integration.env.task_step.task_steps_test", "test_load", 3.0)]) + result = _refresh(table, tmp_path, one, two) + assert result.returncode == 0, result.stdout + written = json.loads(table.read_text()) + assert written["seconds"] == { + "tst/integration/cli/cli_test.py": 42.5, + f"{SPLIT}::test_mode[scale]": 41.0, + f"{SPLIT}::test_other": 7.0, + "tst/integration/env/task_step/task_steps_test.py": 3.0, + } + assert written["split_by_test"] == [SPLIT] and written["_comment"] == "kept" + + +def test_a_case_whose_file_is_gone_is_left_out(tmp_path): + table = _checkout(tmp_path) + report = _report(tmp_path, "r.xml", [("tst.integration.cli.cli_test", "test_deploy", 5.0), + ("tst.integration.renamed_test", "test_x", 99.0)]) + result = _refresh(table, tmp_path, report) + assert result.returncode == 0 + assert "tst.integration.renamed_test" in result.stdout + assert json.loads(table.read_text())["seconds"] == {"tst/integration/cli/cli_test.py": 5.0} + + +def test_reports_with_no_cases_leave_the_table_as_it_was(tmp_path): + table = _checkout(tmp_path) + before = table.read_text() + result = _refresh(table, tmp_path, _report(tmp_path, "empty.xml", [])) + assert result.returncode == 1 + assert table.read_text() == before diff --git a/tst/util/shards.py b/tst/util/shards.py new file mode 100644 index 000000000..a2eede5b6 --- /dev/null +++ b/tst/util/shards.py @@ -0,0 +1,58 @@ +"""Split the selected tests across CI runners: ``AGENTENV_TEST_SHARD=i/N`` keeps the i-th of N groups. + +A file's tests stay together, so its module fixtures (image builds, deployed envs) run once per runner; +a file the durations table names in ``split_by_test`` is split by test instead. The groups are balanced +by the seconds the table records (``tst/integration/slow_durations.json``), a unit it doesn't know +weighs the median of the ones it does, and the same table and selection always give the same groups. +""" + +from __future__ import annotations + +import json +import statistics +from pathlib import Path + +ENV_VAR = "AGENTENV_TEST_SHARD" +DURATIONS = Path(__file__).resolve().parents[1] / "integration" / "slow_durations.json" + + +def parse(value: str) -> tuple[int, int]: + """``"i/N"`` as ``(i, N)``, with ``1 <= i <= N``.""" + try: + index, count = (int(part) for part in value.split("/")) + except ValueError: + raise ValueError(f"{ENV_VAR} must look like 2/4, not {value!r}") from None + if not 1 <= index <= count: + raise ValueError(f"{ENV_VAR}={value}: the shard must be between 1 and {count}") + return index, count + + +def select(nodeids: list[str], index: int, count: int, table: dict) -> list[str]: + """The nodeids that shard ``index`` of ``count`` runs, in their collected order.""" + split = set(table.get("split_by_test", [])) + seconds = table.get("seconds", {}) + units = {_unit(nodeid, split) for nodeid in nodeids} + known = [seconds[unit] for unit in units if seconds.get(unit, 0) > 0] + default = statistics.median(known) if known else 1.0 + shard = _assign({unit: seconds.get(unit, default) for unit in units}, count) + return [nodeid for nodeid in nodeids if shard[_unit(nodeid, split)] == index] + + +def load_table(path: Path = DURATIONS) -> dict: + return json.loads(path.read_text()) + + +def _unit(nodeid: str, split_by_test: set[str]) -> str: + path = nodeid.split("::", 1)[0] + return nodeid if path in split_by_test else path + + +def _assign(weights: dict[str, float], count: int) -> dict[str, int]: + """Each unit's shard, from 1: the heaviest unit first, onto the lightest shard so far.""" + loads = [0.0] * count + shard = {} + for unit in sorted(weights, key=lambda name: (-weights[name], name)): + lightest = min(range(count), key=lambda i: (loads[i], i)) + shard[unit] = lightest + 1 + loads[lightest] += weights[unit] + return shard From c597a5884f11c44e60c11b1b428401b9b7f291e9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 18:15:36 -0700 Subject: [PATCH 2/3] ci(integration): keep tests a dependency links on one shard, and give the CI shards' local commands In a file split by test, tests that pytest.mark.dependency links now share one unit, which weighs their sum, so gateway_test.py's test_load_from_snapshot runs on the same runner as test_snapshot_cli. The Makefile names the exact selections the CI shards use, since a share depends on what is selected. Co-Authored-By: Claude Opus 5.5 (1M context) --- Makefile | 5 ++-- tst/conftest.py | 7 +++++- tst/unit/test_shards.py | 20 ++++++++++++++++ tst/util/shards.py | 53 ++++++++++++++++++++++++++++++++++------- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index e220a4306..bf2ecb416 100644 --- a/Makefile +++ b/Makefile @@ -44,8 +44,9 @@ unit-test: ## Run the unit test suite in parallel (no external services) # look like real bugs but are pure concurrency. Running serial costs wall # clock but is the only way to keep slow runs reliable without rewriting # every module's fixtures with worker-unique tags. CI splits the slow tier -# across runners instead; AGENTENV_TEST_SHARD=2/4 make int-test-slow runs -# one runner's share (tst/util/shards.py). +# across runners instead (tst/util/shards.py). To run one runner's share: +# AGENTENV_TEST_SHARD=2/4 make int-test-slow PYTEST_ARGS=--ignore=tst/integration/env/gateway/gateway_test.py +# AGENTENV_TEST_SHARD=1/2 .venv/bin/python -m pytest -m int_test_slow tst/integration/env/gateway/gateway_test.py INT_PYTEST_ARGS_FAST ?= --dist=loadscope --tb=short --durations=20 -p no:cacheprovider INT_PYTEST_ARGS_SLOW ?= --tb=short --durations=20 -p no:cacheprovider INT_FAST_WORKERS ?= auto diff --git a/tst/conftest.py b/tst/conftest.py index 77692694a..94e263e79 100644 --- a/tst/conftest.py +++ b/tst/conftest.py @@ -53,7 +53,12 @@ def pytest_collection_modifyitems(config, items): index, count = shards.parse(value) except ValueError as e: raise pytest.UsageError(str(e)) from None - keep = set(shards.select([item.nodeid for item in items], index, count, shards.load_table())) + table = shards.load_table() + marked = [(item, item.get_closest_marker("dependency")) for item in items] + linked = [(item.nodeid, mark.kwargs.get("name") or item.name, list(mark.kwargs.get("depends") or [])) + for item, mark in marked if mark is not None] + together = shards.dependency_groups(linked, table) + keep = set(shards.select([item.nodeid for item in items], index, count, table, together)) deselected = [item for item in items if item.nodeid not in keep] if deselected: config.hook.pytest_deselected(items=deselected) diff --git a/tst/unit/test_shards.py b/tst/unit/test_shards.py index 9c854a0f0..bcbf55061 100644 --- a/tst/unit/test_shards.py +++ b/tst/unit/test_shards.py @@ -56,6 +56,26 @@ def test_a_file_named_in_split_by_test_is_split_by_test(): assert [len(group) for group in groups] == [2, 2] +def test_tests_a_dependency_links_stay_on_one_shard_and_weigh_their_sum(): + nodeids = [f"{SPLIT}::test_{name}" for name in ("save", "load", "a", "b")] + table = _table({f"{SPLIT}::test_save": 50, f"{SPLIT}::test_load": 50, f"{SPLIT}::test_a": 60, f"{SPLIT}::test_b": 40}) + linked = [(f"{SPLIT}::test_save", "save", []), (f"{SPLIT}::test_load", "test_load", ["save"])] + together = shards.dependency_groups(linked, table) + assert together == {f"{SPLIT}::test_save": f"{SPLIT}::test_load", f"{SPLIT}::test_load": f"{SPLIT}::test_load"} + groups = [shards.select(nodeids, index, 2, table, together) for index in (1, 2)] + # save+load (100) on one shard, a (60) and b (40) on the other + assert sorted(groups, key=len) == [[f"{SPLIT}::test_save", f"{SPLIT}::test_load"], + [f"{SPLIT}::test_a", f"{SPLIT}::test_b"]] + + +def test_a_dependency_outside_a_split_file_or_on_an_unknown_name_links_nothing(): + table = _table({}) + linked = [("tst/integration/whole_test.py::test_load", "test_load", ["save"]), + ("tst/integration/whole_test.py::test_save", "save", []), + (f"{SPLIT}::test_load", "test_load", ["missing"])] + assert shards.dependency_groups(linked, table) == {} + + def test_a_unit_the_table_does_not_know_weighs_the_median_of_the_known_ones(): seconds = {"tst/integration/a_test.py": 10, "tst/integration/b_test.py": 20, "tst/integration/c_test.py": 900, "tst/integration/skipped_test.py": 0} diff --git a/tst/util/shards.py b/tst/util/shards.py index a2eede5b6..70c0288c5 100644 --- a/tst/util/shards.py +++ b/tst/util/shards.py @@ -1,15 +1,17 @@ """Split the selected tests across CI runners: ``AGENTENV_TEST_SHARD=i/N`` keeps the i-th of N groups. A file's tests stay together, so its module fixtures (image builds, deployed envs) run once per runner; -a file the durations table names in ``split_by_test`` is split by test instead. The groups are balanced -by the seconds the table records (``tst/integration/slow_durations.json``), a unit it doesn't know -weighs the median of the ones it does, and the same table and selection always give the same groups. +a file the durations table names in ``split_by_test`` is split by test instead, except that the tests +``pytest.mark.dependency`` links stay on one runner. The groups are balanced by the seconds the table +records (``tst/integration/slow_durations.json``), a unit it doesn't know weighs the median of the ones it +does, and the same table and selection always give the same groups. """ from __future__ import annotations import json import statistics +from collections import defaultdict from pathlib import Path ENV_VAR = "AGENTENV_TEST_SHARD" @@ -27,15 +29,48 @@ def parse(value: str) -> tuple[int, int]: return index, count -def select(nodeids: list[str], index: int, count: int, table: dict) -> list[str]: - """The nodeids that shard ``index`` of ``count`` runs, in their collected order.""" +def select(nodeids: list[str], index: int, count: int, table: dict, together: dict[str, str] | None = None) -> list[str]: + """The nodeids that shard ``index`` of ``count`` runs, in their collected order. ``together`` maps a nodeid + to a unit it shares with others (``dependency_groups``); that unit weighs the sum of its tests.""" split = set(table.get("split_by_test", [])) seconds = table.get("seconds", {}) - units = {_unit(nodeid, split) for nodeid in nodeids} - known = [seconds[unit] for unit in units if seconds.get(unit, 0) > 0] + base = {nodeid: _unit(nodeid, split) for nodeid in nodeids} + unit = {nodeid: (together or {}).get(nodeid, base[nodeid]) for nodeid in nodeids} + known = [seconds[name] for name in set(base.values()) if seconds.get(name, 0) > 0] default = statistics.median(known) if known else 1.0 - shard = _assign({unit: seconds.get(unit, default) for unit in units}, count) - return [nodeid for nodeid in nodeids if shard[_unit(nodeid, split)] == index] + weights: dict[str, float] = defaultdict(float) + for base_unit, shared in {base[nodeid]: unit[nodeid] for nodeid in nodeids}.items(): + weights[shared] += seconds.get(base_unit, default) + shard = _assign(weights, count) + return [nodeid for nodeid in nodeids if shard[unit[nodeid]] == index] + + +def dependency_groups(tests: list[tuple[str, str, list[str]]], table: dict) -> dict[str, str]: + """The unit each test of a ``split_by_test`` file shares with the tests ``pytest.mark.dependency`` links + it to, as ``select``'s ``together``. ``tests`` are (nodeid, the name its marker gives it, the names it + depends on), for the tests with the marker; a dependency is looked up in the test's own file.""" + split = set(table.get("split_by_test", [])) + parent: dict[str, str] = {} + + def root(nodeid: str) -> str: + while parent.get(nodeid, nodeid) != nodeid: + nodeid = parent[nodeid] + return nodeid + + named = {(nodeid.split("::", 1)[0], name): nodeid for nodeid, name, _ in tests} + for nodeid, _, depends in tests: + path = nodeid.split("::", 1)[0] + if path not in split: + continue + for name in depends: + other = named.get((path, name)) + if other is not None: + parent[root(nodeid)] = root(other) + linked = {nodeid for nodeid, _, _ in tests if nodeid.split("::", 1)[0] in split} + groups = defaultdict(list) + for nodeid in linked: + groups[root(nodeid)].append(nodeid) + return {nodeid: min(members) for members in groups.values() if len(members) > 1 for nodeid in members} def load_table(path: Path = DURATIONS) -> dict: From 0a7976e3b50ea8868634f7bbf17a6a06150474e1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 19:07:13 -0700 Subject: [PATCH 3/3] test: close the grant servers after each module, so one module's CA can't reach the next A module that runs under its own state root (run_role_models_test.py) starts the process's grant server with that root's CA and leaves it running. The next module's containers trust the session's CA, so its agents fail to reach the server (transfer_unavailable). The old serial order hid this, because a module that closes the server happened to run in between; the shards run other orders. The per-module fixture now closes every grant server, and the next grant starts one afresh. Co-Authored-By: Claude Opus 5.5 (1M context) --- tst/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tst/conftest.py b/tst/conftest.py index 94e263e79..e33f3a589 100644 --- a/tst/conftest.py +++ b/tst/conftest.py @@ -10,6 +10,7 @@ from agent_env.config import reset_config from agent_env.store.document_store import document_store as document_store_module +from agent_env.store.object_store.local import grant_server as grant_server_module from agent_env.store.routing import disable_namespace_routing from tst.util import shards @@ -107,8 +108,12 @@ def fresh_config_document(): @pytest.fixture(scope="module", autouse=True) def setup_config(): - """Per-module store-singleton sweep; the config itself is reset per test above.""" + """Per-module store-singleton sweep; the config itself is reset per test above. The grant servers close too: + one a module started under its own state root presents that root's CA, which the next module's containers + don't trust, and the next grant starts a server afresh.""" _reset_store_singletons() yield reset_config() _reset_store_singletons() + for server in list(grant_server_module._servers.values()): + server.close()