ci(integration): split the slow tier and the gateway tests across runners

.github/scripts/slow_test_durations.py

@@ -0,0 +1,62 @@
+"""Rewrite tst/integration/slow_durations.json from the slow tier's JUnit reports, so the CI shards stay balanced.
+
+    python .github/scripts/slow_test_durations.py junit-integration-slow-*.xml
+
+Download the reports from a green ``main`` run (the ``junit-integration-local-*`` artifacts of every
+``integration-local-slow`` and ``integration-local-gateway`` shard). Each test's time counts toward its file,
+or toward the test itself for a file the table names in ``split_by_test``; that list is kept as it is.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+TABLE = ROOT / "tst" / "integration" / "slow_durations.json"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("reports", nargs="+", help="JUnit XML reports written by pytest --junitxml")
+    parser.add_argument("--table", type=Path, default=TABLE, help="the durations table to rewrite")
+    parser.add_argument("--root", type=Path, default=ROOT, help="the checkout the reports' test files are in")
+    args = parser.parse_args()
+
+    table = json.loads(args.table.read_text())
+    split = set(table.get("split_by_test", []))
+    seconds: dict[str, float] = defaultdict(float)
+    for report in args.reports:
+        for case in ET.parse(report).getroot().iter("testcase"):
+            nodeid = _nodeid(case.get("classname", ""), case.get("name", ""), args.root)
+            if nodeid is None:
+                print(f"slow test durations: no test file for {case.get('classname')!r}; left out")
+                continue
+            path = nodeid.split("::", 1)[0]
+            seconds[nodeid if path in split else path] += float(case.get("time") or 0)
+    if not seconds:
+        print("slow test durations: the reports have no test cases; the table is unchanged")
+        return 1
+    table["seconds"] = {unit: round(value, 1) for unit, value in sorted(seconds.items())}
+    args.table.write_text(json.dumps(table, indent=1) + "\n")
+    print(f"slow test durations: {len(seconds)} units, {sum(seconds.values()) / 60:.1f} min, written to {args.table}")
+    return 0
+
+
+def _nodeid(classname: str, name: str, root: Path) -> str | None:
+    """The pytest node id of a JUnit test case: the longest dotted prefix of its classname that is a test
+    file, then any classes, then the test's name."""
+    parts = classname.split(".")
+    for cut in range(len(parts), 0, -1):
+        path = "/".join(parts[:cut]) + ".py"
+        if (root / path).is_file():
+            return "::".join([path, *parts[cut:], name])
+    return None
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/local-backends.yml

@@ -108,17 +108,16 @@ jobs:
   # The slow tier: builds the env images and the echo A2A agent (tst/data/a2a_agent) and deploys
   # them on the runner's Docker. The tests that need a real model, a remote sandbox, a registered
   # default A2A agent or MCP server sources from outside this repository skip with the capability reason.
-  # The gateway tests take as long as the rest of the tier, so they run beside it as a check of their own.
-  integration-local-slow:
-    name: ${{ matrix.check }}
+  # Its tests run serially, since they share image tags and artifact ids, so it is split across runners
+  # instead: AGENTENV_TEST_SHARD picks each runner's share, balanced by tst/integration/slow_durations.json
+  # (tst/util/shards.py). The gateway tests are a check of their own, split the same way. Each check passes
+  # when all of its shards do (the integration-local-slow and integration-local-gateway jobs below).
+  integration-local-slow-shard:
+    name: integration-local-slow (${{ matrix.shard }}/4)
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - check: integration-local-slow
-            tests: tst/integration --ignore=tst/integration/env/gateway/gateway_test.py
-          - check: integration-local-gateway
-            tests: tst/integration/env/gateway/gateway_test.py
+        shard: [1, 2, 3, 4]
     runs-on: ubuntu-latest
     timeout-minutes: 40
     steps:
@@ -140,14 +139,85 @@ jobs:
           docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
           timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
       - name: Slow integration tier on the local defaults
+        env:
+          AGENTENV_TEST_SHARD: ${{ matrix.shard }}/4
+        run: |
+          .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \
+            tst/integration --ignore=tst/integration/env/gateway/gateway_test.py
+      - name: Skips must be declared capability gaps
+        if: always()
+        run: |
+          .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \
+            --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources
+      # The tests' times, for .github/scripts/slow_test_durations.py to rebalance the shards with.
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: always()
+        with:
+          name: junit-integration-local-slow-${{ matrix.shard }}
+          path: junit-integration-slow.xml
+          retention-days: 14
+
+  integration-local-gateway-shard:
+    name: integration-local-gateway (${{ matrix.shard }}/2)
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2]
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+      - name: Install from uv.lock
+        run: |
+          python -m pip install --quiet -r .github/ci-requirements.txt
+          uv sync --locked --extra dev --python "$(command -v python)"
+          echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
+      - name: Pull the base images from mirror.gcr.io
+        run: python3 .github/scripts/mirror_base_images.py src tst
+      - name: Start the local registry
+        run: |
+          docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
+          timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
+      - name: Slow integration tier on the local defaults
+        env:
+          AGENTENV_TEST_SHARD: ${{ matrix.shard }}/2
         run: |
           .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \
-            ${{ matrix.tests }}
+            tst/integration/env/gateway/gateway_test.py
       - name: Skips must be declared capability gaps
         if: always()
         run: |
           .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \
             --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: always()
+        with:
+          name: junit-integration-local-gateway-${{ matrix.shard }}
+          path: junit-integration-slow.xml
+          retention-days: 14
+
+  # One check per tier under its own name (the main ruleset requires integration-local-slow): each passes
+  # when every shard of its tier passed.
+  integration-local-slow:
+    if: always()
+    needs: integration-local-slow-shard
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Every shard passed
+        run: test "${{ needs.integration-local-slow-shard.result }}" = success
+
+  integration-local-gateway:
+    if: always()
+    needs: integration-local-gateway-shard
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Every shard passed
+        run: test "${{ needs.integration-local-gateway-shard.result }}" = success
 
   # Whether this change touches the code the installer tier covers; the tier costs minutes, so
   # it runs only then. A skipped job reports as passed to a required check.

AGENTS.md

@@ -252,7 +252,11 @@ orchestrators call it the same way. What callers rely on:
   `_disable_network` fixture in a closer conftest when a loopback server is the point.
 - `tst/integration/` needs Docker and the local registry on `:5000` and runs on the local default
   backends. Minutes-long tests (real image builds, sandbox VMs, gateways, agents) carry
-  `@pytest.mark.int_test_slow` (or module-level `pytestmark`) and run serially.
+  `@pytest.mark.int_test_slow` (or module-level `pytestmark`) and run serially. CI splits them across
+  runners instead: `AGENTENV_TEST_SHARD=i/N` (`tst/util/shards.py`) runs the i-th of N groups, keeping a
+  file's tests together and balancing the groups by `tst/integration/slow_durations.json`. When the
+  shards' times drift apart, rewrite the table from a `main` run's `junit-integration-local-*`
+  artifacts with `.github/scripts/slow_test_durations.py`.
 - `tst/manual/` and `tst/benchmarks/` run only by hand, as each module's docstring says; no CI job
   collects them.
 - Skips are declared capability gaps, decided at collection time: the gates in
@@ -277,7 +281,9 @@ resolved, and the checks the `main` ruleset requires: `unit`, `integration-local
 pass, but a failing one does not block the merge.
 `.github/workflows/local-backends.yml` runs `unit`, `ui`, `integration-local`, `integration-local-slow`,
 `integration-local-gateway` (the slow tier's `gateway_test.py`, run beside the rest of it) and
-`installer`. The Python jobs install from `uv.lock` (public PyPI only), with the uv that
+`installer`. The slow tier runs on four runners (`integration-local-slow (1/4)` to `(4/4)`) and the
+gateway tests on two, and `integration-local-slow` and `integration-local-gateway` each pass when all
+of their shards do. The Python jobs install from `uv.lock` (public PyPI only), with the uv that
 `.github/ci-requirements.txt` pins, and use no secrets and no external services (a `registry:2`
 container for the integration jobs). The `unit` job also fails if `uv.lock` resolves anything from a
 registry other than PyPI, if `agentenv-framework-protocol` is not the editable workspace member, if

Makefile

@@ -43,7 +43,10 @@ unit-test: ## Run the unit test suite in parallel (no external services)
 #   "task-step-test-deploy-multi"), producing ERROR-at-setup cascades that
 #   look like real bugs but are pure concurrency. Running serial costs wall
 #   clock but is the only way to keep slow runs reliable without rewriting
-#   every module's fixtures with worker-unique tags.
+#   every module's fixtures with worker-unique tags. CI splits the slow tier
+#   across runners instead (tst/util/shards.py). To run one runner's share:
+#     AGENTENV_TEST_SHARD=2/4 make int-test-slow PYTEST_ARGS=--ignore=tst/integration/env/gateway/gateway_test.py
+#     AGENTENV_TEST_SHARD=1/2 .venv/bin/python -m pytest -m int_test_slow tst/integration/env/gateway/gateway_test.py
 INT_PYTEST_ARGS_FAST ?= --dist=loadscope --tb=short --durations=20 -p no:cacheprovider
 INT_PYTEST_ARGS_SLOW ?= --tb=short --durations=20 -p no:cacheprovider
 INT_FAST_WORKERS ?= auto

tst/conftest.py

@@ -1,5 +1,6 @@
 """Shared pytest fixtures for all tests."""
 
+import os
 import shutil
 import signal
 import sys
@@ -9,7 +10,9 @@
 
 from agent_env.config import reset_config
 from agent_env.store.document_store import document_store as document_store_module
+from agent_env.store.object_store.local import grant_server as grant_server_module
 from agent_env.store.routing import disable_namespace_routing
+from tst.util import shards
 
 
 def _reset_store_singletons() -> None:
@@ -40,6 +43,29 @@ def pytest_unconfigure(config):
     shutil.rmtree(config.stash[_STATE_DIR], ignore_errors=True)
 
 
+@pytest.hookimpl(trylast=True)
+def pytest_collection_modifyitems(config, items):
+    """With ``AGENTENV_TEST_SHARD=i/N``, run only the i-th of N groups of the selected tests (``tst/util/shards.py``).
+    Last, so it splits what ``-m`` and the other filters left."""
+    value = os.environ.get(shards.ENV_VAR)
+    if not value:
+        return
+    try:
+        index, count = shards.parse(value)
+    except ValueError as e:
+        raise pytest.UsageError(str(e)) from None
+    table = shards.load_table()
+    marked = [(item, item.get_closest_marker("dependency")) for item in items]
+    linked = [(item.nodeid, mark.kwargs.get("name") or item.name, list(mark.kwargs.get("depends") or []))
+              for item, mark in marked if mark is not None]
+    together = shards.dependency_groups(linked, table)
+    keep = set(shards.select([item.nodeid for item in items], index, count, table, together))
+    deselected = [item for item in items if item.nodeid not in keep]
+    if deselected:
+        config.hook.pytest_deselected(items=deselected)
+        items[:] = [item for item in items if item.nodeid in keep]
+
+
 @pytest.fixture
 def sigint_handled():
     """SIGINT handled as Python handles it by default, for a test that sends it: a CI job started in the background
@@ -82,8 +108,12 @@ def fresh_config_document():
 
 @pytest.fixture(scope="module", autouse=True)
 def setup_config():
-    """Per-module store-singleton sweep; the config itself is reset per test above."""
+    """Per-module store-singleton sweep; the config itself is reset per test above. The grant servers close too:
+    one a module started under its own state root presents that root's CA, which the next module's containers
+    don't trust, and the next grant starts a server afresh."""
     _reset_store_singletons()
     yield
     reset_config()
     _reset_store_singletons()
+    for server in list(grant_server_module._servers.values()):
+        server.close()