From fdfbdb616b4440e7a43615891bc652546e8e441d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 4 Nov 2026 11:58:43 -0800 Subject: [PATCH 1/4] ci: cache the slow tier's local registry between runs A computer image or disk is keyed by its inputs, and the composer reuses one the image store already holds for its key. The slow tier's registry starts empty on every runner, so every run builds them again: chromium_app_test.py spends about seven of its eleven minutes building and pushing. Each slow shard now keeps the registry's storage on the runner, restored from and saved to the Actions cache. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/local-backends.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 5bf95c59c..8fc8219e7 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -167,9 +167,19 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst + # The registry keeps its storage on the runner, cached between runs: a computer image or disk is keyed by its + # inputs, so one an earlier run pushed is reused rather than built again. + - name: Restore the local registry's storage + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/registry + key: registry-slow-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: registry-slow-${{ matrix.shard }}- - name: Start the local registry run: | - docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 + mkdir -p "$RUNNER_TEMP/registry" + docker run --detach --name registry --publish 5000:5000 --volume "$RUNNER_TEMP/registry:/var/lib/registry" \ + mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. - name: Let the local provider boot disks with KVM @@ -180,6 +190,15 @@ jobs: run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ tst/integration --ignore=tst/integration/env/gateway/gateway_test.py + - name: Size of the local registry's storage + if: always() + run: du -sh "$RUNNER_TEMP/registry" + - name: Save the local registry's storage + if: always() + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/registry + key: registry-slow-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Skips must be declared capability gaps if: always() run: | From 6dba5e93dde685472a54b641335f7bceaa6f0b5e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 4 Nov 2026 12:41:31 -0800 Subject: [PATCH 2/4] ci: cache only the computer images, saved from main each week The trial cached each shard's whole registry: about 9.4 GB a run, nearly all the repository may keep, for images no later run reuses. Now a shard keeps only the agentenv-computers repository, garbage-collected, and only a push to main saves it, so pull requests restore main's. The key is the shard, the ISO week and a hash of the computer code, falling back to the week's earlier entries, so it starts afresh every week rather than growing. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/local-backends.yml | 38 ++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 8fc8219e7..cadec1011 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -167,14 +167,19 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst - # The registry keeps its storage on the runner, cached between runs: a computer image or disk is keyed by its - # inputs, so one an earlier run pushed is reused rather than built again. - - name: Restore the local registry's storage + # A computer image or disk is keyed by what it is built from, and the composer reuses one the image store already + # holds for its key. The registry keeps its storage on the runner, restored from the computer images main saved + # this week, so a run reuses them rather than building them again. A shard that runs no computer tests misses. + - name: Week of the computer images' cache + id: week + run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" + - name: Restore the computer images + id: computers uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/registry - key: registry-slow-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: registry-slow-${{ matrix.shard }}- + key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-${{ hashFiles('src/agent_env/env/envs/computer/**', 'tst/integration/computer/**', 'packages/agentenv-protocol/src/**') }} + restore-keys: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}- - name: Start the local registry run: | mkdir -p "$RUNNER_TEMP/registry" @@ -190,15 +195,28 @@ jobs: run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ tst/integration --ignore=tst/integration/env/gateway/gateway_test.py - - name: Size of the local registry's storage - if: always() - run: du -sh "$RUNNER_TEMP/registry" - - name: Save the local registry's storage + # Only the computer images are kept, and only main saves them, so pull requests reuse main's without filling the + # repository's cache. + - name: Keep only the computer images + id: kept if: always() + run: | + repositories="$RUNNER_TEMP/registry/docker/registry/v2/repositories" + if [ ! -d "$repositories/agentenv-computers" ]; then echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0; fi + docker stop registry + sudo find "$repositories" -mindepth 1 -maxdepth 1 ! -name agentenv-computers -exec rm -rf {} + + docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \ + garbage-collect --delete-untagged /etc/docker/registry/config.yml > /dev/null + du -sh "$RUNNER_TEMP/registry" + echo "computers=true" >> "$GITHUB_OUTPUT" + - name: Save the computer images + if: >- + always() && github.event_name == 'push' && steps.kept.outputs.computers == 'true' + && steps.computers.outputs.cache-hit != 'true' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/registry - key: registry-slow-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} + key: ${{ steps.computers.outputs.cache-primary-key }} - name: Skips must be declared capability gaps if: always() run: | From d39b933a38c6e2f465853f489dc8ee5b7d525b32 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 4 Nov 2026 12:59:44 -0800 Subject: [PATCH 3/4] ci: garbage-collect the kept registry without deleting untagged manifests The computer images are pushed as image indexes, whose platform manifests carry no tag. With --delete-untagged the registry's garbage collection deleted them and their layers, so each shard kept about 1 MB of tags with nothing behind them. Without it, only what the deleted repositories referenced goes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/local-backends.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index cadec1011..93a128370 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -196,7 +196,8 @@ jobs: .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ tst/integration --ignore=tst/integration/env/gateway/gateway_test.py # Only the computer images are kept, and only main saves them, so pull requests reuse main's without filling the - # repository's cache. + # repository's cache. The garbage collection frees what only the deleted repositories referenced; with + # --delete-untagged it would also delete the untagged platform manifests of an image index, and their layers. - name: Keep only the computer images id: kept if: always() @@ -206,7 +207,7 @@ jobs: docker stop registry sudo find "$repositories" -mindepth 1 -maxdepth 1 ! -name agentenv-computers -exec rm -rf {} + docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \ - garbage-collect --delete-untagged /etc/docker/registry/config.yml > /dev/null + garbage-collect /etc/docker/registry/config.yml > /dev/null du -sh "$RUNNER_TEMP/registry" echo "computers=true" >> "$GITHUB_OUTPUT" - name: Save the computer images From a0ce15b71b5a904e89d079e6b255a8e935b57283 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 4 Nov 2026 13:22:34 -0800 Subject: [PATCH 4/4] ci: name the computer images' cache by the image keys it holds, and save it only when the tests pass The cache was keyed by a hash of files picked by hand, which missed some build inputs (the protocol's pyproject.toml): an image they changed was rebuilt, but the key still matched, so it was never saved. The registry's tags are the images' own keys, of everything each was built from, so the entry is now named by a hash of those tags: a run that builds any new image saves a new entry, and restores take the week's latest. A shard whose tests fail no longer saves, so a partial set can't become the entry later runs match and don't replace. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/local-backends.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 93a128370..ec4e12ce1 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -167,9 +167,10 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst - # A computer image or disk is keyed by what it is built from, and the composer reuses one the image store already - # holds for its key. The registry keeps its storage on the runner, restored from the computer images main saved - # this week, so a run reuses them rather than building them again. A shard that runs no computer tests misses. + # A computer image or disk is tagged with a key of everything it is built from, and the composer reuses one the + # image store already holds for its key. The registry keeps its storage on the runner, restored from the computer + # images main saved latest this week, so a run reuses them rather than building them again. A shard that runs no + # computer tests misses. - name: Week of the computer images' cache id: week run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" @@ -178,7 +179,7 @@ jobs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/registry - key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-${{ hashFiles('src/agent_env/env/envs/computer/**', 'tst/integration/computer/**', 'packages/agentenv-protocol/src/**') }} + key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }} restore-keys: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}- - name: Start the local registry run: | @@ -190,14 +191,17 @@ jobs: - name: Let the local provider boot disks with KVM run: sudo chmod 666 /dev/kvm - name: Slow integration tier on the local defaults + id: tests env: AGENTENV_TEST_SHARD: ${{ matrix.shard }}/12 run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ tst/integration --ignore=tst/integration/env/gateway/gateway_test.py - # Only the computer images are kept, and only main saves them, so pull requests reuse main's without filling the - # repository's cache. The garbage collection frees what only the deleted repositories referenced; with - # --delete-untagged it would also delete the untagged platform manifests of an image index, and their layers. + # Only the computer images are kept, and only main saves them, once its tests pass, so pull requests reuse main's + # without filling the repository's cache. The entry is named by the image keys the registry holds, so a run that + # built any new image saves a new one. The garbage collection frees what only the deleted repositories + # referenced; with --delete-untagged it would also delete the untagged platform manifests of an image index, and + # their layers. - name: Keep only the computer images id: kept if: always() @@ -209,15 +213,17 @@ jobs: docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \ garbage-collect /etc/docker/registry/config.yml > /dev/null du -sh "$RUNNER_TEMP/registry" + tags=$(sudo ls "$repositories/agentenv-computers/_manifests/tags" | sort | sha256sum | cut -c1-16) + echo "key=computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$tags" >> "$GITHUB_OUTPUT" echo "computers=true" >> "$GITHUB_OUTPUT" - name: Save the computer images if: >- - always() && github.event_name == 'push' && steps.kept.outputs.computers == 'true' - && steps.computers.outputs.cache-hit != 'true' + always() && github.event_name == 'push' && steps.tests.outcome == 'success' + && steps.kept.outputs.computers == 'true' && steps.kept.outputs.key != steps.computers.outputs.cache-matched-key uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/registry - key: ${{ steps.computers.outputs.cache-primary-key }} + key: ${{ steps.kept.outputs.key }} - name: Skips must be declared capability gaps if: always() run: |