diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 5bf95c59c..ec4e12ce1 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -167,19 +167,63 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst + # A computer image or disk is tagged with a key of everything it is built from, and the composer reuses one the + # image store already holds for its key. The registry keeps its storage on the runner, restored from the computer + # images main saved latest this week, so a run reuses them rather than building them again. A shard that runs no + # computer tests misses. + - name: Week of the computer images' cache + id: week + run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" + - name: Restore the computer images + id: computers + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/registry + key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }} + restore-keys: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}- - name: Start the local registry run: | - docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 + mkdir -p "$RUNNER_TEMP/registry" + docker run --detach --name registry --publish 5000:5000 --volume "$RUNNER_TEMP/registry:/var/lib/registry" \ + mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. - name: Let the local provider boot disks with KVM run: sudo chmod 666 /dev/kvm - name: Slow integration tier on the local defaults + id: tests env: AGENTENV_TEST_SHARD: ${{ matrix.shard }}/12 run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ tst/integration --ignore=tst/integration/env/gateway/gateway_test.py + # Only the computer images are kept, and only main saves them, once its tests pass, so pull requests reuse main's + # without filling the repository's cache. The entry is named by the image keys the registry holds, so a run that + # built any new image saves a new one. The garbage collection frees what only the deleted repositories + # referenced; with --delete-untagged it would also delete the untagged platform manifests of an image index, and + # their layers. + - name: Keep only the computer images + id: kept + if: always() + run: | + repositories="$RUNNER_TEMP/registry/docker/registry/v2/repositories" + if [ ! -d "$repositories/agentenv-computers" ]; then echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0; fi + docker stop registry + sudo find "$repositories" -mindepth 1 -maxdepth 1 ! -name agentenv-computers -exec rm -rf {} + + docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \ + garbage-collect /etc/docker/registry/config.yml > /dev/null + du -sh "$RUNNER_TEMP/registry" + tags=$(sudo ls "$repositories/agentenv-computers/_manifests/tags" | sort | sha256sum | cut -c1-16) + echo "key=computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$tags" >> "$GITHUB_OUTPUT" + echo "computers=true" >> "$GITHUB_OUTPUT" + - name: Save the computer images + if: >- + always() && github.event_name == 'push' && steps.tests.outcome == 'success' + && steps.kept.outputs.computers == 'true' && steps.kept.outputs.key != steps.computers.outputs.cache-matched-key + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/registry + key: ${{ steps.kept.outputs.key }} - name: Skips must be declared capability gaps if: always() run: |