perf(computer): mount the lock's .debs and wheels into the RUNs that install them, keeping them out of the image

src/agent_env/env/envs/computer/utils/composer.py

@@ -845,27 +845,27 @@ def _build_settings(env: ComputerEnv, base: ComputerArtifact) -> dict[str, Any]:
 def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build_settings: dict,
                 lock: packages.Lock | None) -> str:
     """The composed image's Dockerfile but its key's label, which the key is a digest of: the build-phase settings, the
-    lock's .debs, then each layer's files under its prefix, its venv beside them from the lock's wheels, its run scripts
+    lock's .debs, mounted into the RUN that installs them, then each layer's files under its prefix, its venv beside
+    them from the lock's wheels, mounted the same way, its run scripts
     and an app's install steps, each run as root or the session user, every install with the network closed; then its
     programs, and an app's server's folder owned by the session user."""
     user = base.session.user
     lines = [f"FROM {source}", "USER root",
              *(f"ENV AGENTENV_SETTING_{name.upper()}={json.dumps(str(value))}" for name, value in build_settings.items())]
     if lock is not None and lock.apt:
         debs = f"/tmp/agentenv-{packages.STAGED_APT}"
-        lines += [f"COPY {packages.STAGED_APT}/ {debs}/",
-                  f"RUN --network=none apt-get install -y --no-install-recommends {debs}/*.deb && rm -rf {debs}"]
+        lines.append(f"RUN --network=none {_mounted(packages.STAGED_APT, debs)} apt-get install -y "
+                     f"--no-install-recommends {debs}/*.deb")
     for layer in layers:
         prefix = layer.prefix
         lines.append(f"COPY {layer.folder}/ {prefix}/")
         if _venv(layer):
             staged = packages.staged_wheels(layer.folder)
             wheels = f"/tmp/agentenv-{staged}"
             if lock is not None and lock.wheels.get(layer.folder):
-                lines += [f"COPY {staged}/ {wheels}/",
-                          f"RUN --network=none python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install "
-                          f"--no-index --no-deps {wheels}/*.whl && {prefix}/venv/bin/pip list --format=json > "
-                          f"{prefix}/installed.json && rm -rf {wheels}"]
+                lines.append(f"RUN --network=none {_mounted(staged, wheels)} python3 -m venv {prefix}/venv && "
+                             f"{prefix}/venv/bin/pip install --no-index --no-deps {wheels}/*.whl && {prefix}/venv/bin/pip "
+                             f"list --format=json > {prefix}/installed.json")
             else:
                 lines.append(f"RUN --network=none python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip list "
                              f"--format=json > {prefix}/installed.json")
@@ -882,6 +882,12 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build
     return "\n".join(lines) + "\n"
 
 
+def _mounted(source: str, target: str) -> str:
+    """A RUN's mount of the build context's ``source`` at ``target``, read-only, so files an install reads once never
+    stay in a layer of the image, as a COPY's would after the next RUN removed them."""
+    return f"--mount=type=bind,source={source},target={target}"
+
+
 def _runs(layer: _Layer) -> list[tuple[str, str]]:
     """The commands of ``layer``'s run scripts and its install steps, each with the scope it runs as: the only steps of
     a build that write outside the layer's folder."""

tst/integration/computer/computer_app_test.py

@@ -10,7 +10,8 @@
 apps' paths out of its own, and each app's data/reset, sent through the driver's route to it, puts the app back as
 installed. And through the driver's route from outside the computer, a WebSocket reaches the clock's own server both
 ways until it closes, only with the control token, a 20 MB answer streams through whole, and the clock sees the
-address the driver was reached at. Needs Docker and the local registry on localhost:5000."""
+address the driver was reached at. And the lock's .debs and wheels, mounted into the RUNs that install them, are in
+no layer of the composed image. Needs Docker and the local registry on localhost:5000."""
 
 import hashlib
 import json
@@ -30,7 +31,7 @@
 from agent_env.store.routing import namespace_routing
 from agent_env.task.teardown import teardown_run
 from agent_env.task_step.context import TaskStepContext
-from tst.integration.computer.conftest import _composed, run_context
+from tst.integration.computer.conftest import _composed, docker, run_context
 
 pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow]
 
@@ -145,6 +146,24 @@ async def test_through_the_drivers_route_a_websocket_reaches_an_apps_server_and_
     assert not report.failed
 
 
+def test_the_locks_debs_and_wheels_install_from_mounts_and_stay_in_no_layer_of_the_image(desk):
+    image = desk.metadata["composed"]["container"][ARCH]["image"]
+    docker("pull", "-q", image)
+
+    made = docker("history", "--no-trunc", "--format", "{{.CreatedBy}}", image).stdout.decode().splitlines()
+    installed = docker("run", "--rm", "--entrypoint", "sh", image, "-c",
+                       "dpkg-query -W -f '${Package} ${Status}\\n' hello fastfetch && "
+                       "/opt/agentenv/apps/clock/venv/bin/python -c 'import websockets; print(\"websockets\")' && "
+                       "if [ -e /tmp/agentenv-packages ]; then echo left; else echo clean; fi"
+                       ).stdout.decode().splitlines()
+
+    assert [line for line in made if "packages/" in line and line.startswith("COPY")] == []
+    assert any(line.startswith("RUN /bin/sh -c apt-get install -y --no-install-recommends /tmp/agentenv-packages/apt/")
+               for line in made)
+    assert sorted(installed[:2]) == ["fastfetch install ok installed", "hello install ok installed"]
+    assert installed[2:] == ["websockets", "clean"]
+
+
 def _kept(snapshot: dict) -> dict:
     """Each environment of a snapshot: the computer's and notes's files by their paths, and the clock's export."""
     with namespace_routing():

tst/unit/env/envs/_computer.py

@@ -127,7 +127,7 @@ def save(self, tag, path):
         layers = [(None, {"etc/os-release": b"ubuntu"})]
         for line in dockerfile.splitlines():
             instruction, _, rest = line.partition(" ")
-            command = rest.removeprefix("--network=none ")
+            command = re.sub(r"^(--\S+ )+", "", rest)  # BuildKit records a RUN without its flags
             if instruction == "RUN":
                 layers.append((f"RUN /bin/sh -c {command} # buildkit", {
                     path: content for key, files in self.writes.items() if key in command for path, content in files.items()}))

tst/unit/env/envs/test_computer_env.py

@@ -99,10 +99,10 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_
     assert arm64["dockerfile"].splitlines() == [
         f"FROM {base.image().image_name}", "USER root", 'ENV AGENTENV_SETTING_THEME="dark"',
         "COPY driver/ /opt/agentenv/drivers/desk/",
-        "COPY packages/wheels/driver/ /tmp/agentenv-packages/wheels/driver/",
-        "RUN --network=none python3 -m venv /opt/agentenv/drivers/desk/venv && /opt/agentenv/drivers/desk/venv/bin/pip "
-        "install --no-index --no-deps /tmp/agentenv-packages/wheels/driver/*.whl && /opt/agentenv/drivers/desk/venv/bin/pip list "
-        "--format=json > /opt/agentenv/drivers/desk/installed.json && rm -rf /tmp/agentenv-packages/wheels/driver",
+        "RUN --network=none --mount=type=bind,source=packages/wheels/driver,target=/tmp/agentenv-packages/wheels/driver "
+        "python3 -m venv /opt/agentenv/drivers/desk/venv && /opt/agentenv/drivers/desk/venv/bin/pip install --no-index "
+        "--no-deps /tmp/agentenv-packages/wheels/driver/*.whl && /opt/agentenv/drivers/desk/venv/bin/pip list "
+        "--format=json > /opt/agentenv/drivers/desk/installed.json",
         "COPY agentenv-driver-desk.conf /etc/supervisor/conf.d/agentenv-driver-desk.conf",
         "RUN test -d /opt/agentenv/drivers/desk && test -f /etc/supervisor/conf.d/agentenv-driver-desk.conf && test -x "
         "/opt/agentenv/drivers/desk/venv/bin/python",
@@ -1449,7 +1449,8 @@ def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_unde
         "name": "apps-mail", "folder": "/ctx/apps/mail", "requirements": "requirements.txt",
         "add": [f"agentenv-framework-protocol[app]=={PROTOCOL_VERSION}"]}
     for line in ("COPY apps/mail/ /opt/agentenv/apps/mail/",
-                 "RUN --network=none python3 -m venv /opt/agentenv/apps/mail/venv && /opt/agentenv/apps/mail/venv/bin/pip"
+                 "RUN --network=none --mount=type=bind,source=packages/wheels/apps-mail,target=/tmp/agentenv-packages/"
+                 "wheels/apps-mail python3 -m venv /opt/agentenv/apps/mail/venv && /opt/agentenv/apps/mail/venv/bin/pip"
                  " install --no-index --no-deps /tmp/agentenv-packages/wheels/apps-mail/*.whl",
                  "RUN mkdir -p /opt/agentenv/apps/mail/.agentenv && chown user /opt/agentenv/apps/mail/.agentenv",
                  "COPY agentenv-app-mail.conf /etc/supervisor/conf.d/agentenv-app-mail.conf"):
@@ -1483,6 +1484,28 @@ def test_an_app_that_requires_nothing_still_gets_a_venv_for_its_own_server(base,
     assert boot.ready_timeout([app], "windows") == boot.READY_TIMEOUT_S
 
 
+def test_the_locks_debs_and_wheels_are_mounted_into_the_runs_that_install_them_never_copied_into_a_layer(
+        base, driver, docker, tmp_path):
+    """A COPY's files stay in its layer of the image though the RUN after it removes them."""
+    _app(base, driver, tmp_path / "mail", install={"python": "requirements.txt", "packages": {"apt": ["hello"]},
+                                                   "services": []})
+
+    build = next(build for build in docker.builds if build["platform"] == "linux/arm64")
+    lines = build["dockerfile"].splitlines()
+    installs = [line for line in lines if "/tmp/agentenv-packages" in line]
+
+    assert [line for line in lines if line.startswith("COPY packages/")] == []
+    assert installs == [
+        "RUN --network=none --mount=type=bind,source=packages/apt,target=/tmp/agentenv-packages/apt apt-get install -y "
+        "--no-install-recommends /tmp/agentenv-packages/apt/*.deb",
+        *(f"RUN --network=none --mount=type=bind,source=packages/wheels/{name},target=/tmp/agentenv-packages/wheels/"
+          f"{name} python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install --no-index --no-deps "
+          f"/tmp/agentenv-packages/wheels/{name}/*.whl && {prefix}/venv/bin/pip list --format=json > {prefix}/installed.json"
+          for name, prefix in (("driver", "/opt/agentenv/drivers/desk"), ("apps-mail", "/opt/agentenv/apps/mail")))]
+    assert {"packages/apt/hello_1.0_arm64.deb", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"} <= set(
+        build["files"])
+
+
 def test_an_apps_packages_downloads_scripts_and_steps_are_fetched_at_put_and_installed_offline(base, driver, docker,
                                                                                                tmp_path, monkeypatch):
     tool = b"#!/bin/sh\necho tool\n"
@@ -1503,11 +1526,10 @@ def test_an_apps_packages_downloads_scripts_and_steps_are_fetched_at_put_and_ins
         {"apt": "https://ppa.example/ubuntu", "suite": "noble", "components": ["main"], "key": "/ctx/apps/mail/ppa.asc"}]
     builds = {build["platform"]: build for build in docker.builds}
     lines = builds["linux/arm64"]["dockerfile"].splitlines()
-    first = lines.index("COPY packages/apt/ /tmp/agentenv-packages/apt/")
-    assert lines[first:first + 3] == [
-        "COPY packages/apt/ /tmp/agentenv-packages/apt/",
-        "RUN --network=none apt-get install -y --no-install-recommends /tmp/agentenv-packages/apt/*.deb && rm -rf "
-        "/tmp/agentenv-packages/apt", "COPY driver/ /opt/agentenv/drivers/desk/"]
+    first = lines.index("USER root") + 1
+    assert lines[first:first + 2] == [
+        "RUN --network=none --mount=type=bind,source=packages/apt,target=/tmp/agentenv-packages/apt apt-get install -y "
+        "--no-install-recommends /tmp/agentenv-packages/apt/*.deb", "COPY driver/ /opt/agentenv/drivers/desk/"]
     steps = lines.index("RUN mkdir -p /opt/agentenv/apps/mail/.agentenv && chown user /opt/agentenv/apps/mail/.agentenv")
     assert lines[steps + 1:steps + 8] == [
         "RUN --network=none cd /opt/agentenv/apps/mail && './set up.sh'",