diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 6caa6d94c..f0d8cf878 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -845,7 +845,8 @@ def _build_settings(env: ComputerEnv, base: ComputerArtifact) -> dict[str, Any]: def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build_settings: dict, lock: packages.Lock | None) -> str: """The composed image's Dockerfile but its key's label, which the key is a digest of: the build-phase settings, the - lock's .debs, then each layer's files under its prefix, its venv beside them from the lock's wheels, its run scripts + lock's .debs, mounted into the RUN that installs them, then each layer's files under its prefix, its venv beside + them from the lock's wheels, mounted the same way, its run scripts and an app's install steps, each run as root or the session user, every install with the network closed; then its programs, and an app's server's folder owned by the session user.""" user = base.session.user @@ -853,8 +854,8 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build *(f"ENV AGENTENV_SETTING_{name.upper()}={json.dumps(str(value))}" for name, value in build_settings.items())] if lock is not None and lock.apt: debs = f"/tmp/agentenv-{packages.STAGED_APT}" - lines += [f"COPY {packages.STAGED_APT}/ {debs}/", - f"RUN --network=none apt-get install -y --no-install-recommends {debs}/*.deb && rm -rf {debs}"] + lines.append(f"RUN --network=none {_mounted(packages.STAGED_APT, debs)} apt-get install -y " + f"--no-install-recommends {debs}/*.deb") for layer in layers: prefix = layer.prefix lines.append(f"COPY {layer.folder}/ {prefix}/") @@ -862,10 +863,9 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build staged = packages.staged_wheels(layer.folder) wheels = f"/tmp/agentenv-{staged}" if lock is not None and lock.wheels.get(layer.folder): - lines += [f"COPY {staged}/ {wheels}/", - f"RUN --network=none python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install " - f"--no-index --no-deps {wheels}/*.whl && {prefix}/venv/bin/pip list --format=json > " - f"{prefix}/installed.json && rm -rf {wheels}"] + lines.append(f"RUN --network=none {_mounted(staged, wheels)} python3 -m venv {prefix}/venv && " + f"{prefix}/venv/bin/pip install --no-index --no-deps {wheels}/*.whl && {prefix}/venv/bin/pip " + f"list --format=json > {prefix}/installed.json") else: lines.append(f"RUN --network=none python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip list " f"--format=json > {prefix}/installed.json") @@ -882,6 +882,12 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build return "\n".join(lines) + "\n" +def _mounted(source: str, target: str) -> str: + """A RUN's mount of the build context's ``source`` at ``target``, read-only, so files an install reads once never + stay in a layer of the image, as a COPY's would after the next RUN removed them.""" + return f"--mount=type=bind,source={source},target={target}" + + def _runs(layer: _Layer) -> list[tuple[str, str]]: """The commands of ``layer``'s run scripts and its install steps, each with the scope it runs as: the only steps of a build that write outside the layer's folder.""" diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index ead25cab4..60c597d21 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -10,7 +10,8 @@ apps' paths out of its own, and each app's data/reset, sent through the driver's route to it, puts the app back as installed. And through the driver's route from outside the computer, a WebSocket reaches the clock's own server both ways until it closes, only with the control token, a 20 MB answer streams through whole, and the clock sees the -address the driver was reached at. Needs Docker and the local registry on localhost:5000.""" +address the driver was reached at. And the lock's .debs and wheels, mounted into the RUNs that install them, are in +no layer of the composed image. Needs Docker and the local registry on localhost:5000.""" import hashlib import json @@ -30,7 +31,7 @@ from agent_env.store.routing import namespace_routing from agent_env.task.teardown import teardown_run from agent_env.task_step.context import TaskStepContext -from tst.integration.computer.conftest import _composed, run_context +from tst.integration.computer.conftest import _composed, docker, run_context pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -145,6 +146,24 @@ async def test_through_the_drivers_route_a_websocket_reaches_an_apps_server_and_ assert not report.failed +def test_the_locks_debs_and_wheels_install_from_mounts_and_stay_in_no_layer_of_the_image(desk): + image = desk.metadata["composed"]["container"][ARCH]["image"] + docker("pull", "-q", image) + + made = docker("history", "--no-trunc", "--format", "{{.CreatedBy}}", image).stdout.decode().splitlines() + installed = docker("run", "--rm", "--entrypoint", "sh", image, "-c", + "dpkg-query -W -f '${Package} ${Status}\\n' hello fastfetch && " + "/opt/agentenv/apps/clock/venv/bin/python -c 'import websockets; print(\"websockets\")' && " + "if [ -e /tmp/agentenv-packages ]; then echo left; else echo clean; fi" + ).stdout.decode().splitlines() + + assert [line for line in made if "packages/" in line and line.startswith("COPY")] == [] + assert any(line.startswith("RUN /bin/sh -c apt-get install -y --no-install-recommends /tmp/agentenv-packages/apt/") + for line in made) + assert sorted(installed[:2]) == ["fastfetch install ok installed", "hello install ok installed"] + assert installed[2:] == ["websockets", "clean"] + + def _kept(snapshot: dict) -> dict: """Each environment of a snapshot: the computer's and notes's files by their paths, and the clock's export.""" with namespace_routing(): diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index e6153865e..768cc6157 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -127,7 +127,7 @@ def save(self, tag, path): layers = [(None, {"etc/os-release": b"ubuntu"})] for line in dockerfile.splitlines(): instruction, _, rest = line.partition(" ") - command = rest.removeprefix("--network=none ") + command = re.sub(r"^(--\S+ )+", "", rest) # BuildKit records a RUN without its flags if instruction == "RUN": layers.append((f"RUN /bin/sh -c {command} # buildkit", { path: content for key, files in self.writes.items() if key in command for path, content in files.items()})) diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index a8a363820..9a1bcf674 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -99,10 +99,10 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ assert arm64["dockerfile"].splitlines() == [ f"FROM {base.image().image_name}", "USER root", 'ENV AGENTENV_SETTING_THEME="dark"', "COPY driver/ /opt/agentenv/drivers/desk/", - "COPY packages/wheels/driver/ /tmp/agentenv-packages/wheels/driver/", - "RUN --network=none python3 -m venv /opt/agentenv/drivers/desk/venv && /opt/agentenv/drivers/desk/venv/bin/pip " - "install --no-index --no-deps /tmp/agentenv-packages/wheels/driver/*.whl && /opt/agentenv/drivers/desk/venv/bin/pip list " - "--format=json > /opt/agentenv/drivers/desk/installed.json && rm -rf /tmp/agentenv-packages/wheels/driver", + "RUN --network=none --mount=type=bind,source=packages/wheels/driver,target=/tmp/agentenv-packages/wheels/driver " + "python3 -m venv /opt/agentenv/drivers/desk/venv && /opt/agentenv/drivers/desk/venv/bin/pip install --no-index " + "--no-deps /tmp/agentenv-packages/wheels/driver/*.whl && /opt/agentenv/drivers/desk/venv/bin/pip list " + "--format=json > /opt/agentenv/drivers/desk/installed.json", "COPY agentenv-driver-desk.conf /etc/supervisor/conf.d/agentenv-driver-desk.conf", "RUN test -d /opt/agentenv/drivers/desk && test -f /etc/supervisor/conf.d/agentenv-driver-desk.conf && test -x " "/opt/agentenv/drivers/desk/venv/bin/python", @@ -1449,7 +1449,8 @@ def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_unde "name": "apps-mail", "folder": "/ctx/apps/mail", "requirements": "requirements.txt", "add": [f"agentenv-framework-protocol[app]=={PROTOCOL_VERSION}"]} for line in ("COPY apps/mail/ /opt/agentenv/apps/mail/", - "RUN --network=none python3 -m venv /opt/agentenv/apps/mail/venv && /opt/agentenv/apps/mail/venv/bin/pip" + "RUN --network=none --mount=type=bind,source=packages/wheels/apps-mail,target=/tmp/agentenv-packages/" + "wheels/apps-mail python3 -m venv /opt/agentenv/apps/mail/venv && /opt/agentenv/apps/mail/venv/bin/pip" " install --no-index --no-deps /tmp/agentenv-packages/wheels/apps-mail/*.whl", "RUN mkdir -p /opt/agentenv/apps/mail/.agentenv && chown user /opt/agentenv/apps/mail/.agentenv", "COPY agentenv-app-mail.conf /etc/supervisor/conf.d/agentenv-app-mail.conf"): @@ -1483,6 +1484,28 @@ def test_an_app_that_requires_nothing_still_gets_a_venv_for_its_own_server(base, assert boot.ready_timeout([app], "windows") == boot.READY_TIMEOUT_S +def test_the_locks_debs_and_wheels_are_mounted_into_the_runs_that_install_them_never_copied_into_a_layer( + base, driver, docker, tmp_path): + """A COPY's files stay in its layer of the image though the RUN after it removes them.""" + _app(base, driver, tmp_path / "mail", install={"python": "requirements.txt", "packages": {"apt": ["hello"]}, + "services": []}) + + build = next(build for build in docker.builds if build["platform"] == "linux/arm64") + lines = build["dockerfile"].splitlines() + installs = [line for line in lines if "/tmp/agentenv-packages" in line] + + assert [line for line in lines if line.startswith("COPY packages/")] == [] + assert installs == [ + "RUN --network=none --mount=type=bind,source=packages/apt,target=/tmp/agentenv-packages/apt apt-get install -y " + "--no-install-recommends /tmp/agentenv-packages/apt/*.deb", + *(f"RUN --network=none --mount=type=bind,source=packages/wheels/{name},target=/tmp/agentenv-packages/wheels/" + f"{name} python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install --no-index --no-deps " + f"/tmp/agentenv-packages/wheels/{name}/*.whl && {prefix}/venv/bin/pip list --format=json > {prefix}/installed.json" + for name, prefix in (("driver", "/opt/agentenv/drivers/desk"), ("apps-mail", "/opt/agentenv/apps/mail")))] + assert {"packages/apt/hello_1.0_arm64.deb", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"} <= set( + build["files"]) + + def test_an_apps_packages_downloads_scripts_and_steps_are_fetched_at_put_and_installed_offline(base, driver, docker, tmp_path, monkeypatch): tool = b"#!/bin/sh\necho tool\n" @@ -1503,11 +1526,10 @@ def test_an_apps_packages_downloads_scripts_and_steps_are_fetched_at_put_and_ins {"apt": "https://ppa.example/ubuntu", "suite": "noble", "components": ["main"], "key": "/ctx/apps/mail/ppa.asc"}] builds = {build["platform"]: build for build in docker.builds} lines = builds["linux/arm64"]["dockerfile"].splitlines() - first = lines.index("COPY packages/apt/ /tmp/agentenv-packages/apt/") - assert lines[first:first + 3] == [ - "COPY packages/apt/ /tmp/agentenv-packages/apt/", - "RUN --network=none apt-get install -y --no-install-recommends /tmp/agentenv-packages/apt/*.deb && rm -rf " - "/tmp/agentenv-packages/apt", "COPY driver/ /opt/agentenv/drivers/desk/"] + first = lines.index("USER root") + 1 + assert lines[first:first + 2] == [ + "RUN --network=none --mount=type=bind,source=packages/apt,target=/tmp/agentenv-packages/apt apt-get install -y " + "--no-install-recommends /tmp/agentenv-packages/apt/*.deb", "COPY driver/ /opt/agentenv/drivers/desk/"] steps = lines.index("RUN mkdir -p /opt/agentenv/apps/mail/.agentenv && chown user /opt/agentenv/apps/mail/.agentenv") assert lines[steps + 1:steps + 8] == [ "RUN --network=none cd /opt/agentenv/apps/mail && './set up.sh'",