From 62c6ee71abfa280e741b9d15cc12c982d5456073 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 5 Nov 2026 14:52:26 -0800 Subject: [PATCH 1/6] feat(run): --set KEY=VALUE sets env keys for one run; computer envs take tools agent-env run --set KEY=VALUE sets a key in every env the bundle's tasks deploy whose type takes it, or in one env as envs/NAME.KEY=VALUE, the value TOML or a bare word and the key dotted into tables. The envs it changes, the envs and tasks naming them and the evals naming those tasks are written under ids ending in the settings, leaving the bundle as written. Computer envs take tools = "standard" | "driver" | "both", which a deploy, and a reset, applies to the agents' role through the driver's tool access. Refs FD-4362 Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/__init__.py | 2 + src/agent_env/bundle/run.py | 46 +++-- src/agent_env/bundle/variants.py | 208 +++++++++++++++++++++ src/agent_env/cli/run.py | 32 +++- src/agent_env/computers/README.md | 2 + src/agent_env/drivers/README.md | 5 + src/agent_env/drivers/aliases.toml | 2 +- src/agent_env/env/envs/_deployment.py | 7 +- src/agent_env/env/envs/computer/app_env.py | 5 +- src/agent_env/env/envs/computer/env.py | 57 ++++-- tst/unit/bundle/installed_test.py | 6 +- tst/unit/bundle/variants_test.py | 175 +++++++++++++++++ tst/unit/env/envs/test_computer_env.py | 49 ++++- 13 files changed, 550 insertions(+), 46 deletions(-) create mode 100644 src/agent_env/bundle/variants.py create mode 100644 tst/unit/bundle/variants_test.py diff --git a/src/agent_env/bundle/__init__.py b/src/agent_env/bundle/__init__.py index 57290f61d..5c736a577 100644 --- a/src/agent_env/bundle/__init__.py +++ b/src/agent_env/bundle/__init__.py @@ -3,6 +3,7 @@ from .parse import Bundle, BundleEntry, BundleError, BundleKind, is_ignored, parse_bundle from .run import (BundleRun, DryRun, EvalRun, Outcome, PreparedBundle, RunInterrupted, TaskRun, dry_run_bundle, prepare_bundle, run_bundle) +from .variants import Variant __all__ = [ "Bundle", @@ -16,6 +17,7 @@ "PreparedBundle", "RunInterrupted", "TaskRun", + "Variant", "dry_run_bundle", "is_ignored", "parse_bundle", diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 3d5d94f25..a9e485fb3 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -18,6 +18,7 @@ import shlex import time import uuid +from collections import Counter from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace from enum import Enum @@ -45,6 +46,7 @@ from .requires import Declaring, declare, declaring_for from .resolve import (BuiltImage, BundleImage, Reference, ReferencedImage, ResolvedEntry, resolve_bundle, without_played_agents) +from .variants import Variant, with_variant logger = logging.getLogger(__name__) @@ -220,6 +222,7 @@ def run_bundle( keep: bool = False, emulate: bool = True, source: HubSource | None = None, + variant: Variant | None = None, ) -> BundleRun: """Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the @@ -231,7 +234,8 @@ def run_bundle( at each write and step. ``id_root`` roots the ids of a bundle an installed package provides (``InstalledBundle.id_root``) or one downloaded from the Hub (``HubBundle.id_root``); a folder's ids otherwise come from its path. ``source`` says where a Hub bundle came from (``HubBundle.source``), which each run's context - records under ``metadata["bundle_source"]``. + records under ``metadata["bundle_source"]``. ``variant`` sets env keys for this run (``with_variant``), writing + what it changes under ids of their own and leaving the bundle as written. Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone. ``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``. @@ -253,7 +257,8 @@ def run_bundle( say = _progress(on_progress) with namespace_routing(): prepared = _prepare(root, tasks, evals, sandbox, id_root, agent, False, say, - declaring_for(Path(root), installed=id_root is not None, dry_run=False), composing) + declaring_for(Path(root), installed=id_root is not None, dry_run=False), composing, + variant=variant) plan, entries = prepared.plan, [entry for entry, _ in prepared.tasks] with Interrupts() as interrupts: overrides = _overrides(sandbox, user_model, judge_model) @@ -339,6 +344,7 @@ def dry_run_bundle( on_progress: Callable[[str], None] | None = None, id_root: str | None = None, emulate: bool = True, + variant: Variant | None = None, ) -> DryRun: """Check what ``run_bundle`` with these arguments checks before its first task runs, and say what it would write and run, writing and running nothing: no entity, ledger row, lock or instance. Each write's version is @@ -352,8 +358,8 @@ def dry_run_bundle( composing = _composing(sandbox, emulate) say = _progress(on_progress) with namespace_routing(): - plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent) - _say_plan(plan, say, declaring_for(Path(root), installed=id_root is not None, dry_run=True)) + plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, variant=variant) + _say_plan(plan, say, declaring_for(Path(root), installed=id_root is not None, dry_run=True), variant) materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done)), composing=composing, sandbox=sandbox, on_progress=lambda write, line: say(f"{_label(plan, write)}: {line}")) @@ -385,10 +391,11 @@ def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRu def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandbox: str | None, id_root: str | None, agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring, - composing: Composing = Composing(), *, solver_endpoint: bool = False) -> PreparedBundle: + composing: Composing = Composing(), *, solver_endpoint: bool = False, + variant: Variant | None = None) -> PreparedBundle: plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, played=played, - solver_endpoint=solver_endpoint) - _say_plan(plan, say, declaring) + solver_endpoint=solver_endpoint, variant=variant) + _say_plan(plan, say, declaring, variant) materialization = _written_with_infra(plan, preflight, say, composing, sandbox) to_run = tuple((entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in _to_run(plan)) return PreparedBundle(plan, materialization, to_run) @@ -396,11 +403,14 @@ def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandb def _planned(root: Path | str, tasks: Sequence[str], evals: Sequence[str], id_root: str | None, sandbox: str | None, agent: str | None = None, *, played: bool = False, - solver_endpoint: bool = False) -> tuple[Plan, Preflight]: - resolved = resolve_bundle(parse_bundle(Path(root), id_root=id_root)) + solver_endpoint: bool = False, variant: Variant | None = None) -> tuple[Plan, Preflight]: + bundle = parse_bundle(Path(root), id_root=id_root) + resolved = resolve_bundle(with_variant(bundle, variant) if variant else bundle) plan = plan_bundle(without_played_agents(resolved) if played else resolved, tasks=tasks, evals=evals, installed=installed_bundles, agent=agent) refuse_store_tasks(plan) + if variant is not None and not _changed(plan, variant): + raise BundleError([f"{variant.typed}: none of the tasks this run selects deploys an env it changes"]) to_run = set(_to_run(plan)) return plan, preflight_run(plan, [write.source for write in plan.writes if write.kind is BundleKind.TASK and write.source.entry in to_run], sandbox, @@ -451,9 +461,10 @@ def refuse_store_tasks(plan: Plan) -> None: raise BundleError(problems) -def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> None: - """Say what each agent, env and artifact alias names, what the solver deploys in place of the tasks' agents, and - what plugin packages the bundle uses that its bundle.toml doesn't require, as ``declaring`` says (``declare``).""" +def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring, variant: Variant | None = None) -> None: + """Say what each agent, env and artifact alias names, what the solver deploys in place of the tasks' agents, the + ids ``variant``'s entries are written under, and what plugin packages the bundle uses that its bundle.toml doesn't + require, as ``declaring`` says (``declare``).""" for use in (use for uses in plan.alias_uses.values() for use in uses.values()): say(_aliased(use)) if plan.agent is not None: @@ -461,10 +472,21 @@ def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> N solved = sum(entry.id in plan.agent.steps for entry in runs) say(f"--agent {plan.agent.name}: the solver deploys {plan.agent.agent_id}, in {solved} of {len(runs)} " f"task{'s' if len(runs) != 1 else ''}") + if variant is not None: + say(f"{variant.typed}: {_changed(plan, variant)} written under ids ending in ({variant.label})") for line in declare(plan.bundle, declaring): say(line) +def _changed(plan: Plan, variant: Variant) -> str: + """How many envs, tasks and evals ``variant`` changes the run writes: ``1 env and 2 tasks``, or ``""``.""" + counts = Counter(write.kind for write in plan.writes + if isinstance(write.source, ResolvedEntry) and write.id.endswith(f"({variant.label})")) + named = [f"{counts[kind]} {noun}{'s' if counts[kind] != 1 else ''}" for kind, noun in ( + (BundleKind.ENV, "env"), (BundleKind.TASK, "task"), (BundleKind.EVAL, "eval")) if counts[kind]] + return " and ".join([", ".join(named[:-1]), named[-1]] if len(named) > 1 else named) + + def _hint(id_root: str | None) -> Declaring: """How a service writing a bundle declares its plugins: nothing for an installed bundle, else a hint.""" return Declaring.QUIET if id_root is not None else Declaring.HINT diff --git a/src/agent_env/bundle/variants.py b/src/agent_env/bundle/variants.py new file mode 100644 index 000000000..4320625d0 --- /dev/null +++ b/src/agent_env/bundle/variants.py @@ -0,0 +1,208 @@ +"""Run-only variants for ``agent-env run --set KEY=VALUE``: the bundle as parsed, with env keys set for one run. Each env +changed, each env and task that names one, and each eval that names such a task is written under a derived id, so a +variant is written and composed beside the bundle's own. Entries keep their names, so every reference to one by name +reaches its variant; the bundle's files stay as written.""" +from __future__ import annotations + +import copy +import hashlib +import tomllib +from dataclasses import dataclass, replace +from typing import Any + +from agent_env.entity.refs import EntityKind, ref_sites +from agent_env.env.registry import get_env_registry +from agent_env.task_step.registry import get_task_step_registry + +from .parse import Bundle, BundleEntry, BundleError, BundleKind, segment_problem + +_TARGET = f"{BundleKind.ENV.value}/" + + +@dataclass(frozen=True) +class Variant: + """The ``--set`` settings of a run, as typed: ``KEY=VALUE`` for every env the bundle's tasks deploy whose type takes + KEY, or ``envs/NAME.KEY=VALUE`` for one env; KEY may be dotted into a table, VALUE is TOML or a bare word.""" + + settings: tuple[str, ...] + + @property + def label(self) -> str: + """The settings as ids end in them: ``driver cua-driver, display 3f2a9c1e``, a value an id can't hold hashed.""" + return ", ".join(f"{key.removeprefix(_TARGET)} {_shown(raw)}" + for key, raw in (setting.partition("=")[::2] for setting in self.settings)) + + @property + def typed(self) -> str: + """The settings as the command line gives them.""" + return " ".join(f"--set {setting}" for setting in self.settings) + + +@dataclass(frozen=True) +class _Setting: + text: str + env: str | None # the env it names, by name; None for every env deployed that takes path[0] + path: tuple[str, ...] + value: Any + + +def with_variant(bundle: Bundle, variant: Variant) -> Bundle: + """``bundle`` with ``variant``'s settings in its envs' tomls, and each entry that changes under an id ending in + ``(