diff --git a/docs/deploy.md b/docs/deploy.md index 5ffe8288..5da2140a 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -86,6 +86,7 @@ sandboxd reads one JSON file (`-config`, default | `cocoon_bin` | `cocoon` | cocoon CLI binary | | `restore_mode` | unset | clone and wake-restore memory mode: `copy`, `ondemand`, or `mmap`; use `mmap` for dense pools | | `no_direct_io` | false | use buffered writable disks for Cloud Hypervisor cold boots and clones; recommended for dense ephemeral pools to avoid direct-I/O CoW journal contention | +| `no_balloon` | false | boot pool and template VMs without the virtio-balloon (cocoon otherwise returns 25% of guest memory to the host); clones inherit it from the golden. A guest that thrashes before deflate-on-OOM fires — a 16G build tier running a large typecheck — needs its whole memory | | `advertise_addr` | = `listen` | the host:port clients reach this node at; returned as a claim's owner address and gossiped to peers. Must be routable when `listen` is a wildcard | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) needs `bridges` and rejects a CNI network at load | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, while every listed name must exist in the node's `tenants` config; root always has access. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | @@ -104,6 +105,7 @@ sandboxd reads one JSON file (`-config`, default | `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the expiry eligibility point for a healed replica a delete broadcast missed, after which its next successful hourly sweep removes it; persistent sweep failure extends retention until one succeeds, so it is not a hard ceiling | | `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | +| `warmup` (pool entry) | unset | argv run in the golden VM after readiness and before its snapshot, so the files it touches are page-cache-resident in every clone — e.g. `["node", "-e", "0"]` on a Node flavor. It runs under the engine's 2-minute command timeout with only `PATH` in its environment; a non-zero exit or a timeout fails the golden build, so the pool stays unfilled until the config is fixed. Config-owned like `egress`: `PUT /v1/pools` rejects it, and a golden built with a different warmup is rebuilt | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, normal warm-candidate placement applies, with volume claims limited to candidates holding every requested volume) | | `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. A request frame whose first line exceeds 4 KiB is skipped, never truncated | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim with no data-plane connection for this long is hibernated; the next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index d9b89bd0..d4dda8ec 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -325,7 +325,8 @@ targets online — no restart, live claims untouched: Pools omitted from the list are drained: their unclaimed warm VMs are destroyed and the pool entry retires. `net`/`size` default like a claim's. Answers the fresh `GET /v1/info` payload. 400 bad key, negative warm/idle, -`warm_max` below `warm`, or duplicate pool; 401 bad api token; 409 egress +`warm_max` below `warm`, duplicate pool, or a config-owned `egress`/`warmup` +field; 401 bad api token; 409 egress pool on a node without an egress attachment. ## POST /v1/drain @@ -515,6 +516,25 @@ the connection is a byte-for-byte relay to the guest's silkd (one silkd RPC per connection — see [silkd](silkd.md)). 426 without the upgrade header, 404 unknown sandbox, 502 guest unreachable. +## POST /v1/sandboxes/{id}/exec + +Auth: the sandbox's own token. A buffered exec for clients that cannot hold +an upgraded connection — plain JSON in, plain JSON out, so it multiplexes over +HTTP/2 through a TLS proxy: + +```json +{"argv": ["node", "-v"], "cwd": "/work", "env": {"CI": "1"}, "timeout_seconds": 60} +``` + +→ `200 {"exit_code": 0, "stdout": "v22.23.2\n", "stderr": ""}`. The command +runs to completion (no stdin, no streaming, no detach — use the relay for +those); `timeout_seconds` 0 means no limit beyond the request itself, and a +timeout closes the guest connection, which kills the command, then answers +504. 400 empty `argv`, negative `timeout_seconds`, or a silkd `bad_request`; +401 missing bearer token; 404 unknown sandbox or wrong token; 413 when +stdout+stderr exceed 8 MiB; 502 guest unreachable or any other silkd error. +A hibernated sandbox wakes transparently like on the relay. + ## GET /v1/sandboxes/{id}/owner Auth: the sandbox's own token. Answers `{"owner_addr": "host:port"}` when diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index f5f05914..ca7909a3 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -30,7 +30,7 @@ type fakeEngine struct { func newFakeEngine(dir string) *fakeEngine { return &fakeEngine{ - real: engine.New("cocoon", nil, nil, false, ""), + real: engine.New("cocoon", nil, nil, false, false, ""), dir: dir, listeners: map[string]io.Closer{}, socks: map[string]string{}, @@ -110,6 +110,8 @@ func (f *fakeEngine) DialGuestPort(context.Context, string, uint16) (net.Conn, e func (f *fakeEngine) InstallCACert(context.Context, string, []byte) error { return nil } +func (f *fakeEngine) Warmup(context.Context, string, []string) error { return nil } + func (f *fakeEngine) DiskAttach(_ context.Context, vmName string, spec engine.VolumeSpec) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 2d4cd98d..0fcc72c4 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -45,6 +45,9 @@ type PoolSpec struct { // Egress is this pool's allow-list, intersected with the tenant's; nil denies all egress. Egress *egress.Policy `json:"egress,omitempty"` + // Warmup runs in the golden VM before its snapshot, so every clone starts with its page cache. + Warmup []string `json:"warmup,omitempty"` + // IdleHibernateSeconds, when >0, hibernates idle claims after that many seconds. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` @@ -69,6 +72,9 @@ func (s PoolSpec) ValidateLimits() error { if s.Net == types.NetEgress && s.IdleHibernateSeconds > 0 { return fmt.Errorf("idle_hibernate_seconds is not supported for egress pools") } + if slices.Contains(s.Warmup, "") { + return fmt.Errorf("warmup must not contain an empty argument") + } return validateArchiveWindow(s.IdleHibernateSeconds, s.ArchiveAfterSeconds, s.ArchiveDeleteAfterSeconds) } @@ -171,6 +177,9 @@ type Config struct { // NoDirectIO enables buffered writable disks for cold boots and clones. NoDirectIO bool `json:"no_direct_io,omitempty"` + // NoBalloon boots VMs without the virtio-balloon, so a guest keeps its whole memory. + NoBalloon bool `json:"no_balloon,omitempty"` + // APIToken, when set, guards claim and info. APIToken string `json:"api_token,omitempty"` //nolint:gosec // config field, not a hardcoded credential diff --git a/sandboxd/engine/cloneargs_test.go b/sandboxd/engine/cloneargs_test.go index fe333429..dc4e9b48 100644 --- a/sandboxd/engine/cloneargs_test.go +++ b/sandboxd/engine/cloneargs_test.go @@ -23,7 +23,7 @@ func TestCloneArgsRestoreMode(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - e := New("cocoon", []string{"br0"}, nil, false, tc.mode) + e := New("cocoon", []string{"br0"}, nil, false, false, tc.mode) for _, args := range [][]string{ e.cloneArgs("/goldens/g1", "sbx-1", tc.key), e.cloneSnapArgs("ck_1", "sbx-1", tc.key), @@ -45,7 +45,7 @@ func TestLifecycleArgsApplyDirectIOPolicy(t *testing.T) { key := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeMedium} for _, noDirectIO := range []bool{false, true} { t.Run(strconv.FormatBool(noDirectIO), func(t *testing.T) { - e := New("cocoon", nil, nil, noDirectIO, "") + e := New("cocoon", nil, nil, noDirectIO, false, "") want := "--no-direct-io=" + strconv.FormatBool(noDirectIO) cold := e.runColdArgs("sbx-1", key) for _, args := range [][]string{ @@ -71,8 +71,8 @@ func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { e *Engine flag string }{ - "networks": {New("cocoon", nil, shards, false, ""), "--network"}, - "bridges": {New("cocoon", shards, nil, false, ""), "--bridge"}, + "networks": {New("cocoon", nil, shards, false, false, ""), "--network"}, + "bridges": {New("cocoon", shards, nil, false, false, ""), "--bridge"}, } { t.Run(name, func(t *testing.T) { counts := map[string]int{} @@ -107,13 +107,28 @@ func TestNetArgsHonorsTheLaneAndTheAttachment(t *testing.T) { none := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeMedium} egress := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeMedium} - if args := New("cocoon", nil, []string{"cni"}, false, "").netArgs("sbx-1", none, false); len(args) != 0 { + if args := New("cocoon", nil, []string{"cni"}, false, false, "").netArgs("sbx-1", none, false); len(args) != 0 { t.Errorf("none lane took an attachment: %v", args) } - if args := New("cocoon", []string{"br0"}, nil, false, "").netArgs("sbx-1", egress, false); !slices.Equal(args, []string{"--bridge", "br0"}) { + if args := New("cocoon", []string{"br0"}, nil, false, false, "").netArgs("sbx-1", egress, false); !slices.Equal(args, []string{"--bridge", "br0"}) { t.Errorf("bridge lane args = %v", args) } - if args := New("cocoon", nil, []string{"cocoon-dhcp"}, false, "").netArgs("sbx-1", egress, false); !slices.Equal(args, []string{"--network", "cocoon-dhcp"}) { + if args := New("cocoon", nil, []string{"cocoon-dhcp"}, false, false, "").netArgs("sbx-1", egress, false); !slices.Equal(args, []string{"--network", "cocoon-dhcp"}) { t.Errorf("single-network args = %v", args) } } + +func TestNoBalloonReachesColdBootsOnly(t *testing.T) { + key := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} + for _, noBalloon := range []bool{false, true} { + t.Run(strconv.FormatBool(noBalloon), func(t *testing.T) { + e := New("cocoon", nil, nil, false, noBalloon, "") + if got := slices.Contains(e.runColdArgs("sbx-1", key), "--no-balloon"); got != noBalloon { + t.Errorf("cold args carry --no-balloon = %v, want %v", got, noBalloon) + } + if slices.Contains(e.cloneArgs("/goldens/g1", "sbx-1", key), "--no-balloon") { + t.Error("clone args carry --no-balloon; clones inherit it from the golden") + } + }) + } +} diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 330264d2..fa22e392 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -65,12 +65,13 @@ type Engine struct { bridges []string networks []string noDirectIO bool + noBalloon bool restoreMode types.RestoreMode } // New returns a cocoon engine with node-wide network and disk policy. -func New(bin string, bridges, networks []string, noDirectIO bool, restoreMode types.RestoreMode) *Engine { - return &Engine{bin: bin, bridges: bridges, networks: networks, noDirectIO: noDirectIO, restoreMode: restoreMode} +func New(bin string, bridges, networks []string, noDirectIO, noBalloon bool, restoreMode types.RestoreMode) *Engine { + return &Engine{bin: bin, bridges: bridges, networks: networks, noDirectIO: noDirectIO, noBalloon: noBalloon, restoreMode: restoreMode} } // Version reports cocoon's version string: a "vX.Y.Z" release or a "master-" dev build. @@ -338,6 +339,9 @@ func (e *Engine) restoreArgs() []string { func (e *Engine) runColdArgs(name string, key types.PoolKey) []string { spec, _ := key.Size.Spec() args := []string{"vm", "run", argName, name, argOutput, formatJSON, "--cpu", strconv.Itoa(spec.CPU), "--memory", spec.Memory, e.directIOArg()} + if e.noBalloon { + args = append(args, "--no-balloon") + } args = append(args, e.netArgs(name, key, true)...) return append(args, key.Template) } diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index 790a9bfb..7143625a 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -23,7 +23,7 @@ func TestDialSilkdConsumesOnlyHandshake(t *testing.T) { listenMuxer(t, path, "OK 2048\nX") - conn, err := New("cocoon", nil, nil, false, "").DialSilkd(t.Context(), path) + conn, err := New("cocoon", nil, nil, false, false, "").DialSilkd(t.Context(), path) if err != nil { t.Fatalf("DialSilkd: %v", err) } @@ -41,7 +41,7 @@ func TestDialSilkdRejectedHandshake(t *testing.T) { path := sockPath(t) listenMuxer(t, path, "ERR no guest listener\n") - _, err := New("cocoon", nil, nil, false, "").DialSilkd(t.Context(), path) + _, err := New("cocoon", nil, nil, false, false, "").DialSilkd(t.Context(), path) if err == nil || !strings.Contains(err.Error(), "no guest listener") { t.Errorf("got %v, want handshake rejection", err) } @@ -51,7 +51,7 @@ func TestProbeSucceeds(t *testing.T) { path := sockPath(t) listenMuxer(t, path, "OK 2048\n", infoFrame) - if err := New("cocoon", nil, nil, false, "").Probe(t.Context(), path, 2*time.Second); err != nil { + if err := New("cocoon", nil, nil, false, false, "").Probe(t.Context(), path, 2*time.Second); err != nil { t.Errorf("Probe: %v", err) } } @@ -60,7 +60,7 @@ func TestInfoRoundTripRejectsErrorFrame(t *testing.T) { path := sockPath(t) listenMuxer(t, path, "OK 2048\n", errFrame) - err := New("cocoon", nil, nil, false, "").infoRoundTrip(t.Context(), path) + err := New("cocoon", nil, nil, false, false, "").infoRoundTrip(t.Context(), path) if err == nil || !strings.Contains(err.Error(), `info reply type "error"`) { t.Errorf("got %v, want error-frame rejection", err) } @@ -70,7 +70,7 @@ func TestProbeRetriesPastFailures(t *testing.T) { path := sockPath(t) listenMuxer(t, path, "OK 2048\n", errFrame, errFrame, infoFrame) - if err := New("cocoon", nil, nil, false, "").Probe(t.Context(), path, 2*time.Second); err != nil { + if err := New("cocoon", nil, nil, false, false, "").Probe(t.Context(), path, 2*time.Second); err != nil { t.Errorf("Probe: %v", err) } } @@ -79,7 +79,7 @@ func TestProbeRetriesUntilListenerAppears(t *testing.T) { path := sockPath(t) done := make(chan error, 1) go func() { - done <- New("cocoon", nil, nil, false, "").Probe(t.Context(), path, 2*time.Second) + done <- New("cocoon", nil, nil, false, false, "").Probe(t.Context(), path, 2*time.Second) }() time.Sleep(60 * time.Millisecond) @@ -117,7 +117,7 @@ func TestDialGuestPortCtxCancel(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 150*time.Millisecond) defer cancel() - if _, err := New("cocoon", nil, nil, false, "").DialGuestPort(ctx, path, 8080); !errors.Is(err, context.DeadlineExceeded) { + if _, err := New("cocoon", nil, nil, false, false, "").DialGuestPort(ctx, path, 8080); !errors.Is(err, context.DeadlineExceeded) { t.Errorf("got %v, want context.DeadlineExceeded", err) } } @@ -125,7 +125,7 @@ func TestDialGuestPortCtxCancel(t *testing.T) { func TestProbeTimeout(t *testing.T) { path := sockPath(t) - err := New("cocoon", nil, nil, false, "").Probe(t.Context(), path, 150*time.Millisecond) + err := New("cocoon", nil, nil, false, false, "").Probe(t.Context(), path, 150*time.Millisecond) if err == nil || !strings.Contains(err.Error(), "silkd probe") { t.Errorf("got %v, want probe timeout", err) } diff --git a/sandboxd/engine/installca.go b/sandboxd/engine/installca.go index 439cebc1..e0d8876a 100644 --- a/sandboxd/engine/installca.go +++ b/sandboxd/engine/installca.go @@ -14,7 +14,7 @@ const ( caCertGuestPath = "/usr/local/share/ca-certificates/sandbox-egress.crt" caBundlePath = "/etc/ssl/certs/ca-certificates.crt" // guestExecPATH is set because silkd starts the guest command with an empty environment. - guestExecPATH = "/usr/sbin:/usr/bin:/sbin:/bin" + guestExecPATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ) // InstallCACert makes the guest trust the cluster root without update-ca-certificates. diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 8c502c43..c9589499 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -18,7 +18,7 @@ import ( func TestInstallCACertWritesCertAndUpdates(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) - if err := New("cocoon", nil, nil, false, "").InstallCACert(t.Context(), path, []byte("CERT-PEM")); err != nil { + if err := New("cocoon", nil, nil, false, false, "").InstallCACert(t.Context(), path, []byte("CERT-PEM")); err != nil { t.Fatalf("InstallCACert: %v", err) } fake.mu.Lock() @@ -48,7 +48,7 @@ func TestInstallCACertNonzeroExitFails(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) fake.execCode = 3 - err := New("cocoon", nil, nil, false, "").InstallCACert(t.Context(), path, []byte("x")) + err := New("cocoon", nil, nil, false, false, "").InstallCACert(t.Context(), path, []byte("x")) if err == nil || !strings.Contains(err.Error(), "exit code 3") { t.Errorf("got %v, want exit code 3 failure", err) } @@ -58,7 +58,7 @@ func TestInstallCACertWriteErrorFrameFails(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) fake.writeErr = "disk full" - err := New("cocoon", nil, nil, false, "").InstallCACert(t.Context(), path, []byte("x")) + err := New("cocoon", nil, nil, false, false, "").InstallCACert(t.Context(), path, []byte("x")) if err == nil || !strings.Contains(err.Error(), "disk full") { t.Errorf("got %v, want fs_write error-frame failure", err) } diff --git a/sandboxd/engine/volume_test.go b/sandboxd/engine/volume_test.go index 687605e1..4aee874a 100644 --- a/sandboxd/engine/volume_test.go +++ b/sandboxd/engine/volume_test.go @@ -24,7 +24,7 @@ func TestDiskAttachArgsModeAndDirectIO(t *testing.T) { {"writable", "", true, "off"}, } { t.Run(tt.name, func(t *testing.T) { - e := New("cocoon", nil, nil, false, "") + e := New("cocoon", nil, nil, false, false, "") args, err := e.diskAttachArgs("sbx-1", VolumeSpec{ Name: "imagenet", Path: "/srv/datasets/imagenet.img", DirectIO: tt.directIO, RW: tt.rw, }) @@ -48,7 +48,7 @@ func TestDiskAttachArgsModeAndDirectIO(t *testing.T) { } func TestDiskAttachArgsRejectBadOptions(t *testing.T) { - _, err := New("cocoon", nil, nil, false, "").diskAttachArgs("sbx-1", VolumeSpec{DirectIO: "maybe"}) + _, err := New("cocoon", nil, nil, false, false, "").diskAttachArgs("sbx-1", VolumeSpec{DirectIO: "maybe"}) if err == nil || !strings.Contains(err.Error(), "on, off, or auto") { t.Errorf("got %v, want directio validation error", err) } @@ -67,7 +67,7 @@ func TestMountVolumeUsesSysfsAndRequestedMode(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) configureVolumeDevices(fake) - if err := New("cocoon", nil, nil, false, "").MountVolume( + if err := New("cocoon", nil, nil, false, false, "").MountVolume( t.Context(), path, "imagenet", "/datasets/training", tt.rw, ); err != nil { t.Fatalf("MountVolume: %v", err) @@ -103,7 +103,7 @@ func TestUnmountVolumeExecsBoundedUmount(t *testing.T) { } path := sockPath(t) fake := serveFakeSilkd(t, path) - e := New("cocoon", nil, nil, false, "") + e := New("cocoon", nil, nil, false, false, "") if err := e.UnmountVolume(t.Context(), path, "/datasets/training"); err != nil { t.Fatalf("UnmountVolume: %v", err) } @@ -124,7 +124,7 @@ func TestUnmountVolumeExecsBoundedUmount(t *testing.T) { func TestSyncGuestExecsPlainSync(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) - e := New("cocoon", nil, nil, false, "") + e := New("cocoon", nil, nil, false, false, "") if err := e.SyncGuest(t.Context(), path); err != nil { t.Fatalf("SyncGuest: %v", err) } @@ -148,7 +148,7 @@ func TestMountVolumeWaitsForDelayedSysfsSerial(t *testing.T) { fake.mu.Lock() fake.readMisses["/sys/block/vdc/serial"] = 1 fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, "").MountVolume( + if err := New("cocoon", nil, nil, false, false, "").MountVolume( t.Context(), path, "imagenet", "/datasets/training", false, ); err != nil { t.Fatalf("MountVolume: %v", err) @@ -171,7 +171,7 @@ func TestMountVolumeWaitsForDelayedDevNode(t *testing.T) { fake.mu.Lock() fake.statMisses["/dev/vdc"] = 3 fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, "").MountVolume( + if err := New("cocoon", nil, nil, false, false, "").MountVolume( t.Context(), path, "imagenet", "/datasets/training", false, ); err != nil { t.Fatalf("MountVolume: %v", err) @@ -207,7 +207,7 @@ func TestMountVolumeStopsAtFailedStage(t *testing.T) { fake.mu.Lock() tt.prepare(fake) fake.mu.Unlock() - err := New("cocoon", nil, nil, false, "").MountVolume( + err := New("cocoon", nil, nil, false, false, "").MountVolume( t.Context(), path, "imagenet", "/datasets/training", false, ) if err == nil || !strings.Contains(err.Error(), tt.wantErr) { @@ -231,7 +231,7 @@ func TestMountVolumeDeviceProbeIsBoundedAndCancelable(t *testing.T) { configureVolumeDevices(fake) ctx, cancel := context.WithCancel(t.Context()) cancel() - err := New("cocoon", nil, nil, false, "").MountVolume( + err := New("cocoon", nil, nil, false, false, "").MountVolume( ctx, path, "missing", "/datasets/training", false, ) if !errors.Is(err, context.Canceled) { @@ -246,7 +246,7 @@ func TestMountVolumeDevNodeNeverAppearsTimesOut(t *testing.T) { fake.mu.Lock() delete(fake.stat, "/dev/vdc") fake.mu.Unlock() - err := New("cocoon", nil, nil, false, "").MountVolume( + err := New("cocoon", nil, nil, false, false, "").MountVolume( t.Context(), path, "imagenet", "/datasets/training", false, ) if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "wait for volume device imagenet") { diff --git a/sandboxd/engine/warmup.go b/sandboxd/engine/warmup.go new file mode 100644 index 00000000..800f8446 --- /dev/null +++ b/sandboxd/engine/warmup.go @@ -0,0 +1,10 @@ +package engine + +import "context" + +// Warmup runs argv in the guest so the golden snapshot carries what it touches. +func (e *Engine) Warmup(ctx context.Context, vsockSocket string, argv []string) error { + ctx, cancel := context.WithTimeout(ctx, cmdTimeout) + defer cancel() + return e.silkdExec(ctx, vsockSocket, argv...) +} diff --git a/sandboxd/main.go b/sandboxd/main.go index 44ddff2d..c7559bdd 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -74,7 +74,7 @@ func main() { logger.Warnf(ctx, "volume %s path %s unavailable at startup: %v", volume.Name, volume.Path, statErr) } } - eng := engine.New(cfg.CocoonBin, cfg.Bridges, cfg.Networks, cfg.NoDirectIO, cfg.RestoreMode) + eng := engine.New(cfg.CocoonBin, cfg.Bridges, cfg.Networks, cfg.NoDirectIO, cfg.NoBalloon, cfg.RestoreMode) if v, warn := eng.VersionWarning(ctx); warn != "" { logger.Warn(ctx, warn) } else { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 3a6c5136..5e921cf3 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -71,7 +71,8 @@ const ( vmStateRunning = "running" vmStateCreating = "creating" - caSidecarSuffix = ".cafp" + caSidecarSuffix = ".cafp" + warmupSidecarSuffix = ".warmup" ) var ( @@ -114,6 +115,7 @@ type Engine interface { Probe(ctx context.Context, vsockSocket string, timeout time.Duration) error DialGuestPort(ctx context.Context, vsockSocket string, port uint16) (net.Conn, error) InstallCACert(ctx context.Context, vsockSocket string, certPEM []byte) error + Warmup(ctx context.Context, vsockSocket string, argv []string) error DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error @@ -273,6 +275,7 @@ type Manager struct { tenantLive map[string]int tenantEgress map[string]*egress.Policy // per-tenant allow-list; nil = no tenant policy poolEgress map[types.PoolKey]*egress.Policy + poolWarmups map[types.PoolKey][]string usage *journal audit *journal counters counters @@ -413,6 +416,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.tenantMax = make(map[string]int, len(cfg.Tenants)) m.tenantEgress = make(map[string]*egress.Policy, len(cfg.Tenants)) m.poolEgress = make(map[types.PoolKey]*egress.Policy, len(cfg.Pools)) + m.poolWarmups = make(map[types.PoolKey][]string, len(cfg.Pools)) for _, tn := range cfg.Tenants { m.tenantMax[tn.Name] = tn.MaxClaims if tn.Egress != nil { @@ -439,6 +443,9 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.poolEgress[spec.PoolKey] = spec.Egress m.guardedEgress = true } + if len(spec.Warmup) > 0 { + m.poolWarmups[spec.PoolKey] = spec.Warmup + } } if slices.ContainsFunc(cfg.Pools, func(s config.PoolSpec) bool { return s.Egress.Intercepts() }) { ca, err := loadEgressCA(cfg.EgressCA) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 3a8780bb..6a45df17 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -794,6 +794,7 @@ type fakeEngine struct { snapSaves, exports, snapshots []string exportContent []byte caInstalls []string + warmups [][]string staleReconciles []string installCAErr error diskAttachErr error @@ -1030,6 +1031,16 @@ func (f *fakeEngine) InstallCACert(_ context.Context, vsockSocket string, _ []by return f.installCAErr } +func (f *fakeEngine) Warmup(_ context.Context, _ string, argv []string) error { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.snapSaves) > 0 { + return fmt.Errorf("warmup after snapshot") + } + f.warmups = append(f.warmups, argv) + return nil +} + func (f *fakeEngine) DiskAttach(_ context.Context, _ string, spec engine.VolumeSpec) error { f.mu.Lock() f.volumeSpecs = append(f.volumeSpecs, spec) diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index a9db122a..c8e642dc 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -112,11 +112,12 @@ func (m *Manager) adoptPersistedPools(ctx context.Context) error { return nil } -// poolSeedHash digests a pool set's warm-target shape, order-independent and egress-excluded. +// poolSeedHash digests a pool set's warm-target shape, order-independent, without the config-owned egress and warmup. func poolSeedHash(specs []config.PoolSpec) string { shaped := slices.Clone(specs) for i := range shaped { shaped[i].Egress = nil + shaped[i].Warmup = nil } slices.SortFunc(shaped, func(a, b config.PoolSpec) int { return strings.Compare(a.Hash(), b.Hash()) }) raw, _ := json.Marshal(shaped) diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index 82ea628e..a3d712b6 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -194,33 +194,44 @@ func (m *Manager) buildGoldenSteps(ctx context.Context, key types.PoolKey, name, return fmt.Errorf("install egress ca: %w", err) } } + warmup := m.poolWarmup(key) + if len(warmup) > 0 { + if err := m.eng.Warmup(ctx, sock, warmup); err != nil { + return fmt.Errorf("warmup: %w", err) + } + } if err := m.eng.SnapshotSave(ctx, name, snap); err != nil { return err } if err := m.exportGolden(ctx, snap, final); err != nil { return err } - return m.writeGoldenCASidecar(final, caBaked) + if err := m.writeGoldenCASidecar(final, caBaked); err != nil { + return err + } + return writeGoldenSidecar(final+warmupSidecarSuffix, warmupStamp(warmup)) +} + +func (m *Manager) poolWarmup(key types.PoolKey) []string { + m.mu.Lock() + defer m.mu.Unlock() + return m.poolWarmups[key] } // writeGoldenCASidecar records or clears the baked-CA fingerprint; a rotated CA forces a rebuild. func (m *Manager) writeGoldenCASidecar(final string, caBaked bool) error { - path := final + caSidecarSuffix - if !caBaked { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("clear ca sidecar: %w", err) - } - return nil - } - if err := os.WriteFile(path, []byte(m.egressCA.Fingerprint()), 0o644); err != nil { //nolint:gosec // public fingerprint - return fmt.Errorf("write ca sidecar: %w", err) + var stamp string + if caBaked { + stamp = m.egressCA.Fingerprint() } - return nil + return writeGoldenSidecar(final+caSidecarSuffix, stamp) } -// adoptGolden points p at an on-disk golden whose baked-CA state still fits. +// adoptGolden points p at an on-disk golden whose baked-CA state and warmup still fit. func (m *Manager) adoptGolden(p *pool) { - if g := filepath.Join(m.goldensDir(), p.hash); dirExists(g) && m.goldenCAMatches(g, m.poolEgress[p.key].Intercepts()) { + g := filepath.Join(m.goldensDir(), p.hash) + if dirExists(g) && m.goldenCAMatches(g, m.poolEgress[p.key].Intercepts()) && + goldenSidecarMatches(g+warmupSidecarSuffix, warmupStamp(m.poolWarmups[p.key])) { p.goldenDir = g } } @@ -228,11 +239,9 @@ func (m *Manager) adoptGolden(p *pool) { // goldenCAMatches reports whether a golden's baked-CA state fits the pool. func (m *Manager) goldenCAMatches(final string, caNeeded bool) bool { if !caNeeded { - _, err := os.Stat(final + caSidecarSuffix) - return errors.Is(err, os.ErrNotExist) + return goldenSidecarMatches(final+caSidecarSuffix, "") } - fp, err := os.ReadFile(final + caSidecarSuffix) //nolint:gosec // node-local golden path - return err == nil && m.egressCA != nil && string(fp) == m.egressCA.Fingerprint() + return m.egressCA != nil && goldenSidecarMatches(final+caSidecarSuffix, m.egressCA.Fingerprint()) } // exportGolden exports snap into final through a unique sibling *.tmp dir. @@ -440,3 +449,28 @@ func (m *Manager) dropSnap(ctx context.Context, snap string) { log.WithFunc("pool.dropSnap").Warnf(ctx, "drop snapshot %s: %v", snap, err) } } + +func writeGoldenSidecar(path, stamp string) error { + if stamp == "" { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("clear golden sidecar %s: %w", filepath.Base(path), err) + } + return nil + } + if err := os.WriteFile(path, []byte(stamp), 0o644); err != nil { //nolint:gosec // public stamp + return fmt.Errorf("write golden sidecar %s: %w", filepath.Base(path), err) + } + return nil +} + +func goldenSidecarMatches(path, stamp string) bool { + got, err := os.ReadFile(path) //nolint:gosec // node-local golden path + if stamp == "" { + return errors.Is(err, os.ErrNotExist) + } + return err == nil && string(got) == stamp +} + +func warmupStamp(argv []string) string { + return strings.Join(argv, "\x00") +} diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index 1a2fa9f9..ab8dd66d 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -27,6 +27,9 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { if spec.Egress != nil { return fmt.Errorf("%w: pool %q: egress is set in the config file, not via the API", ErrBadKey, spec.Template) } + if spec.Warmup != nil { + return fmt.Errorf("%w: pool %q: warmup is set in the config file, not via the API", ErrBadKey, spec.Template) + } if existing, ok := hashes[spec.Hash()]; ok && existing != spec.PoolKey { return fmt.Errorf("%w: pool key hash collision between %q and %q", ErrBadKey, existing.Template, spec.Template) } diff --git a/sandboxd/pool/warmup_test.go b/sandboxd/pool/warmup_test.go new file mode 100644 index 00000000..63620e77 --- /dev/null +++ b/sandboxd/pool/warmup_test.go @@ -0,0 +1,89 @@ +package pool + +import ( + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" +) + +func TestGoldenBuildRunsWarmupBeforeSnapshot(t *testing.T) { + eng := newFakeEngine() + argv := []string{"node", "-e", "0"} + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Warmup: argv}) + final := filepath.Join(m.goldensDir(), testKey.Hash()) + if err := m.buildGoldenSteps(t.Context(), testKey, "sbx-gb", "snap", final); err != nil { + t.Fatalf("buildGoldenSteps: %v", err) + } + if len(eng.warmups) != 1 || !slices.Equal(eng.warmups[0], argv) { + t.Fatalf("warmups = %v, want [%v]", eng.warmups, argv) + } + stamp, err := os.ReadFile(final + warmupSidecarSuffix) + if err != nil { + t.Fatalf("read warmup sidecar: %v", err) + } + if string(stamp) != warmupStamp(argv) { + t.Errorf("sidecar = %q, want %q", stamp, warmupStamp(argv)) + } +} + +func TestGoldenBuildSkipsWarmupWhenUnset(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + final := filepath.Join(m.goldensDir(), testKey.Hash()) + if err := m.buildGoldenSteps(t.Context(), testKey, "sbx-gb", "snap", final); err != nil { + t.Fatalf("buildGoldenSteps: %v", err) + } + if len(eng.warmups) != 0 { + t.Errorf("Warmup called %d times for a pool without one", len(eng.warmups)) + } + if _, err := os.Stat(final + warmupSidecarSuffix); !errors.Is(err, os.ErrNotExist) { + t.Errorf("warmup sidecar present for a pool without one: %v", err) + } +} + +func TestAdoptGoldenRequiresMatchingWarmup(t *testing.T) { + m := newTestManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Warm: 1, Warmup: []string{"node", "-e", "0"}}) + final := filepath.Join(m.goldensDir(), testKey.Hash()) + if err := os.MkdirAll(final, 0o755); err != nil { + t.Fatalf("mkdir golden: %v", err) + } + p := m.pools[testKey] + m.adoptGolden(p) + if p.goldenDir != "" { + t.Error("adopted a golden built without the warmup") + } + if err := os.WriteFile(final+warmupSidecarSuffix, []byte(warmupStamp([]string{"python3", "-c", "0"})), 0o644); err != nil { + t.Fatalf("write sidecar: %v", err) + } + m.adoptGolden(p) + if p.goldenDir != "" { + t.Error("adopted a golden built with a different warmup") + } + if err := os.WriteFile(final+warmupSidecarSuffix, []byte(warmupStamp([]string{"node", "-e", "0"})), 0o644); err != nil { + t.Fatalf("write sidecar: %v", err) + } + m.adoptGolden(p) + if p.goldenDir != final { + t.Errorf("goldenDir = %q, want %q", p.goldenDir, final) + } +} + +func TestSetPoolsRejectsWarmup(t *testing.T) { + m := newTestManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Warm: 1}) + err := m.SetPools(t.Context(), []config.PoolSpec{{PoolKey: testKey, Warm: 1, Warmup: []string{"true"}}}) + if !errors.Is(err, ErrBadKey) || !strings.Contains(err.Error(), "warmup is set in the config file") { + t.Errorf("SetPools error = %v, want ErrBadKey naming warmup as config-owned", err) + } +} + +func TestPoolSpecRejectsEmptyWarmupArgument(t *testing.T) { + spec := config.PoolSpec{PoolKey: testKey, Warm: 1, Warmup: []string{"node", ""}} + if err := spec.ValidateLimits(); err == nil || !strings.Contains(err.Error(), "warmup") { + t.Errorf("ValidateLimits error = %v, want an empty-argument rejection", err) + } +} diff --git a/sandboxd/server/exec.go b/sandboxd/server/exec.go new file mode 100644 index 00000000..e35a0c95 --- /dev/null +++ b/sandboxd/server/exec.go @@ -0,0 +1,147 @@ +package server + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/protocol/wire" +) + +const execOutputCap = 8 << 20 + +var errExecOutputCap = errors.New("command output exceeds the buffered exec cap") + +// ExecRequest is a buffered exec: one command whose whole output comes back in the response. +type ExecRequest struct { + Argv []string `json:"argv"` + Cwd string `json:"cwd,omitempty"` + Env map[string]string `json:"env,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` +} + +// ExecResponse is the exit code and complete output of a buffered exec. +type ExecResponse struct { + ExitCode int32 `json:"exit_code"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` +} + +func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { + token, ok := sandboxToken(w, r) + if !ok { + return + } + req, ok := decodeBody[ExecRequest](w, r) + if !ok { + return + } + if len(req.Argv) == 0 { + writeErr(w, http.StatusBadRequest, "argv must not be empty") + return + } + if req.TimeoutSeconds < 0 { + writeErr(w, http.StatusBadRequest, "timeout_seconds must not be negative") + return + } + ctx := r.Context() + id := r.PathValue("id") + sock, err := s.mgr.WakeAgentSocket(ctx, id, token) + switch { + case writePoolErr(w, err): + return + case err != nil: + log.WithFunc("server.handleExec").Errorf(ctx, err, "agent socket for %s", id) + writeErr(w, http.StatusInternalServerError, "sandbox lookup failed") + return + } + guest, err := s.dialer.DialSilkd(ctx, sock) + if err != nil { + log.WithFunc("server.handleExec").Errorf(ctx, err, "dial silkd for %s", id) + writeErr(w, http.StatusBadGateway, "guest agent unreachable") + return + } + if req.TimeoutSeconds > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(req.TimeoutSeconds)*time.Second) + defer cancel() + } + // silkd kills a non-detached child when its connection drops, so a canceled ctx ends the command + stop := context.AfterFunc(ctx, func() { _ = guest.Close() }) + defer func() { + stop() + _ = guest.Close() + }() + frame, err := wire.EncodeRequest(wire.Exec{Argv: req.Argv, Cwd: req.Cwd, Env: req.Env}) + if err != nil { + writeErr(w, http.StatusBadRequest, "invalid request body") + return + } + frame = append(frame, '\n') + if s.mgr.AuditEnabled() { + s.mgr.Audit(ctx, id, frame) + } + stdinClose, err := wire.EncodeRequest(wire.StdinClose{}) + if err != nil { + writeErr(w, http.StatusInternalServerError, "command setup failed") + return + } + frame = append(frame, stdinClose...) + frame = append(frame, '\n') + if _, err = guest.Write(frame); err != nil { + writeErr(w, http.StatusBadGateway, "guest agent unreachable") + return + } + resp, err := collectExec(guest) + var silkdErr *wire.ErrorResp + switch { + case err == nil: + writeJSON(w, http.StatusOK, resp) + case errors.Is(ctx.Err(), context.DeadlineExceeded): + writeErr(w, http.StatusGatewayTimeout, "command timed out") + case errors.Is(ctx.Err(), context.Canceled): + return + case errors.As(err, &silkdErr) && silkdErr.Kind == "bad_request": + writeErr(w, http.StatusBadRequest, silkdErr.Message) + case errors.As(err, &silkdErr): + writeErr(w, http.StatusBadGateway, silkdErr.Error()) + case errors.Is(err, errExecOutputCap): + writeErr(w, http.StatusRequestEntityTooLarge, err.Error()) + default: + log.WithFunc("server.handleExec").Errorf(ctx, err, "exec relay for %s", id) + writeErr(w, http.StatusBadGateway, "guest agent closed before the command exited") + } +} + +func collectExec(guest net.Conn) (ExecResponse, error) { + var stdout, stderr []byte + sc := wire.NewFrameScanner(guest) + for sc.Scan() { + frame, err := wire.DecodeResponse(sc.Bytes()) + if err != nil { + return ExecResponse{}, err + } + switch f := frame.(type) { + case *wire.Stdout: + stdout = append(stdout, f.Data...) + case *wire.Stderr: + stderr = append(stderr, f.Data...) + case *wire.Exit: + return ExecResponse{ExitCode: f.Code, Stdout: string(stdout), Stderr: string(stderr)}, nil + case *wire.ErrorResp: + return ExecResponse{}, f + } + if len(stdout)+len(stderr) > execOutputCap { + return ExecResponse{}, errExecOutputCap + } + } + if err := sc.Err(); err != nil { + return ExecResponse{}, err + } + return ExecResponse{}, io.ErrUnexpectedEOF +} diff --git a/sandboxd/server/exec_test.go b/sandboxd/server/exec_test.go new file mode 100644 index 00000000..3cd3a648 --- /dev/null +++ b/sandboxd/server/exec_test.go @@ -0,0 +1,178 @@ +package server + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/cocoonstack/sandbox/protocol/wire" +) + +func TestExecReturnsBufferedOutput(t *testing.T) { + var gotExec, gotStdinClose wire.Request + ts, _ := newRelayServer(t, func(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + line, err := r.ReadBytes('\n') + if err != nil { + return + } + gotExec, _ = wire.DecodeRequest(bytes.TrimSpace(line)) + line, err = r.ReadBytes('\n') + if err != nil { + return + } + gotStdinClose, _ = wire.DecodeRequest(bytes.TrimSpace(line)) + _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, `{"type":"stdout","data":"`+base64.StdEncoding.EncodeToString([]byte("v22\n"))+`"}`+"\n") + _, _ = io.WriteString(conn, `{"type":"stderr","data":"`+base64.StdEncoding.EncodeToString([]byte("warn\n"))+`"}`+"\n") + _, _ = io.WriteString(conn, `{"type":"exit","code":3}`+"\n") + }) + status, body := postExec(t, ts, `{"argv":["node","-v"],"cwd":"/work","env":{"A":"1"},"timeout_seconds":5}`) + if status != http.StatusOK { + t.Fatalf("status %d, want 200: %s", status, body) + } + var out ExecResponse + if err := json.Unmarshal(body, &out); err != nil { + t.Fatalf("decode: %v", err) + } + if out.ExitCode != 3 || out.Stdout != "v22\n" || out.Stderr != "warn\n" { + t.Errorf("response = %+v", out) + } + exec, ok := gotExec.(*wire.Exec) + if !ok || strings.Join(exec.Argv, " ") != "node -v" || exec.Cwd != "/work" || exec.Env["A"] != "1" || exec.Detach { + t.Errorf("guest request = %#v", gotExec) + } + if _, ok := gotStdinClose.(*wire.StdinClose); !ok { + t.Errorf("guest stdin request = %#v, want stdin_close", gotStdinClose) + } +} + +func TestExecMapsSilkdErrors(t *testing.T) { + for _, tt := range []struct { + kind string + want int + }{ + {"bad_request", http.StatusBadRequest}, + {"internal", http.StatusBadGateway}, + } { + t.Run(tt.kind, func(t *testing.T) { + ts, _ := newRelayServer(t, func(conn net.Conn) { + defer conn.Close() + _, _ = bufio.NewReader(conn).ReadBytes('\n') + _, _ = io.WriteString(conn, `{"type":"error","kind":"`+tt.kind+`","message":"spawn failed"}`+"\n") + }) + if status, _ := postExec(t, ts, `{"argv":["true"]}`); status != tt.want { + t.Errorf("status %d, want %d", status, tt.want) + } + }) + } +} + +func TestExecTimesOutAndClosesGuest(t *testing.T) { + closed := make(chan struct{}) + ts, _ := newRelayServer(t, func(conn net.Conn) { + defer close(closed) + r := bufio.NewReader(conn) + _, _ = r.ReadBytes('\n') + _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = r.ReadBytes('\n') + _, _ = r.ReadBytes('\n') + }) + if status, _ := postExec(t, ts, `{"argv":["sleep","60"],"timeout_seconds":1}`); status != http.StatusGatewayTimeout { + t.Fatalf("status %d, want 504", status) + } + select { + case <-closed: + case <-time.After(3 * time.Second): + t.Fatal("guest conn still open after the timeout") + } +} + +func TestExecRejectsEmptyArgv(t *testing.T) { + ts, _ := newRelayServer(t, func(conn net.Conn) { _ = conn.Close() }) + if status, _ := postExec(t, ts, `{"argv":[]}`); status != http.StatusBadRequest { + t.Errorf("status %d, want 400", status) + } +} + +func TestExecGuestClosingEarlyIs502(t *testing.T) { + ts, _ := newRelayServer(t, func(conn net.Conn) { + _, _ = bufio.NewReader(conn).ReadBytes('\n') + _ = conn.Close() + }) + if status, _ := postExec(t, ts, `{"argv":["true"]}`); status != http.StatusBadGateway { + t.Errorf("status %d, want 502", status) + } +} + +func TestExecTimeoutStartsAfterWake(t *testing.T) { + wakeHadCommandDeadline := make(chan bool, 1) + mgr := &execManager{ + fakeManager: &fakeManager{}, + wake: func(ctx context.Context, _, _ string) (string, error) { + deadline, ok := ctx.Deadline() + wakeHadCommandDeadline <- ok && time.Until(deadline) < 2*time.Second + return "/v/sock", nil + }, + } + dialer := &fakeDialer{dial: func(context.Context, string) (net.Conn, error) { + relayEnd, guestEnd := net.Pipe() + go func() { + defer guestEnd.Close() + r := bufio.NewReader(guestEnd) + _, _ = r.ReadBytes('\n') + _, _ = r.ReadBytes('\n') + _, _ = io.WriteString(guestEnd, `{"type":"exit","code":0}`+"\n") + }() + return relayEnd, nil + }} + srv := New("", nil, "node:7777", mgr, dialer, nil, nil, nil, nil) + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + + if status, body := postExec(t, ts, `{"argv":["true"],"timeout_seconds":1}`); status != http.StatusOK { + t.Fatalf("status %d, want 200: %s", status, body) + } + if <-wakeHadCommandDeadline { + t.Error("command timeout was active during wake") + } +} + +type execManager struct { + *fakeManager + wake func(context.Context, string, string) (string, error) +} + +func (m *execManager) WakeAgentSocket(ctx context.Context, id, token string) (string, error) { + return m.wake(ctx, id, token) +} + +func postExec(t *testing.T, ts *httptest.Server, body string) (int, []byte) { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/exec", strings.NewReader(body)) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Authorization", "Bearer tok") + req.Header.Set("Content-Type", "application/json") + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("post exec: %v", err) + } + defer resp.Body.Close() + out, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read response: %v", err) + } + return resp.StatusCode, out +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 83f5f42a..02d346ae 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -198,6 +198,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/drain", s.requireRoot(s.handleDrain)) mux.HandleFunc("DELETE /v1/drain", s.requireRoot(s.handleUncordon)) mux.HandleFunc("GET /v1/sandboxes/{id}/agent", s.handleAgent) + mux.HandleFunc("POST /v1/sandboxes/{id}/exec", s.handleExec) mux.HandleFunc("GET /v1/sandboxes/{id}/owner", s.handleOwner) mux.HandleFunc("GET /v1/info", s.requireRoot(s.handleInfo)) mux.HandleFunc("GET /v1/peers", s.requireToken(s.handlePeers))