From 8a08f2a0725f6f9463e70d455531d83b638b18ff Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 16:41:38 +0800 Subject: [PATCH 1/3] test(envdsmoke): the e2b flavor's envd smoke lives with the e2b layer The harness that drives the e2b flavor image's envd through the guest-port relay moves here from the sandbox repo, rewritten on pkg/sandboxd: sandbox keeps no envd knowledge, and what the smoke asserts (the version the compat API reports, the Connect surface, the default user, HTTP/1.1 only) is what this repo's e2b surface and envd proxy depend on. The silkd read-back of an envd upload becomes an envd read-back; the service-active check is envd's own /health. --- test/envdsmoke/main.go | 386 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 386 insertions(+) create mode 100644 test/envdsmoke/main.go diff --git a/test/envdsmoke/main.go b/test/envdsmoke/main.go new file mode 100644 index 0000000..5e69d16 --- /dev/null +++ b/test/envdsmoke/main.go @@ -0,0 +1,386 @@ +//go:build envdsmoke + +// envdsmoke proves the e2b flavor image on real hardware from the operator's side: it claims a +// sandbox from an envd-carrying pool with pkg/sandboxd and drives the real envd through the +// node's guest-port relay, the path the e2b surface and the envd proxy take. It asserts what the +// e2b SDK depends on (the daemon is up, its version is the one the compat API reports, files and +// the ConnectRPC surface answer, the default user has sudo) and that envd serves HTTP/1.1 only. +package main + +import ( + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "slices" + "strings" + "time" + + "github.com/cocoonstack/sandbox-operator/pkg/sandboxd" +) + +const ( + envdPort = 49983 + interpreterPort = 49999 + readyWait = 60 * time.Second + claimTTL = 30 * time.Minute + runTimeout = 300 * time.Second + + uploadPath = "/tmp/envdsmoke-probe.txt" + uploadProbe = "envd-wrote-this" + processProbe = "envd-process-ok" + contentType = "Content-Type" +) + +type relay struct { + client *sandboxd.Client + id string + token string +} + +func (r relay) Dial(ctx context.Context, port uint16) (net.Conn, error) { + return r.client.DialPort(ctx, r.id, r.token, port) +} + +func (r relay) Do(ctx context.Context, port uint16, method, path string, headers http.Header, body io.Reader) (*http.Response, error) { + client := &http.Client{Transport: &http.Transport{ + DisableKeepAlives: true, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return r.Dial(ctx, port) }, + }} + req, err := http.NewRequestWithContext(ctx, method, "http://guest"+path, body) + if err != nil { + return nil, err + } + for k, v := range headers { + req.Header[k] = v + } + return client.Do(req) +} + +type smokeStep struct { + name string + run func(context.Context, relay) error +} + +func main() { + addr := flag.String("addr", "127.0.0.1:7777", "sandboxd address, a host:port or an http(s) origin") + token := flag.String("token", "", "node api token") + template := flag.String("template", "", "e2b flavor template ref") + wantVersion := flag.String("envd-version", "", "version envd must report; empty only prints it") + hold := flag.Duration("hold", 0, "after the steps pass, print the claim and keep it alive for this long") + interpreter := flag.Bool("code-interpreter", false, "also drive the code-interpreter API on 49999 (the e2b-ci flavor)") + size := flag.String("size", "small", "size of the pool to claim from") + flag.Parse() + + if err := run(*addr, *token, *template, *wantVersion, *size, *hold, *interpreter); err != nil { + fmt.Fprintln(os.Stderr, "envdsmoke:", err) + os.Exit(1) + } + fmt.Println("ENVDSMOKE PASS") +} + +func run(addr, token, template, wantVersion, size string, hold time.Duration, interpreter bool) error { + ctx, cancel := context.WithTimeout(context.Background(), runTimeout+hold) + defer cancel() + + base := addr + if !strings.Contains(base, "://") { + base = "http://" + base + } + client := sandboxd.New(base, token) + res, err := client.Claim(ctx, sandboxd.ClaimSpec{Template: template, Net: "none", Size: size, TTLSeconds: int(claimTTL / time.Second)}) + if err != nil { + return fmt.Errorf("claim: %w", err) + } + defer func() { _ = client.Release(context.WithoutCancel(ctx), res.ID, res.Token) }() + fmt.Printf("claimed %s on %s\n", res.ID, res.OwnerAddr) + rt := relay{client: client, id: res.ID, token: res.Token} + + if err := waitReady(ctx, rt); err != nil { + return err + } + version, err := envdVersion(ctx, rt) + if err != nil { + return err + } + fmt.Printf(" envd %s answering through the relay\n", version) + if wantVersion != "" && version != wantVersion { + return fmt.Errorf("envd reports %q, want %q: the compat API's envdVersion would be a lie", version, wantVersion) + } + + steps := []smokeStep{ + {"GET /health", stepHealth}, + {"POST /files then GET /files", stepUploadDownload}, + {"connect unary over http/1.1", stepConnectH1}, + {"process start under -no-cgroups", stepProcessStart}, + {"envd serves http/1.1 only", stepProtocol}, + {"user account with sudo", stepUserAccount}, + } + if interpreter { + steps = append(steps, smokeStep{"interpreter runs 1+1", stepRunCode}) + } + for _, step := range steps { + t0 := time.Now() + if err := step.run(ctx, rt); err != nil { + return fmt.Errorf("%s: %w", step.name, err) + } + fmt.Printf(" ok %-30s %5.1fms\n", step.name, float64(time.Since(t0).Microseconds())/1000) + } + if hold > 0 { + fmt.Printf("SANDBOX %s %s %s %d\n", res.ID, res.Token, res.OwnerAddr, envdPort) + select { + case <-time.After(hold): + case <-ctx.Done(): + } + } + return nil +} + +func waitReady(ctx context.Context, rt relay) error { + deadline := time.Now().Add(readyWait) + var last error + for time.Now().Before(deadline) { + resp, err := rt.Do(ctx, envdPort, http.MethodGet, "/health", nil, nil) + if err == nil { + _ = resp.Body.Close() + return nil + } + last = err + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("envd never answered /health: %w", last) +} + +func envdVersion(ctx context.Context, rt relay) (string, error) { + body, err := download(ctx, rt, "/etc/envd-version") + if err != nil { + return "", fmt.Errorf("GET /files /etc/envd-version: %w", err) + } + version := strings.TrimSpace(body) + if version == "" { + return "", fmt.Errorf("empty /etc/envd-version") + } + return version, nil +} + +func download(ctx context.Context, rt relay, path string) (string, error) { + resp, err := rt.Do(ctx, envdPort, http.MethodGet, "/files?path="+path+"&username=root", nil, nil) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("status %s: %s", resp.Status, body) + } + return string(body), nil +} + +func stepHealth(ctx context.Context, rt relay) error { + resp, err := rt.Do(ctx, envdPort, http.MethodGet, "/health", nil, nil) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("status %s, want 204", resp.Status) + } + return nil +} + +func stepUploadDownload(ctx context.Context, rt relay) error { + const boundary = "envdsmoke" + body := fmt.Sprintf("--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"probe.txt\"\r\n"+ + "Content-Type: text/plain\r\n\r\n%s\r\n--%s--\r\n", boundary, uploadProbe, boundary) + headers := http.Header{contentType: []string{"multipart/form-data; boundary=" + boundary}} + resp, err := rt.Do(ctx, envdPort, http.MethodPost, "/files?path="+uploadPath+"&username=root", headers, strings.NewReader(body)) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + out, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return fmt.Errorf("status %s: %s", resp.Status, out) + } + seen, err := download(ctx, rt, uploadPath) + if err != nil { + return fmt.Errorf("read back: %w", err) + } + if strings.TrimSpace(seen) != uploadProbe { + return fmt.Errorf("envd reads %q at %s, wrote %q", seen, uploadPath, uploadProbe) + } + return nil +} + +func stepConnectH1(ctx context.Context, rt relay) error { + headers := http.Header{ + contentType: []string{"application/json"}, + "Connect-Protocol-Version": []string{"1"}, + "X-User": []string{"root"}, + } + resp, err := rt.Do(ctx, envdPort, http.MethodPost, "/filesystem.Filesystem/Stat", headers, strings.NewReader(`{"path":"/etc/envd-version"}`)) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status %s: %s", resp.Status, body) + } + var out map[string]any + if err := json.Unmarshal(body, &out); err != nil { + return fmt.Errorf("decode %q: %w", body, err) + } + if _, ok := out["entry"]; !ok { + return fmt.Errorf("stat reply carries no entry: %s", body) + } + return nil +} + +func stepProcessStart(ctx context.Context, rt relay) error { + events, err := startProcess(ctx, rt, "root", "/bin/echo", processProbe) + if err != nil { + return err + } + joined := strings.Join(events, "\n") + if !strings.Contains(joined, `"start"`) { + return fmt.Errorf("no start event in the stream:\n%s", joined) + } + if !strings.Contains(joined, base64.StdEncoding.EncodeToString([]byte(processProbe+"\n"))) { + return fmt.Errorf("the command's stdout never arrived:\n%s", joined) + } + return nil +} + +func stepProtocol(ctx context.Context, rt relay) error { + var protocols http.Protocols + protocols.SetUnencryptedHTTP2(true) + client := &http.Client{Transport: &http.Transport{ + Protocols: &protocols, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return rt.Dial(ctx, envdPort) }, + }} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://guest/health", nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err == nil { + defer func() { _ = resp.Body.Close() }() + return fmt.Errorf("envd answered h2c with %s", resp.Status) + } + return stepHealth(ctx, rt) +} + +func stepUserAccount(ctx context.Context, rt relay) error { + events, err := startProcess(ctx, rt, "user", "/bin/sh", "-c", `id -un; echo "$HOME"; sudo -n true && echo sudo`) + if err != nil { + return err + } + out := stdoutOf(events) + if got := strings.Fields(out); !slices.Equal(got, []string{"user", "/home/user", "sudo"}) { + return fmt.Errorf("the user account reports %q, want user, /home/user and passwordless sudo", out) + } + return nil +} + +func stepRunCode(ctx context.Context, rt relay) error { + headers := http.Header{contentType: []string{"application/json"}} + resp, err := rt.Do(ctx, interpreterPort, http.MethodPost, "/execute", headers, strings.NewReader(`{"code":"1+1"}`)) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status %s: %s", resp.Status, body) + } + for line := range strings.Lines(string(body)) { + var out struct { + Type string `json:"type"` + Text string `json:"text"` + } + if json.Unmarshal([]byte(line), &out) == nil && out.Type == "result" && out.Text == "2" { + return nil + } + } + return fmt.Errorf("no result line with text 2 in:\n%s", body) +} + +func startProcess(ctx context.Context, rt relay, user, cmd string, args ...string) ([]string, error) { + headers := http.Header{ + contentType: []string{"application/connect+json"}, + "Connect-Protocol-Version": []string{"1"}, + "X-User": []string{user}, + } + req, _ := json.Marshal(map[string]any{"process": map[string]any{"cmd": cmd, "args": args}}) + resp, err := rt.Do(ctx, envdPort, http.MethodPost, "/process.Process/Start", headers, strings.NewReader(envelope(string(req)))) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("status %s: %s", resp.Status, body) + } + return readEnvelopes(resp.Body) +} + +func stdoutOf(events []string) string { + var out strings.Builder + for _, e := range events { + var ev struct { + Event struct { + Data struct { + Stdout string `json:"stdout"` + } `json:"data"` + } `json:"event"` + } + if json.Unmarshal([]byte(e), &ev) != nil || ev.Event.Data.Stdout == "" { + continue + } + if b, err := base64.StdEncoding.DecodeString(ev.Event.Data.Stdout); err == nil { + out.Write(b) + } + } + return out.String() +} + +func envelope(payload string) string { + head := make([]byte, 5) + binary.BigEndian.PutUint32(head[1:], uint32(len(payload))) //nolint:gosec // a probe payload is tens of bytes + return string(head) + payload +} + +func readEnvelopes(r io.Reader) ([]string, error) { + var out []string + head := make([]byte, 5) + for { + if _, err := io.ReadFull(r, head); err != nil { + return out, fmt.Errorf("stream ended before its end-of-stream envelope: %w", err) + } + body := make([]byte, binary.BigEndian.Uint32(head[1:])) + if _, err := io.ReadFull(r, body); err != nil { + return out, err + } + if head[0]&0x02 == 0 { + out = append(out, string(body)) + continue + } + var end struct { + Error json.RawMessage `json:"error"` + } + if err := json.Unmarshal(body, &end); err != nil { + return out, fmt.Errorf("parse end-of-stream envelope %q: %w", body, err) + } + if len(end.Error) > 0 && string(end.Error) != "null" { + return out, fmt.Errorf("rpc failed: %s", end.Error) + } + return out, nil + } +} From 1fd51229554fd90cd2d9f840188bc4f6befe1d52 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 16:42:07 +0800 Subject: [PATCH 2/3] test(envdsmoke): the harness tag, its node script and the docs section --- Makefile | 2 +- docs/e2b-compat.md | 18 +++++++++++++++ scripts/envd-e2e.sh | 56 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100755 scripts/envd-e2e.sh diff --git a/Makefile b/Makefile index 9efdb64..5e120d0 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ GO_LDFLAGS ?= -s -w \ ## Shipped binaries under cmd/, and the build-tagged harnesses under test/ (one tag per directory) BINARIES := sandbox-apiserver sandbox-envd-proxy sandbox-e2b -TAGGED_HARNESSES := l3bench envdproxysmoke meshinventorysmoke +TAGGED_HARNESSES := l3bench envdproxysmoke meshinventorysmoke envdsmoke ## Target OSes for vet / lint GOOSES ?= linux darwin diff --git a/docs/e2b-compat.md b/docs/e2b-compat.md index 871d2be..ec5d7b6 100644 --- a/docs/e2b-compat.md +++ b/docs/e2b-compat.md @@ -332,6 +332,24 @@ own start, and a publish that deletes an older holder removes a record its siblings still advertise; a create of one fails over to the next advertiser, and the template list lags until those nodes restart. +## Proving envd on hardware + +`scripts/envd-e2e.sh` runs the guest half on a node: it starts a sandboxd with +an e2b flavor pool whose warmup gates on envd's `/health`, then drives +`test/envdsmoke` (`go build -tags envdsmoke ./test/envdsmoke`), which claims a +sandbox through `pkg/sandboxd` and reaches the real envd through the node's +guest-port relay: health, the version the compat API reports, `POST`/`GET +/files`, a ConnectRPC unary, a `process.Process/Start` whose output comes back +over the Connect server stream, the `user` account with passwordless sudo, and +that envd serves HTTP/1.1 only. `CODE_INTERPRETER=1` also gates warmup on 49999 +and runs `1+1` the way the SDK's `runCode` does; `-hold` keeps the sandbox for +an out-of-tree harness that puts an edge proxy in front of it. + +```bash +K= TEMPLATE=ghcr.io/cocoonstack/sandbox/e2b-rt:24.04 bash scripts/envd-e2e.sh +K= TEMPLATE=ghcr.io/cocoonstack/sandbox/e2b-ci:24.04 SIZE=medium CODE_INTERPRETER=1 bash scripts/envd-e2e.sh +``` + ## Limits worth knowing - **Reaching `envd` (the in-sandbox data plane).** The SDK derives the sandbox diff --git a/scripts/envd-e2e.sh b/scripts/envd-e2e.sh new file mode 100755 index 0000000..2baf416 --- /dev/null +++ b/scripts/envd-e2e.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Bare-metal proof for the e2b image flavor: envd inside a cocoon microVM, +# reached only through sandboxd's guest-port relay. The pool's warmup gates a +# warm clone on envd answering, so a claim never hands out a sandbox whose data +# plane is still starting. +# Run as root on a node with cocoon; K names a kit holding bin/sandboxd, +# bin/envdsmoke (go build -tags envdsmoke ./test/envdsmoke), bin/jq and a cocoon on PATH. +set -uo pipefail +K=${K:?kit dir} +ADDR=${ADDR:-127.0.0.1:7996} +TOKEN=${TOKEN:-e2benvd} +TEMPLATE=${TEMPLATE:-e2b-rt:24.04} +ENVD_VERSION=${ENVD_VERSION:-0.8.0} +# CODE_INTERPRETER=1 is the e2b-ci pass: warmup also gates on the interpreter API, and envdsmoke drives it. +CODE_INTERPRETER=${CODE_INTERPRETER:-} +SIZE=${SIZE:-small} +HEALTH="curl -sf -m 1 -o /dev/null http://127.0.0.1:49983/health" +[[ -n $CODE_INTERPRETER ]] && HEALTH="$HEALTH && curl -sf -m 1 -o /dev/null http://127.0.0.1:49999/health" +export PATH=$K/bin:$PATH +DATA=$(mktemp -d /tmp/envd-e2e.XXXXXX); DAEMON_PID="" +cleanup() { + status=$? + echo "== daemon log tail"; tail -25 "$DATA/daemon.log" 2>/dev/null + [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null + wait 2>/dev/null + cocoon vm list --format json 2>/dev/null | + jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | + while read -r vm; do + cocoon vm stop --force "$vm" >/dev/null 2>&1 + cocoon vm rm --force "$vm" >/dev/null 2>&1 + done + rm -rf "$DATA"; exit "$status" +} +trap cleanup EXIT +# warmup gates a warm clone on envd answering: a clone that hands out a sandbox +# whose data plane is not up yet is worse than a slower clone. +cat >"$DATA/config.json" </dev/null)" +"$K/bin/sandboxd" -config "$DATA/config.json" >>"$DATA/daemon.log" 2>&1 & +DAEMON_PID=$! +for _ in $(seq 1 40); do curl -sf "http://$ADDR/healthz" >/dev/null && break; sleep 0.5; done +curl -sf "http://$ADDR/healthz" >/dev/null || { echo "daemon never came up"; exit 1; } +echo "== wait for golden + warm (warmup gates on envd /health)" +for i in $(seq 1 300); do + curl -sf -H "Authorization: Bearer $TOKEN" "http://$ADDR/v1/info" | + jq -e '(.pools|length)>0 and all(.pools[]; .golden and .warm >= .target)' >/dev/null 2>&1 && break + [[ $i == 300 ]] && { echo "pools never became ready"; curl -sf -H "Authorization: Bearer $TOKEN" "http://$ADDR/v1/info" | jq .; exit 1; } + sleep 1 +done +curl -sf -H "Authorization: Bearer $TOKEN" "http://$ADDR/v1/info" | jq -c '.pools' +echo "== envdsmoke" +"$K/bin/envdsmoke" -addr "$ADDR" -token "$TOKEN" -template "$TEMPLATE" -envd-version "$ENVD_VERSION" -size "$SIZE" ${HOLD:+-hold "$HOLD"} ${CODE_INTERPRETER:+-code-interpreter} From 2f0208720e7a4631e674978d8eac4f8625edcf72 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 16:46:31 +0800 Subject: [PATCH 3/3] test(envdsmoke): lint --- test/envdsmoke/main.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/envdsmoke/main.go b/test/envdsmoke/main.go index 5e69d16..f2f6c27 100644 --- a/test/envdsmoke/main.go +++ b/test/envdsmoke/main.go @@ -15,6 +15,7 @@ import ( "flag" "fmt" "io" + "maps" "net" "net/http" "os" @@ -57,9 +58,7 @@ func (r relay) Do(ctx context.Context, port uint16, method, path string, headers if err != nil { return nil, err } - for k, v := range headers { - req.Header[k] = v - } + maps.Copy(req.Header, headers) return client.Do(req) } @@ -102,7 +101,7 @@ func run(addr, token, template, wantVersion, size string, hold time.Duration, in fmt.Printf("claimed %s on %s\n", res.ID, res.OwnerAddr) rt := relay{client: client, id: res.ID, token: res.Token} - if err := waitReady(ctx, rt); err != nil { + if err = waitReady(ctx, rt); err != nil { return err } version, err := envdVersion(ctx, rt) @@ -353,7 +352,7 @@ func stdoutOf(events []string) string { func envelope(payload string) string { head := make([]byte, 5) - binary.BigEndian.PutUint32(head[1:], uint32(len(payload))) //nolint:gosec // a probe payload is tens of bytes + binary.BigEndian.PutUint32(head[1:], uint32(len(payload))) return string(head) + payload }