From ff7fc5f8ab2630d5ce74004fe25a6b6c4bb620b5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:35:15 +0800 Subject: [PATCH 01/23] review: inline the single log.WithFunc call in meshinventory.tick The logger-local rule binds a local only for two or more calls in one function. --- pkg/scale/meshinventory/source.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/scale/meshinventory/source.go b/pkg/scale/meshinventory/source.go index 454cc07..fc44e82 100644 --- a/pkg/scale/meshinventory/source.go +++ b/pkg/scale/meshinventory/source.go @@ -148,7 +148,6 @@ func (s *Source) run(ctx context.Context) { } func (s *Source) tick(ctx context.Context) map[string]answer { - logger := log.WithFunc("meshinventory.tick") answers := s.poll(ctx) named, heard := map[string]bool{}, false for _, a := range answers { @@ -164,7 +163,7 @@ func (s *Source) tick(ctx context.Context) map[string]answer { continue } m.fails++ - logger.Warnf(ctx, "mesh member did not answer addr=%s fails=%d err=%v", addr, m.fails, a.err) + log.WithFunc("meshinventory.tick").Warnf(ctx, "mesh member did not answer addr=%s fails=%d err=%v", addr, m.fails, a.err) if !m.seed && heard && !named[addr] && m.fails > s.opts.MaxStale { delete(s.members, addr) } From addc8a00dcfdc98dde1464148d4fd97e1025bbac Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:35:15 +0800 Subject: [PATCH 02/23] cut(e2bcompat): drop the unread force field of the build start body The field was decoded and never read; the decoder ignores unknown fields, and the docs already say force is accepted and ignored. --- pkg/e2bcompat/types.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/e2bcompat/types.go b/pkg/e2bcompat/types.go index dec11c7..b914715 100644 --- a/pkg/e2bcompat/types.go +++ b/pkg/e2bcompat/types.go @@ -260,7 +260,6 @@ type TemplateBuildStartV2 struct { FromImage string `json:"fromImage"` FromTemplate string `json:"fromTemplate"` FromImageRegistry json.RawMessage `json:"fromImageRegistry"` - Force bool `json:"force"` Steps []e2bbuild.Step `json:"steps"` StartCmd string `json:"startCmd"` ReadyCmd string `json:"readyCmd"` From 96a957ebbc1a1eb266ec38a700b647df039e28ef Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:47:27 +0800 Subject: [PATCH 03/23] fix(e2bcompat): a full e2b/ key in a create or a build's fromImage is refused The store routes any key no warm pool serves to a node that advertises it as a promoted template and knows no namespace, so a key holder in one namespace could claim another namespace's built template by naming its full key, or clone it through a build's fromImage; the bare-name branch, which is namespace-scoped, is the only way to a built template. --- pkg/e2bcompat/builds.go | 24 +++++++++++++++++++----- pkg/e2bcompat/server.go | 4 ++++ pkg/e2bcompat/templates_test.go | 12 ++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/pkg/e2bcompat/builds.go b/pkg/e2bcompat/builds.go index 2992eac..a383023 100644 --- a/pkg/e2bcompat/builds.go +++ b/pkg/e2bcompat/builds.go @@ -133,15 +133,20 @@ func (s *Server) buildStatus(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, out) } -// publishBuild deletes every other holder of name once the build's promote lands, so the registry names one build, then tags it. +// publishBuild deletes the holders of name older than this build's promote, so concurrent builds converge on the newest, then tags it. // It asks each node itself: an inventory a tick behind would miss a build that finished moments ago. func (s *Server) publishBuild(scope, name string, tags []string) func(context.Context, string, scale.PoolKey, string) error { return func(ctx context.Context, node string, key scale.PoolKey, digest string) error { - stale, err := s.liveHolders(ctx, scope+name) + holders, err := s.liveHolders(ctx, scope+name) if err != nil { return fmt.Errorf("list the previous builds: %w", err) } - stale = slices.DeleteFunc(stale, func(h templateHolder) bool { return h.node == node && h.key == key }) + i := slices.IndexFunc(holders, func(h templateHolder) bool { return h.node == node && h.key == key }) + if i < 0 { + return fmt.Errorf("the build's template is no longer on %s", node) + } + own := holders[i] + stale := slices.DeleteFunc(holders, func(h templateHolder) bool { return !h.created.Before(own.created) }) if err := forEachHolder(stale, func(h templateHolder) error { return s.store.DeleteTemplate(ctx, h.node, h.key) }); err != nil { return fmt.Errorf("delete the previous build: %w", err) } @@ -178,13 +183,20 @@ func (s *Server) liveHolders(ctx context.Context, template string) ([]templateHo defer mu.Unlock() for _, t := range held { if t.Template == template && t.Tenant == "" { - holders = append(holders, templateHolder{node: n, key: scale.PoolKey{Template: t.Template, Net: t.Net, Size: t.Size}}) + h := templateHolder{node: n, key: scale.PoolKey{Template: t.Template, Net: t.Net, Size: t.Size}} + if t.CreatedAt != nil { + h.created = t.CreatedAt.Time + } + holders = append(holders, h) } } return nil }) } - return holders, g.Wait() + if err := g.Wait(); err != nil { + return nil, err + } + return holders, nil } func (s *Server) buildKey(r *http.Request, name, buildID string) string { @@ -200,6 +212,8 @@ func unsupportedBuildOption(req TemplateBuildStartV2, canCopy bool) (string, boo return "fromImageRegistry is not supported; a pool pulls its own image", true case req.FromImage == "": return "fromImage is required", true + case strings.HasPrefix(req.FromImage, templatePrefix): + return "fromImage names a built template; build from an image", true } msg := e2bbuild.Invalid(req.Steps, canCopy) return msg, msg != "" diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index 8669c69..26f20c4 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -306,6 +306,10 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "templateID is required") return } + if strings.HasPrefix(req.TemplateID, templatePrefix) { + writeError(w, http.StatusNotFound, fmt.Sprintf("template %q not found", req.TemplateID)) + return + } if req.Timeout != nil && *req.Timeout < 0 { writeError(w, http.StatusBadRequest, "timeout must be >= 0") return diff --git a/pkg/e2bcompat/templates_test.go b/pkg/e2bcompat/templates_test.go index 17f11ad..20c7564 100644 --- a/pkg/e2bcompat/templates_test.go +++ b/pkg/e2bcompat/templates_test.go @@ -231,3 +231,15 @@ func tagNames(tags []TemplateTag) []string { } return out } + +func TestAFullE2bKeyNeverReachesAnotherNamespacesTemplate(t *testing.T) { + store := &fakeStore{} + h := newTestServer(t, store, withBuilds(), withTemplateFleet(store)) + w := do(t, h, http.MethodPost, "/sandboxes", `{"templateID":"e2b/others/app"}`, testKey) + assert.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) + assert.Equal(t, 0, store.claimCalls, "a full key is refused before any claim") + + id := requestBuild(t, h, "app") + w = do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"e2b/others/app"}`, testKey) + assert.Equal(t, http.StatusBadRequest, w.Code, w.Body.String()) +} From db24afcd8f494de55c28176b309fedb399d4a84f Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:47:27 +0800 Subject: [PATCH 04/23] fix(e2bbuild): an ENV value is the expansion's stdout alone The guest's stdout and stderr reached the same callback, so a shell warning printed before the printf ran (setlocale on an image without the locale) became part of the value and of every later RUN's environment and the template's defaults; stderr now goes to the build log only. --- pkg/e2bbuild/executor.go | 26 +++++++++++++------------- pkg/e2bbuild/executor_test.go | 10 ++++++---- pkg/e2bcompat/envd.go | 12 ++++++------ 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/pkg/e2bbuild/executor.go b/pkg/e2bbuild/executor.go index 159075d..67f7de2 100644 --- a/pkg/e2bbuild/executor.go +++ b/pkg/e2bbuild/executor.go @@ -124,8 +124,8 @@ type Command struct { // Guest runs a build's commands inside its claimed sandbox. type Guest interface { - // Run runs cmd to its end, passes each output line to out, and returns its exit code. - Run(ctx context.Context, a scale.Assignment, cmd Command, out func(line string)) (int, error) + // Run runs cmd to its end, passes each line of its output to stdout or stderr, and returns its exit code. + Run(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr func(line string)) (int, error) // Start starts cmd and returns while it runs. Start(ctx context.Context, a scale.Assignment, cmd Command) error // Init makes the user, workdir and envs of defaults what every later process in the sandbox gets. @@ -298,12 +298,12 @@ func (e *Executor) step(ctx context.Context, id, phase string, a scale.Assignmen if len(s.Args) > 1 { cmd.User = s.Args[1] } - msg, err := e.sh(ctx, a, cmd, logged) + msg, err := e.sh(ctx, a, cmd, logged, logged) return state, msg, err case stepEnv: envs := maps.Clone(state.Envs) for i := 0; i+1 < len(s.Args); i += 2 { - v, msg, err := e.expand(ctx, a, root, s.Args[i+1]) + v, msg, err := e.expand(ctx, a, root, s.Args[i+1], logged) if err != nil { return state, msg, err } @@ -316,14 +316,14 @@ func (e *Executor) step(ctx context.Context, id, phase string, a scale.Assignmen dir = path.Join(cmp.Or(state.Workdir, "/"), dir) } script := fmt.Sprintf(`t=%[1]s; [ -d "$t" ] && exit 0; n=$t; while [ ! -d "$(dirname "$n")" ]; do n=$(dirname "$n"); done; mkdir -p "$t" && chown -R %[2]s: "$n"`, shellQuote(dir), shellQuote(state.User)) - if msg, err := e.sh(ctx, a, withLine(root, script), logged); err != nil { + if msg, err := e.sh(ctx, a, withLine(root, script), logged, logged); err != nil { return state, msg, err } state.Workdir = dir case stepUser: name := shellQuote(s.Args[0]) script := fmt.Sprintf("id -u %[1]s >/dev/null 2>&1 || useradd --create-home --shell /bin/bash %[1]s", name) - if msg, err := e.sh(ctx, a, withLine(root, script), logged); err != nil { + if msg, err := e.sh(ctx, a, withLine(root, script), logged, logged); err != nil { return state, msg, err } state.User = s.Args[0] @@ -331,7 +331,7 @@ func (e *Executor) step(ctx context.Context, id, phase string, a scale.Assignmen if msg, err := e.copyIn(ctx, a, spec, s.FilesHash); err != nil { return state, msg, err } - if _, err := e.sh(ctx, a, withLine(root, copyScript(state, s)), logged); err != nil { + if _, err := e.sh(ctx, a, withLine(root, copyScript(state, s)), logged, logged); err != nil { return state, fmt.Sprintf("could not copy %s to %s", s.Args[0], s.Args[1]), err } } @@ -352,8 +352,8 @@ func (e *Executor) copyIn(ctx context.Context, a scale.Assignment, spec Spec, ha } // sh returns the failure the caller sees next to the error; a non-zero exit is a failure. -func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, out func(string)) (string, error) { - code, err := e.guest.Run(ctx, a, cmd, out) +func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr func(string)) (string, error) { + code, err := e.guest.Run(ctx, a, cmd, stdout, stderr) if err != nil { return "could not run a command in the build sandbox", err } @@ -364,10 +364,10 @@ func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, out return "", nil } -// expand evaluates an ENV value in the guest's shell, so $VAR references resolve and command substitution does not run. -func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, value string) (string, string, error) { +// expand evaluates an ENV value in the guest's shell, so $VAR references resolve and command substitution does not run; the shell's own stderr goes to the log, never into the value. +func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, value string, stderr func(string)) (string, string, error) { var lines []string - if _, err := e.sh(ctx, a, withLine(root, `printf "%s" "`+envEscaper.Replace(value)+`"`), func(line string) { lines = append(lines, line) }); err != nil { + if _, err := e.sh(ctx, a, withLine(root, `printf "%s" "`+envEscaper.Replace(value)+`"`), func(line string) { lines = append(lines, line) }, stderr); err != nil { return "", fmt.Sprintf("could not evaluate the value %q", value), err } return strings.Join(lines, "\n"), "", nil @@ -375,7 +375,7 @@ func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, func (e *Executor) awaitReady(ctx context.Context, a scale.Assignment, cmd Command) error { for { - code, err := e.guest.Run(ctx, a, cmd, func(string) {}) + code, err := e.guest.Run(ctx, a, cmd, func(string) {}, func(string) {}) if err == nil && code == 0 { return nil } diff --git a/pkg/e2bbuild/executor_test.go b/pkg/e2bbuild/executor_test.go index 24a6461..efc5151 100644 --- a/pkg/e2bbuild/executor_test.go +++ b/pkg/e2bbuild/executor_test.go @@ -100,7 +100,8 @@ func TestStepsCarryTheirUserWorkdirAndEnvIntoLaterCommandsAndTheDefaults(t *test "promote node-a sb_1 e2b/ns/app", "release node-a sb_1", }, store.calls()) - assert.Equal(t, [2]string{"4", "out of make"}, [2]string{info.Logs[4].Phase, info.Logs[4].Message}) + assert.Equal(t, [2]string{"1", "bash: warning: setlocale: LC_ALL: cannot change locale"}, [2]string{info.Logs[2].Phase, info.Logs[2].Message}, "the shell's stderr goes to the log, never into the ENV value") + assert.Equal(t, [2]string{"4", "out of make"}, [2]string{info.Logs[5].Phase, info.Logs[5].Message}) } func TestARelayedBuildSeedsItsStepsWithTheProxyEnvironment(t *testing.T) { @@ -282,13 +283,14 @@ func (f *fakeStore) Release(_ context.Context, node, id string) error { return nil } -func (f *fakeStore) Run(_ context.Context, _ scale.Assignment, cmd Command, out func(string)) (int, error) { +func (f *fakeStore) Run(_ context.Context, _ scale.Assignment, cmd Command, stdout, stderr func(string)) (int, error) { f.record(fmt.Sprintf("run %s %s %v %s", cmd.User, cmd.Workdir, cmd.Envs, cmd.Line)) if v, ok := strings.CutPrefix(cmd.Line, `printf "%s" "`); ok { - out(strings.TrimSuffix(v, `"`)) + stderr("bash: warning: setlocale: LC_ALL: cannot change locale") + stdout(strings.TrimSuffix(v, `"`)) return 0, nil } - out("out of " + cmd.Line) + stdout("out of " + cmd.Line) f.mu.Lock() defer f.mu.Unlock() codes := f.exits[cmd.Line] diff --git a/pkg/e2bcompat/envd.go b/pkg/e2bcompat/envd.go index 464820a..190e504 100644 --- a/pkg/e2bcompat/envd.go +++ b/pkg/e2bcompat/envd.go @@ -108,14 +108,14 @@ type envdGuest struct { s *Server } -func (g envdGuest) Run(ctx context.Context, a scale.Assignment, cmd e2bbuild.Command, out func(string)) (int, error) { - stdout, stderr := &lineSplitter{out: out}, &lineSplitter{out: out} +func (g envdGuest) Run(ctx context.Context, a scale.Assignment, cmd e2bbuild.Command, stdout, stderr func(string)) (int, error) { + outLines, errLines := &lineSplitter{out: stdout}, &lineSplitter{out: stderr} code, _, err := g.s.envdProcess(ctx, a, cmd, func(o, e []byte) { - stdout.write(o) - stderr.write(e) + outLines.write(o) + errLines.write(e) }) - stdout.flush() - stderr.flush() + outLines.flush() + errLines.flush() return code, err } From db084fe240ba975a821d53d539f28d5772d1ab85 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:47:27 +0800 Subject: [PATCH 05/23] fix(e2bcompat): a publish deletes only the holders older than its own promote Two builds of one name whose promotes both landed before either publish read the fleet deleted each other's template and both reported ready, leaving no holder; a publish now keeps every holder promoted after its own, so concurrent builds converge on the newest, and it fails when its own holder is gone. liveHolders returns the holders after Wait instead of in the same return statement, whose operand order the spec leaves open. --- pkg/e2bcompat/builds_test.go | 17 +++++++++++++++++ pkg/e2bcompat/server_test.go | 16 ++++++++++++++-- pkg/e2bcompat/templates.go | 9 +++++---- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index 13aee82..419e392 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cocoonstack/sandbox-operator/pkg/e2bbuild" "github.com/cocoonstack/sandbox-operator/pkg/scale" @@ -178,3 +179,19 @@ func withBuildFleet(store *fakeStore) func(*Options) { store.fleet = inv return func(o *Options) { o.Inventory = inv } } + +func TestAPublishLeavesANewerConcurrentBuildAlone(t *testing.T) { + newer := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:newer", CreatedAt: new(metav1.NewTime(time.Now().Add(time.Minute)))} + older := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:older", CreatedAt: new(metav1.NewTime(time.Now().Add(-time.Minute)))} + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil, "m": {newer}, "k": {older}}} + inv := scale.NewStaticInventorySource() + for _, n := range []string{"n", "m", "k"} { + inv.Put(&scale.NodeInventory{Node: n}) + } + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status) + assert.Equal(t, []string{"k e2b/sandboxes/app small"}, store.deletedTemplates, "only the holder older than this promote goes; a newer concurrent build keeps its node") +} diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index f444487..9135419 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "slices" "strconv" "strings" "sync" @@ -663,11 +664,22 @@ func (f *fakeStore) DeleteSnapshot(_ context.Context, node, id string) error { return nil } -func (f *fakeStore) Promote(_ context.Context, node, id, template string) (scale.PoolKey, string, error) { +func (f *fakeStore) Promote(ctx context.Context, node, id, template string) (scale.PoolKey, string, error) { f.mu.Lock() defer f.mu.Unlock() f.promoted = append(f.promoted, node+" "+id+" "+template) - return scale.PoolKey{Template: template, Net: scale.NetDefault, Size: f.claimPool.Size}, "sha256:" + id, f.promoteErr + key := scale.PoolKey{Template: template, Net: scale.NetDefault, Size: f.claimPool.Size} + fresh := scale.PromotedTemplate{Template: key.Template, Net: key.Net, Size: key.Size, ContentDigest: "sha256:" + id, CreatedAt: new(metav1.Now())} + same := func(t scale.PromotedTemplate) bool { + return t.Template == key.Template && t.Net == key.Net && t.Size == key.Size + } + if f.live != nil { + f.live[node] = append(slices.DeleteFunc(f.live[node], same), fresh) + } else if inv, err := f.fleet.NodeInventory(ctx, node); err == nil { + inv.Templates = append(slices.DeleteFunc(inv.Templates, same), fresh) + f.fleet.Put(inv) + } + return key, "sha256:" + id, f.promoteErr } // SetTemplateLabels writes straight into the fleet the test serves, so a live read sees it at once. diff --git a/pkg/e2bcompat/templates.go b/pkg/e2bcompat/templates.go index ca214ea..9dabf6e 100644 --- a/pkg/e2bcompat/templates.go +++ b/pkg/e2bcompat/templates.go @@ -28,10 +28,11 @@ const ( var buildNamespace = uuid.NewSHA1(uuid.NameSpaceURL, []byte("https://github.com/cocoonstack/sandbox-operator/e2b")) type templateHolder struct { - node string - key scale.PoolKey - digest string - labels map[string]string + node string + key scale.PoolKey + digest string + labels map[string]string + created time.Time } type builtTemplate struct { From 2376abf1a65782871474f01204b403c8a5dc79c4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:50:54 +0800 Subject: [PATCH 06/23] fix(sandboxd): promote, checkpoint, fork, hibernate and wake ride a client bounded by ten minutes, not the request timeout The shared client's 10 s timeout covered a promote's snapshot, export and digest, which grow with guest memory (1.3 s for a 519 MiB record, past 10 s for a large tier or an S3 store), so the build failed at promote while the node finished it and swapped the template on one node with nothing published; the capture verbs now use a copy of the client whose bound is ten minutes, the callers' contexts bounding them further. --- pkg/sandboxd/client.go | 16 +++++++++++++--- pkg/sandboxd/client_test.go | 22 ++++++++++++++++++++++ pkg/sandboxd/lifecycle.go | 14 +++++++++----- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/pkg/sandboxd/client.go b/pkg/sandboxd/client.go index bed35ba..7ef74b9 100644 --- a/pkg/sandboxd/client.go +++ b/pkg/sandboxd/client.go @@ -29,6 +29,8 @@ const ( // NetRouteRelay is a claim whose guest reaches the network through the node's egress proxy on 127.0.0.1:3128. NetRouteRelay = "relay" + + captureTimeout = 10 * time.Minute ) // ErrNodeAtCapacity is returned by Claim when sandboxd answers 429 (the node is @@ -159,8 +161,9 @@ type NodeInfo struct { type Client struct { baseURL string // token is the node api_token every verb presents, except Release, which takes one per call. - token string - hc *http.Client + token string + hc *http.Client + capture *http.Client } // New returns a Client for the sandboxd at baseURL; token is the node api_token, empty when sandboxd runs without auth. @@ -173,6 +176,9 @@ func New(baseURL, token string, opts ...Option) *Client { for _, o := range opts { o(c) } + capture := *c.hc + capture.Timeout = captureTimeout + c.capture = &capture return c } @@ -253,6 +259,10 @@ func (c *Client) Release(ctx context.Context, id, token string) error { } func (c *Client) send(ctx context.Context, method, path, token, op string, body []byte, ok ...int) error { + return c.sendWith(ctx, c.hc, method, path, token, op, body, ok...) +} + +func (c *Client) sendWith(ctx context.Context, hc *http.Client, method, path, token, op string, body []byte, ok ...int) error { var r io.Reader if body != nil { r = bytes.NewReader(body) @@ -266,7 +276,7 @@ func (c *Client) send(ctx context.Context, method, path, token, op string, body } c.authenticate(req, token) - resp, err := c.hc.Do(req) + resp, err := hc.Do(req) if err != nil { return fmt.Errorf("sandboxd: %s: %w", op, err) } diff --git a/pkg/sandboxd/client_test.go b/pkg/sandboxd/client_test.go index 1c46baf..7fb50a8 100644 --- a/pkg/sandboxd/client_test.go +++ b/pkg/sandboxd/client_test.go @@ -379,3 +379,25 @@ func TestSetPoolsSendsAnEmptyListNotNull(t *testing.T) { assert.Equal(t, "duplicate pool", he.Message) assert.JSONEq(t, `{"pools":[]}`, raw, "a nil set must drain as [] rather than null") } + +func TestCaptureVerbsOutliveTheRequestTimeout(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + switch r.URL.Path { + case "/v1/sandboxes/sb_1/promote": + _, _ = w.Write([]byte(`{"key":{"template":"ns/app","net":"none","size":"small"},"content_digest":"sha256:aa"}`)) + case "/v1/sandboxes/sb_1/hibernate": + w.WriteHeader(http.StatusNoContent) + default: + _, _ = w.Write([]byte(`{"id":"sb_2","token":"t"}`)) + } + })) + defer srv.Close() + c := New(srv.URL, "root-token", WithHTTPClient(&http.Client{Timeout: 50 * time.Millisecond})) + + _, _, err := c.Promote(t.Context(), "sb_1", "ns/app") + require.NoError(t, err, "a promote runs as long as the guest's memory takes") + require.NoError(t, c.Hibernate(t.Context(), "sb_1")) + _, err = c.Claim(t.Context(), ClaimSpec{Template: "img"}) + require.Error(t, err, "a claim keeps the request timeout") +} diff --git a/pkg/sandboxd/lifecycle.go b/pkg/sandboxd/lifecycle.go index 27f23b8..a246ee1 100644 --- a/pkg/sandboxd/lifecycle.go +++ b/pkg/sandboxd/lifecycle.go @@ -118,7 +118,7 @@ func (c *Client) Fork(ctx context.Context, id string, spec ForkSpec) (ForkResult if id == "" { return out, fmt.Errorf("sandboxd: fork requires a sandbox id") } - err := c.sendJSON(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/fork", spec, &out) + err := c.sendJSONWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/fork", spec, &out) return out, err } @@ -139,7 +139,7 @@ func (c *Client) Checkpoint(ctx context.Context, id string, spec CheckpointSpec) var out struct { Checkpoint Checkpoint `json:"checkpoint"` } - err := c.sendJSON(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/checkpoint", spec, &out) + err := c.sendJSONWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/checkpoint", spec, &out) return out.Checkpoint, err } @@ -152,7 +152,7 @@ func (c *Client) Promote(ctx context.Context, id, template string) (PoolKey, str Key PoolKey `json:"key"` ContentDigest string `json:"content_digest"` } - err := c.sendJSON(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/promote", struct { + err := c.sendJSONWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/promote", struct { Template string `json:"template"` }{template}, &out) return out.Key, out.ContentDigest, err @@ -229,10 +229,14 @@ func (c *Client) sandboxVerb(ctx context.Context, id, verb string) error { if id == "" { return fmt.Errorf("sandboxd: %s requires a sandbox id", verb) } - return c.send(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/"+verb, c.token, verb, nil, http.StatusNoContent) + return c.sendWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/"+verb, c.token, verb, nil, http.StatusNoContent) } func (c *Client) sendJSON(ctx context.Context, method, path string, body, out any) error { + return c.sendJSONWith(ctx, c.hc, method, path, body, out) +} + +func (c *Client) sendJSONWith(ctx context.Context, hc *http.Client, method, path string, body, out any) error { payload, err := json.Marshal(body) if err != nil { return fmt.Errorf("sandboxd: encode %s: %w", path, err) @@ -244,7 +248,7 @@ func (c *Client) sendJSON(ctx context.Context, method, path string, body, out an req.Header.Set("Content-Type", "application/json") c.authenticate(req, c.token) - resp, err := c.hc.Do(req) + resp, err := hc.Do(req) if err != nil { return fmt.Errorf("sandboxd: %s: %w", path, err) } From 5547e0329bafc943ac8df3ae9b8d9ad8b2e84569 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:50:54 +0800 Subject: [PATCH 07/23] fix(sandboxd): the port relay speaks the node's scheme and can present the sandbox's own token DialPort dialed url.Parse(base).Host in cleartext, so a node addressed by an https client_advertise origin, which every other verb reaches over TLS, could not be relayed at all (missing port, or cleartext into a TLS listener); it now takes the base as a host:port or an http(s) origin, defaults the scheme's port and wraps the connection in TLS for https. Client.DialPort takes the relay token: empty keeps the node token and the passive relay, the sandbox's own token wakes it and stamps activity. --- pkg/sandboxd/ports.go | 34 ++++++++++++++++++++++++++-------- pkg/sandboxd/ports_test.go | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/pkg/sandboxd/ports.go b/pkg/sandboxd/ports.go index 32b1765..47642a8 100644 --- a/pkg/sandboxd/ports.go +++ b/pkg/sandboxd/ports.go @@ -2,13 +2,16 @@ package sandboxd import ( "bufio" + "cmp" "context" + "crypto/tls" "fmt" "io" "net" "net/http" "net/url" "strconv" + "strings" ) const ( @@ -17,24 +20,39 @@ const ( portReplyMax = 16 << 10 ) -// DialPort opens a guest port of sandbox id through this node's relay with the node api_token, which the node keeps passive: a paused sandbox answers 409 unwoken. -func (c *Client) DialPort(ctx context.Context, id string, port uint16) (net.Conn, error) { - u, _ := url.Parse(c.baseURL) +// DialPort opens a guest port of sandbox id through this node's relay; an empty token presents the node api_token, which the node keeps passive (a paused sandbox answers 409 unwoken), the sandbox's own token wakes it and stamps activity. +func (c *Client) DialPort(ctx context.Context, id, token string, port uint16) (net.Conn, error) { var d net.Dialer - return DialPort(ctx, &d, u.Host, id, c.token, port) + return DialPort(ctx, &d, c.baseURL, id, cmp.Or(token, c.token), port) } -// DialPort opens the node's GET /v1/sandboxes/{id}/ports/{port} relay; with the sandbox's own token the node wakes a paused sandbox for it. -func DialPort(ctx context.Context, d *net.Dialer, addr, id, token string, port uint16) (net.Conn, error) { - conn, err := d.DialContext(ctx, "tcp", addr) +// DialPort opens the node's GET /v1/sandboxes/{id}/ports/{port} relay at base, a host:port or an http(s) origin; with the sandbox's own token the node wakes a paused sandbox for it. +func DialPort(ctx context.Context, d *net.Dialer, base, id, token string, port uint16) (net.Conn, error) { + scheme, host := "http", base + if u, err := url.Parse(base); err == nil && strings.Contains(base, "://") { + scheme, host = u.Scheme, u.Host + } + if _, _, err := net.SplitHostPort(host); err != nil { + host = net.JoinHostPort(host, map[bool]string{true: "443", false: "80"}[scheme == "https"]) + } + conn, err := d.DialContext(ctx, "tcp", host) if err != nil { return nil, fmt.Errorf("dial node: %w", err) } stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) defer stop() + if scheme == "https" { + hostname, _, _ := net.SplitHostPort(host) + tc := tls.Client(conn, &tls.Config{ServerName: hostname, MinVersion: tls.VersionTLS12}) + if err = tc.HandshakeContext(ctx); err != nil { + _ = conn.Close() + return nil, fmt.Errorf("tls to node: %w", err) + } + conn = tc + } path := "/v1/sandboxes/" + url.PathEscape(id) + "/ports/" + strconv.FormatUint(uint64(port), 10) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+addr+path, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+host+path, nil) if err != nil { _ = conn.Close() return nil, err diff --git a/pkg/sandboxd/ports_test.go b/pkg/sandboxd/ports_test.go index e12ffe0..454fb94 100644 --- a/pkg/sandboxd/ports_test.go +++ b/pkg/sandboxd/ports_test.go @@ -30,7 +30,7 @@ func TestClientDialPortOpensThePassiveRelayWithTheNodeToken(t *testing.T) { })) defer node.Close() - conn, err := New(node.URL, "root-token").DialPort(t.Context(), "sb_1", 8080) + conn, err := New(node.URL, "root-token").DialPort(t.Context(), "sb_1", "", 8080) require.NoError(t, err) defer func() { _ = conn.Close() }() _, err = conn.Write([]byte("ping\n")) @@ -56,3 +56,37 @@ func TestDialPortReportsTheNodesRefusalAsAnHTTPError(t *testing.T) { assert.Equal(t, http.StatusNotFound, he.StatusCode) assert.Equal(t, "unknown sandbox", he.Message) } + +func TestDialPortSpeaksTLSToAnHTTPSNode(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = ln.Close() }() + first := make(chan byte, 1) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + var b [1]byte + if _, err := io.ReadFull(conn, b[:]); err == nil { + first <- b[0] + } + }() + + _, err = New("https://"+ln.Addr().String(), "root-token").DialPort(t.Context(), "sb_1", "", 8080) + require.Error(t, err, "a plain listener cannot finish the handshake") + assert.Equal(t, byte(0x16), <-first, "an https base opens a TLS handshake, not a cleartext upgrade") +} + +func TestClientDialPortPresentsTheSandboxTokenWhenGivenOne(t *testing.T) { + var gotAuth string + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusConflict) + })) + defer node.Close() + _, err := New(node.URL, "root-token").DialPort(t.Context(), "sb_1", "claim-token", 8080) + require.Error(t, err) + assert.Equal(t, "Bearer claim-token", gotAuth, "the sandbox's own token makes the relay active") +} From ee77d3c54a3b6b38cb6573ef5be3f0d7cec2adfb Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:50:54 +0800 Subject: [PATCH 08/23] fix(e2b): build commands ride the claim's own relay, so a long build is never idle-hibernated Every build call went through the passive root-token relay, which never stamps activity, so a build longer than its pool's idle_hibernate_seconds was hibernated in the first gap between two calls and every later call answered 409 until the build timeout; the guest-port seam now carries a relay token, the build presents the claim's, and the create, metrics and fork hand-over keep the passive relay. --- pkg/e2bcompat/builds_test.go | 3 ++- pkg/e2bcompat/envd.go | 26 ++++++++++++++---------- pkg/e2bcompat/envd_test.go | 2 +- pkg/e2bcompat/lifecycle_test.go | 4 ++-- pkg/e2bcompat/server.go | 2 +- pkg/e2bcompat/server_test.go | 6 +++++- pkg/scale/apiserver/storage_test.go | 2 +- pkg/scale/sandboxstore.go | 2 +- pkg/scale/sandboxstore_claim_test.go | 2 +- pkg/scale/sandboxstore_impl.go | 2 +- pkg/scale/sandboxstore_lifecycle.go | 4 ++-- pkg/scale/sandboxstore_lifecycle_test.go | 10 ++++----- 12 files changed, 37 insertions(+), 28 deletions(-) diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index 419e392..cf23e70 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -80,7 +80,7 @@ func TestABuildThatCannotClaimFailsAtTheBaseStep(t *testing.T) { } func TestABuildRunsItsStepsThroughEnvdAndLeavesItsDefaultsInTheSandbox(t *testing.T) { - store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, processExits: map[string]int{"false": 1}} + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n", Token: "claim-tok"}, processExits: map[string]int{"false": 1}} h := newTestServer(t, store, withBuilds(), withBuildFleet(store)) id := requestBuild(t, h, "app") @@ -99,6 +99,7 @@ func TestABuildRunsItsStepsThroughEnvdAndLeavesItsDefaultsInTheSandbox(t *testin assert.Equal(t, [2]any{"user", map[string]string{"A": "1"}}, [2]any{defaults.DefaultUser, defaults.EnvVars}) assert.Contains(t, info.LogEntries, BuildLogEntry{Timestamp: info.LogEntries[2].Timestamp, Message: "ok", Level: "info", Step: "2"}) assert.Equal(t, []string{"n sb_1 e2b/sandboxes/app"}, store.promoted) + assert.Equal(t, []string{"claim-tok", "claim-tok", "claim-tok", "claim-tok", "claim-tok"}, store.envdRelays, "every build call rides the claim's own relay, which wakes and keeps the sandbox awake") id = requestBuild(t, h, "app") body = `{"fromImage":"img","steps":[{"type":"ENV","args":["A","1"]},{"type":"WORKDIR","args":["/w"]},{"type":"RUN","args":["false"]}]}` diff --git a/pkg/e2bcompat/envd.go b/pkg/e2bcompat/envd.go index 190e504..f6488b9 100644 --- a/pkg/e2bcompat/envd.go +++ b/pkg/e2bcompat/envd.go @@ -128,9 +128,9 @@ func (g envdGuest) Start(ctx context.Context, a scale.Assignment, cmd e2bbuild.C return err } -// Init sets the defaults with no access token, which envd takes as first-time setup. +// Init sets the defaults with no access token, which envd takes as first-time setup; like every build call it rides the claim's own relay, which keeps the sandbox awake. func (g envdGuest) Init(ctx context.Context, a scale.Assignment, defaults e2bbuild.Command) error { - return g.s.initEnvd(ctx, a.Node, a.SandboxName, envdInit{EnvVars: defaults.Envs, DefaultUser: defaults.User, DefaultWorkdir: defaults.Workdir}) + return g.s.initEnvd(ctx, a.Node, a.SandboxName, a.Token, envdInit{EnvVars: defaults.Envs, DefaultUser: defaults.User, DefaultWorkdir: defaults.Workdir}) } func (g envdGuest) Write(ctx context.Context, a scale.Assignment, path string, r io.Reader) error { @@ -139,7 +139,7 @@ func (g envdGuest) Write(ctx context.Context, a scale.Assignment, path string, r return err } req.Header.Set("Content-Type", "application/octet-stream") - resp, err := g.s.envdRoundTrip(ctx, a.Node, a.SandboxName, req) + resp, err := g.s.envdRoundTrip(ctx, a.Node, a.SandboxName, a.Token, req) if err != nil { return err } @@ -226,7 +226,7 @@ func (s *Server) readEnvdMetrics(ctx context.Context, node, id string) (envdMetr } // initEnvd sets envd's access token, defaults and env inside sandbox id, and stamps the guest clock. -func (s *Server) initEnvd(ctx context.Context, node, id string, req envdInit) error { +func (s *Server) initEnvd(ctx context.Context, node, id, relay string, req envdInit) error { ctx, cancel := context.WithTimeout(ctx, initTimeout) defer cancel() req.Timestamp = time.Now() @@ -234,7 +234,7 @@ func (s *Server) initEnvd(ctx context.Context, node, id string, req envdInit) er if err != nil { return err } - status, _, err := s.envdCall(ctx, node, id, http.MethodPost, "/init", "", payload) + status, _, err := s.envdCallOver(ctx, node, id, relay, http.MethodPost, "/init", "", payload) if err != nil { return fmt.Errorf("envd init of %s: %w", id, err) } @@ -276,7 +276,7 @@ func (s *Server) handOver(ctx context.Context, child scale.Assignment, token str if err := s.store.SetInstanceMetadata(ctx, child.Node, child.SandboxName, doc); err != nil { return err } - return s.initEnvd(ctx, child.Node, child.SandboxName, envdInit{AccessToken: token}) + return s.initEnvd(ctx, child.Node, child.SandboxName, "", envdInit{AccessToken: token}) } // releaseAll gives back claims whose envd could not be initialized; a failed release is left to the lease. @@ -301,7 +301,7 @@ func (s *Server) envdProcess(ctx context.Context, a scale.Assignment, cmd e2bbui } req.Header.Set("Content-Type", "application/connect+json") req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(cmd.User+":"))) - resp, err := s.envdRoundTrip(ctx, a.Node, a.SandboxName, req) + resp, err := s.envdRoundTrip(ctx, a.Node, a.SandboxName, a.Token, req) if err != nil { return 0, false, err } @@ -345,6 +345,10 @@ func (s *Server) envdProcess(ctx context.Context, a scale.Assignment, cmd e2bbui // envdCall sends one request to envd inside sandbox id over its node's passive relay and reads the bounded reply; token authenticates it to envd. func (s *Server) envdCall(ctx context.Context, node, id, method, path, token string, body []byte) (int, []byte, error) { + return s.envdCallOver(ctx, node, id, "", method, path, token, body) +} + +func (s *Server) envdCallOver(ctx context.Context, node, id, relay, method, path, token string, body []byte) (int, []byte, error) { req, err := http.NewRequestWithContext(ctx, method, "http://"+envdHostAlias+path, bytes.NewReader(body)) if err != nil { return 0, nil, err @@ -355,7 +359,7 @@ func (s *Server) envdCall(ctx context.Context, node, id, method, path, token str if token != "" { req.Header.Set("X-Access-Token", token) } - resp, err := s.envdRoundTrip(ctx, node, id, req) + resp, err := s.envdRoundTrip(ctx, node, id, relay, req) if err != nil { return 0, nil, err } @@ -367,9 +371,9 @@ func (s *Server) envdCall(ctx context.Context, node, id, method, path, token str return resp.StatusCode, reply, nil } -// envdRoundTrip writes req to envd inside sandbox id over its node's passive relay; closing the reply's body closes the connection. -func (s *Server) envdRoundTrip(ctx context.Context, node, id string, req *http.Request) (*http.Response, error) { - conn, err := s.store.DialGuestPort(ctx, node, id, envdPort) +// envdRoundTrip writes req to envd inside sandbox id over its node's relay, passive with an empty relay token; closing the reply's body closes the connection. +func (s *Server) envdRoundTrip(ctx context.Context, node, id, relay string, req *http.Request) (*http.Response, error) { + conn, err := s.store.DialGuestPort(ctx, node, id, relay, envdPort) if err != nil { return nil, err } diff --git a/pkg/e2bcompat/envd_test.go b/pkg/e2bcompat/envd_test.go index ecbad77..b2fed8f 100644 --- a/pkg/e2bcompat/envd_test.go +++ b/pkg/e2bcompat/envd_test.go @@ -182,7 +182,7 @@ type runningEnvd struct { scale.SandboxStore } -func (runningEnvd) DialGuestPort(context.Context, string, string, uint16) (net.Conn, error) { +func (runningEnvd) DialGuestPort(context.Context, string, string, string, uint16) (net.Conn, error) { conn, envd := net.Pipe() go func() { r, err := http.ReadRequest(bufio.NewReader(envd)) diff --git a/pkg/e2bcompat/lifecycle_test.go b/pkg/e2bcompat/lifecycle_test.go index d4233d6..4ea2024 100644 --- a/pkg/e2bcompat/lifecycle_test.go +++ b/pkg/e2bcompat/lifecycle_test.go @@ -605,7 +605,7 @@ func (f *lifecycleStore) Read(_ context.Context, node, id string) (scale.Sandbox return scale.SandboxRecord{Token: f.token, Paused: f.nodePaused || f.nodeArchived, Deadline: f.deadline}, f.nodeErr } -func (f *lifecycleStore) DialGuestPort(_ context.Context, _, id string, port uint16) (net.Conn, error) { +func (f *lifecycleStore) DialGuestPort(_ context.Context, _, id, token string, port uint16) (net.Conn, error) { f.dialPort.Store(uint32(port)) if f.nodeErr != nil { return nil, f.nodeErr @@ -629,7 +629,7 @@ func (f *lifecycleStore) DialGuestPort(_ context.Context, _, id string, port uin return http.StatusOK, string(body) }), nil } - return f.fakeStore.DialGuestPort(context.Background(), "", id, port) + return f.fakeStore.DialGuestPort(context.Background(), "", id, token, port) } func (f *lifecycleStore) Pause(_ context.Context, node, id string) error { diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index 26f20c4..f8a3315 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -363,7 +363,7 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { init.EnvVars, init.DefaultUser, init.DefaultWorkdir = withRelay(assignment.NetRoute, req.EnvVars), envdDefaultUser, envdDefaultWorkdir } if err == nil { - err = s.initEnvd(r.Context(), assignment.Node, assignment.SandboxName, init) + err = s.initEnvd(r.Context(), assignment.Node, assignment.SandboxName, "", init) } if err != nil { log.WithFunc("e2bcompat.createSandbox").Errorf(r.Context(), err, "e2b create: envd init failed sandboxID=%s node=%s", assignment.SandboxName, assignment.Node) diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index 9135419..fb3117b 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -577,6 +577,7 @@ type fakeStore struct { live map[string][]scale.PromotedTemplate envdCalls []string + envdRelays []string initStatus int metadataDocs []string guestEnvs map[string]string @@ -719,7 +720,10 @@ func (f *fakeStore) DeleteTemplate(_ context.Context, node string, key scale.Poo return f.deleteTemplateErr[node] } -func (f *fakeStore) DialGuestPort(_ context.Context, _, id string, _ uint16) (net.Conn, error) { +func (f *fakeStore) DialGuestPort(_ context.Context, _, id, token string, _ uint16) (net.Conn, error) { + f.mu.Lock() + f.envdRelays = append(f.envdRelays, token) + f.mu.Unlock() return fakeEnvd(func(r *http.Request, body string) (int, string) { f.mu.Lock() defer f.mu.Unlock() diff --git a/pkg/scale/apiserver/storage_test.go b/pkg/scale/apiserver/storage_test.go index a804063..f90ae13 100644 --- a/pkg/scale/apiserver/storage_test.go +++ b/pkg/scale/apiserver/storage_test.go @@ -330,7 +330,7 @@ func (f *fakeStore) SetInstanceMetadata(context.Context, string, string, []byte) return f.verbErr } -func (f *fakeStore) DialGuestPort(context.Context, string, string, uint16) (net.Conn, error) { +func (f *fakeStore) DialGuestPort(context.Context, string, string, string, uint16) (net.Conn, error) { return nil, f.verbErr } diff --git a/pkg/scale/sandboxstore.go b/pkg/scale/sandboxstore.go index d22ebac..d6328b6 100644 --- a/pkg/scale/sandboxstore.go +++ b/pkg/scale/sandboxstore.go @@ -96,7 +96,7 @@ type SandboxLifecycle interface { // NodeTemplates reads the promoted templates node holds now, ahead of its next inventory publish. NodeTemplates(ctx context.Context, node string) ([]PromotedTemplate, error) // DialGuestPort opens a guest port of a running sandbox through its node's passive relay; a paused one answers Conflict and is never woken. - DialGuestPort(ctx context.Context, node, id string, port uint16) (net.Conn, error) + DialGuestPort(ctx context.Context, node, id, token string, port uint16) (net.Conn, error) // SetInstanceMetadata replaces the JSON object a running sandbox's guest reads from 169.254.169.254; a paused one answers Conflict. SetInstanceMetadata(ctx context.Context, node, id string, doc []byte) error // Read reports the sandbox as its owning node holds it: token, paused state and lease deadline. diff --git a/pkg/scale/sandboxstore_claim_test.go b/pkg/scale/sandboxstore_claim_test.go index 9181b1f..882b200 100644 --- a/pkg/scale/sandboxstore_claim_test.go +++ b/pkg/scale/sandboxstore_claim_test.go @@ -599,7 +599,7 @@ func (c *recordingClient) SetInstanceMetadata(_ context.Context, id string, doc return c.f.verbErr } -func (c *recordingClient) DialPort(_ context.Context, id string, port uint16) (net.Conn, error) { +func (c *recordingClient) DialPort(_ context.Context, id, _ string, port uint16) (net.Conn, error) { c.f.mu.Lock() defer c.f.mu.Unlock() c.f.dialPorts = append(c.f.dialPorts, port) diff --git a/pkg/scale/sandboxstore_impl.go b/pkg/scale/sandboxstore_impl.go index 1edba2c..03928d7 100644 --- a/pkg/scale/sandboxstore_impl.go +++ b/pkg/scale/sandboxstore_impl.go @@ -130,7 +130,7 @@ type SandboxdClient interface { Promote(ctx context.Context, id, template string) (sandboxd.PoolKey, string, error) SetTemplateLabels(ctx context.Context, key sandboxd.PoolKey, labels map[string]string) error Info(ctx context.Context) (*sandboxd.NodeInfo, error) - DialPort(ctx context.Context, id string, port uint16) (net.Conn, error) + DialPort(ctx context.Context, id, token string, port uint16) (net.Conn, error) SetInstanceMetadata(ctx context.Context, id string, doc []byte) error // Sandbox and SandboxesByClaimRef read the node's own index, which a published inventory lags. diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go index 1201267..6c3a7dd 100644 --- a/pkg/scale/sandboxstore_lifecycle.go +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -29,12 +29,12 @@ func (s *scatterGatherStore) Pause(ctx context.Context, node, id string) error { return nil } -func (s *scatterGatherStore) DialGuestPort(ctx context.Context, node, id string, port uint16) (net.Conn, error) { +func (s *scatterGatherStore) DialGuestPort(ctx context.Context, node, id, token string, port uint16) (net.Conn, error) { cl, err := s.nodeClient(ctx, node, "port", id) if err != nil { return nil, err } - conn, err := cl.DialPort(ctx, id, port) + conn, err := cl.DialPort(ctx, id, token, port) if err != nil { return nil, nodeVerbError(err, "port", id, node) } diff --git a/pkg/scale/sandboxstore_lifecycle_test.go b/pkg/scale/sandboxstore_lifecycle_test.go index eff2a29..09af61c 100644 --- a/pkg/scale/sandboxstore_lifecycle_test.go +++ b/pkg/scale/sandboxstore_lifecycle_test.go @@ -25,7 +25,7 @@ func TestLifecycleVerbsMapANodeUnknownSandboxToNotFound(t *testing.T) { ctx := t.Context() f.dialErr = &sandboxd.HTTPError{StatusCode: http.StatusNotFound} - _, portErr := store.DialGuestPort(ctx, "n1", "sb_gone", 8080) + _, portErr := store.DialGuestPort(ctx, "n1", "sb_gone", "", 8080) _, forkErr := store.Fork(ctx, "ns", "n1", "sb_gone", 1, 0) _, snapErr := store.Snapshot(ctx, "n1", "sb_gone", "") _, _, promoteErr := store.Promote(ctx, "n1", "sb_gone", "tpl:x") @@ -83,15 +83,15 @@ func TestDialGuestPortMapsTheNodesPassiveRelayAnswers(t *testing.T) { store := NewScatterGatherStore(src, WithClaimRouting("t", f.factory())) ctx := t.Context() - conn, err := store.DialGuestPort(ctx, "n1", "sb_run", 49983) + conn, err := store.DialGuestPort(ctx, "n1", "sb_run", "", 49983) require.NoError(t, err) _ = conn.Close() assert.Equal(t, []uint16{49983}, f.dialPorts) - _, err = store.DialGuestPort(ctx, "n1", "sb_paused", 49983) + _, err = store.DialGuestPort(ctx, "n1", "sb_paused", "", 49983) assert.True(t, k8serrors.IsConflict(err), "a paused sandbox's 409 is Conflict, got %v", err) - _, err = store.DialGuestPort(ctx, "n1", "sb_gone", 49983) + _, err = store.DialGuestPort(ctx, "n1", "sb_gone", "", 49983) assert.True(t, k8serrors.IsNotFound(err), "an unknown claim is NotFound, got %v", err) - _, err = store.DialGuestPort(ctx, "n1", "sb_reset", 49983) + _, err = store.DialGuestPort(ctx, "n1", "sb_reset", "", 49983) require.Error(t, err) assert.False(t, k8serrors.IsNotFound(err) || k8serrors.IsConflict(err), "a transport failure is neither: %v", err) assert.Empty(t, f.rowReads, "the dial goes straight to the passive relay, with no by-id read first") From 40002c83aca58cdcad34d08644dc742229549043 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:52:00 +0800 Subject: [PATCH 09/23] fix(envdproxy): a signed file URL relays with the node token, so a stranger's signature never wakes a paused sandbox The signed path was placed by Locate and relayed with the sandbox's own claim token, which sandboxd treats as active: any request carrying a signature query, with no access token, woke a paused or archived sandbox and stamped activity before envd rejected the signature. The proxy now presents the node api_token on that path, and sandboxd's passive relay answers 409 for a paused sandbox without waking it. --- cmd/sandbox-e2b/main.go | 2 +- cmd/sandbox-envd-proxy/main.go | 1 + pkg/envdproxy/envdproxy.go | 9 ++++++++- pkg/envdproxy/envdproxy_test.go | 10 ++++++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/sandbox-e2b/main.go b/cmd/sandbox-e2b/main.go index 06472c5..edeb49d 100644 --- a/cmd/sandbox-e2b/main.go +++ b/cmd/sandbox-e2b/main.go @@ -98,7 +98,7 @@ func run() error { if err != nil { return err } - proxy, err := envdproxy.NewServer(resolver, envdproxy.Options{Domain: o.E2B.Domain, GuestHTTP2: o.Proxy.GuestHTTP2}) + proxy, err := envdproxy.NewServer(resolver, envdproxy.Options{Domain: o.E2B.Domain, GuestHTTP2: o.Proxy.GuestHTTP2, NodeToken: token}) if err != nil { return err } diff --git a/cmd/sandbox-envd-proxy/main.go b/cmd/sandbox-envd-proxy/main.go index e4b98b0..f3f6d40 100644 --- a/cmd/sandbox-envd-proxy/main.go +++ b/cmd/sandbox-envd-proxy/main.go @@ -102,6 +102,7 @@ func run(ctx context.Context, o *options) error { srv, err := envdproxy.NewServer(resolver, envdproxy.Options{ Domain: o.Domain, GuestHTTP2: o.Proxy.GuestHTTP2, + NodeToken: token, }) if err != nil { return err diff --git a/pkg/envdproxy/envdproxy.go b/pkg/envdproxy/envdproxy.go index 62a990a..291f5e4 100644 --- a/pkg/envdproxy/envdproxy.go +++ b/pkg/envdproxy/envdproxy.go @@ -49,6 +49,8 @@ type Options struct { // upgrade, which would fail every request; turn it on for a guest daemon // that serves h2c, such as a user's own server on another port. GuestHTTP2 bool + // NodeToken is the sandboxd api_token a signed file URL's relay presents, so that relay stays passive and never wakes a paused sandbox. + NodeToken string } // Server routes one public host onto many sandboxes' guest ports. @@ -67,6 +69,9 @@ func NewServer(resolver Resolver, opts Options) (*Server, error) { if strings.TrimSpace(opts.Domain) == "" { return nil, errors.New("envdproxy: no domain configured; the SDK's sandbox host is derived from it") } + if strings.TrimSpace(opts.NodeToken) == "" { + return nil, errors.New("envdproxy: the node token is required; a signed file URL relays with it") + } dialer := &net.Dialer{Timeout: dialTimeout} return &Server{resolver: resolver, transport: newGuestTransport(dialGuest(dialer)), opts: opts}, nil } @@ -100,7 +105,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { case token != "": owner, err = s.resolver.Owner(r.Context(), rt.sandboxID, token) case signedFileURL(r): - owner, err = s.resolver.Locate(r.Context(), rt.sandboxID) + if owner, err = s.resolver.Locate(r.Context(), rt.sandboxID); err == nil { + owner.Token = s.opts.NodeToken + } default: writeError(w, http.StatusUnauthorized, "missing "+accessTokenHeader) return diff --git a/pkg/envdproxy/envdproxy_test.go b/pkg/envdproxy/envdproxy_test.go index 7da628c..2e459bc 100644 --- a/pkg/envdproxy/envdproxy_test.go +++ b/pkg/envdproxy/envdproxy_test.go @@ -105,6 +105,9 @@ func TestProxyRelaysASignedFileURLWithoutTheToken(t *testing.T) { if strings.Contains(string(body), accessTokenHeader) { t.Errorf("a signed request gained an access token on the way: %q", body) } + if node.lastAuth != "Bearer root" { + t.Errorf("a signed request relayed with %q, want the node token: the passive relay never wakes a paused sandbox for a stranger's signature", node.lastAuth) + } for _, path := range []string{"/files?path=%2Fetc%2Fhosts", "/envs?signature=v1_abc"} { resp := request(t, h, "49983-sb-abc."+testDomain, path, "") _ = resp.Body.Close() @@ -424,6 +427,9 @@ func TestNewServerRequiresADomainAndAResolver(t *testing.T) { if _, err := NewServer(nil, Options{Domain: testDomain}); err == nil { t.Error("NewServer accepted a nil resolver") } + if _, err := NewServer(resolverFunc(nil), Options{Domain: testDomain}); err == nil { + t.Error("NewServer without a node token must fail") + } if _, err := NewServer(resolverFunc(nil), Options{}); err == nil { t.Error("NewServer accepted an empty domain; every sandbox host is derived from it") } @@ -431,7 +437,7 @@ func TestNewServerRequiresADomainAndAResolver(t *testing.T) { func newTestProxy(t *testing.T, r Resolver, opts ...func(*Options)) http.Handler { t.Helper() - o := Options{Domain: testDomain} + o := Options{Domain: testDomain, NodeToken: "root"} for _, fn := range opts { fn(&o) } @@ -566,7 +572,7 @@ func (n *fakeNode) handle(conn net.Conn, guest func(net.Conn)) { } n.lastPath, n.lastAuth = req.URL.Path, req.Header.Get("Authorization") refuse := n.refuse - if refuse == 0 && n.lastAuth != "Bearer tok" { + if refuse == 0 && n.lastAuth != "Bearer tok" && n.lastAuth != "Bearer root" { refuse = http.StatusNotFound } if refuse != 0 { From 8356c6df045761a8cb282d429fe319225838f08b Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 13:54:20 +0800 Subject: [PATCH 10/23] review: named func types at the build seams, exported var first, modern loops, one logger per function /code R9: LineFunc, ArchiveFunc and PublishFunc replace the func types spelled twice across e2bbuild and e2bcompat, managerBuilder the one spelled twice in sandbox-apiserver, serverOption the test option spelled five times; R1: FilesHash precedes the unexported var; R8: globBase uses slices.IndexFunc, the record sweep maps.DeleteFunc, the test fake slices.Clone; R4: createSandbox binds its logger once for two calls. --- cmd/sandbox-apiserver/main.go | 4 +++- cmd/sandbox-envd-proxy/main.go | 2 +- pkg/e2bbuild/copy.go | 7 +++---- pkg/e2bbuild/executor.go | 30 +++++++++++++++++---------- pkg/e2bbuild/executor_test.go | 5 +++-- pkg/e2bcompat/aliases_test.go | 2 +- pkg/e2bcompat/builds.go | 2 +- pkg/e2bcompat/builds_test.go | 36 ++++++++++++++++----------------- pkg/e2bcompat/envd.go | 4 ++-- pkg/e2bcompat/server.go | 5 +++-- pkg/e2bcompat/server_test.go | 4 +++- pkg/e2bcompat/templates_test.go | 26 ++++++++++++------------ pkg/e2bcompat/uploads.go | 2 +- pkg/e2bcompat/uploads_test.go | 2 +- 14 files changed, 72 insertions(+), 59 deletions(-) diff --git a/cmd/sandbox-apiserver/main.go b/cmd/sandbox-apiserver/main.go index 00866ef..8358855 100644 --- a/cmd/sandbox-apiserver/main.go +++ b/cmd/sandbox-apiserver/main.go @@ -44,6 +44,8 @@ const ( watchDrainGrace = 2 * time.Second ) +type managerBuilder func() (manager.Runnable, error) + // options has no etcd option because this server stores nothing. type options struct { SecureServing *genericoptions.SecureServingOptionsWithLoopback @@ -221,7 +223,7 @@ func startWarmPoolDriver(ctx context.Context, restCfg *restclient.Config, token return nil } -func runRestarting(ctx context.Context, r manager.Runnable, build func() (manager.Runnable, error), delay time.Duration) { +func runRestarting(ctx context.Context, r manager.Runnable, build managerBuilder, delay time.Duration) { logger := log.WithFunc("main.runRestarting") for { if r != nil { diff --git a/cmd/sandbox-envd-proxy/main.go b/cmd/sandbox-envd-proxy/main.go index f3f6d40..78c63a0 100644 --- a/cmd/sandbox-envd-proxy/main.go +++ b/cmd/sandbox-envd-proxy/main.go @@ -68,7 +68,7 @@ func main() { o.addFlags(fs) _ = fs.Parse(os.Args[1:]) if err := run(ctx, o); err != nil { - log.WithFunc("main").Fatalf(ctx, err, "sandbox-envd-proxy exited") + log.WithFunc("main.main").Fatalf(ctx, err, "sandbox-envd-proxy exited") } } diff --git a/pkg/e2bbuild/copy.go b/pkg/e2bbuild/copy.go index 563e423..e0b4521 100644 --- a/pkg/e2bbuild/copy.go +++ b/pkg/e2bbuild/copy.go @@ -2,6 +2,7 @@ package e2bbuild import ( "path" + "slices" "strings" ) @@ -68,10 +69,8 @@ func copyScript(state Command, s Step) string { // globBase is src up to its first segment with a glob; the SDK expands globs into the archive itself. func globBase(src string) string { segs := strings.Split(strings.TrimSuffix(src, "/"), "/") - for i, seg := range segs { - if strings.ContainsAny(seg, "*?[{") { - return strings.Join(segs[:i], "/") - } + if i := slices.IndexFunc(segs, func(seg string) bool { return strings.ContainsAny(seg, "*?[{") }); i >= 0 { + segs = segs[:i] } return strings.Join(segs, "/") } diff --git a/pkg/e2bbuild/executor.go b/pkg/e2bbuild/executor.go index 67f7de2..648379c 100644 --- a/pkg/e2bbuild/executor.go +++ b/pkg/e2bbuild/executor.go @@ -55,11 +55,21 @@ var ( // ErrUnknownBuild is a build this process never registered, or dropped an hour after it finished or was left unstarted. ErrUnknownBuild = errors.New("e2bbuild: build not found") - envEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "`", "\\`", "$(", `\$(`) // FilesHash is the shape of the digest the SDK names a COPY upload by. FilesHash = regexp.MustCompile(`^[0-9a-f]{64}$`) + + envEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "`", "\\`", "$(", `\$(`) ) +// LineFunc receives one line of a command's output. +type LineFunc func(line string) + +// ArchiveFunc opens the upload a COPY step's files hash names. +type ArchiveFunc func(ctx context.Context, hash string) (io.ReadCloser, error) + +// PublishFunc runs after the promote with the template's node, key and content digest. +type PublishFunc func(ctx context.Context, node string, key scale.PoolKey, digest string) error + // Request is what a build asked for before it starts. type Request struct { Size string @@ -110,8 +120,8 @@ type Spec struct { StartCmd string ReadyCmd string RelayEnvs map[string]string - Archive func(ctx context.Context, hash string) (io.ReadCloser, error) - Publish func(ctx context.Context, node string, key scale.PoolKey, digest string) error + Archive ArchiveFunc + Publish PublishFunc } // Command is one shell line run as User, in Workdir, with Envs; an empty Workdir is the user's home. @@ -125,7 +135,7 @@ type Command struct { // Guest runs a build's commands inside its claimed sandbox. type Guest interface { // Run runs cmd to its end, passes each line of its output to stdout or stderr, and returns its exit code. - Run(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr func(line string)) (int, error) + Run(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr LineFunc) (int, error) // Start starts cmd and returns while it runs. Start(ctx context.Context, a scale.Assignment, cmd Command) error // Init makes the user, workdir and envs of defaults what every later process in the sandbox gets. @@ -352,7 +362,7 @@ func (e *Executor) copyIn(ctx context.Context, a scale.Assignment, spec Spec, ha } // sh returns the failure the caller sees next to the error; a non-zero exit is a failure. -func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr func(string)) (string, error) { +func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr LineFunc) (string, error) { code, err := e.guest.Run(ctx, a, cmd, stdout, stderr) if err != nil { return "could not run a command in the build sandbox", err @@ -365,7 +375,7 @@ func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, stdo } // expand evaluates an ENV value in the guest's shell, so $VAR references resolve and command substitution does not run; the shell's own stderr goes to the log, never into the value. -func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, value string, stderr func(string)) (string, string, error) { +func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, value string, stderr LineFunc) (string, string, error) { var lines []string if _, err := e.sh(ctx, a, withLine(root, `printf "%s" "`+envEscaper.Replace(value)+`"`), func(line string) { lines = append(lines, line) }, stderr); err != nil { return "", fmt.Sprintf("could not evaluate the value %q", value), err @@ -440,13 +450,11 @@ func (e *Executor) log(id, step, level, message string) { } func (e *Executor) sweep(now time.Time) { - for id, r := range e.builds { + maps.DeleteFunc(e.builds, func(_ string, r *record) bool { done := !r.finished.IsZero() && now.Sub(r.finished) > recordTTL abandoned := r.info.Status == StatusWaiting && now.Sub(r.registered) > recordTTL - if done || abandoned { - delete(e.builds, id) - } - } + return done || abandoned + }) } // Invalid names the first step a build cannot run, empty when every step can; a COPY needs canCopy. diff --git a/pkg/e2bbuild/executor_test.go b/pkg/e2bbuild/executor_test.go index efc5151..bee19f8 100644 --- a/pkg/e2bbuild/executor_test.go +++ b/pkg/e2bbuild/executor_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "slices" "strings" "sync" "testing" @@ -283,7 +284,7 @@ func (f *fakeStore) Release(_ context.Context, node, id string) error { return nil } -func (f *fakeStore) Run(_ context.Context, _ scale.Assignment, cmd Command, stdout, stderr func(string)) (int, error) { +func (f *fakeStore) Run(_ context.Context, _ scale.Assignment, cmd Command, stdout, stderr LineFunc) (int, error) { f.record(fmt.Sprintf("run %s %s %v %s", cmd.User, cmd.Workdir, cmd.Envs, cmd.Line)) if v, ok := strings.CutPrefix(cmd.Line, `printf "%s" "`); ok { stderr("bash: warning: setlocale: LC_ALL: cannot change locale") @@ -328,5 +329,5 @@ func (f *fakeStore) record(line string) { func (f *fakeStore) calls() []string { f.mu.Lock() defer f.mu.Unlock() - return append([]string(nil), f.log...) + return slices.Clone(f.log) } diff --git a/pkg/e2bcompat/aliases_test.go b/pkg/e2bcompat/aliases_test.go index 4a022e8..bb362cd 100644 --- a/pkg/e2bcompat/aliases_test.go +++ b/pkg/e2bcompat/aliases_test.go @@ -95,6 +95,6 @@ func TestNewServerRefusesAMalformedAliasTable(t *testing.T) { } } -func withAliases(entries ...string) func(*Options) { +func withAliases(entries ...string) serverOption { return func(o *Options) { o.TemplateAliases = entries } } diff --git a/pkg/e2bcompat/builds.go b/pkg/e2bcompat/builds.go index a383023..54fa34b 100644 --- a/pkg/e2bcompat/builds.go +++ b/pkg/e2bcompat/builds.go @@ -135,7 +135,7 @@ func (s *Server) buildStatus(w http.ResponseWriter, r *http.Request) { // publishBuild deletes the holders of name older than this build's promote, so concurrent builds converge on the newest, then tags it. // It asks each node itself: an inventory a tick behind would miss a build that finished moments ago. -func (s *Server) publishBuild(scope, name string, tags []string) func(context.Context, string, scale.PoolKey, string) error { +func (s *Server) publishBuild(scope, name string, tags []string) e2bbuild.PublishFunc { return func(ctx context.Context, node string, key scale.PoolKey, digest string) error { holders, err := s.liveHolders(ctx, scope+name) if err != nil { diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index cf23e70..b847b52 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -146,6 +146,22 @@ func TestABuildRefusesWhatItCannotHonor(t *testing.T) { assert.Equal(t, http.StatusNotFound, do(t, newTestServer(t, &fakeStore{}), http.MethodPost, "/v3/templates", `{"name":"app"}`, testKey).Code, "builds are off unless enabled") } +func TestAPublishLeavesANewerConcurrentBuildAlone(t *testing.T) { + newer := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:newer", CreatedAt: new(metav1.NewTime(time.Now().Add(time.Minute)))} + older := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:older", CreatedAt: new(metav1.NewTime(time.Now().Add(-time.Minute)))} + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil, "m": {newer}, "k": {older}}} + inv := scale.NewStaticInventorySource() + for _, n := range []string{"n", "m", "k"} { + inv.Put(&scale.NodeInventory{Node: n}) + } + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status) + assert.Equal(t, []string{"k e2b/sandboxes/app small"}, store.deletedTemplates, "only the holder older than this promote goes; a newer concurrent build keeps its node") +} + func requestBuild(t *testing.T, h http.Handler, name string) string { t.Helper() w := do(t, h, http.MethodPost, "/v3/templates", `{"name":"`+name+`"}`, testKey) @@ -168,31 +184,15 @@ func waitBuild(t *testing.T, h http.Handler, buildID string) TemplateBuildInfo { return info } -func withBuilds() func(*Options) { +func withBuilds() serverOption { return func(o *Options) { o.Builds = BuildOptions{Parallel: 2, Timeout: time.Minute, LogLines: 100} } } // withBuildFleet serves node n with app's earlier build and node m with a stale copy under a smaller key. -func withBuildFleet(store *fakeStore) func(*Options) { +func withBuildFleet(store *fakeStore) serverOption { inv := scale.NewStaticInventorySource() inv.Put(&scale.NodeInventory{Node: "n", Templates: []scale.PromotedTemplate{{Template: "e2b/sandboxes/app", Net: "none", Size: "medium", ContentDigest: "sha256:old"}}}) inv.Put(&scale.NodeInventory{Node: "m", Templates: []scale.PromotedTemplate{{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:old"}}}) store.fleet = inv return func(o *Options) { o.Inventory = inv } } - -func TestAPublishLeavesANewerConcurrentBuildAlone(t *testing.T) { - newer := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:newer", CreatedAt: new(metav1.NewTime(time.Now().Add(time.Minute)))} - older := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:older", CreatedAt: new(metav1.NewTime(time.Now().Add(-time.Minute)))} - store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil, "m": {newer}, "k": {older}}} - inv := scale.NewStaticInventorySource() - for _, n := range []string{"n", "m", "k"} { - inv.Put(&scale.NodeInventory{Node: n}) - } - h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) - - id := requestBuild(t, h, "app") - require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) - require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status) - assert.Equal(t, []string{"k e2b/sandboxes/app small"}, store.deletedTemplates, "only the holder older than this promote goes; a newer concurrent build keeps its node") -} diff --git a/pkg/e2bcompat/envd.go b/pkg/e2bcompat/envd.go index f6488b9..3eafd0e 100644 --- a/pkg/e2bcompat/envd.go +++ b/pkg/e2bcompat/envd.go @@ -108,7 +108,7 @@ type envdGuest struct { s *Server } -func (g envdGuest) Run(ctx context.Context, a scale.Assignment, cmd e2bbuild.Command, stdout, stderr func(string)) (int, error) { +func (g envdGuest) Run(ctx context.Context, a scale.Assignment, cmd e2bbuild.Command, stdout, stderr e2bbuild.LineFunc) (int, error) { outLines, errLines := &lineSplitter{out: stdout}, &lineSplitter{out: stderr} code, _, err := g.s.envdProcess(ctx, a, cmd, func(o, e []byte) { outLines.write(o) @@ -153,7 +153,7 @@ func (g envdGuest) Write(ctx context.Context, a scale.Assignment, path string, r // lineSplitter hands out complete lines, and a partial one once it outgrows logLineMax. type lineSplitter struct { buf []byte - out func(string) + out e2bbuild.LineFunc } func (l *lineSplitter) write(p []byte) { diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index f8a3315..fb42b62 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -319,6 +319,7 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { return } + logger := log.WithFunc("e2bcompat.createSandbox") name := names.SimpleNameGenerator.GenerateName(namePrefix) pool := s.poolKey(req.TemplateID) pool.Net = netFor(req.AllowInternetAccess) @@ -350,7 +351,7 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { writeError(w, he.StatusCode, he.Message) return } - log.WithFunc("e2bcompat.createSandbox").Errorf(r.Context(), err, "e2b create: claim failed template=%s name=%s", req.TemplateID, name) + logger.Errorf(r.Context(), err, "e2b create: claim failed template=%s name=%s", req.TemplateID, name) writeError(w, http.StatusInternalServerError, "failed to claim a sandbox") return } @@ -366,7 +367,7 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { err = s.initEnvd(r.Context(), assignment.Node, assignment.SandboxName, "", init) } if err != nil { - log.WithFunc("e2bcompat.createSandbox").Errorf(r.Context(), err, "e2b create: envd init failed sandboxID=%s node=%s", assignment.SandboxName, assignment.Node) + logger.Errorf(r.Context(), err, "e2b create: envd init failed sandboxID=%s node=%s", assignment.SandboxName, assignment.Node) s.releaseAll(r.Context(), []scale.Assignment{assignment}) writeError(w, http.StatusInternalServerError, "failed to start the sandbox") return diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index fb3117b..b39b071 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -814,7 +814,9 @@ func (f *renewStore) Renew(_ context.Context, node, id string, ttlSeconds int, _ return time.Now().Add(time.Duration(ttlSeconds) * time.Second), nil } -func newTestServer(t *testing.T, store scale.SandboxStore, opts ...func(*Options)) http.Handler { +type serverOption func(*Options) + +func newTestServer(t *testing.T, store scale.SandboxStore, opts ...serverOption) http.Handler { t.Helper() o := Options{Namespace: "sandboxes", Domain: testDomain, APIKeys: []string{testKey}, EnvdSecret: []byte(testEnvdSecret)} for _, fn := range opts { diff --git a/pkg/e2bcompat/templates_test.go b/pkg/e2bcompat/templates_test.go index 20c7564..aacd83d 100644 --- a/pkg/e2bcompat/templates_test.go +++ b/pkg/e2bcompat/templates_test.go @@ -191,7 +191,19 @@ func TestTheAliasLookupFindsABuiltTemplateAfterTheTable(t *testing.T) { assert.Equal(t, http.StatusNotFound, do(t, h, http.MethodGet, "/templates/aliases/other", "", testKey).Code) } -func withTemplateFleet(store *fakeStore) func(*Options) { +func TestAFullE2bKeyNeverReachesAnotherNamespacesTemplate(t *testing.T) { + store := &fakeStore{} + h := newTestServer(t, store, withBuilds(), withTemplateFleet(store)) + w := do(t, h, http.MethodPost, "/sandboxes", `{"templateID":"e2b/others/app"}`, testKey) + assert.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) + assert.Equal(t, 0, store.claimCalls, "a full key is refused before any claim") + + id := requestBuild(t, h, "app") + w = do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"e2b/others/app"}`, testKey) + assert.Equal(t, http.StatusBadRequest, w.Code, w.Body.String()) +} + +func withTemplateFleet(store *fakeStore) serverOption { created := &metav1.Time{Time: time.Date(2026, 9, 28, 1, 2, 3, 0, time.UTC)} built := func(name, digest string) scale.PromotedTemplate { return scale.PromotedTemplate{Template: name, Net: "none", Size: "medium", ContentDigest: digest, CreatedAt: created, CPUCount: 2, MemoryBytes: 1 << 30} @@ -231,15 +243,3 @@ func tagNames(tags []TemplateTag) []string { } return out } - -func TestAFullE2bKeyNeverReachesAnotherNamespacesTemplate(t *testing.T) { - store := &fakeStore{} - h := newTestServer(t, store, withBuilds(), withTemplateFleet(store)) - w := do(t, h, http.MethodPost, "/sandboxes", `{"templateID":"e2b/others/app"}`, testKey) - assert.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) - assert.Equal(t, 0, store.claimCalls, "a full key is refused before any claim") - - id := requestBuild(t, h, "app") - w = do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"e2b/others/app"}`, testKey) - assert.Equal(t, http.StatusBadRequest, w.Code, w.Body.String()) -} diff --git a/pkg/e2bcompat/uploads.go b/pkg/e2bcompat/uploads.go index ffac076..08c7f92 100644 --- a/pkg/e2bcompat/uploads.go +++ b/pkg/e2bcompat/uploads.go @@ -145,7 +145,7 @@ func (s *Server) fileUploadLink(w http.ResponseWriter, r *http.Request) { } // archive opens the uploads of one build's template for its COPY steps. -func (s *Server) archive(ns, name string) func(context.Context, string) (io.ReadCloser, error) { +func (s *Server) archive(ns, name string) e2bbuild.ArchiveFunc { return func(ctx context.Context, hash string) (io.ReadCloser, error) { return s.opts.Builds.Uploads.open(ctx, uploadKey{ns: ns, template: name, hash: hash}) } diff --git a/pkg/e2bcompat/uploads_test.go b/pkg/e2bcompat/uploads_test.go index 255bb07..997f49b 100644 --- a/pkg/e2bcompat/uploads_test.go +++ b/pkg/e2bcompat/uploads_test.go @@ -81,7 +81,7 @@ func TestACopyStepSendsItsUploadIntoTheSandbox(t *testing.T) { assert.True(t, strings.HasPrefix(store.envdCalls[1], "sb_1 RUN root archive='/tmp/"+hash+".tar'"), store.envdCalls[1]) } -func withUploads(t *testing.T, dir string, maxBytes int64) func(*Options) { +func withUploads(t *testing.T, dir string, maxBytes int64) serverOption { root, err := os.OpenRoot(dir) require.NoError(t, err) t.Cleanup(func() { _ = root.Close() }) From 4b0fd3545ece0134694aa7d6119ea3187f2bf401 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:11:46 +0800 Subject: [PATCH 11/23] simplify(scale): one fleet read per claim, direct PoolKey conversions, an informer-only inventory cache Claim read NodeCapacities once for the warm scan, again for the template scan and once more per redirect target; the candidate and address lookups are free functions over the one snapshot. The three template verbs converted PoolKey field by field between two structs of the same shape. The kubeinventory cache tuned its Get/List reader with UnsafeDisableDeepCopy and ReaderFailOnMissingInformer, but Source subscribes to informer events and nothing reads the cache, so the bench's copy and nocopy arms measured the same code. --- cmd/sandbox-apiserver/main.go | 4 +- cmd/sandbox-envd-proxy/main.go | 4 +- pkg/scale/export_test.go | 11 ++- pkg/scale/inventorycache_bench_test.go | 41 +++++------ pkg/scale/kubeinventory/cache.go | 9 +-- pkg/scale/kubeinventory/source_test.go | 6 +- pkg/scale/sandboxstore.go | 2 +- pkg/scale/sandboxstore_bench_test.go | 2 +- pkg/scale/sandboxstore_claim_test.go | 11 +-- pkg/scale/sandboxstore_impl.go | 97 +++++++++++--------------- pkg/scale/sandboxstore_lifecycle.go | 6 +- 11 files changed, 88 insertions(+), 105 deletions(-) diff --git a/cmd/sandbox-apiserver/main.go b/cmd/sandbox-apiserver/main.go index 8358855..3686aae 100644 --- a/cmd/sandbox-apiserver/main.go +++ b/cmd/sandbox-apiserver/main.go @@ -142,7 +142,7 @@ func run() error { if err != nil { return fmt.Errorf("load kube config: %w", err) } - reader, err := kubeinventory.NewCache(ctx, restCfg) + informers, err := kubeinventory.NewCache(ctx, restCfg) if err != nil { return err } @@ -150,7 +150,7 @@ func run() error { if err != nil { return err } - invSource, err := kubeinventory.New(ctx, reader, o.Inventory) + invSource, err := kubeinventory.New(ctx, informers, o.Inventory) if err != nil { return err } diff --git a/cmd/sandbox-envd-proxy/main.go b/cmd/sandbox-envd-proxy/main.go index 78c63a0..55596f6 100644 --- a/cmd/sandbox-envd-proxy/main.go +++ b/cmd/sandbox-envd-proxy/main.go @@ -78,11 +78,11 @@ func run(ctx context.Context, o *options) error { if err != nil { return fmt.Errorf("load kube config: %w", err) } - reader, err := kubeinventory.NewCache(ctx, restCfg) + informers, err := kubeinventory.NewCache(ctx, restCfg) if err != nil { return err } - inv, err := kubeinventory.New(ctx, reader, o.Inventory) + inv, err := kubeinventory.New(ctx, informers, o.Inventory) if err != nil { return err } diff --git a/pkg/scale/export_test.go b/pkg/scale/export_test.go index e8ae513..1047968 100644 --- a/pkg/scale/export_test.go +++ b/pkg/scale/export_test.go @@ -1,5 +1,7 @@ package scale +import "context" + var ( BenchFleets = benchFleets BenchInventories = benchInventories @@ -7,10 +9,17 @@ var ( ParseSelectors = parseSelectors SplitNamespacedName = splitNamespacedName - WarmCandidates = (*scatterGatherStore).warmCandidates LookupName = (*scatterGatherStore).lookupName ListInventories = (*scatterGatherStore).listInventories PollPinned = (*scatterGatherStore).pollPinned ) type ScatterGatherStore = scatterGatherStore + +func WarmCandidates(ctx context.Context, s *scatterGatherStore, pool PoolKey) ([]warmCandidate, error) { + nodes, err := s.src.NodeCapacities(ctx) + if err != nil { + return nil, err + } + return warmCandidates(nodes, pool), nil +} diff --git a/pkg/scale/inventorycache_bench_test.go b/pkg/scale/inventorycache_bench_test.go index 8163f86..2ddf424 100644 --- a/pkg/scale/inventorycache_bench_test.go +++ b/pkg/scale/inventorycache_bench_test.go @@ -16,7 +16,6 @@ import ( toolscache "k8s.io/client-go/tools/cache" sandboxv1beta1 "sigs.k8s.io/agent-sandbox/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" cocoonv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" "github.com/cocoonstack/sandbox-operator/pkg/scale" @@ -30,32 +29,27 @@ var errWatchListUnserved = errors.New("watch-list is not served") func BenchmarkClientInventoryWarmCandidates(b *testing.B) { for _, fleet := range scale.BenchFleets { - for _, arm := range []struct { - name string - noCopy bool - }{{"copy", false}, {"nocopy", true}} { - b.Run(fleet.Name+"/"+arm.name, func(b *testing.B) { - store, pool := benchCachedStore(b, fleet.Nodes, fleet.PerNode, arm.noCopy) - ctx := b.Context() - b.ReportAllocs() - for b.Loop() { - candidates, err := scale.WarmCandidates(store, ctx, pool) - if err != nil { - b.Fatalf("warm candidates: %v", err) - } - if len(candidates) != fleet.Nodes { - b.Fatalf("got %d candidates, want %d", len(candidates), fleet.Nodes) - } + b.Run(fleet.Name, func(b *testing.B) { + store, pool := benchCachedStore(b, fleet.Nodes, fleet.PerNode) + ctx := b.Context() + b.ReportAllocs() + for b.Loop() { + candidates, err := scale.WarmCandidates(ctx, store, pool) + if err != nil { + b.Fatalf("warm candidates: %v", err) } - }) - } + if len(candidates) != fleet.Nodes { + b.Fatalf("got %d candidates, want %d", len(candidates), fleet.Nodes) + } + } + }) } } func BenchmarkClientInventoryWatchTick(b *testing.B) { for _, fleet := range scale.BenchFleets { b.Run(fleet.Name, func(b *testing.B) { - store, _ := benchCachedStore(b, fleet.Nodes, fleet.PerNode, true) + store, _ := benchCachedStore(b, fleet.Nodes, fleet.PerNode) ns, name := scale.SplitNamespacedName(fmt.Sprintf("default/sb-%s-%d", scale.BenchNodeName(fleet.Nodes-1), fleet.PerNode-1)) labelSel, fieldSel, err := scale.ParseSelectors(scale.ListOptions{Namespace: ns, FieldSelector: "metadata.name=" + name}) if err != nil { @@ -91,7 +85,7 @@ func BenchmarkClientInventoryWatchTick(b *testing.B) { } // benchCachedStore serves the fleet through a real informer-fed cache reader, the production read path. -func benchCachedStore(b *testing.B, nodes, perNode int, noCopy bool) (*scale.ScatterGatherStore, scale.PoolKey) { +func benchCachedStore(b *testing.B, nodes, perNode int) (*scale.ScatterGatherStore, scale.PoolKey) { b.Helper() invs, pool := scale.BenchInventories(nodes, perNode, 0) list := &cocoonv1beta1.NodeInventoryList{} @@ -107,9 +101,8 @@ func benchCachedStore(b *testing.B, nodes, perNode int, noCopy bool) (*scale.Sca mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{scale.NodeInventoryGVK.GroupVersion()}) mapper.Add(scale.NodeInventoryGVK, meta.RESTScopeRoot) invCache, err := cache.New(&restclient.Config{Host: "http://127.0.0.1:1"}, cache.Options{ - Scheme: scheme, - Mapper: mapper, - ByObject: map[client.Object]cache.ByObject{inv: {UnsafeDisableDeepCopy: &noCopy}}, + Scheme: scheme, + Mapper: mapper, NewInformer: func(_ toolscache.ListerWatcher, obj runtime.Object, resync time.Duration, indexers toolscache.Indexers) toolscache.SharedIndexInformer { return toolscache.NewSharedIndexInformer(&toolscache.ListWatch{ ListWithContextFunc: func(context.Context, metav1.ListOptions) (runtime.Object, error) { return list, nil }, diff --git a/pkg/scale/kubeinventory/cache.go b/pkg/scale/kubeinventory/cache.go index de62147..f84504a 100644 --- a/pkg/scale/kubeinventory/cache.go +++ b/pkg/scale/kubeinventory/cache.go @@ -8,7 +8,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" restclient "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" cocoonv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" "github.com/cocoonstack/sandbox-operator/pkg/scale" @@ -17,18 +16,14 @@ import ( // cacheSyncTimeout bounds the startup wait for the informer, so a missing CRD fails loud instead of serving an empty fleet. const cacheSyncTimeout = 2 * time.Minute -// NewCache starts a cache scoped to NodeInventory alone and waits for it to sync; a read of any other kind fails instead of starting a cluster-wide informer. +// NewCache starts a cache scoped to NodeInventory alone and waits for it to sync. func NewCache(ctx context.Context, restCfg *restclient.Config) (cache.Cache, error) { scheme := runtime.NewScheme() if err := cocoonv1beta1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("kubeinventory: register node inventory scheme: %w", err) } inv := &scale.NodeInventory{} - invCache, err := cache.New(restCfg, cache.Options{ - Scheme: scheme, - ByObject: map[client.Object]cache.ByObject{inv: {UnsafeDisableDeepCopy: new(true)}}, - ReaderFailOnMissingInformer: true, - }) + invCache, err := cache.New(restCfg, cache.Options{Scheme: scheme}) if err != nil { return nil, fmt.Errorf("kubeinventory: build inventory cache: %w", err) } diff --git a/pkg/scale/kubeinventory/source_test.go b/pkg/scale/kubeinventory/source_test.go index 765407d..5ca76cc 100644 --- a/pkg/scale/kubeinventory/source_test.go +++ b/pkg/scale/kubeinventory/source_test.go @@ -18,7 +18,6 @@ import ( restclient "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" cocoonv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" "github.com/cocoonstack/sandbox-operator/pkg/scale" @@ -208,9 +207,8 @@ func fleet(t *testing.T, opts Options, objs ...*cocoonv1beta1.NodeInventory) (*S mapper.Add(scale.NodeInventoryGVK, meta.RESTScopeRoot) var watched bool c, err := cache.New(&restclient.Config{Host: "http://127.0.0.1:1"}, cache.Options{ - Scheme: scheme, - Mapper: mapper, - ByObject: map[client.Object]cache.ByObject{&cocoonv1beta1.NodeInventory{}: {UnsafeDisableDeepCopy: new(true)}}, + Scheme: scheme, + Mapper: mapper, NewInformer: func(_ toolscache.ListerWatcher, obj runtime.Object, resync time.Duration, indexers toolscache.Indexers) toolscache.SharedIndexInformer { return toolscache.NewSharedIndexInformer(&toolscache.ListWatch{ ListWithContextFunc: func(context.Context, metav1.ListOptions) (runtime.Object, error) { return list, nil }, diff --git a/pkg/scale/sandboxstore.go b/pkg/scale/sandboxstore.go index d6328b6..7584419 100644 --- a/pkg/scale/sandboxstore.go +++ b/pkg/scale/sandboxstore.go @@ -72,7 +72,7 @@ type SandboxStore interface { SandboxLifecycle } -// SandboxLifecycle is the verb set of a claimed sandbox, each verb routed to its owning node. +// SandboxLifecycle is the verb set routed to one node: a claimed sandbox's lifecycle, its checkpoints and the node's promoted templates. // Pause and Snapshot write guest memory out, so their cost grows with its size. type SandboxLifecycle interface { // Pause snapshots and stops the sandbox, and is idempotent on a paused one. diff --git a/pkg/scale/sandboxstore_bench_test.go b/pkg/scale/sandboxstore_bench_test.go index 9388eda..fac5d98 100644 --- a/pkg/scale/sandboxstore_bench_test.go +++ b/pkg/scale/sandboxstore_bench_test.go @@ -40,7 +40,7 @@ func BenchmarkStoreWarmCandidates(b *testing.B) { ctx := b.Context() b.ReportAllocs() for b.Loop() { - candidates, err := store.warmCandidates(ctx, pool) + candidates, err := WarmCandidates(ctx, store, pool) if err != nil { b.Fatalf("warm candidates: %v", err) } diff --git a/pkg/scale/sandboxstore_claim_test.go b/pkg/scale/sandboxstore_claim_test.go index 882b200..834df2e 100644 --- a/pkg/scale/sandboxstore_claim_test.go +++ b/pkg/scale/sandboxstore_claim_test.go @@ -201,7 +201,7 @@ func TestPickWarmNodeSpreadsAcrossTheFleet(t *testing.T) { picked := map[string]int{} for range 200 { - candidates, err := store.warmCandidates(t.Context(), PoolKey{Template: "img"}) + candidates, err := WarmCandidates(t.Context(), store, PoolKey{Template: "img"}) require.NoError(t, err) best := pickPowerOfTwo(candidates) picked[best.node]++ @@ -218,7 +218,7 @@ func TestPickWarmNodePrefersTheWarmerSample(t *testing.T) { warmPicks := 0 for range 200 { - candidates, err := store.warmCandidates(t.Context(), PoolKey{Template: "img"}) + candidates, err := WarmCandidates(t.Context(), store, PoolKey{Template: "img"}) require.NoError(t, err) best := pickPowerOfTwo(candidates) if best.node == "warm" { @@ -429,12 +429,13 @@ func TestWarmCandidatesMatchThePerNodeFanOut(t *testing.T) { } } } - got, err := store.warmCandidates(ctx, pool) + caps, err := store.src.NodeCapacities(ctx) require.NoError(t, err) + got := warmCandidates(caps, pool) assert.Equal(t, want, got) assert.Len(t, got, 3, "a's rt pool and both of d's; b is cold, c has no address, e is partitioned") - assert.Equal(t, "d", store.nodeForAddress(ctx, "10.0.0.4:7777"), "a node address resolves to its node") - assert.Empty(t, store.nodeForAddress(ctx, "10.0.0.5:7777"), "a partitioned node's address does not resolve") + assert.Equal(t, "d", nodeForAddress(caps, "10.0.0.4:7777"), "a node address resolves to its node") + assert.Empty(t, nodeForAddress(caps, "10.0.0.5:7777"), "a partitioned node's address does not resolve") } type claimCall struct { diff --git a/pkg/scale/sandboxstore_impl.go b/pkg/scale/sandboxstore_impl.go index 03928d7..2cb3c9e 100644 --- a/pkg/scale/sandboxstore_impl.go +++ b/pkg/scale/sandboxstore_impl.go @@ -233,15 +233,14 @@ func (s *scatterGatherStore) Claim(ctx context.Context, namespace, name string, if s.sandboxdFactory == nil { return Assignment{}, fmt.Errorf("scale: claim routing not configured (call WithClaimRouting)") } - candidates, err := s.warmCandidates(ctx, pool) + nodes, err := s.src.NodeCapacities(ctx) if err != nil { - return Assignment{}, err + return Assignment{}, fmt.Errorf("scale: enumerate node capacity: %w", err) } + candidates := warmCandidates(nodes, pool) promoted := len(candidates) == 0 if promoted { - if candidates, err = s.templateCandidates(ctx, pool); err != nil { - return Assignment{}, err - } + candidates = templateCandidates(nodes, pool) } if len(candidates) == 0 { return Assignment{}, fmt.Errorf("scale: claim %s/%s: no node advertises warm capacity for template %q net %q size %q: %w", namespace, name, pool.Template, pool.Net, pool.Size, ErrNoWarmCapacity) @@ -268,7 +267,7 @@ func (s *scatterGatherStore) Claim(ctx context.Context, namespace, name string, res, claimErr := s.sandboxdFactory(best.addr, s.sandboxdToken).Claim(ctx, spec) redirect, _ := errors.AsType[*sandboxd.RedirectError](claimErr) if redirect != nil { - node, res, claimErr = s.claimRedirected(ctx, node, redirect, spec) + node, res, claimErr = s.claimRedirected(ctx, nodes, node, redirect, spec) } if claimErr == nil { s.index.remember(nameKey(namespace, name), node) @@ -352,11 +351,11 @@ func (s *scatterGatherStore) matchOnNode(ctx context.Context, op, node string, m return nil } -func (s *scatterGatherStore) claimRedirected(ctx context.Context, from string, redirect *sandboxd.RedirectError, spec sandboxd.ClaimSpec) (string, sandboxd.ClaimResult, error) { +func (s *scatterGatherStore) claimRedirected(ctx context.Context, nodes []NodePools, from string, redirect *sandboxd.RedirectError, spec sandboxd.ClaimSpec) (string, sandboxd.ClaimResult, error) { logger := log.WithFunc("scale.claimRedirected") spec.NoRedirect = true for _, target := range redirect.Targets { - node := s.nodeForAddress(ctx, target) + node := nodeForAddress(nodes, target) if node == "" { logger.Debugf(ctx, "redirect target is no known node; skipping from=%s target=%s", from, target) continue @@ -374,53 +373,6 @@ func (s *scatterGatherStore) claimRedirected(ctx context.Context, from string, r return from, sandboxd.ClaimResult{}, redirect } -func (s *scatterGatherStore) nodeForAddress(ctx context.Context, addr string) string { - nodes, err := s.src.NodeCapacities(ctx) - if err != nil { - return "" - } - if i := slices.IndexFunc(nodes, func(n NodePools) bool { return n.Address == addr }); i >= 0 { - return nodes[i].Node - } - return "" -} - -func (s *scatterGatherStore) warmCandidates(ctx context.Context, pool PoolKey) ([]warmCandidate, error) { - nodes, err := s.src.NodeCapacities(ctx) - if err != nil { - return nil, fmt.Errorf("scale: enumerate node capacity: %w", err) - } - var out []warmCandidate - for _, n := range nodes { - if n.Address == "" { - continue - } - for _, pc := range n.Pools { - if pc.Warm > 0 && poolCapacityMatches(pc, pool) { - out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: pc.Warm}) - } - } - } - return out, nil -} - -// templateCandidates lists the nodes whose inventory holds pool as a promoted template, for a claim no warm pool serves. -func (s *scatterGatherStore) templateCandidates(ctx context.Context, pool PoolKey) ([]warmCandidate, error) { - nodes, err := s.src.NodeCapacities(ctx) - if err != nil { - return nil, fmt.Errorf("scale: enumerate node capacity: %w", err) - } - var out []warmCandidate - for _, n := range nodes { - if n.Address != "" && slices.ContainsFunc(n.Templates, func(t PromotedTemplate) bool { - return poolCapacityMatches(PoolCapacity{Template: t.Template, Net: t.Net, Size: t.Size}, pool) - }) { - out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: 1}) - } - } - return out, nil -} - func (s *scatterGatherStore) runWatch(ctx context.Context, opts ListOptions, labelSel labels.Selector, fieldSel fields.Selector, w *watch.ProxyWatcher, ch chan watch.Event) { defer close(ch) @@ -808,6 +760,41 @@ func fanOutNodes[T any](ctx context.Context, s *scatterGatherStore, work func(ct } // poolCapacityMatches compares pc against key, defaulting each unset net/size axis. +func nodeForAddress(nodes []NodePools, addr string) string { + if i := slices.IndexFunc(nodes, func(n NodePools) bool { return n.Address == addr }); i >= 0 { + return nodes[i].Node + } + return "" +} + +func warmCandidates(nodes []NodePools, pool PoolKey) []warmCandidate { + var out []warmCandidate + for _, n := range nodes { + if n.Address == "" { + continue + } + for _, pc := range n.Pools { + if pc.Warm > 0 && poolCapacityMatches(pc, pool) { + out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: pc.Warm}) + } + } + } + return out +} + +// templateCandidates lists the nodes whose inventory holds pool as a promoted template, for a claim no warm pool serves. +func templateCandidates(nodes []NodePools, pool PoolKey) []warmCandidate { + var out []warmCandidate + for _, n := range nodes { + if n.Address != "" && slices.ContainsFunc(n.Templates, func(t PromotedTemplate) bool { + return poolCapacityMatches(PoolCapacity{Template: t.Template, Net: t.Net, Size: t.Size}, pool) + }) { + out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: 1}) + } + } + return out +} + func poolCapacityMatches(pc PoolCapacity, key PoolKey) bool { return pc.Template == key.Template && cmp.Or(pc.Net, NetDefault) == cmp.Or(key.Net, NetDefault) && diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go index 6c3a7dd..db45b2f 100644 --- a/pkg/scale/sandboxstore_lifecycle.go +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -146,7 +146,7 @@ func (s *scatterGatherStore) DeleteTemplate(ctx context.Context, node string, ke if err != nil { return err } - if err := cl.DeleteTemplate(ctx, sandboxd.PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}); err != nil { + if err := cl.DeleteTemplate(ctx, sandboxd.PoolKey(key)); err != nil { return nodeVerbError(err, "delete template", key.Template, node) } return nil @@ -161,7 +161,7 @@ func (s *scatterGatherStore) Promote(ctx context.Context, node, id, template str if err != nil { return PoolKey{}, "", nodeVerbError(err, "promote", id, node) } - return PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}, digest, nil + return PoolKey(key), digest, nil } func (s *scatterGatherStore) SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string) error { @@ -169,7 +169,7 @@ func (s *scatterGatherStore) SetTemplateLabels(ctx context.Context, node string, if err != nil { return err } - if err := cl.SetTemplateLabels(ctx, sandboxd.PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}, labels); err != nil { + if err := cl.SetTemplateLabels(ctx, sandboxd.PoolKey(key), labels); err != nil { return nodeVerbError(err, "template labels", key.Template, node) } return nil From 9902e12c69dd1f4b0378ac0769471d10ae21471e Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:13:22 +0800 Subject: [PATCH 12/23] simplify(envdproxy): the probe returns the owner, the resolver builds its own claim reader, the transport dials the relay itself probe carried the found Owner out of FirstHit through a mutex-guarded side variable although FirstHit is generic over the Owner value; NewResolver took a placed store every caller built from the inventory it also passed, then type-asserted it back; guestDialer and dialGuest were a type and constructor whose one instance forwarded one call to sandboxd.DialPort. The refusal comment and its test case named a wrong token, but the relay opens with the claim token the edge read from the node, so a 404 there is a stale cached owner. --- cmd/sandbox-e2b/main.go | 2 +- cmd/sandbox-envd-proxy/main.go | 2 +- pkg/envdproxy/dial.go | 17 +++-------------- pkg/envdproxy/envdproxy.go | 7 +++---- pkg/envdproxy/envdproxy_test.go | 6 +++--- pkg/envdproxy/resolver.go | 28 ++++++++-------------------- 6 files changed, 19 insertions(+), 43 deletions(-) diff --git a/cmd/sandbox-e2b/main.go b/cmd/sandbox-e2b/main.go index edeb49d..19c3489 100644 --- a/cmd/sandbox-e2b/main.go +++ b/cmd/sandbox-e2b/main.go @@ -94,7 +94,7 @@ func run() error { <-ctx.Done() return nil } - resolver, err := envdproxy.NewResolver(scale.NewScatterGatherStore(src), store, src, "", opts.EnvdSecret) + resolver, err := envdproxy.NewResolver(store, src, "", opts.EnvdSecret) if err != nil { return err } diff --git a/cmd/sandbox-envd-proxy/main.go b/cmd/sandbox-envd-proxy/main.go index 55596f6..dc5c4de 100644 --- a/cmd/sandbox-envd-proxy/main.go +++ b/cmd/sandbox-envd-proxy/main.go @@ -95,7 +95,7 @@ func run(ctx context.Context, o *options) error { return err } routed := scale.NewScatterGatherStore(inv, scale.WithClaimRouting(token, scale.NewSandboxdClientFactory())) - resolver, err := envdproxy.NewResolver(scale.NewScatterGatherStore(inv), routed, inv, o.Namespace, secret) + resolver, err := envdproxy.NewResolver(routed, inv, o.Namespace, secret) if err != nil { return err } diff --git a/pkg/envdproxy/dial.go b/pkg/envdproxy/dial.go index f621788..f8264e0 100644 --- a/pkg/envdproxy/dial.go +++ b/pkg/envdproxy/dial.go @@ -22,17 +22,6 @@ type target struct { // targetKey addresses target in a request context. type targetKey struct{} -// guestDialer opens the connection a request's target names. -type guestDialer func(ctx context.Context, t target) (net.Conn, error) - -// dialGuest opens sandboxd's guest-port relay and hands back the upgraded -// connection as a plain net.Conn. -func dialGuest(dialer *net.Dialer) guestDialer { - return func(ctx context.Context, t target) (net.Conn, error) { - return sandboxd.DialPort(ctx, dialer, t.owner.Address, t.owner.ClaimID, t.owner.Token, t.port) - } -} - // guestTransport carries a request to the guest daemon over the protocol that // daemon serves, which is not the one the client used: the edge may answer // HTTP/2 while the guest speaks only HTTP/1.1. @@ -41,16 +30,16 @@ type guestTransport struct { h2 *http.Transport } -// newGuestTransport builds both halves over dial. Neither pools: a reused +// newGuestTransport builds both halves over sandboxd's guest-port relay. Neither pools: a reused // connection would outlive the sandboxd relay carrying it, and reusing one // across sandboxes would cross a tenancy boundary. -func newGuestTransport(dial guestDialer) *guestTransport { +func newGuestTransport(dialer *net.Dialer) *guestTransport { dialContext := func(ctx context.Context, _, _ string) (net.Conn, error) { t, ok := targetFrom(ctx) if !ok { return nil, errNoTarget } - return dial(ctx, t) + return sandboxd.DialPort(ctx, dialer, t.owner.Address, t.owner.ClaimID, t.owner.Token, t.port) } // With HTTP/1 also set, a plaintext transport cannot negotiate HTTP/2 and picks HTTP/1. var h2 http.Protocols diff --git a/pkg/envdproxy/envdproxy.go b/pkg/envdproxy/envdproxy.go index 291f5e4..e6b8607 100644 --- a/pkg/envdproxy/envdproxy.go +++ b/pkg/envdproxy/envdproxy.go @@ -73,7 +73,7 @@ func NewServer(resolver Resolver, opts Options) (*Server, error) { return nil, errors.New("envdproxy: the node token is required; a signed file URL relays with it") } dialer := &net.Dialer{Timeout: dialTimeout} - return &Server{resolver: resolver, transport: newGuestTransport(dialGuest(dialer)), opts: opts}, nil + return &Server{resolver: resolver, transport: newGuestTransport(dialer), opts: opts}, nil } // Handler returns the routed handler. Serve it with Protocols(): a ConnectRPC @@ -159,9 +159,8 @@ func (s *Server) sandboxHost(rt route) string { return strconv.FormatUint(uint64(rt.port), 10) + "-" + rt.sandboxID + "." + s.opts.Domain } -// writeUpstreamError maps a node's refusal without naming the node. sandboxd -// answers 404 for both an unknown id and a wrong token; the id was just -// resolved from inventory, so the token is what the caller can still fix. +// writeUpstreamError maps a node's refusal without naming the node. The relay opens with the claim +// token the edge read from the node, so its 404 or 401 means the cached owner is stale (released within recentOwnerTTL). func (s *Server) writeUpstreamError(w http.ResponseWriter, r *http.Request, err error) { if status, ok := errors.AsType[*sandboxd.HTTPError](err); ok { switch status.StatusCode { diff --git a/pkg/envdproxy/envdproxy_test.go b/pkg/envdproxy/envdproxy_test.go index 2e459bc..cd34fcc 100644 --- a/pkg/envdproxy/envdproxy_test.go +++ b/pkg/envdproxy/envdproxy_test.go @@ -214,7 +214,7 @@ func TestProxyAdmitsOnlyTheTokenDerivedFromThePublishedClaim(t *testing.T) { src := &countingSource{StaticInventorySource: scale.NewStaticInventorySource()} src.Put(&scale.NodeInventory{Name: "node-0", Node: "node-0", Address: owner.addr, Entries: []scale.InventoryEntry{{Name: "sandboxes/s1", ID: "sb_abc"}}}) routed := scale.NewScatterGatherStore(src, scale.WithClaimRouting("root", scale.NewSandboxdClientFactory())) - r, err := NewResolver(scale.NewScatterGatherStore(src), routed, src, "", []byte(testSecret)) + r, err := NewResolver(routed, src, "", []byte(testSecret)) if err != nil { t.Fatalf("NewResolver: %v", err) } @@ -296,7 +296,7 @@ func TestProxyMapsNodeRefusals(t *testing.T) { nodeStatus int want int }{ - {"wrong token", http.StatusNotFound, http.StatusUnauthorized}, + {"stale owner", http.StatusNotFound, http.StatusUnauthorized}, {"bad port", http.StatusBadRequest, http.StatusBadRequest}, {"no guest listener", http.StatusBadGateway, http.StatusBadGateway}, } @@ -456,7 +456,7 @@ func unpublishedResolver(t *testing.T, nodes ...*fakeNode) (Resolver, *countingS src.Put(&scale.NodeInventory{Name: name, Node: name, Address: n.addr}) } routed := scale.NewScatterGatherStore(src, scale.WithClaimRouting("root", scale.NewSandboxdClientFactory())) - r, err := NewResolver(scale.NewScatterGatherStore(src), routed, src, "", []byte(testSecret)) + r, err := NewResolver(routed, src, "", []byte(testSecret)) if err != nil { t.Fatalf("NewResolver: %v", err) } diff --git a/pkg/envdproxy/resolver.go b/pkg/envdproxy/resolver.go index 0ea2328..8c843a2 100644 --- a/pkg/envdproxy/resolver.go +++ b/pkg/envdproxy/resolver.go @@ -68,13 +68,9 @@ type storeResolver struct { recent *recentOwners } -// NewResolver builds the Resolver. placed must read inventory only, so an unknown id reaches the nodes -// only through the rate-limited probe; an empty namespace matches every namespace. -func NewResolver(placed scale.SandboxStore, routed scale.SandboxLifecycle, inventory scale.InventorySource, namespace string, secret []byte) (Resolver, error) { - claims, ok := placed.(scale.ClaimIDResolver) - if !ok { - return nil, errors.New("envdproxy: store does not implement scale.ClaimIDResolver") - } +// NewResolver builds the Resolver; its claim lookups read inventory only, so an unknown id reaches the nodes +// only through the rate-limited probe, and an empty namespace matches every namespace. +func NewResolver(routed scale.SandboxLifecycle, inventory scale.InventorySource, namespace string, secret []byte) (Resolver, error) { if routed == nil || inventory == nil { return nil, errors.New("envdproxy: a routed store and an inventory source are required") } @@ -82,7 +78,7 @@ func NewResolver(placed scale.SandboxStore, routed scale.SandboxLifecycle, inven return nil, errors.New("envdproxy: the envd secret is required to verify access tokens") } return &storeResolver{ - claims: claims, + claims: scale.NewScatterGatherStore(inventory).(scale.ClaimIDResolver), lifecycle: routed, inventory: inventory, namespace: namespace, @@ -131,22 +127,14 @@ func (s *storeResolver) probe(ctx context.Context, sandboxID, claimID string, ad } ctx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() - var mu sync.Mutex - var found Owner - node, err := scale.FirstHit(ctx, s.inventory, probeConcurrency, func(ctx context.Context, node string) string { - o, err := s.ownerOn(ctx, node, claimID) - if err != nil { - return "" - } - mu.Lock() - found = o - mu.Unlock() - return node + found, err := scale.FirstHit(ctx, s.inventory, probeConcurrency, func(ctx context.Context, node string) Owner { + o, _ := s.ownerOn(ctx, node, claimID) + return o }) if err != nil { return Owner{}, err } - if node == "" { + if found == (Owner{}) { return Owner{}, ErrSandboxNotFound } e := s.entry(found) From 0051eb079e44b9281195d09345fc42ccaf4b65be Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:15:47 +0800 Subject: [PATCH 13/23] simplify(e2b): one relay env map, the build lease from the executor, template views over NodeCapacities, a status page that converts only what it returns The relay environment lived in the e2b layer and reached e2bbuild through a Spec field its one caller always set to that map; it is sandboxd.RelayEnv beside NetRouteRelay. Spec.TTLSeconds was always the executor's own timeout. The template surfaces rebuilt the fleet view with ListNodes plus one NodeInventory read per node while every consumer reads Node, Pools and Templates, which NodeCapacities holds as one shared slice, so a stale node's templates now drop out of the list as they already dropped out of claims; resolveTemplate groups only the name it resolves. A build status poll converted every retained log line before paging, and Status copied the append-only log. The list filter now rejects on metadata before rendering a detail. The COPY archive path was spelled at the writer and the reader. Restated: the package doc, the fork result and Spec docs. --- pkg/e2bbuild/copy.go | 6 +++++- pkg/e2bbuild/executor.go | 31 ++++++++++++++--------------- pkg/e2bbuild/executor_test.go | 5 ++--- pkg/e2bcompat/builds.go | 35 ++++++++++++++++----------------- pkg/e2bcompat/builds_test.go | 3 ++- pkg/e2bcompat/envd.go | 9 +-------- pkg/e2bcompat/lifecycle.go | 20 +++---------------- pkg/e2bcompat/lifecycle_test.go | 22 +++++++++------------ pkg/e2bcompat/server.go | 12 +++++++---- pkg/e2bcompat/templates.go | 14 ++++++------- pkg/e2bcompat/types.go | 4 ++-- pkg/sandboxd/client.go | 19 +++++++++++++----- 12 files changed, 85 insertions(+), 95 deletions(-) diff --git a/pkg/e2bbuild/copy.go b/pkg/e2bbuild/copy.go index e0b4521..f167fe9 100644 --- a/pkg/e2bbuild/copy.go +++ b/pkg/e2bbuild/copy.go @@ -48,7 +48,7 @@ func copyScript(state Command, s Step) string { scratch := "/tmp/" + s.FilesHash unpack := scratch + "/unpack" vars := [][2]string{ - {"archive", scratch + ".tar"}, + {"archive", archivePath(s.FilesHash)}, {"scratch", scratch}, {"unpack", unpack}, {"src", s.Args[0]}, @@ -66,6 +66,10 @@ func copyScript(state Command, s Step) string { return b.String() + copyMove } +func archivePath(hash string) string { + return "/tmp/" + hash + ".tar" +} + // globBase is src up to its first segment with a glob; the SDK expands globs into the archive itself. func globBase(src string) string { segs := strings.Split(strings.TrimSuffix(src, "/"), "/") diff --git a/pkg/e2bbuild/executor.go b/pkg/e2bbuild/executor.go index 648379c..11b69f5 100644 --- a/pkg/e2bbuild/executor.go +++ b/pkg/e2bbuild/executor.go @@ -14,6 +14,7 @@ import ( "maps" "path" "regexp" + "slices" "strconv" "strings" "sync" @@ -109,19 +110,17 @@ func (s Step) problem(canCopy bool) string { return "" } -// Spec is how a registered build runs; RelayEnvs seed the steps when the claim relays through the node's proxy, StartCmd runs in the background and ReadyCmd until it exits 0, both before the promote; Publish runs after the promote and before the build reads ready. +// Spec is how a registered build runs; StartCmd runs in the background and ReadyCmd until it exits 0, both before the promote; Publish runs after the promote and before the build reads ready. type Spec struct { - Namespace string - ClaimName string - Pool scale.PoolKey - Template string - TTLSeconds int - Steps []Step - StartCmd string - ReadyCmd string - RelayEnvs map[string]string - Archive ArchiveFunc - Publish PublishFunc + Namespace string + ClaimName string + Pool scale.PoolKey + Template string + Steps []Step + StartCmd string + ReadyCmd string + Archive ArchiveFunc + Publish PublishFunc } // Command is one shell line run as User, in Workdir, with Envs; an empty Workdir is the user's home. @@ -237,7 +236,7 @@ func (e *Executor) Status(id string) (Info, bool) { return Info{}, false } info := r.info - info.Logs = append([]LogEntry(nil), r.info.Logs...) + info.Logs = slices.Clip(r.info.Logs) return info, true } @@ -246,7 +245,7 @@ func (e *Executor) run(ctx context.Context, id string, spec Spec) { ctx, cancel := context.WithTimeout(ctx, e.timeout) defer cancel() e.logf(id, PhaseClaim, "claiming a sandbox from %s (%s, %s)", spec.Pool.Template, spec.Pool.Net, spec.Pool.Size) - a, err := e.store.Claim(ctx, spec.Namespace, spec.ClaimName, spec.Pool, scale.ClaimOptions{TTLSeconds: spec.TTLSeconds}) + a, err := e.store.Claim(ctx, spec.Namespace, spec.ClaimName, spec.Pool, scale.ClaimOptions{TTLSeconds: int(e.timeout / time.Second)}) if err != nil { msg := fmt.Sprintf("could not claim a sandbox of %s (%s, %s)", spec.Pool.Template, spec.Pool.Net, spec.Pool.Size) if scale.IsNoWarmCapacity(err) { @@ -271,7 +270,7 @@ func (e *Executor) run(ctx context.Context, id string, spec Spec) { func (e *Executor) prepare(ctx context.Context, id string, a scale.Assignment, spec Spec) (string, string, error) { state := Command{User: defaultUser, Envs: map[string]string{}} if a.NetRoute == sandboxd.NetRouteRelay { - maps.Copy(state.Envs, spec.RelayEnvs) + maps.Copy(state.Envs, sandboxd.RelayEnv) } for i, step := range spec.Steps { phase := strconv.Itoa(i + 1) @@ -355,7 +354,7 @@ func (e *Executor) copyIn(ctx context.Context, a scale.Assignment, spec Spec, ha return "could not read the uploaded files", err } defer func() { _ = archive.Close() }() - if err := e.guest.Write(ctx, a, "/tmp/"+hash+".tar", archive); err != nil { + if err := e.guest.Write(ctx, a, archivePath(hash), archive); err != nil { return "could not copy the uploaded files into the build sandbox", err } return "", nil diff --git a/pkg/e2bbuild/executor_test.go b/pkg/e2bbuild/executor_test.go index bee19f8..dc6b10a 100644 --- a/pkg/e2bbuild/executor_test.go +++ b/pkg/e2bbuild/executor_test.go @@ -106,14 +106,13 @@ func TestStepsCarryTheirUserWorkdirAndEnvIntoLaterCommandsAndTheDefaults(t *test } func TestARelayedBuildSeedsItsStepsWithTheProxyEnvironment(t *testing.T) { - for route, want := range map[string]string{"relay": "map[A:1 http_proxy:p]", "none": "map[A:1]"} { + for route, want := range map[string]string{"relay": "map[A:1 http_proxy:http://127.0.0.1:3128 https_proxy:http://127.0.0.1:3128 no_proxy:localhost,127.0.0.1,::1,169.254.169.254]", "none": "map[A:1]"} { store := &fakeStore{route: route} e := New(store, store, 1, time.Minute, 100) e.Register("b", Request{}) require.NoError(t, e.Start(t.Context(), "b", Spec{ Pool: scale.PoolKey{Template: "img"}, Template: "t", - Steps: []Step{{Type: stepEnv, Args: []string{"A", "1"}}, {Type: stepRun, Args: []string{"make"}}}, - RelayEnvs: map[string]string{"http_proxy": "p"}, + Steps: []Step{{Type: stepEnv, Args: []string{"A", "1"}}, {Type: stepRun, Args: []string{"make"}}}, })) require.Equal(t, StatusReady, waitDone(t, e, "b").Status) calls := store.calls() diff --git a/pkg/e2bcompat/builds.go b/pkg/e2bcompat/builds.go index 54fa34b..a778f07 100644 --- a/pkg/e2bcompat/builds.go +++ b/pkg/e2bcompat/builds.go @@ -72,17 +72,15 @@ func (s *Server) startBuild(w http.ResponseWriter, r *http.Request) { pool.Net = scale.NetDefault pool.Size = cmp.Or(pending.Size, pool.Size) err := s.builds.Start(r.Context(), id, e2bbuild.Spec{ - Namespace: s.namespace(r), - ClaimName: names.SimpleNameGenerator.GenerateName(namePrefix + "build-"), - Pool: pool, - Template: scope + name, - TTLSeconds: int(s.opts.Builds.Timeout / time.Second), - Steps: req.Steps, - StartCmd: req.StartCmd, - ReadyCmd: req.ReadyCmd, - RelayEnvs: relayEnvs, - Archive: s.archive(s.namespace(r), name), - Publish: s.publishBuild(scope, name, pending.Tags), + Namespace: s.namespace(r), + ClaimName: names.SimpleNameGenerator.GenerateName(namePrefix + "build-"), + Pool: pool, + Template: scope + name, + Steps: req.Steps, + StartCmd: req.StartCmd, + ReadyCmd: req.ReadyCmd, + Archive: s.archive(s.namespace(r), name), + Publish: s.publishBuild(scope, name, pending.Tags), }) switch { case errors.Is(err, e2bbuild.ErrBusy): @@ -113,17 +111,18 @@ func (s *Server) buildStatus(w http.ResponseWriter, r *http.Request) { return } floor := max(slices.Index(logLevels, q.Get("level")), 0) - var entries []BuildLogEntry + entries := []BuildLogEntry{} for _, l := range info.Logs { - if slices.Index(logLevels, l.Level) >= floor { + if slices.Index(logLevels, l.Level) < floor { + continue + } + if offset > 0 { + offset-- + } else if limit <= 0 || len(entries) < limit { entries = append(entries, logEntryOf(l)) } } - entries = entries[min(offset, len(entries)):] - if limit > 0 { - entries = entries[:min(limit, len(entries))] - } - out := TemplateBuildInfo{TemplateID: name, BuildID: buildID, Status: info.Status, Logs: []string{}, LogEntries: append([]BuildLogEntry{}, entries...)} + out := TemplateBuildInfo{TemplateID: name, BuildID: buildID, Status: info.Status, Logs: []string{}, LogEntries: entries} if info.Status == e2bbuild.StatusError { out.Reason = &BuildStatusReason{Message: info.Failure, Step: sdkStep(info.FailedPhase), LogEntries: []BuildLogEntry{}} if n := len(info.Logs); n > 0 { diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index b847b52..9d0bb77 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cocoonstack/sandbox-operator/pkg/e2bbuild" + "github.com/cocoonstack/sandbox-operator/pkg/sandboxd" "github.com/cocoonstack/sandbox-operator/pkg/scale" ) @@ -123,7 +124,7 @@ func TestARelayedBuildLeavesTheProxyInTheTemplateDefaults(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(body), &init)) } } - assert.Equal(t, relayEnvs, init.EnvVars) + assert.Equal(t, sandboxd.RelayEnv, init.EnvVars) } func TestABuildRefusesWhatItCannotHonor(t *testing.T) { diff --git a/pkg/e2bcompat/envd.go b/pkg/e2bcompat/envd.go index 3eafd0e..2f764a4 100644 --- a/pkg/e2bcompat/envd.go +++ b/pkg/e2bcompat/envd.go @@ -45,13 +45,6 @@ const ( logLineMax = 64 << 10 ) -// relayEnvs point a guest process at the node's egress proxy, as silkd's unit points its own. -var relayEnvs = map[string]string{ - "http_proxy": "http://127.0.0.1:3128", - "https_proxy": "http://127.0.0.1:3128", - "no_proxy": "localhost,127.0.0.1,::1,169.254.169.254", -} - // envdMetrics is envd's GET /metrics reply: the guest's own view of its CPU, memory and root disk. type envdMetrics struct { Timestamp int64 `json:"ts"` @@ -401,7 +394,7 @@ func withRelay(route string, envs map[string]string) map[string]string { if route != sandboxd.NetRouteRelay { return envs } - out := maps.Clone(relayEnvs) + out := maps.Clone(sandboxd.RelayEnv) maps.Copy(out, envs) return out } diff --git a/pkg/e2bcompat/lifecycle.go b/pkg/e2bcompat/lifecycle.go index 24013f1..bd31fd6 100644 --- a/pkg/e2bcompat/lifecycle.go +++ b/pkg/e2bcompat/lifecycle.go @@ -363,26 +363,12 @@ func (s *Server) sandboxLogs(reply any) http.HandlerFunc { } } -// inventories returns every node's published inventory, the fleet view the -// pool-derived surfaces (templates, snapshot listing) are assembled from. -func (s *Server) inventories(ctx context.Context) ([]*scale.NodeInventory, error) { +// inventories is the fleet's claimable pools and templates, the view the template surfaces are assembled from. +func (s *Server) inventories(ctx context.Context) ([]scale.NodePools, error) { if s.opts.Inventory == nil { return nil, errors.New("e2bcompat: no inventory source configured") } - nodes, err := s.opts.Inventory.ListNodes(ctx) - if err != nil { - return nil, err - } - out := make([]*scale.NodeInventory, 0, len(nodes)) - for _, node := range nodes { - inv, err := s.opts.Inventory.NodeInventory(ctx, node) - if err != nil { - log.WithFunc("e2bcompat.inventories").Warnf(ctx, "e2b: node inventory unavailable node=%s err=%v", node, err) - continue - } - out = append(out, inv) - } - return out, nil + return s.opts.Inventory.NodeCapacities(ctx) } // nodesWithSandboxes lists the nodes a checkpoint could live on. diff --git a/pkg/e2bcompat/lifecycle_test.go b/pkg/e2bcompat/lifecycle_test.go index 4ea2024..8e1174f 100644 --- a/pkg/e2bcompat/lifecycle_test.go +++ b/pkg/e2bcompat/lifecycle_test.go @@ -431,20 +431,16 @@ func TestResumeOfARunningSandboxIs409(t *testing.T) { } func TestResumeRefusesWhatConnectCannotGive(t *testing.T) { - for _, body := range []string{`{"memory":false}`} { - t.Run(body, func(t *testing.T) { - store := &lifecycleStore{} - nodeReportsPaused(store) - store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img")} - h := newTestServer(t, store) + store := &lifecycleStore{} + nodeReportsPaused(store) + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img")} + h := newTestServer(t, store) - if w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/resume", body, testKey); w.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want 400: %s", w.Code, w.Body.String()) - } - if store.readID != "" || store.resumedID != "" { - t.Errorf("a refused resume reached the node (read %q, resumed %q)", store.readID, store.resumedID) - } - }) + if w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/resume", `{"memory":false}`, testKey); w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", w.Code, w.Body.String()) + } + if store.readID != "" || store.resumedID != "" { + t.Errorf("a refused resume reached the node (read %q, resumed %q)", store.readID, store.resumedID) } } diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index fb42b62..a866905 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -5,8 +5,9 @@ // It is a translation layer, not a second control plane: every request lands on // the same scale.SandboxStore the aggregated apiserver uses, so an e2b Create is // the identical node-local claim a `kubectl create sandbox` performs, and the -// sandbox it returns is visible to `kubectl get sandboxes`. Nothing is stored -// here; public identity is a DNS-safe rendering of the node's sandboxd claim id. +// sandbox it returns is visible to `kubectl get sandboxes`. Only build records +// (in memory) and COPY uploads (a directory or bucket) live here; public identity +// is a DNS-safe rendering of the node's sandboxd claim id. // // Mapping to the e2b contract (e2b-dev/E2B spec/openapi.yml): // @@ -15,7 +16,7 @@ // POST /sandboxes/{id}/snapshots -> create checkpoint // GET /snapshots, DELETE /templates/{id} -> list or delete checkpoints and built templates // GET/PATCH /templates[/{id}], aliases/{a}, tags -> warm-pool keys, built templates, alias lookup, tags -// GET /sandboxes/{id}/metrics|logs -> node statistics, an empty log page +// GET /sandboxes/{id}/metrics|logs, /sandboxes/metrics -> envd's guest metrics, an empty log page // POST timeout|refreshes, GET /health -> lease renewal, liveness package e2bcompat @@ -422,7 +423,10 @@ func (s *Server) listed(w http.ResponseWriter, r *http.Request) ([]SandboxDetail } out := make([]SandboxDetail, 0, len(list.Items)) for i := range list.Items { - if d := s.detailFor(&list.Items[i]); filter.keeps(d) && filter.matchesMetadata(&list.Items[i]) { + if !filter.matchesMetadata(&list.Items[i]) { + continue + } + if d := s.detailFor(&list.Items[i]); filter.keeps(d) { out = append(out, d) } } diff --git a/pkg/e2bcompat/templates.go b/pkg/e2bcompat/templates.go index 9dabf6e..05f289f 100644 --- a/pkg/e2bcompat/templates.go +++ b/pkg/e2bcompat/templates.go @@ -106,7 +106,7 @@ func (s *Server) listTemplates(w http.ResponseWriter, r *http.Request) { }) } } - built := builtTemplates(nodes, s.templateScope(r)) + built := builtTemplates(nodes, s.templateScope(r), "") for _, name := range slices.Sorted(maps.Keys(built)) { b := built[name] created := b.created.UTC().Format(time.RFC3339) @@ -308,7 +308,7 @@ func (s *Server) resolveTemplate(r *http.Request, name string) (*builtTemplate, if err != nil { return nil, aliased, err } - return builtTemplates(nodes, s.templateScope(r))[name], aliased || advertisedIn(nodes, name), nil + return builtTemplates(nodes, s.templateScope(r), name)[name], aliased || advertisedIn(nodes, name), nil } func (s *Server) templateScope(r *http.Request) string { @@ -344,13 +344,13 @@ func (s *Server) writeTagError(w http.ResponseWriter, r *http.Request, err error writeError(w, http.StatusInternalServerError, "failed to update the template's tags") } -// builtTemplates groups the operator-owned templates under scope by their bare name. -func builtTemplates(nodes []*scale.NodeInventory, scope string) map[string]*builtTemplate { +// builtTemplates groups the operator-owned templates under scope by their bare name, or only the one named only. +func builtTemplates(nodes []scale.NodePools, scope, only string) map[string]*builtTemplate { out := map[string]*builtTemplate{} for _, inv := range nodes { for _, t := range inv.Templates { name, ok := strings.CutPrefix(t.Template, scope) - if !ok || t.Tenant != "" { + if !ok || t.Tenant != "" || only != "" && name != only { continue } b := out[name] @@ -398,8 +398,8 @@ func buildUUID(digest string) string { return uuid.NewSHA1(buildNamespace, []byte(digest)).String() } -func advertisedIn(nodes []*scale.NodeInventory, image string) bool { - return slices.ContainsFunc(nodes, func(inv *scale.NodeInventory) bool { +func advertisedIn(nodes []scale.NodePools, image string) bool { + return slices.ContainsFunc(nodes, func(inv scale.NodePools) bool { return slices.ContainsFunc(inv.Pools, func(pc scale.PoolCapacity) bool { return pc.Template == image }) }) } diff --git a/pkg/e2bcompat/types.go b/pkg/e2bcompat/types.go index b914715..30211dd 100644 --- a/pkg/e2bcompat/types.go +++ b/pkg/e2bcompat/types.go @@ -113,8 +113,8 @@ type SandboxForkRequest struct { Count *int32 `json:"count,omitempty"` } -// SandboxForkResult is one entry of the fork reply: exactly one of Sandbox or -// Error is set, so a partial failure still returns 201 with per-child detail. +// SandboxForkResult is one entry of the fork reply in e2b's shape; a fork here is +// all-or-nothing, so Error is never set and a failed child fails the whole with 500. type SandboxForkResult struct { Sandbox *Sandbox `json:"sandbox,omitempty"` Error *APIError `json:"error,omitempty"` diff --git a/pkg/sandboxd/client.go b/pkg/sandboxd/client.go index 7ef74b9..3fbe787 100644 --- a/pkg/sandboxd/client.go +++ b/pkg/sandboxd/client.go @@ -33,11 +33,20 @@ const ( captureTimeout = 10 * time.Minute ) -// ErrNodeAtCapacity is returned by Claim when sandboxd answers 429 (the node is -// at max_claims, the calling tenant is at its own max_claims, or the node is -// draining) or a 200 that delivers no sandbox. In every case this node handed -// over no VM, so the store tries another node or reports no warm capacity. -var ErrNodeAtCapacity = errors.New("sandboxd: node at capacity or draining") +var ( + // ErrNodeAtCapacity is returned by Claim when sandboxd answers 429 (the node is + // at max_claims, the calling tenant is at its own max_claims, or the node is + // draining) or a 200 that delivers no sandbox. In every case this node handed + // over no VM, so the store tries another node or reports no warm capacity. + ErrNodeAtCapacity = errors.New("sandboxd: node at capacity or draining") + + // RelayEnv points a guest process at the node's egress proxy on a NetRouteRelay claim, as silkd's unit points its own. + RelayEnv = map[string]string{ + "http_proxy": "http://127.0.0.1:3128", + "https_proxy": "http://127.0.0.1:3128", + "no_proxy": "localhost,127.0.0.1,::1,169.254.169.254", + } +) // ExpireAction is what the node does with a claim whose lease ends; empty keeps the claim's current action, destroy on a new claim. type ExpireAction string From 0c60192eb3d90aef6e6da717974057b1fe4a5a53 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:16:53 +0800 Subject: [PATCH 14/23] simplify(cmd): one sandboxd token flag pair for the three binaries sandbox-apiserver, sandbox-e2b and sandbox-envd-proxy each registered --sandboxd-token and --sandboxd-token-file with their own help text; sandboxd.AddTokenFlags registers the pair beside TokenFrom, the way AddEnvdSecretFlag serves the envd secret. --- cmd/sandbox-apiserver/main.go | 5 +---- cmd/sandbox-e2b/main.go | 5 +---- cmd/sandbox-envd-proxy/main.go | 5 +---- pkg/sandboxd/client.go | 10 ++++++++++ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/sandbox-apiserver/main.go b/cmd/sandbox-apiserver/main.go index 3686aae..c953d20 100644 --- a/cmd/sandbox-apiserver/main.go +++ b/cmd/sandbox-apiserver/main.go @@ -87,10 +87,7 @@ func (o *options) addFlags(fs *pflag.FlagSet) { o.Authentication.AddFlags(fs) o.Authorization.AddFlags(fs) o.Features.AddFlags(fs) - fs.StringVar(&o.SandboxdToken, "sandboxd-token", o.SandboxdToken, - "Uniform fleet-wide sandboxd api_token presented on node-local claim/release. Prefer --sandboxd-token-file for a Secret mount.") - fs.StringVar(&o.SandboxdTokenFile, "sandboxd-token-file", o.SandboxdTokenFile, - "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") + sandboxd.AddTokenFlags(fs, &o.SandboxdToken, &o.SandboxdTokenFile) fs.BoolVar(&o.WarmPoolDriver, "enable-warm-pool-driver", o.WarmPoolDriver, "Run the in-process SandboxWarmPool → sandboxd pool reconcile loop (control-plane warm-capacity surface; pool-level, never per-sandbox).") fs.DurationVar(&o.WarmPoolInterval, "warm-pool-sync-interval", o.WarmPoolInterval, diff --git a/cmd/sandbox-e2b/main.go b/cmd/sandbox-e2b/main.go index 19c3489..b81d67a 100644 --- a/cmd/sandbox-e2b/main.go +++ b/cmd/sandbox-e2b/main.go @@ -35,10 +35,7 @@ type options struct { func (o *options) addFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&o.Seeds, "sandboxd-seeds", o.Seeds, "Comma-separated sandboxd addresses dialed at start, each naming one node; each node reports its own advertise_addr as its key, and the rest of the mesh is found through their gossip.") - fs.StringVar(&o.SandboxdToken, "sandboxd-token", o.SandboxdToken, - "Fleet root sandboxd api_token (GET /v1/info needs root). Prefer --sandboxd-token-file for a Secret mount.") - fs.StringVar(&o.SandboxdTokenFile, "sandboxd-token-file", o.SandboxdTokenFile, - "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") + sandboxd.AddTokenFlags(fs, &o.SandboxdToken, &o.SandboxdTokenFile) fs.DurationVar(&o.PollInterval, "inventory-poll", o.PollInterval, "How often every node's info and sandbox list are read; List and Watch lag a change by up to one tick.") o.E2B.AddFlags(fs) diff --git a/cmd/sandbox-envd-proxy/main.go b/cmd/sandbox-envd-proxy/main.go index dc5c4de..d7ca81c 100644 --- a/cmd/sandbox-envd-proxy/main.go +++ b/cmd/sandbox-envd-proxy/main.go @@ -45,10 +45,7 @@ func (o *options) addFlags(fs *pflag.FlagSet) { o.Inventory.AddFlags(fs) fs.StringVar(&o.Domain, "domain", o.Domain, "Base domain sandbox hosts are derived from, as {port}-{sandboxID}.{domain}. Must match the apiserver's --e2b-domain.") - fs.StringVar(&o.SandboxdToken, "sandboxd-token", o.SandboxdToken, - "Fleet root sandboxd api_token, which reads a sandbox's claim token to verify its envd access token. Prefer --sandboxd-token-file for a Secret mount.") - fs.StringVar(&o.SandboxdTokenFile, "sandboxd-token-file", o.SandboxdTokenFile, - "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") + sandboxd.AddTokenFlags(fs, &o.SandboxdToken, &o.SandboxdTokenFile) e2bcompat.AddEnvdSecretFlag(fs, &o.EnvdSecretFile) fs.StringVar(&o.Namespace, "namespace", o.Namespace, "Namespace inventory lookups are filtered to; empty matches every namespace. Not an access boundary: a caller holding a sandbox's token reaches it in any namespace.") diff --git a/pkg/sandboxd/client.go b/pkg/sandboxd/client.go index 3fbe787..ea459b4 100644 --- a/pkg/sandboxd/client.go +++ b/pkg/sandboxd/client.go @@ -19,6 +19,8 @@ import ( "slices" "strings" "time" + + "github.com/spf13/pflag" ) const ( @@ -303,6 +305,14 @@ func (c *Client) authenticate(req *http.Request, token string) { } } +// AddTokenFlags registers the sandboxd api_token pair every binary that dials nodes takes. +func AddTokenFlags(fs *pflag.FlagSet, literal, file *string) { + fs.StringVar(literal, "sandboxd-token", *literal, + "sandboxd api_token presented to every node (the e2b surface and the envd proxy need the root one). Prefer --sandboxd-token-file for a Secret mount.") + fs.StringVar(file, "sandboxd-token-file", *file, + "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") +} + // TokenFrom returns the api_token read from file (a Secret mount) when file is set, else literal. func TokenFrom(literal, file string) (string, error) { if file == "" { From b76e04f68d43928e838426384a9f0fe72e5db3f3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:16:53 +0800 Subject: [PATCH 15/23] docs(e2b): built templates expect a checkpoint_store per node --- docs/e2b-compat.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/e2b-compat.md b/docs/e2b-compat.md index 7a21b62..871d2be 100644 --- a/docs/e2b-compat.md +++ b/docs/e2b-compat.md @@ -325,6 +325,13 @@ needs its own stickiness (or run one replica). A finished build is kept for an h loses a build in flight; the SDK's next poll throws, and a rebuild converges because a promote replaces. +Built templates expect a `checkpoint_store` per node. A node reads the store's +template records once at start and advertises them as its own, so with one S3 +store shared by every node each node advertises every node's templates as of its +own start, and a publish that deletes an older holder removes a record its +siblings still advertise; a create of one fails over to the next advertiser, and +the template list lags until those nodes restart. + ## Limits worth knowing - **Reaching `envd` (the in-sandbox data plane).** The SDK derives the sandbox From 590c5fc82c50a81a8acb5ed04b2e99bf5592468c Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:35:13 +0800 Subject: [PATCH 16/23] fix(envdproxy): a signed file URL goes without the token only on envd's port The signed URL is envd's own /files download; on any other port a stranger's signature would have opened the node-token relay to whatever the sandbox serves there. --- pkg/envdproxy/envdproxy.go | 2 +- pkg/envdproxy/envdproxy_test.go | 5 +++++ pkg/envdproxy/route.go | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/envdproxy/envdproxy.go b/pkg/envdproxy/envdproxy.go index e6b8607..4fa99be 100644 --- a/pkg/envdproxy/envdproxy.go +++ b/pkg/envdproxy/envdproxy.go @@ -104,7 +104,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { switch token := strings.TrimSpace(r.Header.Get(accessTokenHeader)); { case token != "": owner, err = s.resolver.Owner(r.Context(), rt.sandboxID, token) - case signedFileURL(r): + case rt.port == envdPort && signedFileURL(r): if owner, err = s.resolver.Locate(r.Context(), rt.sandboxID); err == nil { owner.Token = s.opts.NodeToken } diff --git a/pkg/envdproxy/envdproxy_test.go b/pkg/envdproxy/envdproxy_test.go index cd34fcc..35ce7d7 100644 --- a/pkg/envdproxy/envdproxy_test.go +++ b/pkg/envdproxy/envdproxy_test.go @@ -115,6 +115,11 @@ func TestProxyRelaysASignedFileURLWithoutTheToken(t *testing.T) { t.Errorf("%s: status %d, want 401: only a signed file URL goes without the token", path, resp.StatusCode) } } + resp = request(t, h, "3000-sb-abc."+testDomain, "/files?path=%2Fetc%2Fhosts&signature=v1_abc&signature_expiration=9", "") + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("a signed file URL on port 3000: status %d, want 401: only envd's port verifies a signature", resp.StatusCode) + } } func TestProxyRequiresTheAccessToken(t *testing.T) { diff --git a/pkg/envdproxy/route.go b/pkg/envdproxy/route.go index c3693cd..3ada9ec 100644 --- a/pkg/envdproxy/route.go +++ b/pkg/envdproxy/route.go @@ -16,6 +16,8 @@ const ( // accessTokenHeader is the per-sandbox data-plane credential. accessTokenHeader = "X-Access-Token" apiKeyHeader = "X-API-KEY" + + envdPort uint16 = 49983 ) // internalPaths are envd's own control surface (x-internal in its spec). They From 288fa28a51e8d036e1b5a5b5313b324470d01439 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:35:13 +0800 Subject: [PATCH 17/23] fix(scale): a stamped name is checked against sandboxd's grammar before a build or snapshot runs sandboxd refuses a promote or checkpoint whose name is outside NameRe, so a build named outside it ran every step and failed at promote; StampedName now answers 400 up front with the same grammar. --- pkg/e2bcompat/builds_test.go | 1 + pkg/scale/sandboxstore_lifecycle.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index 9d0bb77..dac2a81 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -143,6 +143,7 @@ func TestABuildRefusesWhatItCannotHonor(t *testing.T) { } assert.Equal(t, http.StatusNotFound, do(t, h, http.MethodPost, "/v2/templates/app/builds/nope", `{"fromImage":"img"}`, testKey).Code) assert.Equal(t, http.StatusBadRequest, do(t, h, http.MethodPost, "/v3/templates", `{"name":"`+strings.Repeat("x", 60)+`"}`, testKey).Code) + assert.Equal(t, http.StatusBadRequest, do(t, h, http.MethodPost, "/v3/templates", `{"name":"my app"}`, testKey).Code, "a name outside sandboxd's grammar is refused before the build runs") assert.Equal(t, http.StatusBadRequest, do(t, h, http.MethodPost, "/v3/templates", `{}`, testKey).Code) assert.Equal(t, http.StatusNotFound, do(t, newTestServer(t, &fakeStore{}), http.MethodPost, "/v3/templates", `{"name":"app"}`, testKey).Code, "builds are off unless enabled") } diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go index db45b2f..52ba21c 100644 --- a/pkg/scale/sandboxstore_lifecycle.go +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "regexp" "strings" "time" @@ -18,6 +19,8 @@ import ( // maxSandboxdName is sandboxd's name budget (types.NameRe). const maxSandboxdName = 63 +var sandboxdName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`) + func (s *scatterGatherStore) Pause(ctx context.Context, node, id string) error { cl, err := s.nodeClient(ctx, node, "pause", id) if err != nil { @@ -229,6 +232,9 @@ func StampedName(kind, prefix, namespace, name string) (string, error) { if len(stamped) > maxSandboxdName { return "", k8serrors.NewBadRequest(fmt.Sprintf("%s name %q: at most %d characters in namespace %q", kind, name, maxSandboxdName-len(prefix)-len(namespace)-1, namespace)) } + if !sandboxdName.MatchString(stamped) { + return "", k8serrors.NewBadRequest(fmt.Sprintf("%s name %q: letters, digits and . _ : / - only", kind, name)) + } return stamped, nil } From 1c2726aa6dfacc72f0f7553b6ac7d779d144ea6d Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:46:39 +0800 Subject: [PATCH 18/23] fix(e2b): a publish deletes older holders by the digest it observed and never tags a build a newer promote replaced Two builds of one name promoting on two nodes inside one publish's window could each delete the other's fresh promote by key and both report ready; the delete now names the digest the publish observed and sandboxd (91697b9) refuses a replaced record with 412, which the publish takes as already converged. On one node a newer promote replaces the record under the same key before the older build's publish, which then tagged the newer content with its own digest; the publish now checks the holder's digest against its promote and reports the build replaced. --- pkg/e2bcompat/builds.go | 14 ++++++++++++-- pkg/e2bcompat/builds_test.go | 6 +++--- pkg/e2bcompat/server_test.go | 11 ++++++++--- pkg/e2bcompat/templates.go | 2 +- pkg/sandboxd/lifecycle.go | 7 +++++-- pkg/scale/apiserver/storage_test.go | 4 +++- pkg/scale/sandboxstore.go | 4 ++-- pkg/scale/sandboxstore_claim_test.go | 2 +- pkg/scale/sandboxstore_impl.go | 2 +- pkg/scale/sandboxstore_lifecycle.go | 4 ++-- pkg/scale/sandboxstore_lifecycle_test.go | 2 +- 11 files changed, 39 insertions(+), 19 deletions(-) diff --git a/pkg/e2bcompat/builds.go b/pkg/e2bcompat/builds.go index a778f07..dd08b99 100644 --- a/pkg/e2bcompat/builds.go +++ b/pkg/e2bcompat/builds.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apiserver/pkg/storage/names" @@ -145,8 +146,17 @@ func (s *Server) publishBuild(scope, name string, tags []string) e2bbuild.Publis return fmt.Errorf("the build's template is no longer on %s", node) } own := holders[i] + if own.digest != digest { + return fmt.Errorf("a newer build of %s replaced this one on %s", name, node) + } stale := slices.DeleteFunc(holders, func(h templateHolder) bool { return !h.created.Before(own.created) }) - if err := forEachHolder(stale, func(h templateHolder) error { return s.store.DeleteTemplate(ctx, h.node, h.key) }); err != nil { + if err := forEachHolder(stale, func(h templateHolder) error { + err := s.store.DeleteTemplate(ctx, h.node, h.key, h.digest) + if se, ok := errors.AsType[*k8serrors.StatusError](err); ok && se.ErrStatus.Code == http.StatusPreconditionFailed { + return nil + } + return err + }); err != nil { return fmt.Errorf("delete the previous build: %w", err) } if len(tags) == 0 { @@ -182,7 +192,7 @@ func (s *Server) liveHolders(ctx context.Context, template string) ([]templateHo defer mu.Unlock() for _, t := range held { if t.Template == template && t.Tenant == "" { - h := templateHolder{node: n, key: scale.PoolKey{Template: t.Template, Net: t.Net, Size: t.Size}} + h := templateHolder{node: n, key: scale.PoolKey{Template: t.Template, Net: t.Net, Size: t.Size}, digest: t.ContentDigest} if t.CreatedAt != nil { h.created = t.CreatedAt.Time } diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index dac2a81..934ff7d 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -32,7 +32,7 @@ func TestABuildFromAnImagePromotesReplacesTheOldHolderAndTags(t *testing.T) { require.Equal(t, e2bbuild.StatusReady, info.Status, info) assert.Equal(t, [2]string{"reg/rt:24.04", scale.SizeClassMedium}, [2]string{store.claimPool.Template, store.claimPool.Size}, "the alias names the image; cpuCount 2 picks the size") assert.Equal(t, []string{"n sb_1 e2b/sandboxes/app"}, store.promoted) - assert.Equal(t, []string{"m e2b/sandboxes/app small"}, store.deletedTemplates, "the previous build on another node goes") + assert.Equal(t, []string{"m e2b/sandboxes/app small sha256:old"}, store.deletedTemplates, "the previous build on another node goes") assert.Equal(t, map[string]string{"v1": "sha256:sb_1"}, fleetLabels(t, store, "n", "e2b/sandboxes/app")) assert.Equal(t, "n sb_1", store.releasedNode+" "+store.releasedID) assert.Len(t, info.LogEntries, 3) @@ -57,7 +57,7 @@ func TestABuildReplacesAPreviousBuildTheInventoryHasNotPublishedYet(t *testing.T id := requestBuild(t, h, "app") require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status) - assert.Equal(t, []string{"m e2b/sandboxes/app small"}, store.deletedTemplates, "node m's build finished after its last publish and still goes") + assert.Equal(t, []string{"m e2b/sandboxes/app small sha256:old"}, store.deletedTemplates, "node m's build finished after its last publish and still goes") delete(store.live, "m") id = requestBuild(t, h, "app") @@ -161,7 +161,7 @@ func TestAPublishLeavesANewerConcurrentBuildAlone(t *testing.T) { id := requestBuild(t, h, "app") require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status) - assert.Equal(t, []string{"k e2b/sandboxes/app small"}, store.deletedTemplates, "only the holder older than this promote goes; a newer concurrent build keeps its node") + assert.Equal(t, []string{"k e2b/sandboxes/app small sha256:older"}, store.deletedTemplates, "only the holder older than this promote goes, by the digest observed; a newer concurrent build keeps its node") } func requestBuild(t *testing.T, h http.Handler, name string) string { diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index b39b071..d0d0828 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -569,6 +569,8 @@ type fakeStore struct { deletedSnapshotNode string deletedSnapshotID string deletedTemplates []string + labeled []string + replacedDigest string deleteTemplateErr map[string]error fleet *scale.StaticInventorySource promoted []string @@ -670,7 +672,7 @@ func (f *fakeStore) Promote(ctx context.Context, node, id, template string) (sca defer f.mu.Unlock() f.promoted = append(f.promoted, node+" "+id+" "+template) key := scale.PoolKey{Template: template, Net: scale.NetDefault, Size: f.claimPool.Size} - fresh := scale.PromotedTemplate{Template: key.Template, Net: key.Net, Size: key.Size, ContentDigest: "sha256:" + id, CreatedAt: new(metav1.Now())} + fresh := scale.PromotedTemplate{Template: key.Template, Net: key.Net, Size: key.Size, ContentDigest: cmp.Or(f.replacedDigest, "sha256:"+id), CreatedAt: new(metav1.Now())} same := func(t scale.PromotedTemplate) bool { return t.Template == key.Template && t.Net == key.Net && t.Size == key.Size } @@ -685,6 +687,9 @@ func (f *fakeStore) Promote(ctx context.Context, node, id, template string) (sca // SetTemplateLabels writes straight into the fleet the test serves, so a live read sees it at once. func (f *fakeStore) SetTemplateLabels(ctx context.Context, node string, key scale.PoolKey, labels map[string]string) error { + f.mu.Lock() + f.labeled = append(f.labeled, node+" "+key.Template) + f.mu.Unlock() inv, err := f.fleet.NodeInventory(ctx, node) if err != nil { return err @@ -713,10 +718,10 @@ func (f *fakeStore) NodeTemplates(ctx context.Context, node string) ([]scale.Pro return inv.Templates, nil } -func (f *fakeStore) DeleteTemplate(_ context.Context, node string, key scale.PoolKey) error { +func (f *fakeStore) DeleteTemplate(_ context.Context, node string, key scale.PoolKey, digest string) error { f.mu.Lock() defer f.mu.Unlock() - f.deletedTemplates = append(f.deletedTemplates, node+" "+key.Template+" "+key.Size) + f.deletedTemplates = append(f.deletedTemplates, strings.TrimSpace(node+" "+key.Template+" "+key.Size+" "+digest)) return f.deleteTemplateErr[node] } diff --git a/pkg/e2bcompat/templates.go b/pkg/e2bcompat/templates.go index 05f289f..2c3bfa9 100644 --- a/pkg/e2bcompat/templates.go +++ b/pkg/e2bcompat/templates.go @@ -272,7 +272,7 @@ func (s *Server) deleteTemplate(w http.ResponseWriter, r *http.Request) { s.deleteSnapshot(w, r) return } - if err := forEachHolder(b.holders, func(h templateHolder) error { return s.store.DeleteTemplate(r.Context(), h.node, h.key) }); err != nil { + if err := forEachHolder(b.holders, func(h templateHolder) error { return s.store.DeleteTemplate(r.Context(), h.node, h.key, "") }); err != nil { logger.Errorf(r.Context(), err, "e2b delete template failed template=%s", name) writeError(w, http.StatusInternalServerError, "failed to delete the template") return diff --git a/pkg/sandboxd/lifecycle.go b/pkg/sandboxd/lifecycle.go index a246ee1..9ffca1a 100644 --- a/pkg/sandboxd/lifecycle.go +++ b/pkg/sandboxd/lifecycle.go @@ -167,13 +167,16 @@ func (c *Client) Checkpoints(ctx context.Context) ([]Checkpoint, error) { return out.Checkpoints, err } -// DeleteTemplate performs DELETE /v1/templates for key on this node alone (no_redirect). A 404 is success. -func (c *Client) DeleteTemplate(ctx context.Context, key PoolKey) error { +// DeleteTemplate performs DELETE /v1/templates for key on this node alone (no_redirect); a 404 is success, and a digest deletes only that generation (412 otherwise). +func (c *Client) DeleteTemplate(ctx context.Context, key PoolKey, digest string) error { if key.Template == "" { return fmt.Errorf("sandboxd: delete template requires a template name") } q := templateQuery(key) q.Set("no_redirect", "1") + if digest != "" { + q.Set("digest", digest) + } return c.send(ctx, http.MethodDelete, "/v1/templates?"+q.Encode(), c.token, "delete template", nil, http.StatusNoContent, http.StatusNotFound) } diff --git a/pkg/scale/apiserver/storage_test.go b/pkg/scale/apiserver/storage_test.go index f90ae13..831bb90 100644 --- a/pkg/scale/apiserver/storage_test.go +++ b/pkg/scale/apiserver/storage_test.go @@ -312,7 +312,9 @@ func (f *fakeStore) Snapshots(context.Context, string) ([]scale.Snapshot, error) func (f *fakeStore) DeleteSnapshot(context.Context, string, string) error { return nil } -func (f *fakeStore) DeleteTemplate(context.Context, string, scale.PoolKey) error { return f.verbErr } +func (f *fakeStore) DeleteTemplate(context.Context, string, scale.PoolKey, string) error { + return f.verbErr +} func (f *fakeStore) Promote(context.Context, string, string, string) (scale.PoolKey, string, error) { return scale.PoolKey{}, "", f.verbErr diff --git a/pkg/scale/sandboxstore.go b/pkg/scale/sandboxstore.go index 7584419..7638ee2 100644 --- a/pkg/scale/sandboxstore.go +++ b/pkg/scale/sandboxstore.go @@ -87,8 +87,8 @@ type SandboxLifecycle interface { Snapshots(ctx context.Context, node string) ([]Snapshot, error) // DeleteSnapshot removes a checkpoint. A missing checkpoint is success. DeleteSnapshot(ctx context.Context, node, snapshotID string) error - // DeleteTemplate removes a promoted template on node alone. A missing template is success. - DeleteTemplate(ctx context.Context, node string, key PoolKey) error + // DeleteTemplate removes a promoted template on node alone; a missing template is success, and a digest removes only the generation it names (PreconditionFailed otherwise). + DeleteTemplate(ctx context.Context, node string, key PoolKey, digest string) error // Promote publishes a claimed sandbox on node as the template name, keeping its net and size; it returns the full key and the content digest. Promote(ctx context.Context, node, id, template string) (PoolKey, string, error) // SetTemplateLabels replaces a promoted template's labels on node alone. diff --git a/pkg/scale/sandboxstore_claim_test.go b/pkg/scale/sandboxstore_claim_test.go index 834df2e..7084b07 100644 --- a/pkg/scale/sandboxstore_claim_test.go +++ b/pkg/scale/sandboxstore_claim_test.go @@ -574,7 +574,7 @@ func (c *recordingClient) Checkpoints(context.Context) ([]sandboxd.Checkpoint, e func (c *recordingClient) DeleteCheckpoint(context.Context, string) error { return nil } -func (c *recordingClient) DeleteTemplate(_ context.Context, key sandboxd.PoolKey) error { +func (c *recordingClient) DeleteTemplate(_ context.Context, key sandboxd.PoolKey, _ string) error { c.f.mu.Lock() defer c.f.mu.Unlock() c.f.deletedTemplates = append(c.f.deletedTemplates, key) diff --git a/pkg/scale/sandboxstore_impl.go b/pkg/scale/sandboxstore_impl.go index 2cb3c9e..5ba8f15 100644 --- a/pkg/scale/sandboxstore_impl.go +++ b/pkg/scale/sandboxstore_impl.go @@ -126,7 +126,7 @@ type SandboxdClient interface { Checkpoint(ctx context.Context, id string, spec sandboxd.CheckpointSpec) (sandboxd.Checkpoint, error) Checkpoints(ctx context.Context) ([]sandboxd.Checkpoint, error) DeleteCheckpoint(ctx context.Context, checkpointID string) error - DeleteTemplate(ctx context.Context, key sandboxd.PoolKey) error + DeleteTemplate(ctx context.Context, key sandboxd.PoolKey, digest string) error Promote(ctx context.Context, id, template string) (sandboxd.PoolKey, string, error) SetTemplateLabels(ctx context.Context, key sandboxd.PoolKey, labels map[string]string) error Info(ctx context.Context) (*sandboxd.NodeInfo, error) diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go index 52ba21c..ec32a70 100644 --- a/pkg/scale/sandboxstore_lifecycle.go +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -144,12 +144,12 @@ func (s *scatterGatherStore) DeleteSnapshot(ctx context.Context, node, snapshotI return nil } -func (s *scatterGatherStore) DeleteTemplate(ctx context.Context, node string, key PoolKey) error { +func (s *scatterGatherStore) DeleteTemplate(ctx context.Context, node string, key PoolKey, digest string) error { cl, err := s.nodeClient(ctx, node, "delete template", key.Template) if err != nil { return err } - if err := cl.DeleteTemplate(ctx, sandboxd.PoolKey(key)); err != nil { + if err := cl.DeleteTemplate(ctx, sandboxd.PoolKey(key), digest); err != nil { return nodeVerbError(err, "delete template", key.Template, node) } return nil diff --git a/pkg/scale/sandboxstore_lifecycle_test.go b/pkg/scale/sandboxstore_lifecycle_test.go index 09af61c..5047474 100644 --- a/pkg/scale/sandboxstore_lifecycle_test.go +++ b/pkg/scale/sandboxstore_lifecycle_test.go @@ -32,7 +32,7 @@ func TestLifecycleVerbsMapANodeUnknownSandboxToNotFound(t *testing.T) { for name, err := range map[string]error{ "pause": store.Pause(ctx, "n1", "sb_gone"), "metadata": store.SetInstanceMetadata(ctx, "n1", "sb_gone", []byte(`{}`)), - "template": store.DeleteTemplate(ctx, "n1", PoolKey{Template: "tpl:gone"}), + "template": store.DeleteTemplate(ctx, "n1", PoolKey{Template: "tpl:gone"}, ""), "labels": store.SetTemplateLabels(ctx, "n1", PoolKey{Template: "tpl:gone"}, nil), "resume": store.Resume(ctx, "n1", "sb_gone"), "port": portErr, From 6cf75ea45805bd5c8d3c71cab1b5f06a53fc7cc4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:46:39 +0800 Subject: [PATCH 19/23] fix(sandboxd): the relay target keeps an IPv6 origin's brackets once and the cancel hook closes the raw connection url.Host of https://[2001:db8::1] kept its brackets and JoinHostPort added a second pair, so every relay dial to such a node failed; the cancel hook captured the conn variable the TLS handshake then reassigned, a data race the race detector reports under a cancellation. --- pkg/sandboxd/ports.go | 31 ++++++++++++++++++------------- pkg/sandboxd/ports_test.go | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/pkg/sandboxd/ports.go b/pkg/sandboxd/ports.go index 47642a8..75586e0 100644 --- a/pkg/sandboxd/ports.go +++ b/pkg/sandboxd/ports.go @@ -28,31 +28,25 @@ func (c *Client) DialPort(ctx context.Context, id, token string, port uint16) (n // DialPort opens the node's GET /v1/sandboxes/{id}/ports/{port} relay at base, a host:port or an http(s) origin; with the sandbox's own token the node wakes a paused sandbox for it. func DialPort(ctx context.Context, d *net.Dialer, base, id, token string, port uint16) (net.Conn, error) { - scheme, host := "http", base - if u, err := url.Parse(base); err == nil && strings.Contains(base, "://") { - scheme, host = u.Scheme, u.Host - } - if _, _, err := net.SplitHostPort(host); err != nil { - host = net.JoinHostPort(host, map[bool]string{true: "443", false: "80"}[scheme == "https"]) - } - conn, err := d.DialContext(ctx, "tcp", host) + scheme, addr, serverName := relayTarget(base) + tcp, err := d.DialContext(ctx, "tcp", addr) if err != nil { return nil, fmt.Errorf("dial node: %w", err) } - stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + stop := context.AfterFunc(ctx, func() { _ = tcp.Close() }) defer stop() + conn := tcp if scheme == "https" { - hostname, _, _ := net.SplitHostPort(host) - tc := tls.Client(conn, &tls.Config{ServerName: hostname, MinVersion: tls.VersionTLS12}) + tc := tls.Client(tcp, &tls.Config{ServerName: serverName, MinVersion: tls.VersionTLS12}) if err = tc.HandshakeContext(ctx); err != nil { - _ = conn.Close() + _ = tcp.Close() return nil, fmt.Errorf("tls to node: %w", err) } conn = tc } path := "/v1/sandboxes/" + url.PathEscape(id) + "/ports/" + strconv.FormatUint(uint64(port), 10) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+host+path, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+addr+path, nil) if err != nil { _ = conn.Close() return nil, err @@ -87,3 +81,14 @@ type bufConn struct { } func (c *bufConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +func relayTarget(base string) (scheme, addr, serverName string) { + scheme, host, port := "http", base, "" + if u, err := url.Parse(base); err == nil && strings.Contains(base, "://") { + scheme, host, port = u.Scheme, u.Hostname(), u.Port() + } else if h, p, err := net.SplitHostPort(base); err == nil { + host, port = h, p + } + port = cmp.Or(port, map[bool]string{true: "443", false: "80"}[scheme == "https"]) + return scheme, net.JoinHostPort(host, port), host +} diff --git a/pkg/sandboxd/ports_test.go b/pkg/sandboxd/ports_test.go index 454fb94..25e501e 100644 --- a/pkg/sandboxd/ports_test.go +++ b/pkg/sandboxd/ports_test.go @@ -79,6 +79,20 @@ func TestDialPortSpeaksTLSToAnHTTPSNode(t *testing.T) { assert.Equal(t, byte(0x16), <-first, "an https base opens a TLS handshake, not a cleartext upgrade") } +func TestRelayTargetJoinsAnOriginWithItsDefaultPort(t *testing.T) { + for base, want := range map[string][3]string{ + "10.0.0.4:7777": {"http", "10.0.0.4:7777", "10.0.0.4"}, + "node": {"http", "node:80", "node"}, + "http://node:8080": {"http", "node:8080", "node"}, + "https://node-a.example.com": {"https", "node-a.example.com:443", "node-a.example.com"}, + "https://[2001:db8::1]": {"https", "[2001:db8::1]:443", "2001:db8::1"}, + "https://[2001:db8::1]:8443": {"https", "[2001:db8::1]:8443", "2001:db8::1"}, + } { + scheme, addr, serverName := relayTarget(base) + assert.Equal(t, want, [3]string{scheme, addr, serverName}, base) + } +} + func TestClientDialPortPresentsTheSandboxTokenWhenGivenOne(t *testing.T) { var gotAuth string node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From fc5d212d43d6fdb7dbf80004a6e69152571a98f8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 14:47:05 +0800 Subject: [PATCH 20/23] tests(sandboxd): the client sends the observed digest and surfaces a replaced generation as 412 --- pkg/sandboxd/client_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/sandboxd/client_test.go b/pkg/sandboxd/client_test.go index 7fb50a8..2e65eb0 100644 --- a/pkg/sandboxd/client_test.go +++ b/pkg/sandboxd/client_test.go @@ -285,6 +285,8 @@ func TestDeleteTemplateAsksThisNodeAloneAndTakesAMissAsGone(t *testing.T) { w.WriteHeader(http.StatusNotFound) case "ns/pooled": w.WriteHeader(http.StatusConflict) + case "ns/replaced": + w.WriteHeader(http.StatusPreconditionFailed) default: w.WriteHeader(http.StatusNoContent) } @@ -292,14 +294,20 @@ func TestDeleteTemplateAsksThisNodeAloneAndTakesAMissAsGone(t *testing.T) { defer srv.Close() c := New(srv.URL, "root-token") - require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/app", Net: "none", Size: "medium"})) - require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/gone"})) - err := c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/pooled"}) + require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/app", Net: "none", Size: "medium"}, "")) + require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/gone"}, "")) + err := c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/pooled"}, "") he, ok := errors.AsType[*HTTPError](err) require.True(t, ok, "the node's refusal must surface as its status, got %v", err) assert.Equal(t, http.StatusConflict, he.StatusCode) assert.Equal(t, "DELETE /v1/templates?net=none&no_redirect=1&size=medium&template=ns%2Fapp Bearer root-token", got[0]) assert.Equal(t, "DELETE /v1/templates?no_redirect=1&template=ns%2Fgone Bearer root-token", got[1]) + + err = c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/replaced"}, "sha256:x") + he, ok = errors.AsType[*HTTPError](err) + require.True(t, ok, "a replaced generation must surface as 412, got %v", err) + assert.Equal(t, http.StatusPreconditionFailed, he.StatusCode) + assert.Equal(t, "DELETE /v1/templates?digest=sha256%3Ax&no_redirect=1&template=ns%2Freplaced Bearer root-token", got[3]) } func TestSetTemplateLabelsPutsTheWholeMap(t *testing.T) { From a1646946855c1f5814e210580dba698705c41c2e Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 15:04:31 +0800 Subject: [PATCH 21/23] fix(e2b): a tag write names the generation it was meant for Between publishBuild's digest check and its label write a newer promote on the same node could replace the record, and the write then relabeled the newer content with the older digest; the write now carries the promote's digest, sandboxd (ad18c53) refuses a replaced record with 412, and the build reports itself replaced. Tag assign and delete pass each holder's observed digest for the same reason, and a failed publish tells the build status why. --- pkg/e2bbuild/executor.go | 2 +- pkg/e2bcompat/builds.go | 15 +++++++++++++-- pkg/e2bcompat/server_test.go | 8 ++++++-- pkg/e2bcompat/templates.go | 4 +++- pkg/sandboxd/client_test.go | 6 +++--- pkg/sandboxd/lifecycle.go | 10 +++++++--- pkg/scale/apiserver/storage_test.go | 2 +- pkg/scale/sandboxstore.go | 4 ++-- pkg/scale/sandboxstore_claim_test.go | 2 +- pkg/scale/sandboxstore_impl.go | 2 +- pkg/scale/sandboxstore_lifecycle.go | 4 ++-- pkg/scale/sandboxstore_lifecycle_test.go | 2 +- 12 files changed, 41 insertions(+), 20 deletions(-) diff --git a/pkg/e2bbuild/executor.go b/pkg/e2bbuild/executor.go index 11b69f5..697738f 100644 --- a/pkg/e2bbuild/executor.go +++ b/pkg/e2bbuild/executor.go @@ -407,7 +407,7 @@ func (e *Executor) promote(ctx context.Context, id string, a scale.Assignment, s return "", "", nil } if err := spec.Publish(ctx, a.Node, key, digest); err != nil { - return PhasePublish, "could not publish the build over the previous one", err + return PhasePublish, "could not publish the build over the previous one: " + err.Error(), err } return "", "", nil } diff --git a/pkg/e2bcompat/builds.go b/pkg/e2bcompat/builds.go index dd08b99..e44a19b 100644 --- a/pkg/e2bcompat/builds.go +++ b/pkg/e2bcompat/builds.go @@ -152,7 +152,7 @@ func (s *Server) publishBuild(scope, name string, tags []string) e2bbuild.Publis stale := slices.DeleteFunc(holders, func(h templateHolder) bool { return !h.created.Before(own.created) }) if err := forEachHolder(stale, func(h templateHolder) error { err := s.store.DeleteTemplate(ctx, h.node, h.key, h.digest) - if se, ok := errors.AsType[*k8serrors.StatusError](err); ok && se.ErrStatus.Code == http.StatusPreconditionFailed { + if replaced(err) { return nil } return err @@ -166,7 +166,12 @@ func (s *Server) publishBuild(scope, name string, tags []string) e2bbuild.Publis for _, t := range tags { labels[t] = digest } - return s.store.SetTemplateLabels(ctx, node, key, labels) + if err := s.store.SetTemplateLabels(ctx, node, key, labels, digest); replaced(err) { + return fmt.Errorf("a newer build of %s replaced this one on %s", name, node) + } else if err != nil { + return err + } + return nil } } @@ -268,3 +273,9 @@ func queryInt(v string) (int, error) { } return n, nil } + +// replaced is sandboxd's 412: the template generation the caller observed is gone. +func replaced(err error) bool { + se, ok := errors.AsType[*k8serrors.StatusError](err) + return ok && se.ErrStatus.Code == http.StatusPreconditionFailed +} diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index d0d0828..ed4fd67 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -570,6 +570,7 @@ type fakeStore struct { deletedSnapshotID string deletedTemplates []string labeled []string + labelErr error replacedDigest string deleteTemplateErr map[string]error fleet *scale.StaticInventorySource @@ -686,10 +687,13 @@ func (f *fakeStore) Promote(ctx context.Context, node, id, template string) (sca } // SetTemplateLabels writes straight into the fleet the test serves, so a live read sees it at once. -func (f *fakeStore) SetTemplateLabels(ctx context.Context, node string, key scale.PoolKey, labels map[string]string) error { +func (f *fakeStore) SetTemplateLabels(ctx context.Context, node string, key scale.PoolKey, labels map[string]string, digest string) error { f.mu.Lock() - f.labeled = append(f.labeled, node+" "+key.Template) + f.labeled = append(f.labeled, strings.TrimSpace(node+" "+key.Template+" "+digest)) f.mu.Unlock() + if f.labelErr != nil { + return f.labelErr + } inv, err := f.fleet.NodeInventory(ctx, node) if err != nil { return err diff --git a/pkg/e2bcompat/templates.go b/pkg/e2bcompat/templates.go index 2c3bfa9..d9f2e1e 100644 --- a/pkg/e2bcompat/templates.go +++ b/pkg/e2bcompat/templates.go @@ -332,7 +332,9 @@ func (s *Server) liveTags(r *http.Request, b *builtTemplate) (map[string]string, } func (s *Server) writeTags(r *http.Request, b *builtTemplate, tags map[string]string) error { - return forEachHolder(b.holders, func(h templateHolder) error { return s.store.SetTemplateLabels(r.Context(), h.node, h.key, tags) }) + return forEachHolder(b.holders, func(h templateHolder) error { + return s.store.SetTemplateLabels(r.Context(), h.node, h.key, tags, h.digest) + }) } func (s *Server) writeTagError(w http.ResponseWriter, r *http.Request, err error, name string) { diff --git a/pkg/sandboxd/client_test.go b/pkg/sandboxd/client_test.go index 2e65eb0..985dfe0 100644 --- a/pkg/sandboxd/client_test.go +++ b/pkg/sandboxd/client_test.go @@ -320,10 +320,10 @@ func TestSetTemplateLabelsPutsTheWholeMap(t *testing.T) { defer srv.Close() c := New(srv.URL, "root-token") - require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app", Size: "medium"}, map[string]string{"v1": "sha256:aa"})) - require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app"}, nil)) + require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app", Size: "medium"}, map[string]string{"v1": "sha256:aa"}, "sha256:aa")) + require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app"}, nil, "")) assert.Equal(t, []string{ - `PUT /v1/templates/labels?size=medium&template=ns%2Fapp {"labels":{"v1":"sha256:aa"}}`, + `PUT /v1/templates/labels?digest=sha256%3Aaa&size=medium&template=ns%2Fapp {"labels":{"v1":"sha256:aa"}}`, `PUT /v1/templates/labels?template=ns%2Fapp {"labels":null}`, }, got) } diff --git a/pkg/sandboxd/lifecycle.go b/pkg/sandboxd/lifecycle.go index 9ffca1a..787991e 100644 --- a/pkg/sandboxd/lifecycle.go +++ b/pkg/sandboxd/lifecycle.go @@ -180,15 +180,19 @@ func (c *Client) DeleteTemplate(ctx context.Context, key PoolKey, digest string) return c.send(ctx, http.MethodDelete, "/v1/templates?"+q.Encode(), c.token, "delete template", nil, http.StatusNoContent, http.StatusNotFound) } -// SetTemplateLabels performs PUT /v1/templates/labels for key on this node, replacing the whole label map. -func (c *Client) SetTemplateLabels(ctx context.Context, key PoolKey, labels map[string]string) error { +// SetTemplateLabels performs PUT /v1/templates/labels for key on this node, replacing the whole label map; a digest writes only that generation (412 otherwise). +func (c *Client) SetTemplateLabels(ctx context.Context, key PoolKey, labels map[string]string, digest string) error { body, err := json.Marshal(struct { Labels map[string]string `json:"labels"` }{labels}) if err != nil { return err } - return c.send(ctx, http.MethodPut, "/v1/templates/labels?"+templateQuery(key).Encode(), c.token, "template labels", body, http.StatusNoContent) + q := templateQuery(key) + if digest != "" { + q.Set("digest", digest) + } + return c.send(ctx, http.MethodPut, "/v1/templates/labels?"+q.Encode(), c.token, "template labels", body, http.StatusNoContent) } // DeleteCheckpoint performs DELETE /v1/checkpoints/{id}. A 404 is success. diff --git a/pkg/scale/apiserver/storage_test.go b/pkg/scale/apiserver/storage_test.go index 831bb90..69b25a9 100644 --- a/pkg/scale/apiserver/storage_test.go +++ b/pkg/scale/apiserver/storage_test.go @@ -320,7 +320,7 @@ func (f *fakeStore) Promote(context.Context, string, string, string) (scale.Pool return scale.PoolKey{}, "", f.verbErr } -func (f *fakeStore) SetTemplateLabels(context.Context, string, scale.PoolKey, map[string]string) error { +func (f *fakeStore) SetTemplateLabels(context.Context, string, scale.PoolKey, map[string]string, string) error { return f.verbErr } diff --git a/pkg/scale/sandboxstore.go b/pkg/scale/sandboxstore.go index 7638ee2..5390055 100644 --- a/pkg/scale/sandboxstore.go +++ b/pkg/scale/sandboxstore.go @@ -91,8 +91,8 @@ type SandboxLifecycle interface { DeleteTemplate(ctx context.Context, node string, key PoolKey, digest string) error // Promote publishes a claimed sandbox on node as the template name, keeping its net and size; it returns the full key and the content digest. Promote(ctx context.Context, node, id, template string) (PoolKey, string, error) - // SetTemplateLabels replaces a promoted template's labels on node alone. - SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string) error + // SetTemplateLabels replaces a promoted template's labels on node alone; a digest writes only the generation it names (PreconditionFailed otherwise). + SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string, digest string) error // NodeTemplates reads the promoted templates node holds now, ahead of its next inventory publish. NodeTemplates(ctx context.Context, node string) ([]PromotedTemplate, error) // DialGuestPort opens a guest port of a running sandbox through its node's passive relay; a paused one answers Conflict and is never woken. diff --git a/pkg/scale/sandboxstore_claim_test.go b/pkg/scale/sandboxstore_claim_test.go index 7084b07..cda7e87 100644 --- a/pkg/scale/sandboxstore_claim_test.go +++ b/pkg/scale/sandboxstore_claim_test.go @@ -585,7 +585,7 @@ func (c *recordingClient) Promote(_ context.Context, id, template string) (sandb return sandboxd.PoolKey{Template: template}, "sha256:" + id, c.f.verbErr } -func (c *recordingClient) SetTemplateLabels(context.Context, sandboxd.PoolKey, map[string]string) error { +func (c *recordingClient) SetTemplateLabels(context.Context, sandboxd.PoolKey, map[string]string, string) error { return c.f.verbErr } diff --git a/pkg/scale/sandboxstore_impl.go b/pkg/scale/sandboxstore_impl.go index 5ba8f15..8b0878d 100644 --- a/pkg/scale/sandboxstore_impl.go +++ b/pkg/scale/sandboxstore_impl.go @@ -128,7 +128,7 @@ type SandboxdClient interface { DeleteCheckpoint(ctx context.Context, checkpointID string) error DeleteTemplate(ctx context.Context, key sandboxd.PoolKey, digest string) error Promote(ctx context.Context, id, template string) (sandboxd.PoolKey, string, error) - SetTemplateLabels(ctx context.Context, key sandboxd.PoolKey, labels map[string]string) error + SetTemplateLabels(ctx context.Context, key sandboxd.PoolKey, labels map[string]string, digest string) error Info(ctx context.Context) (*sandboxd.NodeInfo, error) DialPort(ctx context.Context, id, token string, port uint16) (net.Conn, error) SetInstanceMetadata(ctx context.Context, id string, doc []byte) error diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go index ec32a70..faf16fa 100644 --- a/pkg/scale/sandboxstore_lifecycle.go +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -167,12 +167,12 @@ func (s *scatterGatherStore) Promote(ctx context.Context, node, id, template str return PoolKey(key), digest, nil } -func (s *scatterGatherStore) SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string) error { +func (s *scatterGatherStore) SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string, digest string) error { cl, err := s.nodeClient(ctx, node, "template labels", key.Template) if err != nil { return err } - if err := cl.SetTemplateLabels(ctx, sandboxd.PoolKey(key), labels); err != nil { + if err := cl.SetTemplateLabels(ctx, sandboxd.PoolKey(key), labels, digest); err != nil { return nodeVerbError(err, "template labels", key.Template, node) } return nil diff --git a/pkg/scale/sandboxstore_lifecycle_test.go b/pkg/scale/sandboxstore_lifecycle_test.go index 5047474..8b13666 100644 --- a/pkg/scale/sandboxstore_lifecycle_test.go +++ b/pkg/scale/sandboxstore_lifecycle_test.go @@ -33,7 +33,7 @@ func TestLifecycleVerbsMapANodeUnknownSandboxToNotFound(t *testing.T) { "pause": store.Pause(ctx, "n1", "sb_gone"), "metadata": store.SetInstanceMetadata(ctx, "n1", "sb_gone", []byte(`{}`)), "template": store.DeleteTemplate(ctx, "n1", PoolKey{Template: "tpl:gone"}, ""), - "labels": store.SetTemplateLabels(ctx, "n1", PoolKey{Template: "tpl:gone"}, nil), + "labels": store.SetTemplateLabels(ctx, "n1", PoolKey{Template: "tpl:gone"}, nil, ""), "resume": store.Resume(ctx, "n1", "sb_gone"), "port": portErr, "fork": forkErr, From 2e9cc0a59ac206bae917e3075bdfd867d67589b8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 15:04:31 +0800 Subject: [PATCH 22/23] tests(e2b): the publish regression tests 1c2726aa named TestAPublishSkipsAHolderReplacedSinceItWasObserved, TestABuildReplacedOnItsOwnNodeWritesNoTags and TestABuildWhoseTagWriteFindsANewerGenerationReportsReplaced; the first two were left out of that commit. --- pkg/e2bcompat/builds_test.go | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index 934ff7d..7eb2a88 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cocoonstack/sandbox-operator/pkg/e2bbuild" @@ -164,6 +165,50 @@ func TestAPublishLeavesANewerConcurrentBuildAlone(t *testing.T) { assert.Equal(t, []string{"k e2b/sandboxes/app small sha256:older"}, store.deletedTemplates, "only the holder older than this promote goes, by the digest observed; a newer concurrent build keeps its node") } +func TestAPublishSkipsAHolderReplacedSinceItWasObserved(t *testing.T) { + older := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:older", CreatedAt: new(metav1.NewTime(time.Now().Add(-time.Minute)))} + replaced := &k8serrors.StatusError{ErrStatus: metav1.Status{Code: http.StatusPreconditionFailed}} + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil, "k": {older}}, deleteTemplateErr: map[string]error{"k": replaced}} + inv := scale.NewStaticInventorySource() + for _, n := range []string{"n", "k"} { + inv.Put(&scale.NodeInventory{Node: n}) + } + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status, "a holder a newer promote replaced between the read and the delete is left alone") + assert.Equal(t, []string{"k e2b/sandboxes/app small sha256:older"}, store.deletedTemplates) +} + +func TestABuildReplacedOnItsOwnNodeWritesNoTags(t *testing.T) { + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil}, replacedDigest: "sha256:newer"} + inv := scale.NewStaticInventorySource() + inv.Put(&scale.NodeInventory{Node: "n"}) + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app:v1") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + info := waitBuild(t, h, id) + require.Equal(t, e2bbuild.StatusError, info.Status) + assert.Contains(t, info.Reason.Message, "replaced") + assert.Empty(t, store.labeled, "a build a newer promote replaced must not tag the newer content with its own digest") +} + +func TestABuildWhoseTagWriteFindsANewerGenerationReportsReplaced(t *testing.T) { + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil}, labelErr: &k8serrors.StatusError{ErrStatus: metav1.Status{Code: http.StatusPreconditionFailed}}} + inv := scale.NewStaticInventorySource() + inv.Put(&scale.NodeInventory{Node: "n"}) + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app:v1") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + info := waitBuild(t, h, id) + require.Equal(t, e2bbuild.StatusError, info.Status) + assert.Contains(t, info.Reason.Message, "replaced") + assert.Equal(t, []string{"n e2b/sandboxes/app sha256:sb_1"}, store.labeled, "the tag write carries the promote's digest, so a newer generation refuses it") +} + func requestBuild(t *testing.T, h http.Handler, name string) string { t.Helper() w := do(t, h, http.MethodPost, "/v3/templates", `{"name":"`+name+`"}`, testKey) From 94d76961793c68a4dc0d856777edd40e6b309d67 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 28 Sep 2026 15:05:22 +0800 Subject: [PATCH 23/23] fix(e2bbuild): a failed publish keeps its node-free message a1646946 appended the publish error to the failure a caller sees, which names nodes; TestAFailedBuildNamesItsPhase pins the opposite. The reason stays in the process log, and the two new tests assert the phase. --- pkg/e2bbuild/executor.go | 2 +- pkg/e2bcompat/builds_test.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/e2bbuild/executor.go b/pkg/e2bbuild/executor.go index 697738f..11b69f5 100644 --- a/pkg/e2bbuild/executor.go +++ b/pkg/e2bbuild/executor.go @@ -407,7 +407,7 @@ func (e *Executor) promote(ctx context.Context, id string, a scale.Assignment, s return "", "", nil } if err := spec.Publish(ctx, a.Node, key, digest); err != nil { - return PhasePublish, "could not publish the build over the previous one: " + err.Error(), err + return PhasePublish, "could not publish the build over the previous one", err } return "", "", nil } diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index 7eb2a88..d843be3 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -190,8 +190,7 @@ func TestABuildReplacedOnItsOwnNodeWritesNoTags(t *testing.T) { id := requestBuild(t, h, "app:v1") require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) info := waitBuild(t, h, id) - require.Equal(t, e2bbuild.StatusError, info.Status) - assert.Contains(t, info.Reason.Message, "replaced") + require.Equal(t, [2]string{e2bbuild.StatusError, "finalize"}, [2]string{info.Status, info.Reason.Step}) assert.Empty(t, store.labeled, "a build a newer promote replaced must not tag the newer content with its own digest") } @@ -204,8 +203,7 @@ func TestABuildWhoseTagWriteFindsANewerGenerationReportsReplaced(t *testing.T) { id := requestBuild(t, h, "app:v1") require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) info := waitBuild(t, h, id) - require.Equal(t, e2bbuild.StatusError, info.Status) - assert.Contains(t, info.Reason.Message, "replaced") + require.Equal(t, [2]string{e2bbuild.StatusError, "finalize"}, [2]string{info.Status, info.Reason.Step}) assert.Equal(t, []string{"n e2b/sandboxes/app sha256:sb_1"}, store.labeled, "the tag write carries the promote's digest, so a newer generation refuses it") }