diff --git a/cmd/sandbox-apiserver/main.go b/cmd/sandbox-apiserver/main.go index 00866ef..c953d20 100644 --- a/cmd/sandbox-apiserver/main.go +++ b/cmd/sandbox-apiserver/main.go @@ -44,6 +44,8 @@ const ( watchDrainGrace = 2 * time.Second ) +type managerBuilder func() (manager.Runnable, error) + // options has no etcd option because this server stores nothing. type options struct { SecureServing *genericoptions.SecureServingOptionsWithLoopback @@ -85,10 +87,7 @@ func (o *options) addFlags(fs *pflag.FlagSet) { o.Authentication.AddFlags(fs) o.Authorization.AddFlags(fs) o.Features.AddFlags(fs) - fs.StringVar(&o.SandboxdToken, "sandboxd-token", o.SandboxdToken, - "Uniform fleet-wide sandboxd api_token presented on node-local claim/release. Prefer --sandboxd-token-file for a Secret mount.") - fs.StringVar(&o.SandboxdTokenFile, "sandboxd-token-file", o.SandboxdTokenFile, - "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") + sandboxd.AddTokenFlags(fs, &o.SandboxdToken, &o.SandboxdTokenFile) fs.BoolVar(&o.WarmPoolDriver, "enable-warm-pool-driver", o.WarmPoolDriver, "Run the in-process SandboxWarmPool → sandboxd pool reconcile loop (control-plane warm-capacity surface; pool-level, never per-sandbox).") fs.DurationVar(&o.WarmPoolInterval, "warm-pool-sync-interval", o.WarmPoolInterval, @@ -140,7 +139,7 @@ func run() error { if err != nil { return fmt.Errorf("load kube config: %w", err) } - reader, err := kubeinventory.NewCache(ctx, restCfg) + informers, err := kubeinventory.NewCache(ctx, restCfg) if err != nil { return err } @@ -148,7 +147,7 @@ func run() error { if err != nil { return err } - invSource, err := kubeinventory.New(ctx, reader, o.Inventory) + invSource, err := kubeinventory.New(ctx, informers, o.Inventory) if err != nil { return err } @@ -221,7 +220,7 @@ func startWarmPoolDriver(ctx context.Context, restCfg *restclient.Config, token return nil } -func runRestarting(ctx context.Context, r manager.Runnable, build func() (manager.Runnable, error), delay time.Duration) { +func runRestarting(ctx context.Context, r manager.Runnable, build managerBuilder, delay time.Duration) { logger := log.WithFunc("main.runRestarting") for { if r != nil { diff --git a/cmd/sandbox-e2b/main.go b/cmd/sandbox-e2b/main.go index 06472c5..b81d67a 100644 --- a/cmd/sandbox-e2b/main.go +++ b/cmd/sandbox-e2b/main.go @@ -35,10 +35,7 @@ type options struct { func (o *options) addFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&o.Seeds, "sandboxd-seeds", o.Seeds, "Comma-separated sandboxd addresses dialed at start, each naming one node; each node reports its own advertise_addr as its key, and the rest of the mesh is found through their gossip.") - fs.StringVar(&o.SandboxdToken, "sandboxd-token", o.SandboxdToken, - "Fleet root sandboxd api_token (GET /v1/info needs root). Prefer --sandboxd-token-file for a Secret mount.") - fs.StringVar(&o.SandboxdTokenFile, "sandboxd-token-file", o.SandboxdTokenFile, - "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") + sandboxd.AddTokenFlags(fs, &o.SandboxdToken, &o.SandboxdTokenFile) fs.DurationVar(&o.PollInterval, "inventory-poll", o.PollInterval, "How often every node's info and sandbox list are read; List and Watch lag a change by up to one tick.") o.E2B.AddFlags(fs) @@ -94,11 +91,11 @@ func run() error { <-ctx.Done() return nil } - resolver, err := envdproxy.NewResolver(scale.NewScatterGatherStore(src), store, src, "", opts.EnvdSecret) + resolver, err := envdproxy.NewResolver(store, src, "", opts.EnvdSecret) if err != nil { return err } - proxy, err := envdproxy.NewServer(resolver, envdproxy.Options{Domain: o.E2B.Domain, GuestHTTP2: o.Proxy.GuestHTTP2}) + proxy, err := envdproxy.NewServer(resolver, envdproxy.Options{Domain: o.E2B.Domain, GuestHTTP2: o.Proxy.GuestHTTP2, NodeToken: token}) if err != nil { return err } diff --git a/cmd/sandbox-envd-proxy/main.go b/cmd/sandbox-envd-proxy/main.go index e4b98b0..d7ca81c 100644 --- a/cmd/sandbox-envd-proxy/main.go +++ b/cmd/sandbox-envd-proxy/main.go @@ -45,10 +45,7 @@ func (o *options) addFlags(fs *pflag.FlagSet) { o.Inventory.AddFlags(fs) fs.StringVar(&o.Domain, "domain", o.Domain, "Base domain sandbox hosts are derived from, as {port}-{sandboxID}.{domain}. Must match the apiserver's --e2b-domain.") - fs.StringVar(&o.SandboxdToken, "sandboxd-token", o.SandboxdToken, - "Fleet root sandboxd api_token, which reads a sandbox's claim token to verify its envd access token. Prefer --sandboxd-token-file for a Secret mount.") - fs.StringVar(&o.SandboxdTokenFile, "sandboxd-token-file", o.SandboxdTokenFile, - "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") + sandboxd.AddTokenFlags(fs, &o.SandboxdToken, &o.SandboxdTokenFile) e2bcompat.AddEnvdSecretFlag(fs, &o.EnvdSecretFile) fs.StringVar(&o.Namespace, "namespace", o.Namespace, "Namespace inventory lookups are filtered to; empty matches every namespace. Not an access boundary: a caller holding a sandbox's token reaches it in any namespace.") @@ -68,7 +65,7 @@ func main() { o.addFlags(fs) _ = fs.Parse(os.Args[1:]) if err := run(ctx, o); err != nil { - log.WithFunc("main").Fatalf(ctx, err, "sandbox-envd-proxy exited") + log.WithFunc("main.main").Fatalf(ctx, err, "sandbox-envd-proxy exited") } } @@ -78,11 +75,11 @@ func run(ctx context.Context, o *options) error { if err != nil { return fmt.Errorf("load kube config: %w", err) } - reader, err := kubeinventory.NewCache(ctx, restCfg) + informers, err := kubeinventory.NewCache(ctx, restCfg) if err != nil { return err } - inv, err := kubeinventory.New(ctx, reader, o.Inventory) + inv, err := kubeinventory.New(ctx, informers, o.Inventory) if err != nil { return err } @@ -95,13 +92,14 @@ func run(ctx context.Context, o *options) error { return err } routed := scale.NewScatterGatherStore(inv, scale.WithClaimRouting(token, scale.NewSandboxdClientFactory())) - resolver, err := envdproxy.NewResolver(scale.NewScatterGatherStore(inv), routed, inv, o.Namespace, secret) + resolver, err := envdproxy.NewResolver(routed, inv, o.Namespace, secret) if err != nil { return err } srv, err := envdproxy.NewServer(resolver, envdproxy.Options{ Domain: o.Domain, GuestHTTP2: o.Proxy.GuestHTTP2, + NodeToken: token, }) if err != nil { return err diff --git a/docs/e2b-compat.md b/docs/e2b-compat.md index 7a21b62..871d2be 100644 --- a/docs/e2b-compat.md +++ b/docs/e2b-compat.md @@ -325,6 +325,13 @@ needs its own stickiness (or run one replica). A finished build is kept for an h loses a build in flight; the SDK's next poll throws, and a rebuild converges because a promote replaces. +Built templates expect a `checkpoint_store` per node. A node reads the store's +template records once at start and advertises them as its own, so with one S3 +store shared by every node each node advertises every node's templates as of its +own start, and a publish that deletes an older holder removes a record its +siblings still advertise; a create of one fails over to the next advertiser, and +the template list lags until those nodes restart. + ## Limits worth knowing - **Reaching `envd` (the in-sandbox data plane).** The SDK derives the sandbox diff --git a/pkg/e2bbuild/copy.go b/pkg/e2bbuild/copy.go index 563e423..f167fe9 100644 --- a/pkg/e2bbuild/copy.go +++ b/pkg/e2bbuild/copy.go @@ -2,6 +2,7 @@ package e2bbuild import ( "path" + "slices" "strings" ) @@ -47,7 +48,7 @@ func copyScript(state Command, s Step) string { scratch := "/tmp/" + s.FilesHash unpack := scratch + "/unpack" vars := [][2]string{ - {"archive", scratch + ".tar"}, + {"archive", archivePath(s.FilesHash)}, {"scratch", scratch}, {"unpack", unpack}, {"src", s.Args[0]}, @@ -65,13 +66,15 @@ func copyScript(state Command, s Step) string { return b.String() + copyMove } +func archivePath(hash string) string { + return "/tmp/" + hash + ".tar" +} + // globBase is src up to its first segment with a glob; the SDK expands globs into the archive itself. func globBase(src string) string { segs := strings.Split(strings.TrimSuffix(src, "/"), "/") - for i, seg := range segs { - if strings.ContainsAny(seg, "*?[{") { - return strings.Join(segs[:i], "/") - } + if i := slices.IndexFunc(segs, func(seg string) bool { return strings.ContainsAny(seg, "*?[{") }); i >= 0 { + segs = segs[:i] } return strings.Join(segs, "/") } diff --git a/pkg/e2bbuild/executor.go b/pkg/e2bbuild/executor.go index 159075d..11b69f5 100644 --- a/pkg/e2bbuild/executor.go +++ b/pkg/e2bbuild/executor.go @@ -14,6 +14,7 @@ import ( "maps" "path" "regexp" + "slices" "strconv" "strings" "sync" @@ -55,11 +56,21 @@ var ( // ErrUnknownBuild is a build this process never registered, or dropped an hour after it finished or was left unstarted. ErrUnknownBuild = errors.New("e2bbuild: build not found") - envEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "`", "\\`", "$(", `\$(`) // FilesHash is the shape of the digest the SDK names a COPY upload by. FilesHash = regexp.MustCompile(`^[0-9a-f]{64}$`) + + envEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "`", "\\`", "$(", `\$(`) ) +// LineFunc receives one line of a command's output. +type LineFunc func(line string) + +// ArchiveFunc opens the upload a COPY step's files hash names. +type ArchiveFunc func(ctx context.Context, hash string) (io.ReadCloser, error) + +// PublishFunc runs after the promote with the template's node, key and content digest. +type PublishFunc func(ctx context.Context, node string, key scale.PoolKey, digest string) error + // Request is what a build asked for before it starts. type Request struct { Size string @@ -99,19 +110,17 @@ func (s Step) problem(canCopy bool) string { return "" } -// Spec is how a registered build runs; RelayEnvs seed the steps when the claim relays through the node's proxy, StartCmd runs in the background and ReadyCmd until it exits 0, both before the promote; Publish runs after the promote and before the build reads ready. +// Spec is how a registered build runs; StartCmd runs in the background and ReadyCmd until it exits 0, both before the promote; Publish runs after the promote and before the build reads ready. type Spec struct { - Namespace string - ClaimName string - Pool scale.PoolKey - Template string - TTLSeconds int - Steps []Step - StartCmd string - ReadyCmd string - RelayEnvs map[string]string - Archive func(ctx context.Context, hash string) (io.ReadCloser, error) - Publish func(ctx context.Context, node string, key scale.PoolKey, digest string) error + Namespace string + ClaimName string + Pool scale.PoolKey + Template string + Steps []Step + StartCmd string + ReadyCmd string + Archive ArchiveFunc + Publish PublishFunc } // Command is one shell line run as User, in Workdir, with Envs; an empty Workdir is the user's home. @@ -124,8 +133,8 @@ type Command struct { // Guest runs a build's commands inside its claimed sandbox. type Guest interface { - // Run runs cmd to its end, passes each output line to out, and returns its exit code. - Run(ctx context.Context, a scale.Assignment, cmd Command, out func(line string)) (int, error) + // Run runs cmd to its end, passes each line of its output to stdout or stderr, and returns its exit code. + Run(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr LineFunc) (int, error) // Start starts cmd and returns while it runs. Start(ctx context.Context, a scale.Assignment, cmd Command) error // Init makes the user, workdir and envs of defaults what every later process in the sandbox gets. @@ -227,7 +236,7 @@ func (e *Executor) Status(id string) (Info, bool) { return Info{}, false } info := r.info - info.Logs = append([]LogEntry(nil), r.info.Logs...) + info.Logs = slices.Clip(r.info.Logs) return info, true } @@ -236,7 +245,7 @@ func (e *Executor) run(ctx context.Context, id string, spec Spec) { ctx, cancel := context.WithTimeout(ctx, e.timeout) defer cancel() e.logf(id, PhaseClaim, "claiming a sandbox from %s (%s, %s)", spec.Pool.Template, spec.Pool.Net, spec.Pool.Size) - a, err := e.store.Claim(ctx, spec.Namespace, spec.ClaimName, spec.Pool, scale.ClaimOptions{TTLSeconds: spec.TTLSeconds}) + a, err := e.store.Claim(ctx, spec.Namespace, spec.ClaimName, spec.Pool, scale.ClaimOptions{TTLSeconds: int(e.timeout / time.Second)}) if err != nil { msg := fmt.Sprintf("could not claim a sandbox of %s (%s, %s)", spec.Pool.Template, spec.Pool.Net, spec.Pool.Size) if scale.IsNoWarmCapacity(err) { @@ -261,7 +270,7 @@ func (e *Executor) run(ctx context.Context, id string, spec Spec) { func (e *Executor) prepare(ctx context.Context, id string, a scale.Assignment, spec Spec) (string, string, error) { state := Command{User: defaultUser, Envs: map[string]string{}} if a.NetRoute == sandboxd.NetRouteRelay { - maps.Copy(state.Envs, spec.RelayEnvs) + maps.Copy(state.Envs, sandboxd.RelayEnv) } for i, step := range spec.Steps { phase := strconv.Itoa(i + 1) @@ -298,12 +307,12 @@ func (e *Executor) step(ctx context.Context, id, phase string, a scale.Assignmen if len(s.Args) > 1 { cmd.User = s.Args[1] } - msg, err := e.sh(ctx, a, cmd, logged) + msg, err := e.sh(ctx, a, cmd, logged, logged) return state, msg, err case stepEnv: envs := maps.Clone(state.Envs) for i := 0; i+1 < len(s.Args); i += 2 { - v, msg, err := e.expand(ctx, a, root, s.Args[i+1]) + v, msg, err := e.expand(ctx, a, root, s.Args[i+1], logged) if err != nil { return state, msg, err } @@ -316,14 +325,14 @@ func (e *Executor) step(ctx context.Context, id, phase string, a scale.Assignmen dir = path.Join(cmp.Or(state.Workdir, "/"), dir) } script := fmt.Sprintf(`t=%[1]s; [ -d "$t" ] && exit 0; n=$t; while [ ! -d "$(dirname "$n")" ]; do n=$(dirname "$n"); done; mkdir -p "$t" && chown -R %[2]s: "$n"`, shellQuote(dir), shellQuote(state.User)) - if msg, err := e.sh(ctx, a, withLine(root, script), logged); err != nil { + if msg, err := e.sh(ctx, a, withLine(root, script), logged, logged); err != nil { return state, msg, err } state.Workdir = dir case stepUser: name := shellQuote(s.Args[0]) script := fmt.Sprintf("id -u %[1]s >/dev/null 2>&1 || useradd --create-home --shell /bin/bash %[1]s", name) - if msg, err := e.sh(ctx, a, withLine(root, script), logged); err != nil { + if msg, err := e.sh(ctx, a, withLine(root, script), logged, logged); err != nil { return state, msg, err } state.User = s.Args[0] @@ -331,7 +340,7 @@ func (e *Executor) step(ctx context.Context, id, phase string, a scale.Assignmen if msg, err := e.copyIn(ctx, a, spec, s.FilesHash); err != nil { return state, msg, err } - if _, err := e.sh(ctx, a, withLine(root, copyScript(state, s)), logged); err != nil { + if _, err := e.sh(ctx, a, withLine(root, copyScript(state, s)), logged, logged); err != nil { return state, fmt.Sprintf("could not copy %s to %s", s.Args[0], s.Args[1]), err } } @@ -345,15 +354,15 @@ func (e *Executor) copyIn(ctx context.Context, a scale.Assignment, spec Spec, ha return "could not read the uploaded files", err } defer func() { _ = archive.Close() }() - if err := e.guest.Write(ctx, a, "/tmp/"+hash+".tar", archive); err != nil { + if err := e.guest.Write(ctx, a, archivePath(hash), archive); err != nil { return "could not copy the uploaded files into the build sandbox", err } return "", nil } // sh returns the failure the caller sees next to the error; a non-zero exit is a failure. -func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, out func(string)) (string, error) { - code, err := e.guest.Run(ctx, a, cmd, out) +func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, stdout, stderr LineFunc) (string, error) { + code, err := e.guest.Run(ctx, a, cmd, stdout, stderr) if err != nil { return "could not run a command in the build sandbox", err } @@ -364,10 +373,10 @@ func (e *Executor) sh(ctx context.Context, a scale.Assignment, cmd Command, out return "", nil } -// expand evaluates an ENV value in the guest's shell, so $VAR references resolve and command substitution does not run. -func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, value string) (string, string, error) { +// expand evaluates an ENV value in the guest's shell, so $VAR references resolve and command substitution does not run; the shell's own stderr goes to the log, never into the value. +func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, value string, stderr LineFunc) (string, string, error) { var lines []string - if _, err := e.sh(ctx, a, withLine(root, `printf "%s" "`+envEscaper.Replace(value)+`"`), func(line string) { lines = append(lines, line) }); err != nil { + if _, err := e.sh(ctx, a, withLine(root, `printf "%s" "`+envEscaper.Replace(value)+`"`), func(line string) { lines = append(lines, line) }, stderr); err != nil { return "", fmt.Sprintf("could not evaluate the value %q", value), err } return strings.Join(lines, "\n"), "", nil @@ -375,7 +384,7 @@ func (e *Executor) expand(ctx context.Context, a scale.Assignment, root Command, func (e *Executor) awaitReady(ctx context.Context, a scale.Assignment, cmd Command) error { for { - code, err := e.guest.Run(ctx, a, cmd, func(string) {}) + code, err := e.guest.Run(ctx, a, cmd, func(string) {}, func(string) {}) if err == nil && code == 0 { return nil } @@ -440,13 +449,11 @@ func (e *Executor) log(id, step, level, message string) { } func (e *Executor) sweep(now time.Time) { - for id, r := range e.builds { + maps.DeleteFunc(e.builds, func(_ string, r *record) bool { done := !r.finished.IsZero() && now.Sub(r.finished) > recordTTL abandoned := r.info.Status == StatusWaiting && now.Sub(r.registered) > recordTTL - if done || abandoned { - delete(e.builds, id) - } - } + return done || abandoned + }) } // Invalid names the first step a build cannot run, empty when every step can; a COPY needs canCopy. diff --git a/pkg/e2bbuild/executor_test.go b/pkg/e2bbuild/executor_test.go index 24a6461..dc6b10a 100644 --- a/pkg/e2bbuild/executor_test.go +++ b/pkg/e2bbuild/executor_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "slices" "strings" "sync" "testing" @@ -100,18 +101,18 @@ func TestStepsCarryTheirUserWorkdirAndEnvIntoLaterCommandsAndTheDefaults(t *test "promote node-a sb_1 e2b/ns/app", "release node-a sb_1", }, store.calls()) - assert.Equal(t, [2]string{"4", "out of make"}, [2]string{info.Logs[4].Phase, info.Logs[4].Message}) + assert.Equal(t, [2]string{"1", "bash: warning: setlocale: LC_ALL: cannot change locale"}, [2]string{info.Logs[2].Phase, info.Logs[2].Message}, "the shell's stderr goes to the log, never into the ENV value") + assert.Equal(t, [2]string{"4", "out of make"}, [2]string{info.Logs[5].Phase, info.Logs[5].Message}) } func TestARelayedBuildSeedsItsStepsWithTheProxyEnvironment(t *testing.T) { - for route, want := range map[string]string{"relay": "map[A:1 http_proxy:p]", "none": "map[A:1]"} { + for route, want := range map[string]string{"relay": "map[A:1 http_proxy:http://127.0.0.1:3128 https_proxy:http://127.0.0.1:3128 no_proxy:localhost,127.0.0.1,::1,169.254.169.254]", "none": "map[A:1]"} { store := &fakeStore{route: route} e := New(store, store, 1, time.Minute, 100) e.Register("b", Request{}) require.NoError(t, e.Start(t.Context(), "b", Spec{ Pool: scale.PoolKey{Template: "img"}, Template: "t", - Steps: []Step{{Type: stepEnv, Args: []string{"A", "1"}}, {Type: stepRun, Args: []string{"make"}}}, - RelayEnvs: map[string]string{"http_proxy": "p"}, + Steps: []Step{{Type: stepEnv, Args: []string{"A", "1"}}, {Type: stepRun, Args: []string{"make"}}}, })) require.Equal(t, StatusReady, waitDone(t, e, "b").Status) calls := store.calls() @@ -282,13 +283,14 @@ func (f *fakeStore) Release(_ context.Context, node, id string) error { return nil } -func (f *fakeStore) Run(_ context.Context, _ scale.Assignment, cmd Command, out func(string)) (int, error) { +func (f *fakeStore) Run(_ context.Context, _ scale.Assignment, cmd Command, stdout, stderr LineFunc) (int, error) { f.record(fmt.Sprintf("run %s %s %v %s", cmd.User, cmd.Workdir, cmd.Envs, cmd.Line)) if v, ok := strings.CutPrefix(cmd.Line, `printf "%s" "`); ok { - out(strings.TrimSuffix(v, `"`)) + stderr("bash: warning: setlocale: LC_ALL: cannot change locale") + stdout(strings.TrimSuffix(v, `"`)) return 0, nil } - out("out of " + cmd.Line) + stdout("out of " + cmd.Line) f.mu.Lock() defer f.mu.Unlock() codes := f.exits[cmd.Line] @@ -326,5 +328,5 @@ func (f *fakeStore) record(line string) { func (f *fakeStore) calls() []string { f.mu.Lock() defer f.mu.Unlock() - return append([]string(nil), f.log...) + return slices.Clone(f.log) } diff --git a/pkg/e2bcompat/aliases_test.go b/pkg/e2bcompat/aliases_test.go index 4a022e8..bb362cd 100644 --- a/pkg/e2bcompat/aliases_test.go +++ b/pkg/e2bcompat/aliases_test.go @@ -95,6 +95,6 @@ func TestNewServerRefusesAMalformedAliasTable(t *testing.T) { } } -func withAliases(entries ...string) func(*Options) { +func withAliases(entries ...string) serverOption { return func(o *Options) { o.TemplateAliases = entries } } diff --git a/pkg/e2bcompat/builds.go b/pkg/e2bcompat/builds.go index 2992eac..e44a19b 100644 --- a/pkg/e2bcompat/builds.go +++ b/pkg/e2bcompat/builds.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apiserver/pkg/storage/names" @@ -72,17 +73,15 @@ func (s *Server) startBuild(w http.ResponseWriter, r *http.Request) { pool.Net = scale.NetDefault pool.Size = cmp.Or(pending.Size, pool.Size) err := s.builds.Start(r.Context(), id, e2bbuild.Spec{ - Namespace: s.namespace(r), - ClaimName: names.SimpleNameGenerator.GenerateName(namePrefix + "build-"), - Pool: pool, - Template: scope + name, - TTLSeconds: int(s.opts.Builds.Timeout / time.Second), - Steps: req.Steps, - StartCmd: req.StartCmd, - ReadyCmd: req.ReadyCmd, - RelayEnvs: relayEnvs, - Archive: s.archive(s.namespace(r), name), - Publish: s.publishBuild(scope, name, pending.Tags), + Namespace: s.namespace(r), + ClaimName: names.SimpleNameGenerator.GenerateName(namePrefix + "build-"), + Pool: pool, + Template: scope + name, + Steps: req.Steps, + StartCmd: req.StartCmd, + ReadyCmd: req.ReadyCmd, + Archive: s.archive(s.namespace(r), name), + Publish: s.publishBuild(scope, name, pending.Tags), }) switch { case errors.Is(err, e2bbuild.ErrBusy): @@ -113,17 +112,18 @@ func (s *Server) buildStatus(w http.ResponseWriter, r *http.Request) { return } floor := max(slices.Index(logLevels, q.Get("level")), 0) - var entries []BuildLogEntry + entries := []BuildLogEntry{} for _, l := range info.Logs { - if slices.Index(logLevels, l.Level) >= floor { + if slices.Index(logLevels, l.Level) < floor { + continue + } + if offset > 0 { + offset-- + } else if limit <= 0 || len(entries) < limit { entries = append(entries, logEntryOf(l)) } } - entries = entries[min(offset, len(entries)):] - if limit > 0 { - entries = entries[:min(limit, len(entries))] - } - out := TemplateBuildInfo{TemplateID: name, BuildID: buildID, Status: info.Status, Logs: []string{}, LogEntries: append([]BuildLogEntry{}, entries...)} + out := TemplateBuildInfo{TemplateID: name, BuildID: buildID, Status: info.Status, Logs: []string{}, LogEntries: entries} if info.Status == e2bbuild.StatusError { out.Reason = &BuildStatusReason{Message: info.Failure, Step: sdkStep(info.FailedPhase), LogEntries: []BuildLogEntry{}} if n := len(info.Logs); n > 0 { @@ -133,16 +133,30 @@ func (s *Server) buildStatus(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, out) } -// publishBuild deletes every other holder of name once the build's promote lands, so the registry names one build, then tags it. +// publishBuild deletes the holders of name older than this build's promote, so concurrent builds converge on the newest, then tags it. // It asks each node itself: an inventory a tick behind would miss a build that finished moments ago. -func (s *Server) publishBuild(scope, name string, tags []string) func(context.Context, string, scale.PoolKey, string) error { +func (s *Server) publishBuild(scope, name string, tags []string) e2bbuild.PublishFunc { return func(ctx context.Context, node string, key scale.PoolKey, digest string) error { - stale, err := s.liveHolders(ctx, scope+name) + holders, err := s.liveHolders(ctx, scope+name) if err != nil { return fmt.Errorf("list the previous builds: %w", err) } - stale = slices.DeleteFunc(stale, func(h templateHolder) bool { return h.node == node && h.key == key }) - if err := forEachHolder(stale, func(h templateHolder) error { return s.store.DeleteTemplate(ctx, h.node, h.key) }); err != nil { + i := slices.IndexFunc(holders, func(h templateHolder) bool { return h.node == node && h.key == key }) + if i < 0 { + return fmt.Errorf("the build's template is no longer on %s", node) + } + own := holders[i] + if own.digest != digest { + return fmt.Errorf("a newer build of %s replaced this one on %s", name, node) + } + stale := slices.DeleteFunc(holders, func(h templateHolder) bool { return !h.created.Before(own.created) }) + if err := forEachHolder(stale, func(h templateHolder) error { + err := s.store.DeleteTemplate(ctx, h.node, h.key, h.digest) + if replaced(err) { + return nil + } + return err + }); err != nil { return fmt.Errorf("delete the previous build: %w", err) } if len(tags) == 0 { @@ -152,7 +166,12 @@ func (s *Server) publishBuild(scope, name string, tags []string) func(context.Co for _, t := range tags { labels[t] = digest } - return s.store.SetTemplateLabels(ctx, node, key, labels) + if err := s.store.SetTemplateLabels(ctx, node, key, labels, digest); replaced(err) { + return fmt.Errorf("a newer build of %s replaced this one on %s", name, node) + } else if err != nil { + return err + } + return nil } } @@ -178,13 +197,20 @@ func (s *Server) liveHolders(ctx context.Context, template string) ([]templateHo defer mu.Unlock() for _, t := range held { if t.Template == template && t.Tenant == "" { - holders = append(holders, templateHolder{node: n, key: scale.PoolKey{Template: t.Template, Net: t.Net, Size: t.Size}}) + h := templateHolder{node: n, key: scale.PoolKey{Template: t.Template, Net: t.Net, Size: t.Size}, digest: t.ContentDigest} + if t.CreatedAt != nil { + h.created = t.CreatedAt.Time + } + holders = append(holders, h) } } return nil }) } - return holders, g.Wait() + if err := g.Wait(); err != nil { + return nil, err + } + return holders, nil } func (s *Server) buildKey(r *http.Request, name, buildID string) string { @@ -200,6 +226,8 @@ func unsupportedBuildOption(req TemplateBuildStartV2, canCopy bool) (string, boo return "fromImageRegistry is not supported; a pool pulls its own image", true case req.FromImage == "": return "fromImage is required", true + case strings.HasPrefix(req.FromImage, templatePrefix): + return "fromImage names a built template; build from an image", true } msg := e2bbuild.Invalid(req.Steps, canCopy) return msg, msg != "" @@ -245,3 +273,9 @@ func queryInt(v string) (int, error) { } return n, nil } + +// replaced is sandboxd's 412: the template generation the caller observed is gone. +func replaced(err error) bool { + se, ok := errors.AsType[*k8serrors.StatusError](err) + return ok && se.ErrStatus.Code == http.StatusPreconditionFailed +} diff --git a/pkg/e2bcompat/builds_test.go b/pkg/e2bcompat/builds_test.go index 13aee82..d843be3 100644 --- a/pkg/e2bcompat/builds_test.go +++ b/pkg/e2bcompat/builds_test.go @@ -9,8 +9,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cocoonstack/sandbox-operator/pkg/e2bbuild" + "github.com/cocoonstack/sandbox-operator/pkg/sandboxd" "github.com/cocoonstack/sandbox-operator/pkg/scale" ) @@ -30,7 +33,7 @@ func TestABuildFromAnImagePromotesReplacesTheOldHolderAndTags(t *testing.T) { require.Equal(t, e2bbuild.StatusReady, info.Status, info) assert.Equal(t, [2]string{"reg/rt:24.04", scale.SizeClassMedium}, [2]string{store.claimPool.Template, store.claimPool.Size}, "the alias names the image; cpuCount 2 picks the size") assert.Equal(t, []string{"n sb_1 e2b/sandboxes/app"}, store.promoted) - assert.Equal(t, []string{"m e2b/sandboxes/app small"}, store.deletedTemplates, "the previous build on another node goes") + assert.Equal(t, []string{"m e2b/sandboxes/app small sha256:old"}, store.deletedTemplates, "the previous build on another node goes") assert.Equal(t, map[string]string{"v1": "sha256:sb_1"}, fleetLabels(t, store, "n", "e2b/sandboxes/app")) assert.Equal(t, "n sb_1", store.releasedNode+" "+store.releasedID) assert.Len(t, info.LogEntries, 3) @@ -55,7 +58,7 @@ func TestABuildReplacesAPreviousBuildTheInventoryHasNotPublishedYet(t *testing.T id := requestBuild(t, h, "app") require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status) - assert.Equal(t, []string{"m e2b/sandboxes/app small"}, store.deletedTemplates, "node m's build finished after its last publish and still goes") + assert.Equal(t, []string{"m e2b/sandboxes/app small sha256:old"}, store.deletedTemplates, "node m's build finished after its last publish and still goes") delete(store.live, "m") id = requestBuild(t, h, "app") @@ -79,7 +82,7 @@ func TestABuildThatCannotClaimFailsAtTheBaseStep(t *testing.T) { } func TestABuildRunsItsStepsThroughEnvdAndLeavesItsDefaultsInTheSandbox(t *testing.T) { - store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, processExits: map[string]int{"false": 1}} + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n", Token: "claim-tok"}, processExits: map[string]int{"false": 1}} h := newTestServer(t, store, withBuilds(), withBuildFleet(store)) id := requestBuild(t, h, "app") @@ -98,6 +101,7 @@ func TestABuildRunsItsStepsThroughEnvdAndLeavesItsDefaultsInTheSandbox(t *testin assert.Equal(t, [2]any{"user", map[string]string{"A": "1"}}, [2]any{defaults.DefaultUser, defaults.EnvVars}) assert.Contains(t, info.LogEntries, BuildLogEntry{Timestamp: info.LogEntries[2].Timestamp, Message: "ok", Level: "info", Step: "2"}) assert.Equal(t, []string{"n sb_1 e2b/sandboxes/app"}, store.promoted) + assert.Equal(t, []string{"claim-tok", "claim-tok", "claim-tok", "claim-tok", "claim-tok"}, store.envdRelays, "every build call rides the claim's own relay, which wakes and keeps the sandbox awake") id = requestBuild(t, h, "app") body = `{"fromImage":"img","steps":[{"type":"ENV","args":["A","1"]},{"type":"WORKDIR","args":["/w"]},{"type":"RUN","args":["false"]}]}` @@ -121,7 +125,7 @@ func TestARelayedBuildLeavesTheProxyInTheTemplateDefaults(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(body), &init)) } } - assert.Equal(t, relayEnvs, init.EnvVars) + assert.Equal(t, sandboxd.RelayEnv, init.EnvVars) } func TestABuildRefusesWhatItCannotHonor(t *testing.T) { @@ -140,10 +144,69 @@ func TestABuildRefusesWhatItCannotHonor(t *testing.T) { } assert.Equal(t, http.StatusNotFound, do(t, h, http.MethodPost, "/v2/templates/app/builds/nope", `{"fromImage":"img"}`, testKey).Code) assert.Equal(t, http.StatusBadRequest, do(t, h, http.MethodPost, "/v3/templates", `{"name":"`+strings.Repeat("x", 60)+`"}`, testKey).Code) + assert.Equal(t, http.StatusBadRequest, do(t, h, http.MethodPost, "/v3/templates", `{"name":"my app"}`, testKey).Code, "a name outside sandboxd's grammar is refused before the build runs") assert.Equal(t, http.StatusBadRequest, do(t, h, http.MethodPost, "/v3/templates", `{}`, testKey).Code) assert.Equal(t, http.StatusNotFound, do(t, newTestServer(t, &fakeStore{}), http.MethodPost, "/v3/templates", `{"name":"app"}`, testKey).Code, "builds are off unless enabled") } +func TestAPublishLeavesANewerConcurrentBuildAlone(t *testing.T) { + newer := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:newer", CreatedAt: new(metav1.NewTime(time.Now().Add(time.Minute)))} + older := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:older", CreatedAt: new(metav1.NewTime(time.Now().Add(-time.Minute)))} + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil, "m": {newer}, "k": {older}}} + inv := scale.NewStaticInventorySource() + for _, n := range []string{"n", "m", "k"} { + inv.Put(&scale.NodeInventory{Node: n}) + } + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status) + assert.Equal(t, []string{"k e2b/sandboxes/app small sha256:older"}, store.deletedTemplates, "only the holder older than this promote goes, by the digest observed; a newer concurrent build keeps its node") +} + +func TestAPublishSkipsAHolderReplacedSinceItWasObserved(t *testing.T) { + older := scale.PromotedTemplate{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:older", CreatedAt: new(metav1.NewTime(time.Now().Add(-time.Minute)))} + replaced := &k8serrors.StatusError{ErrStatus: metav1.Status{Code: http.StatusPreconditionFailed}} + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil, "k": {older}}, deleteTemplateErr: map[string]error{"k": replaced}} + inv := scale.NewStaticInventorySource() + for _, n := range []string{"n", "k"} { + inv.Put(&scale.NodeInventory{Node: n}) + } + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + require.Equal(t, e2bbuild.StatusReady, waitBuild(t, h, id).Status, "a holder a newer promote replaced between the read and the delete is left alone") + assert.Equal(t, []string{"k e2b/sandboxes/app small sha256:older"}, store.deletedTemplates) +} + +func TestABuildReplacedOnItsOwnNodeWritesNoTags(t *testing.T) { + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil}, replacedDigest: "sha256:newer"} + inv := scale.NewStaticInventorySource() + inv.Put(&scale.NodeInventory{Node: "n"}) + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app:v1") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + info := waitBuild(t, h, id) + require.Equal(t, [2]string{e2bbuild.StatusError, "finalize"}, [2]string{info.Status, info.Reason.Step}) + assert.Empty(t, store.labeled, "a build a newer promote replaced must not tag the newer content with its own digest") +} + +func TestABuildWhoseTagWriteFindsANewerGenerationReportsReplaced(t *testing.T) { + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}, live: map[string][]scale.PromotedTemplate{"n": nil}, labelErr: &k8serrors.StatusError{ErrStatus: metav1.Status{Code: http.StatusPreconditionFailed}}} + inv := scale.NewStaticInventorySource() + inv.Put(&scale.NodeInventory{Node: "n"}) + h := newTestServer(t, store, withBuilds(), func(o *Options) { o.Inventory = inv }) + + id := requestBuild(t, h, "app:v1") + require.Equal(t, http.StatusAccepted, do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"img"}`, testKey).Code) + info := waitBuild(t, h, id) + require.Equal(t, [2]string{e2bbuild.StatusError, "finalize"}, [2]string{info.Status, info.Reason.Step}) + assert.Equal(t, []string{"n e2b/sandboxes/app sha256:sb_1"}, store.labeled, "the tag write carries the promote's digest, so a newer generation refuses it") +} + func requestBuild(t *testing.T, h http.Handler, name string) string { t.Helper() w := do(t, h, http.MethodPost, "/v3/templates", `{"name":"`+name+`"}`, testKey) @@ -166,12 +229,12 @@ func waitBuild(t *testing.T, h http.Handler, buildID string) TemplateBuildInfo { return info } -func withBuilds() func(*Options) { +func withBuilds() serverOption { return func(o *Options) { o.Builds = BuildOptions{Parallel: 2, Timeout: time.Minute, LogLines: 100} } } // withBuildFleet serves node n with app's earlier build and node m with a stale copy under a smaller key. -func withBuildFleet(store *fakeStore) func(*Options) { +func withBuildFleet(store *fakeStore) serverOption { inv := scale.NewStaticInventorySource() inv.Put(&scale.NodeInventory{Node: "n", Templates: []scale.PromotedTemplate{{Template: "e2b/sandboxes/app", Net: "none", Size: "medium", ContentDigest: "sha256:old"}}}) inv.Put(&scale.NodeInventory{Node: "m", Templates: []scale.PromotedTemplate{{Template: "e2b/sandboxes/app", Net: "none", Size: "small", ContentDigest: "sha256:old"}}}) diff --git a/pkg/e2bcompat/envd.go b/pkg/e2bcompat/envd.go index 464820a..2f764a4 100644 --- a/pkg/e2bcompat/envd.go +++ b/pkg/e2bcompat/envd.go @@ -45,13 +45,6 @@ const ( logLineMax = 64 << 10 ) -// relayEnvs point a guest process at the node's egress proxy, as silkd's unit points its own. -var relayEnvs = map[string]string{ - "http_proxy": "http://127.0.0.1:3128", - "https_proxy": "http://127.0.0.1:3128", - "no_proxy": "localhost,127.0.0.1,::1,169.254.169.254", -} - // envdMetrics is envd's GET /metrics reply: the guest's own view of its CPU, memory and root disk. type envdMetrics struct { Timestamp int64 `json:"ts"` @@ -108,14 +101,14 @@ type envdGuest struct { s *Server } -func (g envdGuest) Run(ctx context.Context, a scale.Assignment, cmd e2bbuild.Command, out func(string)) (int, error) { - stdout, stderr := &lineSplitter{out: out}, &lineSplitter{out: out} +func (g envdGuest) Run(ctx context.Context, a scale.Assignment, cmd e2bbuild.Command, stdout, stderr e2bbuild.LineFunc) (int, error) { + outLines, errLines := &lineSplitter{out: stdout}, &lineSplitter{out: stderr} code, _, err := g.s.envdProcess(ctx, a, cmd, func(o, e []byte) { - stdout.write(o) - stderr.write(e) + outLines.write(o) + errLines.write(e) }) - stdout.flush() - stderr.flush() + outLines.flush() + errLines.flush() return code, err } @@ -128,9 +121,9 @@ func (g envdGuest) Start(ctx context.Context, a scale.Assignment, cmd e2bbuild.C return err } -// Init sets the defaults with no access token, which envd takes as first-time setup. +// Init sets the defaults with no access token, which envd takes as first-time setup; like every build call it rides the claim's own relay, which keeps the sandbox awake. func (g envdGuest) Init(ctx context.Context, a scale.Assignment, defaults e2bbuild.Command) error { - return g.s.initEnvd(ctx, a.Node, a.SandboxName, envdInit{EnvVars: defaults.Envs, DefaultUser: defaults.User, DefaultWorkdir: defaults.Workdir}) + return g.s.initEnvd(ctx, a.Node, a.SandboxName, a.Token, envdInit{EnvVars: defaults.Envs, DefaultUser: defaults.User, DefaultWorkdir: defaults.Workdir}) } func (g envdGuest) Write(ctx context.Context, a scale.Assignment, path string, r io.Reader) error { @@ -139,7 +132,7 @@ func (g envdGuest) Write(ctx context.Context, a scale.Assignment, path string, r return err } req.Header.Set("Content-Type", "application/octet-stream") - resp, err := g.s.envdRoundTrip(ctx, a.Node, a.SandboxName, req) + resp, err := g.s.envdRoundTrip(ctx, a.Node, a.SandboxName, a.Token, req) if err != nil { return err } @@ -153,7 +146,7 @@ func (g envdGuest) Write(ctx context.Context, a scale.Assignment, path string, r // lineSplitter hands out complete lines, and a partial one once it outgrows logLineMax. type lineSplitter struct { buf []byte - out func(string) + out e2bbuild.LineFunc } func (l *lineSplitter) write(p []byte) { @@ -226,7 +219,7 @@ func (s *Server) readEnvdMetrics(ctx context.Context, node, id string) (envdMetr } // initEnvd sets envd's access token, defaults and env inside sandbox id, and stamps the guest clock. -func (s *Server) initEnvd(ctx context.Context, node, id string, req envdInit) error { +func (s *Server) initEnvd(ctx context.Context, node, id, relay string, req envdInit) error { ctx, cancel := context.WithTimeout(ctx, initTimeout) defer cancel() req.Timestamp = time.Now() @@ -234,7 +227,7 @@ func (s *Server) initEnvd(ctx context.Context, node, id string, req envdInit) er if err != nil { return err } - status, _, err := s.envdCall(ctx, node, id, http.MethodPost, "/init", "", payload) + status, _, err := s.envdCallOver(ctx, node, id, relay, http.MethodPost, "/init", "", payload) if err != nil { return fmt.Errorf("envd init of %s: %w", id, err) } @@ -276,7 +269,7 @@ func (s *Server) handOver(ctx context.Context, child scale.Assignment, token str if err := s.store.SetInstanceMetadata(ctx, child.Node, child.SandboxName, doc); err != nil { return err } - return s.initEnvd(ctx, child.Node, child.SandboxName, envdInit{AccessToken: token}) + return s.initEnvd(ctx, child.Node, child.SandboxName, "", envdInit{AccessToken: token}) } // releaseAll gives back claims whose envd could not be initialized; a failed release is left to the lease. @@ -301,7 +294,7 @@ func (s *Server) envdProcess(ctx context.Context, a scale.Assignment, cmd e2bbui } req.Header.Set("Content-Type", "application/connect+json") req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(cmd.User+":"))) - resp, err := s.envdRoundTrip(ctx, a.Node, a.SandboxName, req) + resp, err := s.envdRoundTrip(ctx, a.Node, a.SandboxName, a.Token, req) if err != nil { return 0, false, err } @@ -345,6 +338,10 @@ func (s *Server) envdProcess(ctx context.Context, a scale.Assignment, cmd e2bbui // envdCall sends one request to envd inside sandbox id over its node's passive relay and reads the bounded reply; token authenticates it to envd. func (s *Server) envdCall(ctx context.Context, node, id, method, path, token string, body []byte) (int, []byte, error) { + return s.envdCallOver(ctx, node, id, "", method, path, token, body) +} + +func (s *Server) envdCallOver(ctx context.Context, node, id, relay, method, path, token string, body []byte) (int, []byte, error) { req, err := http.NewRequestWithContext(ctx, method, "http://"+envdHostAlias+path, bytes.NewReader(body)) if err != nil { return 0, nil, err @@ -355,7 +352,7 @@ func (s *Server) envdCall(ctx context.Context, node, id, method, path, token str if token != "" { req.Header.Set("X-Access-Token", token) } - resp, err := s.envdRoundTrip(ctx, node, id, req) + resp, err := s.envdRoundTrip(ctx, node, id, relay, req) if err != nil { return 0, nil, err } @@ -367,9 +364,9 @@ func (s *Server) envdCall(ctx context.Context, node, id, method, path, token str return resp.StatusCode, reply, nil } -// envdRoundTrip writes req to envd inside sandbox id over its node's passive relay; closing the reply's body closes the connection. -func (s *Server) envdRoundTrip(ctx context.Context, node, id string, req *http.Request) (*http.Response, error) { - conn, err := s.store.DialGuestPort(ctx, node, id, envdPort) +// envdRoundTrip writes req to envd inside sandbox id over its node's relay, passive with an empty relay token; closing the reply's body closes the connection. +func (s *Server) envdRoundTrip(ctx context.Context, node, id, relay string, req *http.Request) (*http.Response, error) { + conn, err := s.store.DialGuestPort(ctx, node, id, relay, envdPort) if err != nil { return nil, err } @@ -397,7 +394,7 @@ func withRelay(route string, envs map[string]string) map[string]string { if route != sandboxd.NetRouteRelay { return envs } - out := maps.Clone(relayEnvs) + out := maps.Clone(sandboxd.RelayEnv) maps.Copy(out, envs) return out } diff --git a/pkg/e2bcompat/envd_test.go b/pkg/e2bcompat/envd_test.go index ecbad77..b2fed8f 100644 --- a/pkg/e2bcompat/envd_test.go +++ b/pkg/e2bcompat/envd_test.go @@ -182,7 +182,7 @@ type runningEnvd struct { scale.SandboxStore } -func (runningEnvd) DialGuestPort(context.Context, string, string, uint16) (net.Conn, error) { +func (runningEnvd) DialGuestPort(context.Context, string, string, string, uint16) (net.Conn, error) { conn, envd := net.Pipe() go func() { r, err := http.ReadRequest(bufio.NewReader(envd)) diff --git a/pkg/e2bcompat/lifecycle.go b/pkg/e2bcompat/lifecycle.go index 24013f1..bd31fd6 100644 --- a/pkg/e2bcompat/lifecycle.go +++ b/pkg/e2bcompat/lifecycle.go @@ -363,26 +363,12 @@ func (s *Server) sandboxLogs(reply any) http.HandlerFunc { } } -// inventories returns every node's published inventory, the fleet view the -// pool-derived surfaces (templates, snapshot listing) are assembled from. -func (s *Server) inventories(ctx context.Context) ([]*scale.NodeInventory, error) { +// inventories is the fleet's claimable pools and templates, the view the template surfaces are assembled from. +func (s *Server) inventories(ctx context.Context) ([]scale.NodePools, error) { if s.opts.Inventory == nil { return nil, errors.New("e2bcompat: no inventory source configured") } - nodes, err := s.opts.Inventory.ListNodes(ctx) - if err != nil { - return nil, err - } - out := make([]*scale.NodeInventory, 0, len(nodes)) - for _, node := range nodes { - inv, err := s.opts.Inventory.NodeInventory(ctx, node) - if err != nil { - log.WithFunc("e2bcompat.inventories").Warnf(ctx, "e2b: node inventory unavailable node=%s err=%v", node, err) - continue - } - out = append(out, inv) - } - return out, nil + return s.opts.Inventory.NodeCapacities(ctx) } // nodesWithSandboxes lists the nodes a checkpoint could live on. diff --git a/pkg/e2bcompat/lifecycle_test.go b/pkg/e2bcompat/lifecycle_test.go index d4233d6..8e1174f 100644 --- a/pkg/e2bcompat/lifecycle_test.go +++ b/pkg/e2bcompat/lifecycle_test.go @@ -431,20 +431,16 @@ func TestResumeOfARunningSandboxIs409(t *testing.T) { } func TestResumeRefusesWhatConnectCannotGive(t *testing.T) { - for _, body := range []string{`{"memory":false}`} { - t.Run(body, func(t *testing.T) { - store := &lifecycleStore{} - nodeReportsPaused(store) - store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img")} - h := newTestServer(t, store) + store := &lifecycleStore{} + nodeReportsPaused(store) + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img")} + h := newTestServer(t, store) - if w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/resume", body, testKey); w.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want 400: %s", w.Code, w.Body.String()) - } - if store.readID != "" || store.resumedID != "" { - t.Errorf("a refused resume reached the node (read %q, resumed %q)", store.readID, store.resumedID) - } - }) + if w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/resume", `{"memory":false}`, testKey); w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", w.Code, w.Body.String()) + } + if store.readID != "" || store.resumedID != "" { + t.Errorf("a refused resume reached the node (read %q, resumed %q)", store.readID, store.resumedID) } } @@ -605,7 +601,7 @@ func (f *lifecycleStore) Read(_ context.Context, node, id string) (scale.Sandbox return scale.SandboxRecord{Token: f.token, Paused: f.nodePaused || f.nodeArchived, Deadline: f.deadline}, f.nodeErr } -func (f *lifecycleStore) DialGuestPort(_ context.Context, _, id string, port uint16) (net.Conn, error) { +func (f *lifecycleStore) DialGuestPort(_ context.Context, _, id, token string, port uint16) (net.Conn, error) { f.dialPort.Store(uint32(port)) if f.nodeErr != nil { return nil, f.nodeErr @@ -629,7 +625,7 @@ func (f *lifecycleStore) DialGuestPort(_ context.Context, _, id string, port uin return http.StatusOK, string(body) }), nil } - return f.fakeStore.DialGuestPort(context.Background(), "", id, port) + return f.fakeStore.DialGuestPort(context.Background(), "", id, token, port) } func (f *lifecycleStore) Pause(_ context.Context, node, id string) error { diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index 8669c69..a866905 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -5,8 +5,9 @@ // It is a translation layer, not a second control plane: every request lands on // the same scale.SandboxStore the aggregated apiserver uses, so an e2b Create is // the identical node-local claim a `kubectl create sandbox` performs, and the -// sandbox it returns is visible to `kubectl get sandboxes`. Nothing is stored -// here; public identity is a DNS-safe rendering of the node's sandboxd claim id. +// sandbox it returns is visible to `kubectl get sandboxes`. Only build records +// (in memory) and COPY uploads (a directory or bucket) live here; public identity +// is a DNS-safe rendering of the node's sandboxd claim id. // // Mapping to the e2b contract (e2b-dev/E2B spec/openapi.yml): // @@ -15,7 +16,7 @@ // POST /sandboxes/{id}/snapshots -> create checkpoint // GET /snapshots, DELETE /templates/{id} -> list or delete checkpoints and built templates // GET/PATCH /templates[/{id}], aliases/{a}, tags -> warm-pool keys, built templates, alias lookup, tags -// GET /sandboxes/{id}/metrics|logs -> node statistics, an empty log page +// GET /sandboxes/{id}/metrics|logs, /sandboxes/metrics -> envd's guest metrics, an empty log page // POST timeout|refreshes, GET /health -> lease renewal, liveness package e2bcompat @@ -306,6 +307,10 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "templateID is required") return } + if strings.HasPrefix(req.TemplateID, templatePrefix) { + writeError(w, http.StatusNotFound, fmt.Sprintf("template %q not found", req.TemplateID)) + return + } if req.Timeout != nil && *req.Timeout < 0 { writeError(w, http.StatusBadRequest, "timeout must be >= 0") return @@ -315,6 +320,7 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { return } + logger := log.WithFunc("e2bcompat.createSandbox") name := names.SimpleNameGenerator.GenerateName(namePrefix) pool := s.poolKey(req.TemplateID) pool.Net = netFor(req.AllowInternetAccess) @@ -346,7 +352,7 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { writeError(w, he.StatusCode, he.Message) return } - log.WithFunc("e2bcompat.createSandbox").Errorf(r.Context(), err, "e2b create: claim failed template=%s name=%s", req.TemplateID, name) + logger.Errorf(r.Context(), err, "e2b create: claim failed template=%s name=%s", req.TemplateID, name) writeError(w, http.StatusInternalServerError, "failed to claim a sandbox") return } @@ -359,10 +365,10 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { init.EnvVars, init.DefaultUser, init.DefaultWorkdir = withRelay(assignment.NetRoute, req.EnvVars), envdDefaultUser, envdDefaultWorkdir } if err == nil { - err = s.initEnvd(r.Context(), assignment.Node, assignment.SandboxName, init) + err = s.initEnvd(r.Context(), assignment.Node, assignment.SandboxName, "", init) } if err != nil { - log.WithFunc("e2bcompat.createSandbox").Errorf(r.Context(), err, "e2b create: envd init failed sandboxID=%s node=%s", assignment.SandboxName, assignment.Node) + logger.Errorf(r.Context(), err, "e2b create: envd init failed sandboxID=%s node=%s", assignment.SandboxName, assignment.Node) s.releaseAll(r.Context(), []scale.Assignment{assignment}) writeError(w, http.StatusInternalServerError, "failed to start the sandbox") return @@ -417,7 +423,10 @@ func (s *Server) listed(w http.ResponseWriter, r *http.Request) ([]SandboxDetail } out := make([]SandboxDetail, 0, len(list.Items)) for i := range list.Items { - if d := s.detailFor(&list.Items[i]); filter.keeps(d) && filter.matchesMetadata(&list.Items[i]) { + if !filter.matchesMetadata(&list.Items[i]) { + continue + } + if d := s.detailFor(&list.Items[i]); filter.keeps(d) { out = append(out, d) } } diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index f444487..ed4fd67 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "slices" "strconv" "strings" "sync" @@ -568,6 +569,9 @@ type fakeStore struct { deletedSnapshotNode string deletedSnapshotID string deletedTemplates []string + labeled []string + labelErr error + replacedDigest string deleteTemplateErr map[string]error fleet *scale.StaticInventorySource promoted []string @@ -576,6 +580,7 @@ type fakeStore struct { live map[string][]scale.PromotedTemplate envdCalls []string + envdRelays []string initStatus int metadataDocs []string guestEnvs map[string]string @@ -663,15 +668,32 @@ func (f *fakeStore) DeleteSnapshot(_ context.Context, node, id string) error { return nil } -func (f *fakeStore) Promote(_ context.Context, node, id, template string) (scale.PoolKey, string, error) { +func (f *fakeStore) Promote(ctx context.Context, node, id, template string) (scale.PoolKey, string, error) { f.mu.Lock() defer f.mu.Unlock() f.promoted = append(f.promoted, node+" "+id+" "+template) - return scale.PoolKey{Template: template, Net: scale.NetDefault, Size: f.claimPool.Size}, "sha256:" + id, f.promoteErr + key := scale.PoolKey{Template: template, Net: scale.NetDefault, Size: f.claimPool.Size} + fresh := scale.PromotedTemplate{Template: key.Template, Net: key.Net, Size: key.Size, ContentDigest: cmp.Or(f.replacedDigest, "sha256:"+id), CreatedAt: new(metav1.Now())} + same := func(t scale.PromotedTemplate) bool { + return t.Template == key.Template && t.Net == key.Net && t.Size == key.Size + } + if f.live != nil { + f.live[node] = append(slices.DeleteFunc(f.live[node], same), fresh) + } else if inv, err := f.fleet.NodeInventory(ctx, node); err == nil { + inv.Templates = append(slices.DeleteFunc(inv.Templates, same), fresh) + f.fleet.Put(inv) + } + return key, "sha256:" + id, f.promoteErr } // SetTemplateLabels writes straight into the fleet the test serves, so a live read sees it at once. -func (f *fakeStore) SetTemplateLabels(ctx context.Context, node string, key scale.PoolKey, labels map[string]string) error { +func (f *fakeStore) SetTemplateLabels(ctx context.Context, node string, key scale.PoolKey, labels map[string]string, digest string) error { + f.mu.Lock() + f.labeled = append(f.labeled, strings.TrimSpace(node+" "+key.Template+" "+digest)) + f.mu.Unlock() + if f.labelErr != nil { + return f.labelErr + } inv, err := f.fleet.NodeInventory(ctx, node) if err != nil { return err @@ -700,14 +722,17 @@ func (f *fakeStore) NodeTemplates(ctx context.Context, node string) ([]scale.Pro return inv.Templates, nil } -func (f *fakeStore) DeleteTemplate(_ context.Context, node string, key scale.PoolKey) error { +func (f *fakeStore) DeleteTemplate(_ context.Context, node string, key scale.PoolKey, digest string) error { f.mu.Lock() defer f.mu.Unlock() - f.deletedTemplates = append(f.deletedTemplates, node+" "+key.Template+" "+key.Size) + f.deletedTemplates = append(f.deletedTemplates, strings.TrimSpace(node+" "+key.Template+" "+key.Size+" "+digest)) return f.deleteTemplateErr[node] } -func (f *fakeStore) DialGuestPort(_ context.Context, _, id string, _ uint16) (net.Conn, error) { +func (f *fakeStore) DialGuestPort(_ context.Context, _, id, token string, _ uint16) (net.Conn, error) { + f.mu.Lock() + f.envdRelays = append(f.envdRelays, token) + f.mu.Unlock() return fakeEnvd(func(r *http.Request, body string) (int, string) { f.mu.Lock() defer f.mu.Unlock() @@ -798,7 +823,9 @@ func (f *renewStore) Renew(_ context.Context, node, id string, ttlSeconds int, _ return time.Now().Add(time.Duration(ttlSeconds) * time.Second), nil } -func newTestServer(t *testing.T, store scale.SandboxStore, opts ...func(*Options)) http.Handler { +type serverOption func(*Options) + +func newTestServer(t *testing.T, store scale.SandboxStore, opts ...serverOption) http.Handler { t.Helper() o := Options{Namespace: "sandboxes", Domain: testDomain, APIKeys: []string{testKey}, EnvdSecret: []byte(testEnvdSecret)} for _, fn := range opts { diff --git a/pkg/e2bcompat/templates.go b/pkg/e2bcompat/templates.go index ca214ea..d9f2e1e 100644 --- a/pkg/e2bcompat/templates.go +++ b/pkg/e2bcompat/templates.go @@ -28,10 +28,11 @@ const ( var buildNamespace = uuid.NewSHA1(uuid.NameSpaceURL, []byte("https://github.com/cocoonstack/sandbox-operator/e2b")) type templateHolder struct { - node string - key scale.PoolKey - digest string - labels map[string]string + node string + key scale.PoolKey + digest string + labels map[string]string + created time.Time } type builtTemplate struct { @@ -105,7 +106,7 @@ func (s *Server) listTemplates(w http.ResponseWriter, r *http.Request) { }) } } - built := builtTemplates(nodes, s.templateScope(r)) + built := builtTemplates(nodes, s.templateScope(r), "") for _, name := range slices.Sorted(maps.Keys(built)) { b := built[name] created := b.created.UTC().Format(time.RFC3339) @@ -271,7 +272,7 @@ func (s *Server) deleteTemplate(w http.ResponseWriter, r *http.Request) { s.deleteSnapshot(w, r) return } - if err := forEachHolder(b.holders, func(h templateHolder) error { return s.store.DeleteTemplate(r.Context(), h.node, h.key) }); err != nil { + if err := forEachHolder(b.holders, func(h templateHolder) error { return s.store.DeleteTemplate(r.Context(), h.node, h.key, "") }); err != nil { logger.Errorf(r.Context(), err, "e2b delete template failed template=%s", name) writeError(w, http.StatusInternalServerError, "failed to delete the template") return @@ -307,7 +308,7 @@ func (s *Server) resolveTemplate(r *http.Request, name string) (*builtTemplate, if err != nil { return nil, aliased, err } - return builtTemplates(nodes, s.templateScope(r))[name], aliased || advertisedIn(nodes, name), nil + return builtTemplates(nodes, s.templateScope(r), name)[name], aliased || advertisedIn(nodes, name), nil } func (s *Server) templateScope(r *http.Request) string { @@ -331,7 +332,9 @@ func (s *Server) liveTags(r *http.Request, b *builtTemplate) (map[string]string, } func (s *Server) writeTags(r *http.Request, b *builtTemplate, tags map[string]string) error { - return forEachHolder(b.holders, func(h templateHolder) error { return s.store.SetTemplateLabels(r.Context(), h.node, h.key, tags) }) + return forEachHolder(b.holders, func(h templateHolder) error { + return s.store.SetTemplateLabels(r.Context(), h.node, h.key, tags, h.digest) + }) } func (s *Server) writeTagError(w http.ResponseWriter, r *http.Request, err error, name string) { @@ -343,13 +346,13 @@ func (s *Server) writeTagError(w http.ResponseWriter, r *http.Request, err error writeError(w, http.StatusInternalServerError, "failed to update the template's tags") } -// builtTemplates groups the operator-owned templates under scope by their bare name. -func builtTemplates(nodes []*scale.NodeInventory, scope string) map[string]*builtTemplate { +// builtTemplates groups the operator-owned templates under scope by their bare name, or only the one named only. +func builtTemplates(nodes []scale.NodePools, scope, only string) map[string]*builtTemplate { out := map[string]*builtTemplate{} for _, inv := range nodes { for _, t := range inv.Templates { name, ok := strings.CutPrefix(t.Template, scope) - if !ok || t.Tenant != "" { + if !ok || t.Tenant != "" || only != "" && name != only { continue } b := out[name] @@ -397,8 +400,8 @@ func buildUUID(digest string) string { return uuid.NewSHA1(buildNamespace, []byte(digest)).String() } -func advertisedIn(nodes []*scale.NodeInventory, image string) bool { - return slices.ContainsFunc(nodes, func(inv *scale.NodeInventory) bool { +func advertisedIn(nodes []scale.NodePools, image string) bool { + return slices.ContainsFunc(nodes, func(inv scale.NodePools) bool { return slices.ContainsFunc(inv.Pools, func(pc scale.PoolCapacity) bool { return pc.Template == image }) }) } diff --git a/pkg/e2bcompat/templates_test.go b/pkg/e2bcompat/templates_test.go index 17f11ad..aacd83d 100644 --- a/pkg/e2bcompat/templates_test.go +++ b/pkg/e2bcompat/templates_test.go @@ -191,7 +191,19 @@ func TestTheAliasLookupFindsABuiltTemplateAfterTheTable(t *testing.T) { assert.Equal(t, http.StatusNotFound, do(t, h, http.MethodGet, "/templates/aliases/other", "", testKey).Code) } -func withTemplateFleet(store *fakeStore) func(*Options) { +func TestAFullE2bKeyNeverReachesAnotherNamespacesTemplate(t *testing.T) { + store := &fakeStore{} + h := newTestServer(t, store, withBuilds(), withTemplateFleet(store)) + w := do(t, h, http.MethodPost, "/sandboxes", `{"templateID":"e2b/others/app"}`, testKey) + assert.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) + assert.Equal(t, 0, store.claimCalls, "a full key is refused before any claim") + + id := requestBuild(t, h, "app") + w = do(t, h, http.MethodPost, "/v2/templates/app/builds/"+id, `{"fromImage":"e2b/others/app"}`, testKey) + assert.Equal(t, http.StatusBadRequest, w.Code, w.Body.String()) +} + +func withTemplateFleet(store *fakeStore) serverOption { created := &metav1.Time{Time: time.Date(2026, 9, 28, 1, 2, 3, 0, time.UTC)} built := func(name, digest string) scale.PromotedTemplate { return scale.PromotedTemplate{Template: name, Net: "none", Size: "medium", ContentDigest: digest, CreatedAt: created, CPUCount: 2, MemoryBytes: 1 << 30} diff --git a/pkg/e2bcompat/types.go b/pkg/e2bcompat/types.go index dec11c7..30211dd 100644 --- a/pkg/e2bcompat/types.go +++ b/pkg/e2bcompat/types.go @@ -113,8 +113,8 @@ type SandboxForkRequest struct { Count *int32 `json:"count,omitempty"` } -// SandboxForkResult is one entry of the fork reply: exactly one of Sandbox or -// Error is set, so a partial failure still returns 201 with per-child detail. +// SandboxForkResult is one entry of the fork reply in e2b's shape; a fork here is +// all-or-nothing, so Error is never set and a failed child fails the whole with 500. type SandboxForkResult struct { Sandbox *Sandbox `json:"sandbox,omitempty"` Error *APIError `json:"error,omitempty"` @@ -260,7 +260,6 @@ type TemplateBuildStartV2 struct { FromImage string `json:"fromImage"` FromTemplate string `json:"fromTemplate"` FromImageRegistry json.RawMessage `json:"fromImageRegistry"` - Force bool `json:"force"` Steps []e2bbuild.Step `json:"steps"` StartCmd string `json:"startCmd"` ReadyCmd string `json:"readyCmd"` diff --git a/pkg/e2bcompat/uploads.go b/pkg/e2bcompat/uploads.go index ffac076..08c7f92 100644 --- a/pkg/e2bcompat/uploads.go +++ b/pkg/e2bcompat/uploads.go @@ -145,7 +145,7 @@ func (s *Server) fileUploadLink(w http.ResponseWriter, r *http.Request) { } // archive opens the uploads of one build's template for its COPY steps. -func (s *Server) archive(ns, name string) func(context.Context, string) (io.ReadCloser, error) { +func (s *Server) archive(ns, name string) e2bbuild.ArchiveFunc { return func(ctx context.Context, hash string) (io.ReadCloser, error) { return s.opts.Builds.Uploads.open(ctx, uploadKey{ns: ns, template: name, hash: hash}) } diff --git a/pkg/e2bcompat/uploads_test.go b/pkg/e2bcompat/uploads_test.go index 255bb07..997f49b 100644 --- a/pkg/e2bcompat/uploads_test.go +++ b/pkg/e2bcompat/uploads_test.go @@ -81,7 +81,7 @@ func TestACopyStepSendsItsUploadIntoTheSandbox(t *testing.T) { assert.True(t, strings.HasPrefix(store.envdCalls[1], "sb_1 RUN root archive='/tmp/"+hash+".tar'"), store.envdCalls[1]) } -func withUploads(t *testing.T, dir string, maxBytes int64) func(*Options) { +func withUploads(t *testing.T, dir string, maxBytes int64) serverOption { root, err := os.OpenRoot(dir) require.NoError(t, err) t.Cleanup(func() { _ = root.Close() }) diff --git a/pkg/envdproxy/dial.go b/pkg/envdproxy/dial.go index f621788..f8264e0 100644 --- a/pkg/envdproxy/dial.go +++ b/pkg/envdproxy/dial.go @@ -22,17 +22,6 @@ type target struct { // targetKey addresses target in a request context. type targetKey struct{} -// guestDialer opens the connection a request's target names. -type guestDialer func(ctx context.Context, t target) (net.Conn, error) - -// dialGuest opens sandboxd's guest-port relay and hands back the upgraded -// connection as a plain net.Conn. -func dialGuest(dialer *net.Dialer) guestDialer { - return func(ctx context.Context, t target) (net.Conn, error) { - return sandboxd.DialPort(ctx, dialer, t.owner.Address, t.owner.ClaimID, t.owner.Token, t.port) - } -} - // guestTransport carries a request to the guest daemon over the protocol that // daemon serves, which is not the one the client used: the edge may answer // HTTP/2 while the guest speaks only HTTP/1.1. @@ -41,16 +30,16 @@ type guestTransport struct { h2 *http.Transport } -// newGuestTransport builds both halves over dial. Neither pools: a reused +// newGuestTransport builds both halves over sandboxd's guest-port relay. Neither pools: a reused // connection would outlive the sandboxd relay carrying it, and reusing one // across sandboxes would cross a tenancy boundary. -func newGuestTransport(dial guestDialer) *guestTransport { +func newGuestTransport(dialer *net.Dialer) *guestTransport { dialContext := func(ctx context.Context, _, _ string) (net.Conn, error) { t, ok := targetFrom(ctx) if !ok { return nil, errNoTarget } - return dial(ctx, t) + return sandboxd.DialPort(ctx, dialer, t.owner.Address, t.owner.ClaimID, t.owner.Token, t.port) } // With HTTP/1 also set, a plaintext transport cannot negotiate HTTP/2 and picks HTTP/1. var h2 http.Protocols diff --git a/pkg/envdproxy/envdproxy.go b/pkg/envdproxy/envdproxy.go index 62a990a..4fa99be 100644 --- a/pkg/envdproxy/envdproxy.go +++ b/pkg/envdproxy/envdproxy.go @@ -49,6 +49,8 @@ type Options struct { // upgrade, which would fail every request; turn it on for a guest daemon // that serves h2c, such as a user's own server on another port. GuestHTTP2 bool + // NodeToken is the sandboxd api_token a signed file URL's relay presents, so that relay stays passive and never wakes a paused sandbox. + NodeToken string } // Server routes one public host onto many sandboxes' guest ports. @@ -67,8 +69,11 @@ func NewServer(resolver Resolver, opts Options) (*Server, error) { if strings.TrimSpace(opts.Domain) == "" { return nil, errors.New("envdproxy: no domain configured; the SDK's sandbox host is derived from it") } + if strings.TrimSpace(opts.NodeToken) == "" { + return nil, errors.New("envdproxy: the node token is required; a signed file URL relays with it") + } dialer := &net.Dialer{Timeout: dialTimeout} - return &Server{resolver: resolver, transport: newGuestTransport(dialGuest(dialer)), opts: opts}, nil + return &Server{resolver: resolver, transport: newGuestTransport(dialer), opts: opts}, nil } // Handler returns the routed handler. Serve it with Protocols(): a ConnectRPC @@ -99,8 +104,10 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { switch token := strings.TrimSpace(r.Header.Get(accessTokenHeader)); { case token != "": owner, err = s.resolver.Owner(r.Context(), rt.sandboxID, token) - case signedFileURL(r): - owner, err = s.resolver.Locate(r.Context(), rt.sandboxID) + case rt.port == envdPort && signedFileURL(r): + if owner, err = s.resolver.Locate(r.Context(), rt.sandboxID); err == nil { + owner.Token = s.opts.NodeToken + } default: writeError(w, http.StatusUnauthorized, "missing "+accessTokenHeader) return @@ -152,9 +159,8 @@ func (s *Server) sandboxHost(rt route) string { return strconv.FormatUint(uint64(rt.port), 10) + "-" + rt.sandboxID + "." + s.opts.Domain } -// writeUpstreamError maps a node's refusal without naming the node. sandboxd -// answers 404 for both an unknown id and a wrong token; the id was just -// resolved from inventory, so the token is what the caller can still fix. +// writeUpstreamError maps a node's refusal without naming the node. The relay opens with the claim +// token the edge read from the node, so its 404 or 401 means the cached owner is stale (released within recentOwnerTTL). func (s *Server) writeUpstreamError(w http.ResponseWriter, r *http.Request, err error) { if status, ok := errors.AsType[*sandboxd.HTTPError](err); ok { switch status.StatusCode { diff --git a/pkg/envdproxy/envdproxy_test.go b/pkg/envdproxy/envdproxy_test.go index 7da628c..35ce7d7 100644 --- a/pkg/envdproxy/envdproxy_test.go +++ b/pkg/envdproxy/envdproxy_test.go @@ -105,6 +105,9 @@ func TestProxyRelaysASignedFileURLWithoutTheToken(t *testing.T) { if strings.Contains(string(body), accessTokenHeader) { t.Errorf("a signed request gained an access token on the way: %q", body) } + if node.lastAuth != "Bearer root" { + t.Errorf("a signed request relayed with %q, want the node token: the passive relay never wakes a paused sandbox for a stranger's signature", node.lastAuth) + } for _, path := range []string{"/files?path=%2Fetc%2Fhosts", "/envs?signature=v1_abc"} { resp := request(t, h, "49983-sb-abc."+testDomain, path, "") _ = resp.Body.Close() @@ -112,6 +115,11 @@ func TestProxyRelaysASignedFileURLWithoutTheToken(t *testing.T) { t.Errorf("%s: status %d, want 401: only a signed file URL goes without the token", path, resp.StatusCode) } } + resp = request(t, h, "3000-sb-abc."+testDomain, "/files?path=%2Fetc%2Fhosts&signature=v1_abc&signature_expiration=9", "") + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("a signed file URL on port 3000: status %d, want 401: only envd's port verifies a signature", resp.StatusCode) + } } func TestProxyRequiresTheAccessToken(t *testing.T) { @@ -211,7 +219,7 @@ func TestProxyAdmitsOnlyTheTokenDerivedFromThePublishedClaim(t *testing.T) { src := &countingSource{StaticInventorySource: scale.NewStaticInventorySource()} src.Put(&scale.NodeInventory{Name: "node-0", Node: "node-0", Address: owner.addr, Entries: []scale.InventoryEntry{{Name: "sandboxes/s1", ID: "sb_abc"}}}) routed := scale.NewScatterGatherStore(src, scale.WithClaimRouting("root", scale.NewSandboxdClientFactory())) - r, err := NewResolver(scale.NewScatterGatherStore(src), routed, src, "", []byte(testSecret)) + r, err := NewResolver(routed, src, "", []byte(testSecret)) if err != nil { t.Fatalf("NewResolver: %v", err) } @@ -293,7 +301,7 @@ func TestProxyMapsNodeRefusals(t *testing.T) { nodeStatus int want int }{ - {"wrong token", http.StatusNotFound, http.StatusUnauthorized}, + {"stale owner", http.StatusNotFound, http.StatusUnauthorized}, {"bad port", http.StatusBadRequest, http.StatusBadRequest}, {"no guest listener", http.StatusBadGateway, http.StatusBadGateway}, } @@ -424,6 +432,9 @@ func TestNewServerRequiresADomainAndAResolver(t *testing.T) { if _, err := NewServer(nil, Options{Domain: testDomain}); err == nil { t.Error("NewServer accepted a nil resolver") } + if _, err := NewServer(resolverFunc(nil), Options{Domain: testDomain}); err == nil { + t.Error("NewServer without a node token must fail") + } if _, err := NewServer(resolverFunc(nil), Options{}); err == nil { t.Error("NewServer accepted an empty domain; every sandbox host is derived from it") } @@ -431,7 +442,7 @@ func TestNewServerRequiresADomainAndAResolver(t *testing.T) { func newTestProxy(t *testing.T, r Resolver, opts ...func(*Options)) http.Handler { t.Helper() - o := Options{Domain: testDomain} + o := Options{Domain: testDomain, NodeToken: "root"} for _, fn := range opts { fn(&o) } @@ -450,7 +461,7 @@ func unpublishedResolver(t *testing.T, nodes ...*fakeNode) (Resolver, *countingS src.Put(&scale.NodeInventory{Name: name, Node: name, Address: n.addr}) } routed := scale.NewScatterGatherStore(src, scale.WithClaimRouting("root", scale.NewSandboxdClientFactory())) - r, err := NewResolver(scale.NewScatterGatherStore(src), routed, src, "", []byte(testSecret)) + r, err := NewResolver(routed, src, "", []byte(testSecret)) if err != nil { t.Fatalf("NewResolver: %v", err) } @@ -566,7 +577,7 @@ func (n *fakeNode) handle(conn net.Conn, guest func(net.Conn)) { } n.lastPath, n.lastAuth = req.URL.Path, req.Header.Get("Authorization") refuse := n.refuse - if refuse == 0 && n.lastAuth != "Bearer tok" { + if refuse == 0 && n.lastAuth != "Bearer tok" && n.lastAuth != "Bearer root" { refuse = http.StatusNotFound } if refuse != 0 { diff --git a/pkg/envdproxy/resolver.go b/pkg/envdproxy/resolver.go index 0ea2328..8c843a2 100644 --- a/pkg/envdproxy/resolver.go +++ b/pkg/envdproxy/resolver.go @@ -68,13 +68,9 @@ type storeResolver struct { recent *recentOwners } -// NewResolver builds the Resolver. placed must read inventory only, so an unknown id reaches the nodes -// only through the rate-limited probe; an empty namespace matches every namespace. -func NewResolver(placed scale.SandboxStore, routed scale.SandboxLifecycle, inventory scale.InventorySource, namespace string, secret []byte) (Resolver, error) { - claims, ok := placed.(scale.ClaimIDResolver) - if !ok { - return nil, errors.New("envdproxy: store does not implement scale.ClaimIDResolver") - } +// NewResolver builds the Resolver; its claim lookups read inventory only, so an unknown id reaches the nodes +// only through the rate-limited probe, and an empty namespace matches every namespace. +func NewResolver(routed scale.SandboxLifecycle, inventory scale.InventorySource, namespace string, secret []byte) (Resolver, error) { if routed == nil || inventory == nil { return nil, errors.New("envdproxy: a routed store and an inventory source are required") } @@ -82,7 +78,7 @@ func NewResolver(placed scale.SandboxStore, routed scale.SandboxLifecycle, inven return nil, errors.New("envdproxy: the envd secret is required to verify access tokens") } return &storeResolver{ - claims: claims, + claims: scale.NewScatterGatherStore(inventory).(scale.ClaimIDResolver), lifecycle: routed, inventory: inventory, namespace: namespace, @@ -131,22 +127,14 @@ func (s *storeResolver) probe(ctx context.Context, sandboxID, claimID string, ad } ctx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() - var mu sync.Mutex - var found Owner - node, err := scale.FirstHit(ctx, s.inventory, probeConcurrency, func(ctx context.Context, node string) string { - o, err := s.ownerOn(ctx, node, claimID) - if err != nil { - return "" - } - mu.Lock() - found = o - mu.Unlock() - return node + found, err := scale.FirstHit(ctx, s.inventory, probeConcurrency, func(ctx context.Context, node string) Owner { + o, _ := s.ownerOn(ctx, node, claimID) + return o }) if err != nil { return Owner{}, err } - if node == "" { + if found == (Owner{}) { return Owner{}, ErrSandboxNotFound } e := s.entry(found) diff --git a/pkg/envdproxy/route.go b/pkg/envdproxy/route.go index c3693cd..3ada9ec 100644 --- a/pkg/envdproxy/route.go +++ b/pkg/envdproxy/route.go @@ -16,6 +16,8 @@ const ( // accessTokenHeader is the per-sandbox data-plane credential. accessTokenHeader = "X-Access-Token" apiKeyHeader = "X-API-KEY" + + envdPort uint16 = 49983 ) // internalPaths are envd's own control surface (x-internal in its spec). They diff --git a/pkg/sandboxd/client.go b/pkg/sandboxd/client.go index bed35ba..ea459b4 100644 --- a/pkg/sandboxd/client.go +++ b/pkg/sandboxd/client.go @@ -19,6 +19,8 @@ import ( "slices" "strings" "time" + + "github.com/spf13/pflag" ) const ( @@ -29,13 +31,24 @@ const ( // NetRouteRelay is a claim whose guest reaches the network through the node's egress proxy on 127.0.0.1:3128. NetRouteRelay = "relay" + + captureTimeout = 10 * time.Minute ) -// ErrNodeAtCapacity is returned by Claim when sandboxd answers 429 (the node is -// at max_claims, the calling tenant is at its own max_claims, or the node is -// draining) or a 200 that delivers no sandbox. In every case this node handed -// over no VM, so the store tries another node or reports no warm capacity. -var ErrNodeAtCapacity = errors.New("sandboxd: node at capacity or draining") +var ( + // ErrNodeAtCapacity is returned by Claim when sandboxd answers 429 (the node is + // at max_claims, the calling tenant is at its own max_claims, or the node is + // draining) or a 200 that delivers no sandbox. In every case this node handed + // over no VM, so the store tries another node or reports no warm capacity. + ErrNodeAtCapacity = errors.New("sandboxd: node at capacity or draining") + + // RelayEnv points a guest process at the node's egress proxy on a NetRouteRelay claim, as silkd's unit points its own. + RelayEnv = map[string]string{ + "http_proxy": "http://127.0.0.1:3128", + "https_proxy": "http://127.0.0.1:3128", + "no_proxy": "localhost,127.0.0.1,::1,169.254.169.254", + } +) // ExpireAction is what the node does with a claim whose lease ends; empty keeps the claim's current action, destroy on a new claim. type ExpireAction string @@ -159,8 +172,9 @@ type NodeInfo struct { type Client struct { baseURL string // token is the node api_token every verb presents, except Release, which takes one per call. - token string - hc *http.Client + token string + hc *http.Client + capture *http.Client } // New returns a Client for the sandboxd at baseURL; token is the node api_token, empty when sandboxd runs without auth. @@ -173,6 +187,9 @@ func New(baseURL, token string, opts ...Option) *Client { for _, o := range opts { o(c) } + capture := *c.hc + capture.Timeout = captureTimeout + c.capture = &capture return c } @@ -253,6 +270,10 @@ func (c *Client) Release(ctx context.Context, id, token string) error { } func (c *Client) send(ctx context.Context, method, path, token, op string, body []byte, ok ...int) error { + return c.sendWith(ctx, c.hc, method, path, token, op, body, ok...) +} + +func (c *Client) sendWith(ctx context.Context, hc *http.Client, method, path, token, op string, body []byte, ok ...int) error { var r io.Reader if body != nil { r = bytes.NewReader(body) @@ -266,7 +287,7 @@ func (c *Client) send(ctx context.Context, method, path, token, op string, body } c.authenticate(req, token) - resp, err := c.hc.Do(req) + resp, err := hc.Do(req) if err != nil { return fmt.Errorf("sandboxd: %s: %w", op, err) } @@ -284,6 +305,14 @@ func (c *Client) authenticate(req *http.Request, token string) { } } +// AddTokenFlags registers the sandboxd api_token pair every binary that dials nodes takes. +func AddTokenFlags(fs *pflag.FlagSet, literal, file *string) { + fs.StringVar(literal, "sandboxd-token", *literal, + "sandboxd api_token presented to every node (the e2b surface and the envd proxy need the root one). Prefer --sandboxd-token-file for a Secret mount.") + fs.StringVar(file, "sandboxd-token-file", *file, + "Path to a file (Secret mount) holding the sandboxd api_token; overrides --sandboxd-token when set.") +} + // TokenFrom returns the api_token read from file (a Secret mount) when file is set, else literal. func TokenFrom(literal, file string) (string, error) { if file == "" { diff --git a/pkg/sandboxd/client_test.go b/pkg/sandboxd/client_test.go index 1c46baf..985dfe0 100644 --- a/pkg/sandboxd/client_test.go +++ b/pkg/sandboxd/client_test.go @@ -285,6 +285,8 @@ func TestDeleteTemplateAsksThisNodeAloneAndTakesAMissAsGone(t *testing.T) { w.WriteHeader(http.StatusNotFound) case "ns/pooled": w.WriteHeader(http.StatusConflict) + case "ns/replaced": + w.WriteHeader(http.StatusPreconditionFailed) default: w.WriteHeader(http.StatusNoContent) } @@ -292,14 +294,20 @@ func TestDeleteTemplateAsksThisNodeAloneAndTakesAMissAsGone(t *testing.T) { defer srv.Close() c := New(srv.URL, "root-token") - require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/app", Net: "none", Size: "medium"})) - require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/gone"})) - err := c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/pooled"}) + require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/app", Net: "none", Size: "medium"}, "")) + require.NoError(t, c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/gone"}, "")) + err := c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/pooled"}, "") he, ok := errors.AsType[*HTTPError](err) require.True(t, ok, "the node's refusal must surface as its status, got %v", err) assert.Equal(t, http.StatusConflict, he.StatusCode) assert.Equal(t, "DELETE /v1/templates?net=none&no_redirect=1&size=medium&template=ns%2Fapp Bearer root-token", got[0]) assert.Equal(t, "DELETE /v1/templates?no_redirect=1&template=ns%2Fgone Bearer root-token", got[1]) + + err = c.DeleteTemplate(t.Context(), PoolKey{Template: "ns/replaced"}, "sha256:x") + he, ok = errors.AsType[*HTTPError](err) + require.True(t, ok, "a replaced generation must surface as 412, got %v", err) + assert.Equal(t, http.StatusPreconditionFailed, he.StatusCode) + assert.Equal(t, "DELETE /v1/templates?digest=sha256%3Ax&no_redirect=1&template=ns%2Freplaced Bearer root-token", got[3]) } func TestSetTemplateLabelsPutsTheWholeMap(t *testing.T) { @@ -312,10 +320,10 @@ func TestSetTemplateLabelsPutsTheWholeMap(t *testing.T) { defer srv.Close() c := New(srv.URL, "root-token") - require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app", Size: "medium"}, map[string]string{"v1": "sha256:aa"})) - require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app"}, nil)) + require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app", Size: "medium"}, map[string]string{"v1": "sha256:aa"}, "sha256:aa")) + require.NoError(t, c.SetTemplateLabels(t.Context(), PoolKey{Template: "ns/app"}, nil, "")) assert.Equal(t, []string{ - `PUT /v1/templates/labels?size=medium&template=ns%2Fapp {"labels":{"v1":"sha256:aa"}}`, + `PUT /v1/templates/labels?digest=sha256%3Aaa&size=medium&template=ns%2Fapp {"labels":{"v1":"sha256:aa"}}`, `PUT /v1/templates/labels?template=ns%2Fapp {"labels":null}`, }, got) } @@ -379,3 +387,25 @@ func TestSetPoolsSendsAnEmptyListNotNull(t *testing.T) { assert.Equal(t, "duplicate pool", he.Message) assert.JSONEq(t, `{"pools":[]}`, raw, "a nil set must drain as [] rather than null") } + +func TestCaptureVerbsOutliveTheRequestTimeout(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + switch r.URL.Path { + case "/v1/sandboxes/sb_1/promote": + _, _ = w.Write([]byte(`{"key":{"template":"ns/app","net":"none","size":"small"},"content_digest":"sha256:aa"}`)) + case "/v1/sandboxes/sb_1/hibernate": + w.WriteHeader(http.StatusNoContent) + default: + _, _ = w.Write([]byte(`{"id":"sb_2","token":"t"}`)) + } + })) + defer srv.Close() + c := New(srv.URL, "root-token", WithHTTPClient(&http.Client{Timeout: 50 * time.Millisecond})) + + _, _, err := c.Promote(t.Context(), "sb_1", "ns/app") + require.NoError(t, err, "a promote runs as long as the guest's memory takes") + require.NoError(t, c.Hibernate(t.Context(), "sb_1")) + _, err = c.Claim(t.Context(), ClaimSpec{Template: "img"}) + require.Error(t, err, "a claim keeps the request timeout") +} diff --git a/pkg/sandboxd/lifecycle.go b/pkg/sandboxd/lifecycle.go index 27f23b8..787991e 100644 --- a/pkg/sandboxd/lifecycle.go +++ b/pkg/sandboxd/lifecycle.go @@ -118,7 +118,7 @@ func (c *Client) Fork(ctx context.Context, id string, spec ForkSpec) (ForkResult if id == "" { return out, fmt.Errorf("sandboxd: fork requires a sandbox id") } - err := c.sendJSON(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/fork", spec, &out) + err := c.sendJSONWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/fork", spec, &out) return out, err } @@ -139,7 +139,7 @@ func (c *Client) Checkpoint(ctx context.Context, id string, spec CheckpointSpec) var out struct { Checkpoint Checkpoint `json:"checkpoint"` } - err := c.sendJSON(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/checkpoint", spec, &out) + err := c.sendJSONWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/checkpoint", spec, &out) return out.Checkpoint, err } @@ -152,7 +152,7 @@ func (c *Client) Promote(ctx context.Context, id, template string) (PoolKey, str Key PoolKey `json:"key"` ContentDigest string `json:"content_digest"` } - err := c.sendJSON(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/promote", struct { + err := c.sendJSONWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/promote", struct { Template string `json:"template"` }{template}, &out) return out.Key, out.ContentDigest, err @@ -167,25 +167,32 @@ func (c *Client) Checkpoints(ctx context.Context) ([]Checkpoint, error) { return out.Checkpoints, err } -// DeleteTemplate performs DELETE /v1/templates for key on this node alone (no_redirect). A 404 is success. -func (c *Client) DeleteTemplate(ctx context.Context, key PoolKey) error { +// DeleteTemplate performs DELETE /v1/templates for key on this node alone (no_redirect); a 404 is success, and a digest deletes only that generation (412 otherwise). +func (c *Client) DeleteTemplate(ctx context.Context, key PoolKey, digest string) error { if key.Template == "" { return fmt.Errorf("sandboxd: delete template requires a template name") } q := templateQuery(key) q.Set("no_redirect", "1") + if digest != "" { + q.Set("digest", digest) + } return c.send(ctx, http.MethodDelete, "/v1/templates?"+q.Encode(), c.token, "delete template", nil, http.StatusNoContent, http.StatusNotFound) } -// SetTemplateLabels performs PUT /v1/templates/labels for key on this node, replacing the whole label map. -func (c *Client) SetTemplateLabels(ctx context.Context, key PoolKey, labels map[string]string) error { +// SetTemplateLabels performs PUT /v1/templates/labels for key on this node, replacing the whole label map; a digest writes only that generation (412 otherwise). +func (c *Client) SetTemplateLabels(ctx context.Context, key PoolKey, labels map[string]string, digest string) error { body, err := json.Marshal(struct { Labels map[string]string `json:"labels"` }{labels}) if err != nil { return err } - return c.send(ctx, http.MethodPut, "/v1/templates/labels?"+templateQuery(key).Encode(), c.token, "template labels", body, http.StatusNoContent) + q := templateQuery(key) + if digest != "" { + q.Set("digest", digest) + } + return c.send(ctx, http.MethodPut, "/v1/templates/labels?"+q.Encode(), c.token, "template labels", body, http.StatusNoContent) } // DeleteCheckpoint performs DELETE /v1/checkpoints/{id}. A 404 is success. @@ -229,10 +236,14 @@ func (c *Client) sandboxVerb(ctx context.Context, id, verb string) error { if id == "" { return fmt.Errorf("sandboxd: %s requires a sandbox id", verb) } - return c.send(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/"+verb, c.token, verb, nil, http.StatusNoContent) + return c.sendWith(ctx, c.capture, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/"+verb, c.token, verb, nil, http.StatusNoContent) } func (c *Client) sendJSON(ctx context.Context, method, path string, body, out any) error { + return c.sendJSONWith(ctx, c.hc, method, path, body, out) +} + +func (c *Client) sendJSONWith(ctx context.Context, hc *http.Client, method, path string, body, out any) error { payload, err := json.Marshal(body) if err != nil { return fmt.Errorf("sandboxd: encode %s: %w", path, err) @@ -244,7 +255,7 @@ func (c *Client) sendJSON(ctx context.Context, method, path string, body, out an req.Header.Set("Content-Type", "application/json") c.authenticate(req, c.token) - resp, err := c.hc.Do(req) + resp, err := hc.Do(req) if err != nil { return fmt.Errorf("sandboxd: %s: %w", path, err) } diff --git a/pkg/sandboxd/ports.go b/pkg/sandboxd/ports.go index 32b1765..75586e0 100644 --- a/pkg/sandboxd/ports.go +++ b/pkg/sandboxd/ports.go @@ -2,13 +2,16 @@ package sandboxd import ( "bufio" + "cmp" "context" + "crypto/tls" "fmt" "io" "net" "net/http" "net/url" "strconv" + "strings" ) const ( @@ -17,24 +20,33 @@ const ( portReplyMax = 16 << 10 ) -// DialPort opens a guest port of sandbox id through this node's relay with the node api_token, which the node keeps passive: a paused sandbox answers 409 unwoken. -func (c *Client) DialPort(ctx context.Context, id string, port uint16) (net.Conn, error) { - u, _ := url.Parse(c.baseURL) +// DialPort opens a guest port of sandbox id through this node's relay; an empty token presents the node api_token, which the node keeps passive (a paused sandbox answers 409 unwoken), the sandbox's own token wakes it and stamps activity. +func (c *Client) DialPort(ctx context.Context, id, token string, port uint16) (net.Conn, error) { var d net.Dialer - return DialPort(ctx, &d, u.Host, id, c.token, port) + return DialPort(ctx, &d, c.baseURL, id, cmp.Or(token, c.token), port) } -// DialPort opens the node's GET /v1/sandboxes/{id}/ports/{port} relay; with the sandbox's own token the node wakes a paused sandbox for it. -func DialPort(ctx context.Context, d *net.Dialer, addr, id, token string, port uint16) (net.Conn, error) { - conn, err := d.DialContext(ctx, "tcp", addr) +// DialPort opens the node's GET /v1/sandboxes/{id}/ports/{port} relay at base, a host:port or an http(s) origin; with the sandbox's own token the node wakes a paused sandbox for it. +func DialPort(ctx context.Context, d *net.Dialer, base, id, token string, port uint16) (net.Conn, error) { + scheme, addr, serverName := relayTarget(base) + tcp, err := d.DialContext(ctx, "tcp", addr) if err != nil { return nil, fmt.Errorf("dial node: %w", err) } - stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + stop := context.AfterFunc(ctx, func() { _ = tcp.Close() }) defer stop() + conn := tcp + if scheme == "https" { + tc := tls.Client(tcp, &tls.Config{ServerName: serverName, MinVersion: tls.VersionTLS12}) + if err = tc.HandshakeContext(ctx); err != nil { + _ = tcp.Close() + return nil, fmt.Errorf("tls to node: %w", err) + } + conn = tc + } path := "/v1/sandboxes/" + url.PathEscape(id) + "/ports/" + strconv.FormatUint(uint64(port), 10) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+addr+path, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+addr+path, nil) if err != nil { _ = conn.Close() return nil, err @@ -69,3 +81,14 @@ type bufConn struct { } func (c *bufConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +func relayTarget(base string) (scheme, addr, serverName string) { + scheme, host, port := "http", base, "" + if u, err := url.Parse(base); err == nil && strings.Contains(base, "://") { + scheme, host, port = u.Scheme, u.Hostname(), u.Port() + } else if h, p, err := net.SplitHostPort(base); err == nil { + host, port = h, p + } + port = cmp.Or(port, map[bool]string{true: "443", false: "80"}[scheme == "https"]) + return scheme, net.JoinHostPort(host, port), host +} diff --git a/pkg/sandboxd/ports_test.go b/pkg/sandboxd/ports_test.go index e12ffe0..25e501e 100644 --- a/pkg/sandboxd/ports_test.go +++ b/pkg/sandboxd/ports_test.go @@ -30,7 +30,7 @@ func TestClientDialPortOpensThePassiveRelayWithTheNodeToken(t *testing.T) { })) defer node.Close() - conn, err := New(node.URL, "root-token").DialPort(t.Context(), "sb_1", 8080) + conn, err := New(node.URL, "root-token").DialPort(t.Context(), "sb_1", "", 8080) require.NoError(t, err) defer func() { _ = conn.Close() }() _, err = conn.Write([]byte("ping\n")) @@ -56,3 +56,51 @@ func TestDialPortReportsTheNodesRefusalAsAnHTTPError(t *testing.T) { assert.Equal(t, http.StatusNotFound, he.StatusCode) assert.Equal(t, "unknown sandbox", he.Message) } + +func TestDialPortSpeaksTLSToAnHTTPSNode(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = ln.Close() }() + first := make(chan byte, 1) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + var b [1]byte + if _, err := io.ReadFull(conn, b[:]); err == nil { + first <- b[0] + } + }() + + _, err = New("https://"+ln.Addr().String(), "root-token").DialPort(t.Context(), "sb_1", "", 8080) + require.Error(t, err, "a plain listener cannot finish the handshake") + assert.Equal(t, byte(0x16), <-first, "an https base opens a TLS handshake, not a cleartext upgrade") +} + +func TestRelayTargetJoinsAnOriginWithItsDefaultPort(t *testing.T) { + for base, want := range map[string][3]string{ + "10.0.0.4:7777": {"http", "10.0.0.4:7777", "10.0.0.4"}, + "node": {"http", "node:80", "node"}, + "http://node:8080": {"http", "node:8080", "node"}, + "https://node-a.example.com": {"https", "node-a.example.com:443", "node-a.example.com"}, + "https://[2001:db8::1]": {"https", "[2001:db8::1]:443", "2001:db8::1"}, + "https://[2001:db8::1]:8443": {"https", "[2001:db8::1]:8443", "2001:db8::1"}, + } { + scheme, addr, serverName := relayTarget(base) + assert.Equal(t, want, [3]string{scheme, addr, serverName}, base) + } +} + +func TestClientDialPortPresentsTheSandboxTokenWhenGivenOne(t *testing.T) { + var gotAuth string + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusConflict) + })) + defer node.Close() + _, err := New(node.URL, "root-token").DialPort(t.Context(), "sb_1", "claim-token", 8080) + require.Error(t, err) + assert.Equal(t, "Bearer claim-token", gotAuth, "the sandbox's own token makes the relay active") +} diff --git a/pkg/scale/apiserver/storage_test.go b/pkg/scale/apiserver/storage_test.go index a804063..69b25a9 100644 --- a/pkg/scale/apiserver/storage_test.go +++ b/pkg/scale/apiserver/storage_test.go @@ -312,13 +312,15 @@ func (f *fakeStore) Snapshots(context.Context, string) ([]scale.Snapshot, error) func (f *fakeStore) DeleteSnapshot(context.Context, string, string) error { return nil } -func (f *fakeStore) DeleteTemplate(context.Context, string, scale.PoolKey) error { return f.verbErr } +func (f *fakeStore) DeleteTemplate(context.Context, string, scale.PoolKey, string) error { + return f.verbErr +} func (f *fakeStore) Promote(context.Context, string, string, string) (scale.PoolKey, string, error) { return scale.PoolKey{}, "", f.verbErr } -func (f *fakeStore) SetTemplateLabels(context.Context, string, scale.PoolKey, map[string]string) error { +func (f *fakeStore) SetTemplateLabels(context.Context, string, scale.PoolKey, map[string]string, string) error { return f.verbErr } @@ -330,7 +332,7 @@ func (f *fakeStore) SetInstanceMetadata(context.Context, string, string, []byte) return f.verbErr } -func (f *fakeStore) DialGuestPort(context.Context, string, string, uint16) (net.Conn, error) { +func (f *fakeStore) DialGuestPort(context.Context, string, string, string, uint16) (net.Conn, error) { return nil, f.verbErr } diff --git a/pkg/scale/export_test.go b/pkg/scale/export_test.go index e8ae513..1047968 100644 --- a/pkg/scale/export_test.go +++ b/pkg/scale/export_test.go @@ -1,5 +1,7 @@ package scale +import "context" + var ( BenchFleets = benchFleets BenchInventories = benchInventories @@ -7,10 +9,17 @@ var ( ParseSelectors = parseSelectors SplitNamespacedName = splitNamespacedName - WarmCandidates = (*scatterGatherStore).warmCandidates LookupName = (*scatterGatherStore).lookupName ListInventories = (*scatterGatherStore).listInventories PollPinned = (*scatterGatherStore).pollPinned ) type ScatterGatherStore = scatterGatherStore + +func WarmCandidates(ctx context.Context, s *scatterGatherStore, pool PoolKey) ([]warmCandidate, error) { + nodes, err := s.src.NodeCapacities(ctx) + if err != nil { + return nil, err + } + return warmCandidates(nodes, pool), nil +} diff --git a/pkg/scale/inventorycache_bench_test.go b/pkg/scale/inventorycache_bench_test.go index 8163f86..2ddf424 100644 --- a/pkg/scale/inventorycache_bench_test.go +++ b/pkg/scale/inventorycache_bench_test.go @@ -16,7 +16,6 @@ import ( toolscache "k8s.io/client-go/tools/cache" sandboxv1beta1 "sigs.k8s.io/agent-sandbox/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" cocoonv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" "github.com/cocoonstack/sandbox-operator/pkg/scale" @@ -30,32 +29,27 @@ var errWatchListUnserved = errors.New("watch-list is not served") func BenchmarkClientInventoryWarmCandidates(b *testing.B) { for _, fleet := range scale.BenchFleets { - for _, arm := range []struct { - name string - noCopy bool - }{{"copy", false}, {"nocopy", true}} { - b.Run(fleet.Name+"/"+arm.name, func(b *testing.B) { - store, pool := benchCachedStore(b, fleet.Nodes, fleet.PerNode, arm.noCopy) - ctx := b.Context() - b.ReportAllocs() - for b.Loop() { - candidates, err := scale.WarmCandidates(store, ctx, pool) - if err != nil { - b.Fatalf("warm candidates: %v", err) - } - if len(candidates) != fleet.Nodes { - b.Fatalf("got %d candidates, want %d", len(candidates), fleet.Nodes) - } + b.Run(fleet.Name, func(b *testing.B) { + store, pool := benchCachedStore(b, fleet.Nodes, fleet.PerNode) + ctx := b.Context() + b.ReportAllocs() + for b.Loop() { + candidates, err := scale.WarmCandidates(ctx, store, pool) + if err != nil { + b.Fatalf("warm candidates: %v", err) } - }) - } + if len(candidates) != fleet.Nodes { + b.Fatalf("got %d candidates, want %d", len(candidates), fleet.Nodes) + } + } + }) } } func BenchmarkClientInventoryWatchTick(b *testing.B) { for _, fleet := range scale.BenchFleets { b.Run(fleet.Name, func(b *testing.B) { - store, _ := benchCachedStore(b, fleet.Nodes, fleet.PerNode, true) + store, _ := benchCachedStore(b, fleet.Nodes, fleet.PerNode) ns, name := scale.SplitNamespacedName(fmt.Sprintf("default/sb-%s-%d", scale.BenchNodeName(fleet.Nodes-1), fleet.PerNode-1)) labelSel, fieldSel, err := scale.ParseSelectors(scale.ListOptions{Namespace: ns, FieldSelector: "metadata.name=" + name}) if err != nil { @@ -91,7 +85,7 @@ func BenchmarkClientInventoryWatchTick(b *testing.B) { } // benchCachedStore serves the fleet through a real informer-fed cache reader, the production read path. -func benchCachedStore(b *testing.B, nodes, perNode int, noCopy bool) (*scale.ScatterGatherStore, scale.PoolKey) { +func benchCachedStore(b *testing.B, nodes, perNode int) (*scale.ScatterGatherStore, scale.PoolKey) { b.Helper() invs, pool := scale.BenchInventories(nodes, perNode, 0) list := &cocoonv1beta1.NodeInventoryList{} @@ -107,9 +101,8 @@ func benchCachedStore(b *testing.B, nodes, perNode int, noCopy bool) (*scale.Sca mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{scale.NodeInventoryGVK.GroupVersion()}) mapper.Add(scale.NodeInventoryGVK, meta.RESTScopeRoot) invCache, err := cache.New(&restclient.Config{Host: "http://127.0.0.1:1"}, cache.Options{ - Scheme: scheme, - Mapper: mapper, - ByObject: map[client.Object]cache.ByObject{inv: {UnsafeDisableDeepCopy: &noCopy}}, + Scheme: scheme, + Mapper: mapper, NewInformer: func(_ toolscache.ListerWatcher, obj runtime.Object, resync time.Duration, indexers toolscache.Indexers) toolscache.SharedIndexInformer { return toolscache.NewSharedIndexInformer(&toolscache.ListWatch{ ListWithContextFunc: func(context.Context, metav1.ListOptions) (runtime.Object, error) { return list, nil }, diff --git a/pkg/scale/kubeinventory/cache.go b/pkg/scale/kubeinventory/cache.go index de62147..f84504a 100644 --- a/pkg/scale/kubeinventory/cache.go +++ b/pkg/scale/kubeinventory/cache.go @@ -8,7 +8,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" restclient "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" cocoonv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" "github.com/cocoonstack/sandbox-operator/pkg/scale" @@ -17,18 +16,14 @@ import ( // cacheSyncTimeout bounds the startup wait for the informer, so a missing CRD fails loud instead of serving an empty fleet. const cacheSyncTimeout = 2 * time.Minute -// NewCache starts a cache scoped to NodeInventory alone and waits for it to sync; a read of any other kind fails instead of starting a cluster-wide informer. +// NewCache starts a cache scoped to NodeInventory alone and waits for it to sync. func NewCache(ctx context.Context, restCfg *restclient.Config) (cache.Cache, error) { scheme := runtime.NewScheme() if err := cocoonv1beta1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("kubeinventory: register node inventory scheme: %w", err) } inv := &scale.NodeInventory{} - invCache, err := cache.New(restCfg, cache.Options{ - Scheme: scheme, - ByObject: map[client.Object]cache.ByObject{inv: {UnsafeDisableDeepCopy: new(true)}}, - ReaderFailOnMissingInformer: true, - }) + invCache, err := cache.New(restCfg, cache.Options{Scheme: scheme}) if err != nil { return nil, fmt.Errorf("kubeinventory: build inventory cache: %w", err) } diff --git a/pkg/scale/kubeinventory/source_test.go b/pkg/scale/kubeinventory/source_test.go index 765407d..5ca76cc 100644 --- a/pkg/scale/kubeinventory/source_test.go +++ b/pkg/scale/kubeinventory/source_test.go @@ -18,7 +18,6 @@ import ( restclient "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" cocoonv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" "github.com/cocoonstack/sandbox-operator/pkg/scale" @@ -208,9 +207,8 @@ func fleet(t *testing.T, opts Options, objs ...*cocoonv1beta1.NodeInventory) (*S mapper.Add(scale.NodeInventoryGVK, meta.RESTScopeRoot) var watched bool c, err := cache.New(&restclient.Config{Host: "http://127.0.0.1:1"}, cache.Options{ - Scheme: scheme, - Mapper: mapper, - ByObject: map[client.Object]cache.ByObject{&cocoonv1beta1.NodeInventory{}: {UnsafeDisableDeepCopy: new(true)}}, + Scheme: scheme, + Mapper: mapper, NewInformer: func(_ toolscache.ListerWatcher, obj runtime.Object, resync time.Duration, indexers toolscache.Indexers) toolscache.SharedIndexInformer { return toolscache.NewSharedIndexInformer(&toolscache.ListWatch{ ListWithContextFunc: func(context.Context, metav1.ListOptions) (runtime.Object, error) { return list, nil }, diff --git a/pkg/scale/meshinventory/source.go b/pkg/scale/meshinventory/source.go index 454cc07..fc44e82 100644 --- a/pkg/scale/meshinventory/source.go +++ b/pkg/scale/meshinventory/source.go @@ -148,7 +148,6 @@ func (s *Source) run(ctx context.Context) { } func (s *Source) tick(ctx context.Context) map[string]answer { - logger := log.WithFunc("meshinventory.tick") answers := s.poll(ctx) named, heard := map[string]bool{}, false for _, a := range answers { @@ -164,7 +163,7 @@ func (s *Source) tick(ctx context.Context) map[string]answer { continue } m.fails++ - logger.Warnf(ctx, "mesh member did not answer addr=%s fails=%d err=%v", addr, m.fails, a.err) + log.WithFunc("meshinventory.tick").Warnf(ctx, "mesh member did not answer addr=%s fails=%d err=%v", addr, m.fails, a.err) if !m.seed && heard && !named[addr] && m.fails > s.opts.MaxStale { delete(s.members, addr) } diff --git a/pkg/scale/sandboxstore.go b/pkg/scale/sandboxstore.go index d22ebac..5390055 100644 --- a/pkg/scale/sandboxstore.go +++ b/pkg/scale/sandboxstore.go @@ -72,7 +72,7 @@ type SandboxStore interface { SandboxLifecycle } -// SandboxLifecycle is the verb set of a claimed sandbox, each verb routed to its owning node. +// SandboxLifecycle is the verb set routed to one node: a claimed sandbox's lifecycle, its checkpoints and the node's promoted templates. // Pause and Snapshot write guest memory out, so their cost grows with its size. type SandboxLifecycle interface { // Pause snapshots and stops the sandbox, and is idempotent on a paused one. @@ -87,16 +87,16 @@ type SandboxLifecycle interface { Snapshots(ctx context.Context, node string) ([]Snapshot, error) // DeleteSnapshot removes a checkpoint. A missing checkpoint is success. DeleteSnapshot(ctx context.Context, node, snapshotID string) error - // DeleteTemplate removes a promoted template on node alone. A missing template is success. - DeleteTemplate(ctx context.Context, node string, key PoolKey) error + // DeleteTemplate removes a promoted template on node alone; a missing template is success, and a digest removes only the generation it names (PreconditionFailed otherwise). + DeleteTemplate(ctx context.Context, node string, key PoolKey, digest string) error // Promote publishes a claimed sandbox on node as the template name, keeping its net and size; it returns the full key and the content digest. Promote(ctx context.Context, node, id, template string) (PoolKey, string, error) - // SetTemplateLabels replaces a promoted template's labels on node alone. - SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string) error + // SetTemplateLabels replaces a promoted template's labels on node alone; a digest writes only the generation it names (PreconditionFailed otherwise). + SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string, digest string) error // NodeTemplates reads the promoted templates node holds now, ahead of its next inventory publish. NodeTemplates(ctx context.Context, node string) ([]PromotedTemplate, error) // DialGuestPort opens a guest port of a running sandbox through its node's passive relay; a paused one answers Conflict and is never woken. - DialGuestPort(ctx context.Context, node, id string, port uint16) (net.Conn, error) + DialGuestPort(ctx context.Context, node, id, token string, port uint16) (net.Conn, error) // SetInstanceMetadata replaces the JSON object a running sandbox's guest reads from 169.254.169.254; a paused one answers Conflict. SetInstanceMetadata(ctx context.Context, node, id string, doc []byte) error // Read reports the sandbox as its owning node holds it: token, paused state and lease deadline. diff --git a/pkg/scale/sandboxstore_bench_test.go b/pkg/scale/sandboxstore_bench_test.go index 9388eda..fac5d98 100644 --- a/pkg/scale/sandboxstore_bench_test.go +++ b/pkg/scale/sandboxstore_bench_test.go @@ -40,7 +40,7 @@ func BenchmarkStoreWarmCandidates(b *testing.B) { ctx := b.Context() b.ReportAllocs() for b.Loop() { - candidates, err := store.warmCandidates(ctx, pool) + candidates, err := WarmCandidates(ctx, store, pool) if err != nil { b.Fatalf("warm candidates: %v", err) } diff --git a/pkg/scale/sandboxstore_claim_test.go b/pkg/scale/sandboxstore_claim_test.go index 9181b1f..cda7e87 100644 --- a/pkg/scale/sandboxstore_claim_test.go +++ b/pkg/scale/sandboxstore_claim_test.go @@ -201,7 +201,7 @@ func TestPickWarmNodeSpreadsAcrossTheFleet(t *testing.T) { picked := map[string]int{} for range 200 { - candidates, err := store.warmCandidates(t.Context(), PoolKey{Template: "img"}) + candidates, err := WarmCandidates(t.Context(), store, PoolKey{Template: "img"}) require.NoError(t, err) best := pickPowerOfTwo(candidates) picked[best.node]++ @@ -218,7 +218,7 @@ func TestPickWarmNodePrefersTheWarmerSample(t *testing.T) { warmPicks := 0 for range 200 { - candidates, err := store.warmCandidates(t.Context(), PoolKey{Template: "img"}) + candidates, err := WarmCandidates(t.Context(), store, PoolKey{Template: "img"}) require.NoError(t, err) best := pickPowerOfTwo(candidates) if best.node == "warm" { @@ -429,12 +429,13 @@ func TestWarmCandidatesMatchThePerNodeFanOut(t *testing.T) { } } } - got, err := store.warmCandidates(ctx, pool) + caps, err := store.src.NodeCapacities(ctx) require.NoError(t, err) + got := warmCandidates(caps, pool) assert.Equal(t, want, got) assert.Len(t, got, 3, "a's rt pool and both of d's; b is cold, c has no address, e is partitioned") - assert.Equal(t, "d", store.nodeForAddress(ctx, "10.0.0.4:7777"), "a node address resolves to its node") - assert.Empty(t, store.nodeForAddress(ctx, "10.0.0.5:7777"), "a partitioned node's address does not resolve") + assert.Equal(t, "d", nodeForAddress(caps, "10.0.0.4:7777"), "a node address resolves to its node") + assert.Empty(t, nodeForAddress(caps, "10.0.0.5:7777"), "a partitioned node's address does not resolve") } type claimCall struct { @@ -573,7 +574,7 @@ func (c *recordingClient) Checkpoints(context.Context) ([]sandboxd.Checkpoint, e func (c *recordingClient) DeleteCheckpoint(context.Context, string) error { return nil } -func (c *recordingClient) DeleteTemplate(_ context.Context, key sandboxd.PoolKey) error { +func (c *recordingClient) DeleteTemplate(_ context.Context, key sandboxd.PoolKey, _ string) error { c.f.mu.Lock() defer c.f.mu.Unlock() c.f.deletedTemplates = append(c.f.deletedTemplates, key) @@ -584,7 +585,7 @@ func (c *recordingClient) Promote(_ context.Context, id, template string) (sandb return sandboxd.PoolKey{Template: template}, "sha256:" + id, c.f.verbErr } -func (c *recordingClient) SetTemplateLabels(context.Context, sandboxd.PoolKey, map[string]string) error { +func (c *recordingClient) SetTemplateLabels(context.Context, sandboxd.PoolKey, map[string]string, string) error { return c.f.verbErr } @@ -599,7 +600,7 @@ func (c *recordingClient) SetInstanceMetadata(_ context.Context, id string, doc return c.f.verbErr } -func (c *recordingClient) DialPort(_ context.Context, id string, port uint16) (net.Conn, error) { +func (c *recordingClient) DialPort(_ context.Context, id, _ string, port uint16) (net.Conn, error) { c.f.mu.Lock() defer c.f.mu.Unlock() c.f.dialPorts = append(c.f.dialPorts, port) diff --git a/pkg/scale/sandboxstore_impl.go b/pkg/scale/sandboxstore_impl.go index 1edba2c..8b0878d 100644 --- a/pkg/scale/sandboxstore_impl.go +++ b/pkg/scale/sandboxstore_impl.go @@ -126,11 +126,11 @@ type SandboxdClient interface { Checkpoint(ctx context.Context, id string, spec sandboxd.CheckpointSpec) (sandboxd.Checkpoint, error) Checkpoints(ctx context.Context) ([]sandboxd.Checkpoint, error) DeleteCheckpoint(ctx context.Context, checkpointID string) error - DeleteTemplate(ctx context.Context, key sandboxd.PoolKey) error + DeleteTemplate(ctx context.Context, key sandboxd.PoolKey, digest string) error Promote(ctx context.Context, id, template string) (sandboxd.PoolKey, string, error) - SetTemplateLabels(ctx context.Context, key sandboxd.PoolKey, labels map[string]string) error + SetTemplateLabels(ctx context.Context, key sandboxd.PoolKey, labels map[string]string, digest string) error Info(ctx context.Context) (*sandboxd.NodeInfo, error) - DialPort(ctx context.Context, id string, port uint16) (net.Conn, error) + DialPort(ctx context.Context, id, token string, port uint16) (net.Conn, error) SetInstanceMetadata(ctx context.Context, id string, doc []byte) error // Sandbox and SandboxesByClaimRef read the node's own index, which a published inventory lags. @@ -233,15 +233,14 @@ func (s *scatterGatherStore) Claim(ctx context.Context, namespace, name string, if s.sandboxdFactory == nil { return Assignment{}, fmt.Errorf("scale: claim routing not configured (call WithClaimRouting)") } - candidates, err := s.warmCandidates(ctx, pool) + nodes, err := s.src.NodeCapacities(ctx) if err != nil { - return Assignment{}, err + return Assignment{}, fmt.Errorf("scale: enumerate node capacity: %w", err) } + candidates := warmCandidates(nodes, pool) promoted := len(candidates) == 0 if promoted { - if candidates, err = s.templateCandidates(ctx, pool); err != nil { - return Assignment{}, err - } + candidates = templateCandidates(nodes, pool) } if len(candidates) == 0 { return Assignment{}, fmt.Errorf("scale: claim %s/%s: no node advertises warm capacity for template %q net %q size %q: %w", namespace, name, pool.Template, pool.Net, pool.Size, ErrNoWarmCapacity) @@ -268,7 +267,7 @@ func (s *scatterGatherStore) Claim(ctx context.Context, namespace, name string, res, claimErr := s.sandboxdFactory(best.addr, s.sandboxdToken).Claim(ctx, spec) redirect, _ := errors.AsType[*sandboxd.RedirectError](claimErr) if redirect != nil { - node, res, claimErr = s.claimRedirected(ctx, node, redirect, spec) + node, res, claimErr = s.claimRedirected(ctx, nodes, node, redirect, spec) } if claimErr == nil { s.index.remember(nameKey(namespace, name), node) @@ -352,11 +351,11 @@ func (s *scatterGatherStore) matchOnNode(ctx context.Context, op, node string, m return nil } -func (s *scatterGatherStore) claimRedirected(ctx context.Context, from string, redirect *sandboxd.RedirectError, spec sandboxd.ClaimSpec) (string, sandboxd.ClaimResult, error) { +func (s *scatterGatherStore) claimRedirected(ctx context.Context, nodes []NodePools, from string, redirect *sandboxd.RedirectError, spec sandboxd.ClaimSpec) (string, sandboxd.ClaimResult, error) { logger := log.WithFunc("scale.claimRedirected") spec.NoRedirect = true for _, target := range redirect.Targets { - node := s.nodeForAddress(ctx, target) + node := nodeForAddress(nodes, target) if node == "" { logger.Debugf(ctx, "redirect target is no known node; skipping from=%s target=%s", from, target) continue @@ -374,53 +373,6 @@ func (s *scatterGatherStore) claimRedirected(ctx context.Context, from string, r return from, sandboxd.ClaimResult{}, redirect } -func (s *scatterGatherStore) nodeForAddress(ctx context.Context, addr string) string { - nodes, err := s.src.NodeCapacities(ctx) - if err != nil { - return "" - } - if i := slices.IndexFunc(nodes, func(n NodePools) bool { return n.Address == addr }); i >= 0 { - return nodes[i].Node - } - return "" -} - -func (s *scatterGatherStore) warmCandidates(ctx context.Context, pool PoolKey) ([]warmCandidate, error) { - nodes, err := s.src.NodeCapacities(ctx) - if err != nil { - return nil, fmt.Errorf("scale: enumerate node capacity: %w", err) - } - var out []warmCandidate - for _, n := range nodes { - if n.Address == "" { - continue - } - for _, pc := range n.Pools { - if pc.Warm > 0 && poolCapacityMatches(pc, pool) { - out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: pc.Warm}) - } - } - } - return out, nil -} - -// templateCandidates lists the nodes whose inventory holds pool as a promoted template, for a claim no warm pool serves. -func (s *scatterGatherStore) templateCandidates(ctx context.Context, pool PoolKey) ([]warmCandidate, error) { - nodes, err := s.src.NodeCapacities(ctx) - if err != nil { - return nil, fmt.Errorf("scale: enumerate node capacity: %w", err) - } - var out []warmCandidate - for _, n := range nodes { - if n.Address != "" && slices.ContainsFunc(n.Templates, func(t PromotedTemplate) bool { - return poolCapacityMatches(PoolCapacity{Template: t.Template, Net: t.Net, Size: t.Size}, pool) - }) { - out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: 1}) - } - } - return out, nil -} - func (s *scatterGatherStore) runWatch(ctx context.Context, opts ListOptions, labelSel labels.Selector, fieldSel fields.Selector, w *watch.ProxyWatcher, ch chan watch.Event) { defer close(ch) @@ -808,6 +760,41 @@ func fanOutNodes[T any](ctx context.Context, s *scatterGatherStore, work func(ct } // poolCapacityMatches compares pc against key, defaulting each unset net/size axis. +func nodeForAddress(nodes []NodePools, addr string) string { + if i := slices.IndexFunc(nodes, func(n NodePools) bool { return n.Address == addr }); i >= 0 { + return nodes[i].Node + } + return "" +} + +func warmCandidates(nodes []NodePools, pool PoolKey) []warmCandidate { + var out []warmCandidate + for _, n := range nodes { + if n.Address == "" { + continue + } + for _, pc := range n.Pools { + if pc.Warm > 0 && poolCapacityMatches(pc, pool) { + out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: pc.Warm}) + } + } + } + return out +} + +// templateCandidates lists the nodes whose inventory holds pool as a promoted template, for a claim no warm pool serves. +func templateCandidates(nodes []NodePools, pool PoolKey) []warmCandidate { + var out []warmCandidate + for _, n := range nodes { + if n.Address != "" && slices.ContainsFunc(n.Templates, func(t PromotedTemplate) bool { + return poolCapacityMatches(PoolCapacity{Template: t.Template, Net: t.Net, Size: t.Size}, pool) + }) { + out = append(out, warmCandidate{node: n.Node, addr: n.Address, warm: 1}) + } + } + return out +} + func poolCapacityMatches(pc PoolCapacity, key PoolKey) bool { return pc.Template == key.Template && cmp.Or(pc.Net, NetDefault) == cmp.Or(key.Net, NetDefault) && diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go index 1201267..faf16fa 100644 --- a/pkg/scale/sandboxstore_lifecycle.go +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "regexp" "strings" "time" @@ -18,6 +19,8 @@ import ( // maxSandboxdName is sandboxd's name budget (types.NameRe). const maxSandboxdName = 63 +var sandboxdName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`) + func (s *scatterGatherStore) Pause(ctx context.Context, node, id string) error { cl, err := s.nodeClient(ctx, node, "pause", id) if err != nil { @@ -29,12 +32,12 @@ func (s *scatterGatherStore) Pause(ctx context.Context, node, id string) error { return nil } -func (s *scatterGatherStore) DialGuestPort(ctx context.Context, node, id string, port uint16) (net.Conn, error) { +func (s *scatterGatherStore) DialGuestPort(ctx context.Context, node, id, token string, port uint16) (net.Conn, error) { cl, err := s.nodeClient(ctx, node, "port", id) if err != nil { return nil, err } - conn, err := cl.DialPort(ctx, id, port) + conn, err := cl.DialPort(ctx, id, token, port) if err != nil { return nil, nodeVerbError(err, "port", id, node) } @@ -141,12 +144,12 @@ func (s *scatterGatherStore) DeleteSnapshot(ctx context.Context, node, snapshotI return nil } -func (s *scatterGatherStore) DeleteTemplate(ctx context.Context, node string, key PoolKey) error { +func (s *scatterGatherStore) DeleteTemplate(ctx context.Context, node string, key PoolKey, digest string) error { cl, err := s.nodeClient(ctx, node, "delete template", key.Template) if err != nil { return err } - if err := cl.DeleteTemplate(ctx, sandboxd.PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}); err != nil { + if err := cl.DeleteTemplate(ctx, sandboxd.PoolKey(key), digest); err != nil { return nodeVerbError(err, "delete template", key.Template, node) } return nil @@ -161,15 +164,15 @@ func (s *scatterGatherStore) Promote(ctx context.Context, node, id, template str if err != nil { return PoolKey{}, "", nodeVerbError(err, "promote", id, node) } - return PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}, digest, nil + return PoolKey(key), digest, nil } -func (s *scatterGatherStore) SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string) error { +func (s *scatterGatherStore) SetTemplateLabels(ctx context.Context, node string, key PoolKey, labels map[string]string, digest string) error { cl, err := s.nodeClient(ctx, node, "template labels", key.Template) if err != nil { return err } - if err := cl.SetTemplateLabels(ctx, sandboxd.PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}, labels); err != nil { + if err := cl.SetTemplateLabels(ctx, sandboxd.PoolKey(key), labels, digest); err != nil { return nodeVerbError(err, "template labels", key.Template, node) } return nil @@ -229,6 +232,9 @@ func StampedName(kind, prefix, namespace, name string) (string, error) { if len(stamped) > maxSandboxdName { return "", k8serrors.NewBadRequest(fmt.Sprintf("%s name %q: at most %d characters in namespace %q", kind, name, maxSandboxdName-len(prefix)-len(namespace)-1, namespace)) } + if !sandboxdName.MatchString(stamped) { + return "", k8serrors.NewBadRequest(fmt.Sprintf("%s name %q: letters, digits and . _ : / - only", kind, name)) + } return stamped, nil } diff --git a/pkg/scale/sandboxstore_lifecycle_test.go b/pkg/scale/sandboxstore_lifecycle_test.go index eff2a29..8b13666 100644 --- a/pkg/scale/sandboxstore_lifecycle_test.go +++ b/pkg/scale/sandboxstore_lifecycle_test.go @@ -25,15 +25,15 @@ func TestLifecycleVerbsMapANodeUnknownSandboxToNotFound(t *testing.T) { ctx := t.Context() f.dialErr = &sandboxd.HTTPError{StatusCode: http.StatusNotFound} - _, portErr := store.DialGuestPort(ctx, "n1", "sb_gone", 8080) + _, portErr := store.DialGuestPort(ctx, "n1", "sb_gone", "", 8080) _, forkErr := store.Fork(ctx, "ns", "n1", "sb_gone", 1, 0) _, snapErr := store.Snapshot(ctx, "n1", "sb_gone", "") _, _, promoteErr := store.Promote(ctx, "n1", "sb_gone", "tpl:x") for name, err := range map[string]error{ "pause": store.Pause(ctx, "n1", "sb_gone"), "metadata": store.SetInstanceMetadata(ctx, "n1", "sb_gone", []byte(`{}`)), - "template": store.DeleteTemplate(ctx, "n1", PoolKey{Template: "tpl:gone"}), - "labels": store.SetTemplateLabels(ctx, "n1", PoolKey{Template: "tpl:gone"}, nil), + "template": store.DeleteTemplate(ctx, "n1", PoolKey{Template: "tpl:gone"}, ""), + "labels": store.SetTemplateLabels(ctx, "n1", PoolKey{Template: "tpl:gone"}, nil, ""), "resume": store.Resume(ctx, "n1", "sb_gone"), "port": portErr, "fork": forkErr, @@ -83,15 +83,15 @@ func TestDialGuestPortMapsTheNodesPassiveRelayAnswers(t *testing.T) { store := NewScatterGatherStore(src, WithClaimRouting("t", f.factory())) ctx := t.Context() - conn, err := store.DialGuestPort(ctx, "n1", "sb_run", 49983) + conn, err := store.DialGuestPort(ctx, "n1", "sb_run", "", 49983) require.NoError(t, err) _ = conn.Close() assert.Equal(t, []uint16{49983}, f.dialPorts) - _, err = store.DialGuestPort(ctx, "n1", "sb_paused", 49983) + _, err = store.DialGuestPort(ctx, "n1", "sb_paused", "", 49983) assert.True(t, k8serrors.IsConflict(err), "a paused sandbox's 409 is Conflict, got %v", err) - _, err = store.DialGuestPort(ctx, "n1", "sb_gone", 49983) + _, err = store.DialGuestPort(ctx, "n1", "sb_gone", "", 49983) assert.True(t, k8serrors.IsNotFound(err), "an unknown claim is NotFound, got %v", err) - _, err = store.DialGuestPort(ctx, "n1", "sb_reset", 49983) + _, err = store.DialGuestPort(ctx, "n1", "sb_reset", "", 49983) require.Error(t, err) assert.False(t, k8serrors.IsNotFound(err) || k8serrors.IsConflict(err), "a transport failure is neither: %v", err) assert.Empty(t, f.rowReads, "the dial goes straight to the passive relay, with no by-id read first")