From 91337e5697fdfd92588dd233eb914918be9516c9 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Mon, 19 Oct 2026 10:58:21 +0200 Subject: [PATCH 1/3] Clarify executor isolation and certificate renewal diagnostics --- cmd/dbl/doctor.go | 35 +++++++++++--- cmd/dbl/doctor_test.go | 105 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 7 deletions(-) diff --git a/cmd/dbl/doctor.go b/cmd/dbl/doctor.go index 59989c16..e79ba4e0 100644 --- a/cmd/dbl/doctor.go +++ b/cmd/dbl/doctor.go @@ -31,6 +31,8 @@ identity, which may differ from the service account. No measurement is launched. The clock check reads the kernel's own synchronization state and error estimate (Linux adjtimex, read only) against the executor's clock.max_error_ms; no time source is queried. +Certificates expiring within 14 days include a renewal reminder; they remain +valid and do not make this command fail. Statuses: pass, failure, unavailable, not_checked. Exit 1 means a failure or unavailable check; not_checked is inconclusive. This does not certify isolation, @@ -169,15 +171,16 @@ func localDaemonChecks(ctx context.Context, role string, cfg daemonConfig, offli } else { checks = append(checks, doctorCheck{"schema", "not_checked", "offline database inspection was not requested", "stop the daemon and pass --offline to inspect its schema without writing SQLite state"}) } - checks = append(checks, certificateCheck(tlsDisabled, cert, key, authority)) + checks = append(checks, certificateCheck(role, tlsDisabled, cert, key, authority)) if cfg.executor != nil { c := cfg.executor + checks = append(checks, isolationCheck(c.Isolation.Shared())) checks = append(checks, executorHostChecks(c.Network.Interface, c.Network.PacketCounter)...) } return checks } -func certificateCheck(disabled bool, cert, key, authority string) doctorCheck { +func certificateCheck(role string, disabled bool, cert, key, authority string) doctorCheck { if disabled { return doctorCheck{"tls_files", "not_checked", "TLS is disabled in the selected configuration", ""} } @@ -190,13 +193,31 @@ func certificateCheck(disabled bool, cert, key, authority string) doctorCheck { } now := time.Now() identity, err := tlsfiles.KeyPair("certificate", cert, "key", key, now) - if err == nil && authority != "" { - _, err = tlsfiles.TrustRoots("CA", authority, now) - } if err != nil { - return doctorCheck{"tls_files", "failure", "configured TLS identity or authority is invalid at the local time", "check the key/certificate pair, validity dates, CA bundle and host clock"} + return doctorCheck{"tls_files", "failure", "configured TLS identity is invalid at the local time", "check the key/certificate pair, validity dates and host clock; renew an expired certificate before starting"} + } + if authority != "" { + if _, err := tlsfiles.TrustRoots("CA", authority, now); err != nil { + return doctorCheck{"tls_files", "failure", "configured CA bundle is invalid at the local time", "obtain a valid CA bundle from the dispatcher operator and check the host clock; renewing the executor certificate alone does not replace its trust roots"} + } + } + check := doctorCheck{"tls_files", "pass", "key matches certificate; certificate and configured CA dates are valid; leaf expires " + identity.Leaf.NotAfter.UTC().Format(time.RFC3339), "peer trust and hostname verification require the actual control handshake"} + if identity.Leaf.NotAfter.Sub(now) <= 14*24*time.Hour { + check.Detail += "; renewal is due within 14 days" + if role == "executor" { + check.Next += "; for a managed executor, drain and back up its state, obtain a fresh setup token from My Nodes, then use dbl executor join --renew and resume explicitly; see docs/operations/executor-onboarding.md" + } else { + check.Next += "; arrange certificate renewal with the dispatcher operator before the expiry date" + } + } + return check +} + +func isolationCheck(shared bool) doctorCheck { + if !shared { + return doctorCheck{"isolation", "not_checked", "trusted profile: guests run inside the executor process without hard CPU or process-memory limits", "use only trusted guests, or configure the shared Linux profile before accepting untrusted work; see docs/operations/guest-isolation.md"} } - return doctorCheck{"tls_files", "pass", "key matches certificate; certificate and configured CA dates are valid; leaf expires " + identity.Leaf.NotAfter.UTC().Format(time.RFC3339), "peer trust and hostname verification require the actual control handshake"} + return doctorCheck{"isolation", "not_checked", "shared profile selected: compiler and guest workers require cgroup v2 resource limits; runtime enforcement was not checked", "the service needs a writable memory/cpu/pids delegation with finite parent budgets and a separate control-process leaf; check service startup for delegation errors; see docs/operations/guest-isolation.md"} } func schemaCheck(ctx context.Context, role storagecheck.Role, path string) doctorCheck { diff --git a/cmd/dbl/doctor_test.go b/cmd/dbl/doctor_test.go index 1ee466f6..019027cc 100644 --- a/cmd/dbl/doctor_test.go +++ b/cmd/dbl/doctor_test.go @@ -3,7 +3,12 @@ package main import ( "bytes" "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" "encoding/json" + "encoding/pem" + "math/big" "net/http" "os" "path/filepath" @@ -204,3 +209,103 @@ func TestDoctorLiveSchemaRequiresOfflineSelection(t *testing.T) { t.Fatalf("unexpected schema check: %+v", check) } } + +func TestDoctorCertificateRenewalGuidance(t *testing.T) { + for _, tc := range []struct { + name, role string + days int + status string + renewal bool + }{ + {"not due", "executor", 30, "pass", false}, + {"due soon", "executor", 13, "pass", true}, + {"dispatcher", "dispatcher", 13, "pass", true}, + {"expired", "executor", -1, "failure", false}, + } { + t.Run(tc.name, func(t *testing.T) { + cert, key := doctorCertificate(t, time.Now().Add(time.Duration(tc.days)*24*time.Hour)) + check := certificateCheck(tc.role, false, cert, key, "") + if check.Status != tc.status || strings.Contains(check.Detail, "renewal is due") != tc.renewal { + t.Fatalf("unexpected certificate result: %+v", check) + } + if strings.Contains(check.Next, "join --renew") != (tc.renewal && tc.role == "executor") { + t.Fatalf("incorrect role-specific renewal advice: %+v", check) + } + if tc.renewal && !strings.Contains(check.Next, "actual control handshake") { + t.Fatalf("renewal reminder lost trust-verification limitation: %+v", check) + } + }) + } + cert, key := doctorCertificate(t, time.Now().Add(13*24*time.Hour)) + expiredCA, _ := doctorCertificate(t, time.Now().Add(-time.Hour)) + check := certificateCheck("executor", false, cert, key, expiredCA) + if check.Status != "failure" || !strings.Contains(check.Detail, "CA bundle") || strings.Contains(check.Next, "join --renew") { + t.Fatalf("CA failure incorrectly recommends leaf-only renewal: %+v", check) + } + if check := certificateCheck("executor", true, cert, key, expiredCA); check.Status != "not_checked" { + t.Fatalf("disabled TLS checked certificate: %+v", check) + } +} + +func doctorCertificate(t *testing.T, expires time.Time) (string, string) { + t.Helper() + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{SerialNumber: big.NewInt(1), NotBefore: time.Now().Add(-30 * 24 * time.Hour), NotAfter: expires, + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign} + der, err := x509.CreateCertificate(rand.Reader, template, template, public, private) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + t.Fatal(err) + } + cert, key := filepath.Join(t.TempDir(), "certificate.pem"), filepath.Join(t.TempDir(), "private.pem") + for path, block := range map[string]*pem.Block{cert: {Type: "CERTIFICATE", Bytes: der}, key: {Type: "PRIVATE KEY", Bytes: keyDER}} { + if err := os.WriteFile(path, pem.EncodeToMemory(block), 0600); err != nil { + t.Fatal(err) + } + } + return cert, key +} + +func TestDoctorReportsIsolationWithoutClaimingRuntimeReadiness(t *testing.T) { + for _, profile := range []string{"", "trusted", "shared"} { + t.Run(profile, func(t *testing.T) { + extra := "[isolation]\nprofile='" + profile + "'\n" + if profile == "shared" { + extra += `cgroup_root='/unavailable-doctor-fixture' +node_memory_bytes=536870912 +node_cpu_quota_us=100000 +node_pids=128 +control_memory_reserve_bytes=134217728 +control_cpu_reserve_us=25000 +compile_memory_bytes=268435456 +compile_cpu_quota_us=50000 +compile_wall_ms=10000 +compile_concurrency=1 +run_memory_bytes=268435456 +run_cpu_quota_us=50000 +run_wall_ms=30000 +worker_pids=64 +memory_pages=1024 +` + } + path := writeOperatorConfig(t, "executor", extra) + _, out, _ := runCLI(context.Background(), "--output", outputJSON, "doctor", "--role", "executor", "--file", path) + check := doctorResult(t, out, "isolation") + if check.Status != "not_checked" || check.Next == "" { + t.Fatalf("configuration was treated as proof of isolation: %+v", check) + } + if profile == "shared" && !strings.Contains(check.Detail, "runtime enforcement was not checked") { + t.Fatalf("shared profile lacks its runtime limitation: %+v", check) + } + if profile != "shared" && !strings.Contains(check.Detail, "without hard CPU or process-memory limits") { + t.Fatalf("trusted profile lacks its isolation limitation: %+v", check) + } + }) + } +} From 2b88af46ab6ea2333faccb56770e2e1b4df00901 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Mon, 19 Oct 2026 11:01:31 +0200 Subject: [PATCH 2/3] Prioritize expired trust roots in certificate diagnostics --- cmd/dbl/doctor.go | 10 +++++----- cmd/dbl/doctor_test.go | 12 +++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/dbl/doctor.go b/cmd/dbl/doctor.go index e79ba4e0..8e67f103 100644 --- a/cmd/dbl/doctor.go +++ b/cmd/dbl/doctor.go @@ -192,20 +192,20 @@ func certificateCheck(role string, disabled bool, cert, key, authority string) d } } now := time.Now() - identity, err := tlsfiles.KeyPair("certificate", cert, "key", key, now) - if err != nil { - return doctorCheck{"tls_files", "failure", "configured TLS identity is invalid at the local time", "check the key/certificate pair, validity dates and host clock; renew an expired certificate before starting"} - } if authority != "" { if _, err := tlsfiles.TrustRoots("CA", authority, now); err != nil { return doctorCheck{"tls_files", "failure", "configured CA bundle is invalid at the local time", "obtain a valid CA bundle from the dispatcher operator and check the host clock; renewing the executor certificate alone does not replace its trust roots"} } } + identity, err := tlsfiles.KeyPair("certificate", cert, "key", key, now) + if err != nil { + return doctorCheck{"tls_files", "failure", "configured TLS identity is invalid at the local time", "check the key/certificate pair, validity dates and host clock; renew an expired certificate before starting"} + } check := doctorCheck{"tls_files", "pass", "key matches certificate; certificate and configured CA dates are valid; leaf expires " + identity.Leaf.NotAfter.UTC().Format(time.RFC3339), "peer trust and hostname verification require the actual control handshake"} if identity.Leaf.NotAfter.Sub(now) <= 14*24*time.Hour { check.Detail += "; renewal is due within 14 days" if role == "executor" { - check.Next += "; for a managed executor, drain and back up its state, obtain a fresh setup token from My Nodes, then use dbl executor join --renew and resume explicitly; see docs/operations/executor-onboarding.md" + check.Next += "; for a managed executor, drain and back up its state, obtain a fresh setup token from My Nodes, then use dbl executor join --renew and resume explicitly; this retains its CA bundle, so contact the dispatcher operator if the CA also needs renewal; see docs/operations/executor-onboarding.md" } else { check.Next += "; arrange certificate renewal with the dispatcher operator before the expiry date" } diff --git a/cmd/dbl/doctor_test.go b/cmd/dbl/doctor_test.go index 019027cc..9e5f5cee 100644 --- a/cmd/dbl/doctor_test.go +++ b/cmd/dbl/doctor_test.go @@ -234,13 +234,19 @@ func TestDoctorCertificateRenewalGuidance(t *testing.T) { if tc.renewal && !strings.Contains(check.Next, "actual control handshake") { t.Fatalf("renewal reminder lost trust-verification limitation: %+v", check) } + if tc.renewal && tc.role == "executor" && !strings.Contains(check.Next, "contact the dispatcher operator if the CA also needs renewal") { + t.Fatalf("renewal reminder lost issuer-expiry guidance: %+v", check) + } }) } cert, key := doctorCertificate(t, time.Now().Add(13*24*time.Hour)) expiredCA, _ := doctorCertificate(t, time.Now().Add(-time.Hour)) - check := certificateCheck("executor", false, cert, key, expiredCA) - if check.Status != "failure" || !strings.Contains(check.Detail, "CA bundle") || strings.Contains(check.Next, "join --renew") { - t.Fatalf("CA failure incorrectly recommends leaf-only renewal: %+v", check) + expiredCert, expiredKey := doctorCertificate(t, time.Now().Add(-time.Hour)) + for _, identity := range [][2]string{{cert, key}, {expiredCert, expiredKey}} { + check := certificateCheck("executor", false, identity[0], identity[1], expiredCA) + if check.Status != "failure" || !strings.Contains(check.Detail, "CA bundle") || strings.Contains(check.Next, "join --renew") { + t.Fatalf("CA failure incorrectly recommends leaf-only renewal: %+v", check) + } } if check := certificateCheck("executor", true, cert, key, expiredCA); check.Status != "not_checked" { t.Fatalf("disabled TLS checked certificate: %+v", check) From 0e357d2b15d1ec3675d186b8fc461534af1712d8 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Mon, 19 Oct 2026 11:04:20 +0200 Subject: [PATCH 3/3] Add opt-in managed shared executor setup --- cmd/dbl/service.go | 10 +- docs/operations/guest-isolation.md | 34 +++- internal/demo/service/enrolled.go | 17 +- internal/demo/service/install.go | 12 +- internal/demo/service/profile.go | 8 + internal/demo/service/replace.go | 8 + internal/demo/service/replace_test.go | 2 +- internal/demo/service/shared.go | 104 +++++++++++ internal/demo/service/shared_test.go | 164 ++++++++++++++++++ internal/demo/service/unit.go | 15 +- .../executor/isolation/supervisor_linux.go | 25 +-- 11 files changed, 379 insertions(+), 20 deletions(-) create mode 100644 internal/demo/service/shared.go create mode 100644 internal/demo/service/shared_test.go diff --git a/cmd/dbl/service.go b/cmd/dbl/service.go index 7187ab54..02080c4e 100644 --- a/cmd/dbl/service.go +++ b/cmd/dbl/service.go @@ -18,7 +18,7 @@ const serviceUsage = `Usage: dbl service install (--role dispatcher|executor) [--name NAME] [--user debuglet] [--port 9000] [--grpc-port 9001] [--dispatcher 127.0.0.1:9001] [--dispatcher-http 127.0.0.1:9000] [--start=false] [--enable=false] [--root DIR] - [--enrolled-state /var/lib/debuglet/executors/NAME] + [--enrolled-state /var/lib/debuglet/executors/NAME] [--isolation shared] [--replace-package /absolute/path/to/installed/full-package] dbl service start|stop|status (--role ROLE) [--name NAME] dbl service uninstall (--role ROLE) [--name NAME] [--purge] @@ -45,6 +45,11 @@ retention is configured by the host. database already at the named instance's managed directory. Stop its foreground daemon first and do not launch it concurrently with installation. See docs/operations/executor-onboarding.md for the system-account setup sequence. +--isolation shared opts into a finite shared-worker preset on first enrolled +service installation only. It needs Linux 5.14+, cgroup v2 and systemd 254+. +The service is capped at 1.25 GiB memory and 2.25 CPUs, with up to four workers; +these ceilings do not reserve host capacity. Existing isolation configuration +is never replaced. See docs/operations/guest-isolation.md for the limits. --replace-package selects a verified full package for an already installed, enrolled executor. Stop and disable its unit first. Both --start=false and --enable=false are required; replacement never starts, enrolls or migrates it. @@ -204,6 +209,7 @@ func serviceFlags(fs *flag.FlagSet, local *serviceOptions, subcommand string) { fs.StringVar(&local.Request.DispatcherGRPC, "dispatcher", "", "executor: dispatcher control address as host:port") fs.StringVar(&local.Request.DispatcherHTTP, "dispatcher-http", "", "executor: dispatcher HTTP address as host:port") fs.StringVar(&local.Request.EnrolledState, "enrolled-state", "", "executor: preserve enrolled state at this instance's canonical managed path") + fs.StringVar(&local.Request.Isolation, "isolation", "", "executor: shared worker preset on first enrolled installation") fs.StringVar(&local.ReplacePackage, "replace-package", "", "enrolled executor: select an installed full-package directory while stopped and disabled") fs.BoolVar(&local.Start, "start", true, "start the unit and observe its readiness record") fs.BoolVar(&local.Enable, "enable", true, "let the service manager start the unit at boot") @@ -233,7 +239,7 @@ func serviceRole(name string, local *serviceOptions, fs *flag.FlagSet, options g blank := false fs.Visit(func(f *flag.Flag) { switch f.Name { - case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state", "replace-package": + case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state", "replace-package", "isolation": if strings.TrimSpace(f.Value.String()) == "" { blank = true } diff --git a/docs/operations/guest-isolation.md b/docs/operations/guest-isolation.md index 04591d74..480888e5 100644 --- a/docs/operations/guest-isolation.md +++ b/docs/operations/guest-isolation.md @@ -13,6 +13,38 @@ uses the parent’s network policy and quotas for every guest network import. A synchronous inherited socket carries fixed import calls and bounded memory transfers; there is no extra TCP listener. +## Managed enrolled executor + +On Linux 5.14+ with cgroup v2 and systemd 254+ (including Ubuntu 24.04), +`dbl service install --isolation shared` prepares the delegation and a bounded +four-worker preset. After `dbl executor join --install-only` finishes, append +`--isolation shared` to the service-install command it prints, **before the first +service installation**. Check that the installed CLI's `service install --help` +lists this option. The usual unprivileged service account and canonical +`--enrolled-state` directory are still required. + +The preset limits the whole service to **1.25 GiB memory and 2.25 CPUs**, with +swap disabled. Worker groups share at most 1 GiB and 2 CPUs; the remaining +configured allowance is for the control process. This does not reserve physical +host resources. Provision the host accordingly. Each worker has 256 MiB process +memory, half a CPU and 64 MiB WASM memory. Compilation is serialized, with three +queue places and a 60-second deadline including queueing. The execution ceiling +is 24 hours; the submitted run's policy timeout still applies first. An executor +configured for more than four concurrent runs is refused, without changing its +admission setting. Larger programs may need a manually provisioned profile. + +The installer appends the preset only when the enrolled configuration has no +`[isolation]` table. It preserves identity, credentials, network policy, pricing +and storage. A custom isolation table is never overwritten. Reinstallation and +package replacement retain the recorded profile; this option does not convert +an existing trusted service. The generated unit runs the daemon in its own +`control` leaf and delegates the sibling worker groups. Startup must validate +the actual delegation and publish readiness; an unsupported host never falls +back to the trusted engine. Check `dbl service status --role executor --name NAME` +and the unit's journal if installation reports incomplete readiness. + +The manual setup below remains available for other budgets or service managers. + ## Required host setup The shared profile requires Linux with cgroup v2, `CLONE_INTO_CGROUP` and @@ -20,7 +52,7 @@ The shared profile requires Linux with cgroup v2, `CLONE_INTO_CGROUP` and `pids` controllers. Mount cgroup v2 at `/sys/fs/cgroup`. The executor must run inside the configured delegation, in a different leaf from its worker groups. Use an exclusive delegation sized within the host resources available to this -service. Its root must already enable these controllers for its children and +service. Its root must permit enabling these controllers for its children and have finite `memory.max` and `cpu.max` limits. An unlimited user service root is insufficient; a separately limited delegated service or scope is required. Startup refuses an absent, read-only or incomplete delegation; it does not fall diff --git a/internal/demo/service/enrolled.go b/internal/demo/service/enrolled.go index 872715cc..7e28b31d 100644 --- a/internal/demo/service/enrolled.go +++ b/internal/demo/service/enrolled.go @@ -17,8 +17,9 @@ import ( ) // prepareEnrolledState adopts only the canonical, already enrolled directory. -// Configuration, credentials and database are never regenerated or relocated. -func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, account Account) error { +// Credentials and database are never regenerated or relocated. The explicit +// shared preset may append isolation settings before first adoption. +func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, account Account, report *Report) error { cfg, err := validateEnrolledState(*p) if err != nil { return err @@ -42,6 +43,15 @@ func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, accoun if err := storagecheck.Check(ctx, storagecheck.Executor, p.DatabasePath); err != nil { return err } + if p.SharedIsolation { + changed, err := prepareSharedConfig(*p) + if err != nil { + return err + } + if changed { + report.Changed = append(report.Changed, "shared worker configuration") + } + } if err := i.own(p.StateDir, account); err != nil { return err } @@ -87,6 +97,9 @@ func readEnrolledConfig(p Profile) (*executorconfig.ExecutorConfig, error) { cfg.Credentials.ClientKey != filepath.Join(p.StateDir, "executor.key") || cfg.Credentials.EnrollmentToken != "" { return nil, errors.New("enrolled configuration must retain TLS and its original credentials and database inside the managed directory") } + if p.SharedIsolation && p.ExecutorID != "" && (cfg.Isolation != sharedPreset(p) || cfg.Resources.MaxDebuglets > 4) { + return nil, errors.New("managed shared isolation configuration changed; restore the installed preset before restarting or replacing it") + } return cfg, nil } diff --git a/internal/demo/service/install.go b/internal/demo/service/install.go index 1d0904fb..1c5b1444 100644 --- a/internal/demo/service/install.go +++ b/internal/demo/service/install.go @@ -162,6 +162,10 @@ func (i *Installer) Install(ctx context.Context, request Request, assets demo.As if existing.Enrolled != p.Enrolled { return report, errors.New("installation mode differs from retained state; reinstall enrolled executors with --enrolled-state and never replace their configuration") } + if request.Isolation != "" && p.SharedIsolation != existing.SharedIsolation { + return report, errors.New("--isolation shared is only available on first adoption; existing services retain their isolation profile") + } + p.SharedIsolation = existing.SharedIsolation if existing.Enrolled { if err := i.verifyRemoval(ctx, existing); err != nil { // A same-package counter-mode reinstall records its capability @@ -188,6 +192,11 @@ func (i *Installer) Install(ctx context.Context, request Request, assets demo.As return report, err } } + if p.SharedIsolation && (i.root == "" || i.root == "/") { + if err := checkSharedHost(ctx); err != nil { + return report, err + } + } if err := i.prepareState(ctx, &p, &report, account); err != nil { return report, err } @@ -304,7 +313,7 @@ func (i *Installer) observeReady(ctx context.Context, p Profile, state UnitState // account could already write to. func (i *Installer) prepareState(ctx context.Context, p *Profile, report *Report, account Account) error { if p.Enrolled { - return i.prepareEnrolledState(ctx, p, account) + return i.prepareEnrolledState(ctx, p, account, report) } if err := os.MkdirAll(filepath.Dir(p.StateDir), 0755); err != nil { return err @@ -761,6 +770,7 @@ func (i *Installer) load(ctx context.Context, operation string, role storagechec p.ExecutorID = record.ExecutorID p.Enrolled = record.Enrolled p.PacketCapabilities = record.PacketCapabilities + p.SharedIsolation = record.SharedIsolation p.HTTPPort, p.GRPCPort = record.HTTPPort, record.GRPCPort p.DispatcherGRPC, p.DispatcherHTTP = record.DispatcherGRPC, record.DispatcherHTTP report.Version, report.ExecutorID = p.Version, p.ExecutorID diff --git a/internal/demo/service/profile.go b/internal/demo/service/profile.go index c32dc5f3..f4d7fc38 100644 --- a/internal/demo/service/profile.go +++ b/internal/demo/service/profile.go @@ -82,6 +82,8 @@ type Profile struct { // PacketCapabilities records the capability set installed for an enrolled // executor. Missing in older records means the original capability-free unit. PacketCapabilities bool `json:"packet_capabilities,omitempty"` + // SharedIsolation retains the managed shared-worker preset across replacements. + SharedIsolation bool `json:"shared_isolation,omitempty"` // HTTPPort and GRPCPort are the dispatcher's loopback listeners. HTTPPort int `json:"http_port,omitempty"` GRPCPort int `json:"grpc_port,omitempty"` @@ -118,6 +120,8 @@ type Request struct { DispatcherGRPC, DispatcherHTTP string // EnrolledState adopts an enrolled executor at its canonical managed path. EnrolledState string + // Isolation selects the shared preset on first enrolled service installation. + Isolation string } // DefaultAccount is the unprivileged service account the generated units use. @@ -164,6 +168,10 @@ func Resolve(request Request, assets demo.Assets) (Profile, error) { if err := systemPayload(assets.Root); err != nil { return p, err } + if request.Isolation != "" && (request.Isolation != "shared" || request.Role != storagecheck.Executor || request.EnrolledState == "") { + return p, errors.New("--isolation shared requires an enrolled executor") + } + p.SharedIsolation = request.Isolation == "shared" p.User, p.Group = user, group p.Version, p.SourceSHA, p.PayloadRoot = assets.Manifest.Version, assets.Manifest.SourceSHA, assets.Root if request.EnrolledState != "" { diff --git a/internal/demo/service/replace.go b/internal/demo/service/replace.go index 8df71ecc..1a118d1d 100644 --- a/internal/demo/service/replace.go +++ b/internal/demo/service/replace.go @@ -28,6 +28,9 @@ func (i *Installer) ReplacePackage(ctx context.Context, request Request, target if request.Role != storagecheck.Executor || !old.Enrolled || old.ExecutorID == "" || request.EnrolledState != old.StateDir { return report, errors.New("package replacement requires the existing enrolled executor and its canonical --enrolled-state") } + if request.Isolation != "" { + return report, errors.New("package replacement preserves isolation; omit --isolation") + } if (request.User != "" && request.User != old.User) || (request.Group != "" && request.Group != old.Group) { return report, errors.New("package replacement cannot change the service account or group") } @@ -139,6 +142,11 @@ func (i *Installer) ReplacePackage(ctx context.Context, request Request, target // marker removal. Reload also prevents the manager starting a stale definition // after the files were committed but a previous process was interrupted. func (i *Installer) prepareStart(ctx context.Context, p Profile) error { + if p.SharedIsolation { + if _, err := readEnrolledConfig(p); err != nil { + return err + } + } if _, err := os.Lstat(p.UnitPath); err != nil { return err } diff --git a/internal/demo/service/replace_test.go b/internal/demo/service/replace_test.go index e6ba69b8..a758dada 100644 --- a/internal/demo/service/replace_test.go +++ b/internal/demo/service/replace_test.go @@ -333,7 +333,7 @@ func TestEnrolledInstallRetriesCapabilityOnlyInterruption(t *testing.T) { } // Exercise the actual ordinary reinstall stage, then interrupt it // before writeUnit: new capability record, previous capability unit. - if err := f.installer.prepareEnrolledState(t.Context(), &p, Account{UID: 4242, GID: 4343}); err != nil { + if err := f.installer.prepareEnrolledState(t.Context(), &p, Account{UID: 4242, GID: 4343}, &Report{}); err != nil { t.Fatal(err) } if p.PacketCapabilities != automatic { diff --git a/internal/demo/service/shared.go b/internal/demo/service/shared.go new file mode 100644 index 00000000..8dcdd9ff --- /dev/null +++ b/internal/demo/service/shared.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package service + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + executorconfig "github.com/netsec-ethz/debuglet/internal/executor/config" + "github.com/netsec-ethz/debuglet/internal/executor/isolation" + "github.com/netsec-ethz/debuglet/internal/fsutil" + "github.com/pelletier/go-toml/v2" +) + +// The preset bounds four workers without changing the enrolled executor's +// admission setting. Its service limits also leave room for the control process. +func sharedPreset(p Profile) isolation.Config { + return isolation.Config{ + Profile: "shared", CgroupRoot: filepath.Join("/sys/fs/cgroup/system.slice", p.Unit), + NodeMemoryBytes: 1 << 30, NodeCPUQuotaUS: 200000, NodePIDs: 256, + ControlMemoryReserveBytes: 256 << 20, ControlCPUReserveUS: 25000, + CompileMemoryBytes: 256 << 20, CompileCPUQuotaUS: 50000, + CompileWallMS: 60000, CompileConcurrency: 1, CompileQueue: 3, + RunMemoryBytes: 256 << 20, RunCPUQuotaUS: 50000, + RunWallMS: 86400000, WorkerPIDs: 64, MemoryPages: 1024, + } +} + +func prepareSharedConfig(p Profile) (bool, error) { + data, err := os.ReadFile(p.ConfigPath) + if err != nil { + return false, err + } + cfg, document, err := executorconfig.DecodeConfig(data) + if err != nil { + return false, err + } + if cfg.Resources.MaxDebuglets > 4 { + return false, errors.New("the managed shared preset supports resources.max_debuglets up to 4; review that setting before installation") + } + preset := sharedPreset(p) + if document.Set("isolation") { + if cfg.Isolation == preset { + return false, nil // Includes recovery after configuration was published. + } + return false, errors.New("--isolation shared does not replace existing isolation configuration; retain the custom host setup described in docs/operations/guest-isolation.md") + } + if p.ExecutorID != "" { + return false, errors.New("managed shared isolation configuration was removed; restore it before reinstalling") + } + section, err := toml.Marshal(struct { + Isolation isolation.Config `toml:"isolation"` + }{preset}) + if err != nil { + return false, err + } + data = append(append(data, '\n'), section...) + if _, _, err := executorconfig.DecodeConfig(data); err != nil { + return false, fmt.Errorf("shared isolation configuration: %w", err) + } + return true, fsutil.WriteFile(p.ConfigPath, data, 0600) +} + +// Check prerequisites before modifying a real installation. The daemon checks +// the actual delegation, ancestor capacity and kernel operations at startup. +func checkSharedHost(ctx context.Context) error { + if runtime.GOOS != "linux" { + return errors.New("managed shared isolation requires Linux 5.14+, cgroup v2 and systemd 254+") + } + kernel, err := os.ReadFile("/proc/sys/kernel/osrelease") + if err != nil { + return err + } + var major, minor int + if _, err := fmt.Sscanf(string(kernel), "%d.%d", &major, &minor); err != nil || major < 5 || major == 5 && minor < 14 { + return errors.New("managed shared isolation requires Linux 5.14 or newer") + } + version, err := exec.CommandContext(ctx, "systemctl", "--version").Output() + if err != nil { + return fmt.Errorf("managed shared isolation requires systemd 254+: %w", err) + } + var systemd int + if _, err := fmt.Sscanf(string(version), "systemd %d", &systemd); err != nil || systemd < 254 { + return errors.New("managed shared isolation requires systemd 254+ for DelegateSubgroup") + } + controllers, err := os.ReadFile("/sys/fs/cgroup/cgroup.controllers") + if err != nil { + return fmt.Errorf("managed shared isolation requires cgroup v2 mounted at /sys/fs/cgroup: %w", err) + } + available := " " + strings.Join(strings.Fields(string(controllers)), " ") + " " + for _, name := range []string{"cpu", "memory", "pids"} { + if !strings.Contains(available, " "+name+" ") { + return fmt.Errorf("managed shared isolation requires the cgroup v2 %s controller", name) + } + } + return nil +} diff --git a/internal/demo/service/shared_test.go b/internal/demo/service/shared_test.go new file mode 100644 index 00000000..387faffe --- /dev/null +++ b/internal/demo/service/shared_test.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package service + +import ( + "os" + "reflect" + "strings" + "testing" + + "github.com/netsec-ethz/debuglet/internal/storagecheck" +) + +func freshSharedFixture(t *testing.T) (*fixture, Profile, Request) { + t.Helper() + f, p, r := enrolledReplacementFixture(t) + if err := os.Remove(RecordPath(f.root, p.Role, p.Name)); err != nil { + t.Fatal(err) + } + if err := os.Remove(p.UnitPath); err != nil { + t.Fatal(err) + } + p.ExecutorID = "" + p.SharedIsolation = true + r.User, r.Group, r.Isolation = p.User, p.Group, "shared" + return f, p, r +} + +func TestManagedSharedAdoptionAndReplacementPreserveState(t *testing.T) { + f, p, r := freshSharedFixture(t) + before := replacementSnapshot(t, p, false) + for n := 0; n < 2; n++ { + report, err := f.installer.Install(t.Context(), r, f.assets, false, false) + if err != nil { + t.Fatal(err) + } + if n == 1 && len(report.Changed) != 0 { + t.Fatalf("reinstall changed: %+v", report) + } + p, err = ReadRecord(f.root, p.Role, p.Name) + if err != nil || !p.SharedIsolation { + t.Fatalf("profile lost: %+v %v", p, err) + } + if n == 0 { + cfg, err := readEnrolledConfig(p) + if err != nil || cfg.Isolation != sharedPreset(p) { + t.Fatalf("preset: %+v %v", cfg, err) + } + after := replacementSnapshot(t, p, false) + delete(before, p.ConfigPath) + delete(after, p.ConfigPath) + if !reflect.DeepEqual(before, after) { + t.Fatal("enrollment or retained data changed") + } + } + r.Isolation = "" // The installed profile, not the new CLI default, wins. + } + unit, err := Unit(p) + if err != nil { + t.Fatal(err) + } + for _, setting := range []string{"Delegate=cpu memory pids", "DelegateSubgroup=control", "MemoryMax=1342177280", "CPUQuota=225%", "TasksMax=384", "ProtectControlGroups=no"} { + if !strings.Contains(unit, setting+"\n") { + t.Fatalf("missing %s", setting) + } + } + retained := replacementSnapshot(t, p, false) + target := replacementTarget(t, f, p, "1.4.3", "exit 0") + if _, err := f.installer.ReplacePackage(t.Context(), r, target); err != nil { + t.Fatal(err) + } + p, err = ReadRecord(f.root, p.Role, p.Name) + if err != nil || !p.SharedIsolation { + t.Fatalf("replacement lost shared: %v", err) + } + if !reflect.DeepEqual(retained, replacementSnapshot(t, p, false)) { + t.Fatal("replacement rewrote shared configuration") + } + data, err := os.ReadFile(p.ConfigPath) + if err != nil { + t.Fatal(err) + } + changed := strings.Replace(string(data), "profile = 'shared'", "profile = 'trusted'", 1) + if changed == string(data) { + t.Fatal("missing shared profile in fixture") + } + if err := os.WriteFile(p.ConfigPath, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + if err := f.installer.prepareStart(t.Context(), p); err == nil { + t.Fatal("start accepted downgraded isolation") + } + if _, err := f.installer.ReplacePackage(t.Context(), r, target); err == nil { + t.Fatal("replacement accepted downgraded isolation") + } + if _, err := readEnrolledConfig(p); err == nil { + t.Fatal("retained config accepted downgraded isolation") + } + loaded, _, err := f.installer.load(t.Context(), "status", p.Role, p.Name) + if err != nil || !loaded.SharedIsolation { + t.Fatalf("load lost shared: %v", err) + } +} + +func TestManagedSharedRefusesConfigurationChanges(t *testing.T) { + for _, section := range []string{"[isolation]\nprofile = 'trusted'\n", "[isolation]\n", "[Isolation]\nprofile = 'trusted'\n"} { + t.Run(section, func(t *testing.T) { + f, p, r := freshSharedFixture(t) + data, _ := os.ReadFile(p.ConfigPath) + if err := os.WriteFile(p.ConfigPath, append(data, []byte("\n"+section)...), 0600); err != nil { + t.Fatal(err) + } + before := replacementSnapshot(t, p, false) + if _, err := f.installer.Install(t.Context(), r, f.assets, false, false); err == nil { + t.Fatal("custom isolation accepted") + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, false)) { + t.Fatal("refusal changed state") + } + }) + } + f, p, r := enrolledReplacementFixture(t) + r.User, r.Group, r.Isolation = p.User, p.Group, "shared" + before := replacementSnapshot(t, p, true) + if _, err := f.installer.Install(t.Context(), r, f.assets, false, false); err == nil { + t.Fatal("existing profile changed") + } + if _, err := f.installer.ReplacePackage(t.Context(), r, f.assets); err == nil { + t.Fatal("replacement accepted profile change") + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, true)) { + t.Fatal("refusal changed existing installation") + } +} + +func TestManagedSharedRecoversConfigurationBeforeRecord(t *testing.T) { + f, p, r := freshSharedFixture(t) + changed, err := prepareSharedConfig(p) + if err != nil || !changed { + t.Fatalf("prepare: %v %v", changed, err) + } + before := replacementSnapshot(t, p, false) + if _, err := f.installer.Install(t.Context(), r, f.assets, false, false); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, false)) { + t.Fatal("retry changed configuration") + } +} + +func TestManagedSharedRequiresEnrolledExecutor(t *testing.T) { + f := newFixture(t) + for _, r := range []Request{ + {Role: storagecheck.Executor, Name: "worker", Isolation: "shared"}, + {Role: storagecheck.Dispatcher, Name: "local", Isolation: "shared"}, + {Role: storagecheck.Executor, Name: "worker", Isolation: "trusted", EnrolledState: StateDirectory(f.root, storagecheck.Executor, "worker")}, + } { + r.Root = f.root + if _, err := Resolve(r, f.assets); err == nil { + t.Fatalf("accepted %+v", r) + } + } +} diff --git a/internal/demo/service/unit.go b/internal/demo/service/unit.go index 50bd2f44..40da6973 100644 --- a/internal/demo/service/unit.go +++ b/internal/demo/service/unit.go @@ -17,8 +17,8 @@ import ( // working-directory lookup and no shell are involved. // - It runs as an existing unprivileged account with no new privileges. Only // enrolled automatic-counter executors receive the four packet capabilities. -// The only writable location is the role's own -// persistent state directory plus its runtime directory. +// Writes stay within persistent/runtime state and, for shared executors, +// the delegated cgroup subtree. // - Stopping it sends SIGTERM to the daemon alone and gives it the whole // stop budget to revoke admission and join its owned local work. Only // after that budget does the service manager kill the remaining processes. @@ -36,6 +36,9 @@ func Unit(p Profile) (string, error) { return "", err } } + if p.SharedIsolation && (p.Role != storagecheck.Executor || !p.Enrolled) { + return "", fmt.Errorf("shared isolation requires an enrolled executor") + } description := fmt.Sprintf("Debuglet %s %s (%s)", p.Role, p.Name, p.Version) var b strings.Builder fmt.Fprintf(&b, "[Unit]\n") @@ -89,7 +92,13 @@ func Unit(p Profile) (string, error) { fmt.Fprintf(&b, "PrivateTmp=yes\n") fmt.Fprintf(&b, "ProtectSystem=strict\n") fmt.Fprintf(&b, "ProtectHome=yes\n") - fmt.Fprintf(&b, "ProtectControlGroups=yes\n") + if p.SharedIsolation { + fmt.Fprintf(&b, "ProtectControlGroups=no\n") + fmt.Fprintf(&b, "Slice=system.slice\nDelegate=cpu memory pids\nDelegateSubgroup=control\n") + fmt.Fprintf(&b, "MemoryMax=1342177280\nMemorySwapMax=0\nCPUQuota=225%%\nTasksMax=384\n") + } else { + fmt.Fprintf(&b, "ProtectControlGroups=yes\n") + } fmt.Fprintf(&b, "ProtectKernelModules=yes\n") fmt.Fprintf(&b, "ReadWritePaths=%s\n", p.StateDir) fmt.Fprintf(&b, "StandardOutput=journal\n") diff --git a/internal/executor/isolation/supervisor_linux.go b/internal/executor/isolation/supervisor_linux.go index 3c665772..6e97e449 100644 --- a/internal/executor/isolation/supervisor_linux.go +++ b/internal/executor/isolation/supervisor_linux.go @@ -49,16 +49,6 @@ func New(c Config) (result *Supervisor, resultErr error) { if fs.Type != unix.CGROUP2_SUPER_MAGIC { return nil, errors.New("shared isolation requires cgroup v2") } - controls, err := os.ReadFile(filepath.Join(c.CgroupRoot, "cgroup.subtree_control")) - if err != nil { - return nil, err - } - available := " " + strings.TrimSpace(string(controls)) + " " - for _, name := range []string{"memory", "cpu", "pids"} { - if !strings.Contains(available, " "+name+" ") { - return nil, fmt.Errorf("delegated cgroup lacks enabled %s control", name) - } - } // The supervisor must run within the same delegation as its children. // This also catches an SSH/session scope which can create directories but // lacks permission to migrate processes into the delegated subtree. @@ -81,6 +71,21 @@ func New(c Config) (result *Supervisor, resultErr error) { if err := checkControlCapacity(c, "/sys/fs/cgroup"); err != nil { return nil, err } + // A delegated systemd service starts in its control leaf. Enable only + // its configured delegation, after checking membership and finite capacity. + if err := write(c.CgroupRoot, "cgroup.subtree_control", "+memory +cpu +pids"); err != nil { + return nil, fmt.Errorf("enable shared worker controllers (the executor must run in a separate control leaf): %w", err) + } + controls, err := os.ReadFile(filepath.Join(c.CgroupRoot, "cgroup.subtree_control")) + if err != nil { + return nil, err + } + available := " " + strings.TrimSpace(string(controls)) + " " + for _, name := range []string{"memory", "cpu", "pids"} { + if !strings.Contains(available, " "+name+" ") { + return nil, fmt.Errorf("delegated cgroup lacks enabled %s control", name) + } + } root := filepath.Join(c.CgroupRoot, "debuglet-"+uuid.NewString()) if err = os.Mkdir(root, 0700); err != nil { return nil, err