Simplify shared executor setup and health diagnostics

cmd/dbl/doctor.go

@@ -31,6 +31,8 @@ identity, which may differ from the service account. No measurement is launched.
 The clock check reads the kernel's own synchronization state and error estimate
 (Linux adjtimex, read only) against the executor's clock.max_error_ms; no time
 source is queried.
+Certificates expiring within 14 days include a renewal reminder; they remain
+valid and do not make this command fail.
 
 Statuses: pass, failure, unavailable, not_checked. Exit 1 means a failure or
 unavailable check; not_checked is inconclusive. This does not certify isolation,
@@ -169,15 +171,16 @@ func localDaemonChecks(ctx context.Context, role string, cfg daemonConfig, offli
 	} else {
 		checks = append(checks, doctorCheck{"schema", "not_checked", "offline database inspection was not requested", "stop the daemon and pass --offline to inspect its schema without writing SQLite state"})
 	}
-	checks = append(checks, certificateCheck(tlsDisabled, cert, key, authority))
+	checks = append(checks, certificateCheck(role, tlsDisabled, cert, key, authority))
 	if cfg.executor != nil {
 		c := cfg.executor
+		checks = append(checks, isolationCheck(c.Isolation.Shared()))
 		checks = append(checks, executorHostChecks(c.Network.Interface, c.Network.PacketCounter)...)
 	}
 	return checks
 }
 
-func certificateCheck(disabled bool, cert, key, authority string) doctorCheck {
+func certificateCheck(role string, disabled bool, cert, key, authority string) doctorCheck {
 	if disabled {
 		return doctorCheck{"tls_files", "not_checked", "TLS is disabled in the selected configuration", ""}
 	}
@@ -189,14 +192,32 @@ func certificateCheck(disabled bool, cert, key, authority string) doctorCheck {
 		}
 	}
 	now := time.Now()
-	identity, err := tlsfiles.KeyPair("certificate", cert, "key", key, now)
-	if err == nil && authority != "" {
-		_, err = tlsfiles.TrustRoots("CA", authority, now)
+	if authority != "" {
+		if _, err := tlsfiles.TrustRoots("CA", authority, now); err != nil {
+			return doctorCheck{"tls_files", "failure", "configured CA bundle is invalid at the local time", "obtain a valid CA bundle from the dispatcher operator and check the host clock; renewing the executor certificate alone does not replace its trust roots"}
+		}
 	}
+	identity, err := tlsfiles.KeyPair("certificate", cert, "key", key, now)
 	if err != nil {
-		return doctorCheck{"tls_files", "failure", "configured TLS identity or authority is invalid at the local time", "check the key/certificate pair, validity dates, CA bundle and host clock"}
+		return doctorCheck{"tls_files", "failure", "configured TLS identity is invalid at the local time", "check the key/certificate pair, validity dates and host clock; renew an expired certificate before starting"}
+	}
+	check := doctorCheck{"tls_files", "pass", "key matches certificate; certificate and configured CA dates are valid; leaf expires " + identity.Leaf.NotAfter.UTC().Format(time.RFC3339), "peer trust and hostname verification require the actual control handshake"}
+	if identity.Leaf.NotAfter.Sub(now) <= 14*24*time.Hour {
+		check.Detail += "; renewal is due within 14 days"
+		if role == "executor" {
+			check.Next += "; for a managed executor, drain and back up its state, obtain a fresh setup token from My Nodes, then use dbl executor join --renew and resume explicitly; this retains its CA bundle, so contact the dispatcher operator if the CA also needs renewal; see docs/operations/executor-onboarding.md"
+		} else {
+			check.Next += "; arrange certificate renewal with the dispatcher operator before the expiry date"
+		}
+	}
+	return check
+}
+
+func isolationCheck(shared bool) doctorCheck {
+	if !shared {
+		return doctorCheck{"isolation", "not_checked", "trusted profile: guests run inside the executor process without hard CPU or process-memory limits", "use only trusted guests, or configure the shared Linux profile before accepting untrusted work; see docs/operations/guest-isolation.md"}
 	}
-	return doctorCheck{"tls_files", "pass", "key matches certificate; certificate and configured CA dates are valid; leaf expires " + identity.Leaf.NotAfter.UTC().Format(time.RFC3339), "peer trust and hostname verification require the actual control handshake"}
+	return doctorCheck{"isolation", "not_checked", "shared profile selected: compiler and guest workers require cgroup v2 resource limits; runtime enforcement was not checked", "the service needs a writable memory/cpu/pids delegation with finite parent budgets and a separate control-process leaf; check service startup for delegation errors; see docs/operations/guest-isolation.md"}
 }
 
 func schemaCheck(ctx context.Context, role storagecheck.Role, path string) doctorCheck {

cmd/dbl/doctor_test.go

@@ -3,7 +3,12 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
+	"math/big"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -204,3 +209,109 @@ func TestDoctorLiveSchemaRequiresOfflineSelection(t *testing.T) {
 		t.Fatalf("unexpected schema check: %+v", check)
 	}
 }
+
+func TestDoctorCertificateRenewalGuidance(t *testing.T) {
+	for _, tc := range []struct {
+		name, role string
+		days       int
+		status     string
+		renewal    bool
+	}{
+		{"not due", "executor", 30, "pass", false},
+		{"due soon", "executor", 13, "pass", true},
+		{"dispatcher", "dispatcher", 13, "pass", true},
+		{"expired", "executor", -1, "failure", false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cert, key := doctorCertificate(t, time.Now().Add(time.Duration(tc.days)*24*time.Hour))
+			check := certificateCheck(tc.role, false, cert, key, "")
+			if check.Status != tc.status || strings.Contains(check.Detail, "renewal is due") != tc.renewal {
+				t.Fatalf("unexpected certificate result: %+v", check)
+			}
+			if strings.Contains(check.Next, "join --renew") != (tc.renewal && tc.role == "executor") {
+				t.Fatalf("incorrect role-specific renewal advice: %+v", check)
+			}
+			if tc.renewal && !strings.Contains(check.Next, "actual control handshake") {
+				t.Fatalf("renewal reminder lost trust-verification limitation: %+v", check)
+			}
+			if tc.renewal && tc.role == "executor" && !strings.Contains(check.Next, "contact the dispatcher operator if the CA also needs renewal") {
+				t.Fatalf("renewal reminder lost issuer-expiry guidance: %+v", check)
+			}
+		})
+	}
+	cert, key := doctorCertificate(t, time.Now().Add(13*24*time.Hour))
+	expiredCA, _ := doctorCertificate(t, time.Now().Add(-time.Hour))
+	expiredCert, expiredKey := doctorCertificate(t, time.Now().Add(-time.Hour))
+	for _, identity := range [][2]string{{cert, key}, {expiredCert, expiredKey}} {
+		check := certificateCheck("executor", false, identity[0], identity[1], expiredCA)
+		if check.Status != "failure" || !strings.Contains(check.Detail, "CA bundle") || strings.Contains(check.Next, "join --renew") {
+			t.Fatalf("CA failure incorrectly recommends leaf-only renewal: %+v", check)
+		}
+	}
+	if check := certificateCheck("executor", true, cert, key, expiredCA); check.Status != "not_checked" {
+		t.Fatalf("disabled TLS checked certificate: %+v", check)
+	}
+}
+
+func doctorCertificate(t *testing.T, expires time.Time) (string, string) {
+	t.Helper()
+	public, private, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), NotBefore: time.Now().Add(-30 * 24 * time.Hour), NotAfter: expires,
+		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, public, private)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keyDER, err := x509.MarshalPKCS8PrivateKey(private)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cert, key := filepath.Join(t.TempDir(), "certificate.pem"), filepath.Join(t.TempDir(), "private.pem")
+	for path, block := range map[string]*pem.Block{cert: {Type: "CERTIFICATE", Bytes: der}, key: {Type: "PRIVATE KEY", Bytes: keyDER}} {
+		if err := os.WriteFile(path, pem.EncodeToMemory(block), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return cert, key
+}
+
+func TestDoctorReportsIsolationWithoutClaimingRuntimeReadiness(t *testing.T) {
+	for _, profile := range []string{"", "trusted", "shared"} {
+		t.Run(profile, func(t *testing.T) {
+			extra := "[isolation]\nprofile='" + profile + "'\n"
+			if profile == "shared" {
+				extra += `cgroup_root='/unavailable-doctor-fixture'
+node_memory_bytes=536870912
+node_cpu_quota_us=100000
+node_pids=128
+control_memory_reserve_bytes=134217728
+control_cpu_reserve_us=25000
+compile_memory_bytes=268435456
+compile_cpu_quota_us=50000
+compile_wall_ms=10000
+compile_concurrency=1
+run_memory_bytes=268435456
+run_cpu_quota_us=50000
+run_wall_ms=30000
+worker_pids=64
+memory_pages=1024
+`
+			}
+			path := writeOperatorConfig(t, "executor", extra)
+			_, out, _ := runCLI(context.Background(), "--output", outputJSON, "doctor", "--role", "executor", "--file", path)
+			check := doctorResult(t, out, "isolation")
+			if check.Status != "not_checked" || check.Next == "" {
+				t.Fatalf("configuration was treated as proof of isolation: %+v", check)
+			}
+			if profile == "shared" && !strings.Contains(check.Detail, "runtime enforcement was not checked") {
+				t.Fatalf("shared profile lacks its runtime limitation: %+v", check)
+			}
+			if profile != "shared" && !strings.Contains(check.Detail, "without hard CPU or process-memory limits") {
+				t.Fatalf("trusted profile lacks its isolation limitation: %+v", check)
+			}
+		})
+	}
+}

cmd/dbl/service.go

@@ -18,7 +18,7 @@ const serviceUsage = `Usage:
   dbl service install (--role dispatcher|executor) [--name NAME] [--user debuglet]
       [--port 9000] [--grpc-port 9001] [--dispatcher 127.0.0.1:9001]
       [--dispatcher-http 127.0.0.1:9000] [--start=false] [--enable=false] [--root DIR]
-      [--enrolled-state /var/lib/debuglet/executors/NAME]
+      [--enrolled-state /var/lib/debuglet/executors/NAME] [--isolation shared]
       [--replace-package /absolute/path/to/installed/full-package]
   dbl service start|stop|status (--role ROLE) [--name NAME]
   dbl service uninstall (--role ROLE) [--name NAME] [--purge]
@@ -45,6 +45,11 @@ retention is configured by the host.
 database already at the named instance's managed directory. Stop its foreground
 daemon first and do not launch it concurrently with installation. See
 docs/operations/executor-onboarding.md for the system-account setup sequence.
+--isolation shared opts into a finite shared-worker preset on first enrolled
+service installation only. It needs Linux 5.14+, cgroup v2 and systemd 254+.
+The service is capped at 1.25 GiB memory and 2.25 CPUs, with up to four workers;
+these ceilings do not reserve host capacity. Existing isolation configuration
+is never replaced. See docs/operations/guest-isolation.md for the limits.
 --replace-package selects a verified full package for an already installed,
 enrolled executor. Stop and disable its unit first. Both --start=false and
 --enable=false are required; replacement never starts, enrolls or migrates it.
@@ -204,6 +209,7 @@ func serviceFlags(fs *flag.FlagSet, local *serviceOptions, subcommand string) {
 		fs.StringVar(&local.Request.DispatcherGRPC, "dispatcher", "", "executor: dispatcher control address as host:port")
 		fs.StringVar(&local.Request.DispatcherHTTP, "dispatcher-http", "", "executor: dispatcher HTTP address as host:port")
 		fs.StringVar(&local.Request.EnrolledState, "enrolled-state", "", "executor: preserve enrolled state at this instance's canonical managed path")
+		fs.StringVar(&local.Request.Isolation, "isolation", "", "executor: shared worker preset on first enrolled installation")
 		fs.StringVar(&local.ReplacePackage, "replace-package", "", "enrolled executor: select an installed full-package directory while stopped and disabled")
 		fs.BoolVar(&local.Start, "start", true, "start the unit and observe its readiness record")
 		fs.BoolVar(&local.Enable, "enable", true, "let the service manager start the unit at boot")
@@ -233,7 +239,7 @@ func serviceRole(name string, local *serviceOptions, fs *flag.FlagSet, options g
 	blank := false
 	fs.Visit(func(f *flag.Flag) {
 		switch f.Name {
-		case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state", "replace-package":
+		case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state", "replace-package", "isolation":
 			if strings.TrimSpace(f.Value.String()) == "" {
 				blank = true
 			}

docs/operations/guest-isolation.md

@@ -13,14 +13,46 @@ uses the parent’s network policy and quotas for every guest network import.
 A synchronous inherited socket carries
 fixed import calls and bounded memory transfers; there is no extra TCP listener.
 
+## Managed enrolled executor
+
+On Linux 5.14+ with cgroup v2 and systemd 254+ (including Ubuntu 24.04),
+`dbl service install --isolation shared` prepares the delegation and a bounded
+four-worker preset. After `dbl executor join --install-only` finishes, append
+`--isolation shared` to the service-install command it prints, **before the first
+service installation**. Check that the installed CLI's `service install --help`
+lists this option. The usual unprivileged service account and canonical
+`--enrolled-state` directory are still required.
+
+The preset limits the whole service to **1.25 GiB memory and 2.25 CPUs**, with
+swap disabled. Worker groups share at most 1 GiB and 2 CPUs; the remaining
+configured allowance is for the control process. This does not reserve physical
+host resources. Provision the host accordingly. Each worker has 256 MiB process
+memory, half a CPU and 64 MiB WASM memory. Compilation is serialized, with three
+queue places and a 60-second deadline including queueing. The execution ceiling
+is 24 hours; the submitted run's policy timeout still applies first. An executor
+configured for more than four concurrent runs is refused, without changing its
+admission setting. Larger programs may need a manually provisioned profile.
+
+The installer appends the preset only when the enrolled configuration has no
+`[isolation]` table. It preserves identity, credentials, network policy, pricing
+and storage. A custom isolation table is never overwritten. Reinstallation and
+package replacement retain the recorded profile; this option does not convert
+an existing trusted service. The generated unit runs the daemon in its own
+`control` leaf and delegates the sibling worker groups. Startup must validate
+the actual delegation and publish readiness; an unsupported host never falls
+back to the trusted engine. Check `dbl service status --role executor --name NAME`
+and the unit's journal if installation reports incomplete readiness.
+
+The manual setup below remains available for other budgets or service managers.
+
 ## Required host setup
 
 The shared profile requires Linux with cgroup v2, `CLONE_INTO_CGROUP` and
 `cgroup.kill` (Linux 5.14 or newer), and writable delegated `memory`, `cpu` and
 `pids` controllers. Mount cgroup v2 at `/sys/fs/cgroup`. The executor must run
 inside the configured delegation, in a different leaf from its worker groups.
 Use an exclusive delegation sized within the host resources available to this
-service. Its root must already enable these controllers for its children and
+service. Its root must permit enabling these controllers for its children and
 have finite `memory.max` and `cpu.max` limits. An unlimited user service root
 is insufficient; a separately limited delegated service or scope is required.
 Startup refuses an absent, read-only or incomplete delegation; it does not fall

internal/demo/service/enrolled.go

@@ -17,8 +17,9 @@ import (
 )
 
 // prepareEnrolledState adopts only the canonical, already enrolled directory.
-// Configuration, credentials and database are never regenerated or relocated.
-func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, account Account) error {
+// Credentials and database are never regenerated or relocated. The explicit
+// shared preset may append isolation settings before first adoption.
+func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, account Account, report *Report) error {
 	cfg, err := validateEnrolledState(*p)
 	if err != nil {
 		return err
@@ -42,6 +43,15 @@ func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, accoun
 	if err := storagecheck.Check(ctx, storagecheck.Executor, p.DatabasePath); err != nil {
 		return err
 	}
+	if p.SharedIsolation {
+		changed, err := prepareSharedConfig(*p)
+		if err != nil {
+			return err
+		}
+		if changed {
+			report.Changed = append(report.Changed, "shared worker configuration")
+		}
+	}
 	if err := i.own(p.StateDir, account); err != nil {
 		return err
 	}
@@ -87,6 +97,9 @@ func readEnrolledConfig(p Profile) (*executorconfig.ExecutorConfig, error) {
 		cfg.Credentials.ClientKey != filepath.Join(p.StateDir, "executor.key") || cfg.Credentials.EnrollmentToken != "" {
 		return nil, errors.New("enrolled configuration must retain TLS and its original credentials and database inside the managed directory")
 	}
+	if p.SharedIsolation && p.ExecutorID != "" && (cfg.Isolation != sharedPreset(p) || cfg.Resources.MaxDebuglets > 4) {
+		return nil, errors.New("managed shared isolation configuration changed; restore the installed preset before restarting or replacing it")
+	}
 	return cfg, nil
 }
 

internal/demo/service/install.go

@@ -162,6 +162,10 @@ func (i *Installer) Install(ctx context.Context, request Request, assets demo.As
 		if existing.Enrolled != p.Enrolled {
 			return report, errors.New("installation mode differs from retained state; reinstall enrolled executors with --enrolled-state and never replace their configuration")
 		}
+		if request.Isolation != "" && p.SharedIsolation != existing.SharedIsolation {
+			return report, errors.New("--isolation shared is only available on first adoption; existing services retain their isolation profile")
+		}
+		p.SharedIsolation = existing.SharedIsolation
 		if existing.Enrolled {
 			if err := i.verifyRemoval(ctx, existing); err != nil {
 				// A same-package counter-mode reinstall records its capability
@@ -188,6 +192,11 @@ func (i *Installer) Install(ctx context.Context, request Request, assets demo.As
 			return report, err
 		}
 	}
+	if p.SharedIsolation && (i.root == "" || i.root == "/") {
+		if err := checkSharedHost(ctx); err != nil {
+			return report, err
+		}
+	}
 	if err := i.prepareState(ctx, &p, &report, account); err != nil {
 		return report, err
 	}
@@ -304,7 +313,7 @@ func (i *Installer) observeReady(ctx context.Context, p Profile, state UnitState
 // account could already write to.
 func (i *Installer) prepareState(ctx context.Context, p *Profile, report *Report, account Account) error {
 	if p.Enrolled {
-		return i.prepareEnrolledState(ctx, p, account)
+		return i.prepareEnrolledState(ctx, p, account, report)
 	}
 	if err := os.MkdirAll(filepath.Dir(p.StateDir), 0755); err != nil {
 		return err
@@ -761,6 +770,7 @@ func (i *Installer) load(ctx context.Context, operation string, role storagechec
 	p.ExecutorID = record.ExecutorID
 	p.Enrolled = record.Enrolled
 	p.PacketCapabilities = record.PacketCapabilities
+	p.SharedIsolation = record.SharedIsolation
 	p.HTTPPort, p.GRPCPort = record.HTTPPort, record.GRPCPort
 	p.DispatcherGRPC, p.DispatcherHTTP = record.DispatcherGRPC, record.DispatcherHTTP
 	report.Version, report.ExecutorID = p.Version, p.ExecutorID