diff --git a/cmd/gvisor-start-probe/main_linux.go b/cmd/gvisor-start-probe/main_linux.go new file mode 100644 index 0000000000..348e688420 --- /dev/null +++ b/cmd/gvisor-start-probe/main_linux.go @@ -0,0 +1,78 @@ +//go:build linux + +// Command gvisor-start-probe is a controlled experiment for the SecCheck +// container/start point. It does not enable gVisor collection in node-agent. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/kubescape/node-agent/pkg/gvisor" +) + +func main() { + os.Exit(run()) +} + +func run() (exitCode int) { + socket := flag.String("socket", "/run/kubescape/gvisor-events.sock", "private Unix socket for the SecCheck remote sink") + containerID := flag.String("container-id", "", "exact container ID obtained independently from the local runtime before start") + flag.Parse() + if *containerID == "" { + fmt.Fprintln(os.Stderr, "container-id is required") + return 2 + } + output, err := newJSONOutput(os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "gvisor start probe output: %v\n", err) + return 1 + } + defer func() { + if err := output.Close(); err != nil { + fmt.Fprintf(os.Stderr, "gvisor start probe output cleanup: %v\n", err) + exitCode = 1 + } + }() + signalCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithCancel(signalCtx) + defer cancel() + outputErrors := make(chan error, 1) + receiver := &gvisor.Receiver{ + SocketPath: *socket, + Resolve: func(id string) bool { + return id == *containerID + }, + OnStart: func(ctx context.Context, start gvisor.Start) { + if err := output.Encode(ctx, start); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || + (ctx.Err() != nil && errors.Is(err, os.ErrDeadlineExceeded)) { + return + } + select { + case outputErrors <- err: + default: + } + cancel() + } + }, + } + runErr := receiver.Run(ctx) + select { + case err := <-outputErrors: + fmt.Fprintf(os.Stderr, "gvisor start probe output: %v\n", err) + return 1 + default: + } + if runErr != nil { + fmt.Fprintf(os.Stderr, "gvisor start probe: %v\n", runErr) + return 1 + } + return 0 +} diff --git a/cmd/gvisor-start-probe/main_linux_test.go b/cmd/gvisor-start-probe/main_linux_test.go new file mode 100644 index 0000000000..4854300265 --- /dev/null +++ b/cmd/gvisor-start-probe/main_linux_test.go @@ -0,0 +1,133 @@ +//go:build linux + +package main + +import ( + "bytes" + "context" + "encoding/binary" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" +) + +func TestProbeRestoresInheritedOutputFlags(t *testing.T) { + probeBinary := filepath.Join(t.TempDir(), "gvisor-start-probe") + build := exec.Command("go", "build", "-buildvcs=false", "-o", probeBinary, ".") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("building probe: %v\n%s", err, output) + } + for _, mode := range []struct { + name string + flags int + }{{"blocking", 0}, {"nonblocking", unix.O_NONBLOCK}} { + for _, scenario := range []string{"signal_exit", "invalid_socket_directory", "closed_output"} { + name := mode.name + "/" + scenario + t.Run(name, func(t *testing.T) { + fds := make([]int, 2) + if err := unix.Pipe2(fds, unix.O_CLOEXEC|mode.flags); err != nil { + t.Fatal(err) + } + reader := os.NewFile(uintptr(fds[0]), "reader") + writer := os.NewFile(uintptr(fds[1]), "writer") + defer reader.Close() + defer writer.Close() + if scenario == "closed_output" { + if err := reader.Close(); err != nil { + t.Fatal(err) + } + } + original, err := unix.FcntlInt(uintptr(fds[1]), unix.F_GETFL, 0) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + if scenario == "invalid_socket_directory" { + path = filepath.Join(dir, "missing", "events.sock") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, probeBinary, "--socket", path, "--container-id", "known") + cmd.Stdout = writer + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + if scenario != "invalid_socket_directory" { + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Lstat(path); err == nil { + break + } + if time.Now().After(deadline) { + _ = cmd.Process.Kill() + _ = cmd.Wait() + t.Fatalf("probe did not create socket: %s", stderr.String()) + } + time.Sleep(20 * time.Millisecond) + } + if scenario == "closed_output" { + client := connectProbe(t, path) + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + if _, err := unix.Write(client, handshake); err != nil { + t.Fatal(err) + } + if _, err := unix.Read(client, make([]byte, 32)); err != nil { + t.Fatal(err) + } + frame := make([]byte, 8) + binary.LittleEndian.PutUint16(frame[:2], 8) + binary.LittleEndian.PutUint16(frame[2:4], 1) + frame = protowire.AppendTag(frame, 2, protowire.BytesType) + frame = protowire.AppendString(frame, "known") + if _, err := unix.Write(client, frame); err != nil { + t.Fatal(err) + } + unix.Close(client) + } else if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + } + err = cmd.Wait() + if ctx.Err() != nil { + t.Fatalf("probe did not exit: %v", ctx.Err()) + } + if scenario != "signal_exit" { + if cmd.ProcessState.ExitCode() != 1 || stderr.Len() == 0 { + t.Fatalf("expected probe failure: exit=%d stderr=%s", cmd.ProcessState.ExitCode(), stderr.String()) + } + if scenario == "closed_output" && !strings.Contains(stderr.String(), "gvisor start probe output:") { + t.Fatalf("missing output failure diagnostic: %s", stderr.String()) + } + } else if err != nil { + t.Fatalf("normal probe exit: %v\n%s", err, stderr.String()) + } + after, err := unix.FcntlInt(uintptr(fds[1]), unix.F_GETFL, 0) + if err != nil { + t.Fatal(err) + } + if after != original { + t.Fatalf("probe changed parent's output flags: before=%#x after=%#x", original, after) + } + if scenario != "invalid_socket_directory" { + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("socket remains after exit: %v", err) + } + } + }) + } + } +} diff --git a/cmd/gvisor-start-probe/output_linux.go b/cmd/gvisor-start-probe/output_linux.go new file mode 100644 index 0000000000..fb8f37377a --- /dev/null +++ b/cmd/gvisor-start-probe/output_linux.go @@ -0,0 +1,80 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync" + "time" + + "github.com/kubescape/node-agent/pkg/gvisor" + "golang.org/x/sys/unix" +) + +type jsonOutput struct { + file *os.File + encoder *json.Encoder + fd uintptr + flags int + closeOnce sync.Once + closeErr error +} + +func newJSONOutput(output *os.File) (*jsonOutput, error) { + // Inherited stdout may be a blocking pipe. Register a nonblocking duplicate + // with Go's poller so a write deadline can interrupt a full pipe. The probe + // must restore the shared flags before exiting. SyscallConn avoids Fd(), + // which can itself switch a Go-managed pipe back to blocking mode. + raw, err := output.SyscallConn() + if err != nil { + return nil, err + } + var flags, fd int + var setupErr error + if err := raw.Control(func(original uintptr) { + flags, setupErr = unix.FcntlInt(original, unix.F_GETFL, 0) + if setupErr == nil { + fd, setupErr = unix.FcntlInt(original, unix.F_DUPFD_CLOEXEC, 0) + } + }); err != nil { + return nil, err + } + if setupErr != nil { + return nil, setupErr + } + if err := unix.SetNonblock(fd, true); err != nil { + unix.Close(fd) + return nil, err + } + file := os.NewFile(uintptr(fd), "probe-output") + return &jsonOutput{file: file, encoder: json.NewEncoder(file), fd: uintptr(fd), flags: flags}, nil +} + +// Close restores the inherited file description after all output has finished. +func (o *jsonOutput) Close() error { + o.closeOnce.Do(func() { + _, restoreErr := unix.FcntlInt(o.fd, unix.F_SETFL, o.flags) + o.closeErr = errors.Join(restoreErr, o.file.Close()) + }) + return o.closeErr +} + +func (o *jsonOutput) Encode(ctx context.Context, start gvisor.Start) error { + if err := ctx.Err(); err != nil { + return err + } + interrupted := make(chan struct{}) + stop := context.AfterFunc(ctx, func() { + // Regular files do not support deadlines; pipes and sockets do. + _ = o.file.SetWriteDeadline(time.Now()) + close(interrupted) + }) + err := o.encoder.Encode(start) + if !stop() { + <-interrupted + } + return err +} diff --git a/cmd/gvisor-start-probe/output_linux_test.go b/cmd/gvisor-start-probe/output_linux_test.go new file mode 100644 index 0000000000..9f638b7263 --- /dev/null +++ b/cmd/gvisor-start-probe/output_linux_test.go @@ -0,0 +1,165 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kubescape/node-agent/pkg/gvisor" + "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" +) + +func TestReceiverCancellationInterruptsFullOutputPipe(t *testing.T) { + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer reader.Close() // Keep the reader open throughout cancellation. + defer writer.Close() + fd := int(writer.Fd()) + if _, err := unix.FcntlInt(uintptr(fd), unix.F_SETPIPE_SZ, 4096); err != nil { + t.Fatal(err) + } + if err := unix.SetNonblock(fd, true); err != nil { + t.Fatal(err) + } + for { + _, err := unix.Write(fd, make([]byte, 4096)) + if err == unix.EAGAIN { + break + } + if err != nil { + t.Fatal(err) + } + } + // Mimic inherited stdout, which starts as a blocking descriptor. + if err := unix.SetNonblock(fd, false); err != nil { + t.Fatal(err) + } + output, err := newJSONOutput(writer) + if err != nil { + t.Fatal(err) + } + defer output.Close() + + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + entered := make(chan struct{}) + writeDone := make(chan error, 1) + receiver := &gvisor.Receiver{ + SocketPath: path, + Resolve: func(id string) bool { return id == "known" }, + OnStart: func(ctx context.Context, start gvisor.Start) { + close(entered) + writeDone <- output.Encode(ctx, start) + }, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- receiver.Run(ctx) }() + client := connectProbe(t, path) + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + if _, err := unix.Write(client, handshake); err != nil { + t.Fatal(err) + } + if _, err := unix.Read(client, make([]byte, 32)); err != nil { + t.Fatal(err) + } + frame := make([]byte, 8) + binary.LittleEndian.PutUint16(frame[:2], 8) + binary.LittleEndian.PutUint16(frame[2:4], 1) + frame = protowire.AppendTag(frame, 2, protowire.BytesType) + frame = protowire.AppendString(frame, "known") + if _, err := unix.Write(client, frame); err != nil { + t.Fatal(err) + } + unix.Close(client) + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("output callback did not start") + } + select { + case err := <-writeDone: + t.Fatalf("output did not block on the full pipe: %v", err) + case <-time.After(100 * time.Millisecond): + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("Run remained blocked after cancellation with pipe reader open") + } + if err := <-writeDone; !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("blocked write was not interrupted by deadline: %v", err) + } + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("receiver socket remains after shutdown: %v", err) + } +} + +func TestJSONOutputWritesNormally(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "events") + if err != nil { + t.Fatal(err) + } + defer file.Close() + output, err := newJSONOutput(file) + if err != nil { + t.Fatal(err) + } + defer output.Close() + for _, id := range []string{"first", "second", "third"} { + if err := output.Encode(context.Background(), gvisor.Start{ContainerID: id}); err != nil { + t.Fatal(err) + } + } + if _, err := file.Seek(0, 0); err != nil { + t.Fatal(err) + } + decoder := json.NewDecoder(file) + for _, id := range []string{"first", "second", "third"} { + var start gvisor.Start + if err := decoder.Decode(&start); err != nil || start.ContainerID != id { + t.Fatalf("normal output: got %q, want %q, err=%v", start.ContainerID, id, err) + } + } +} + +func connectProbe(t *testing.T, path string) int { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + if unix.Connect(fd, &unix.SockaddrUnix{Name: path}) == nil { + timeout := unix.NsecToTimeval((3 * time.Second).Nanoseconds()) + if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &timeout); err != nil { + t.Fatal(err) + } + return fd + } + unix.Close(fd) + time.Sleep(20 * time.Millisecond) + } + t.Fatal("could not connect to probe receiver") + return -1 +} diff --git a/docs/gvisor-start-probe.md b/docs/gvisor-start-probe.md new file mode 100644 index 0000000000..eaf4a60c39 --- /dev/null +++ b/docs/gvisor-start-probe.md @@ -0,0 +1,96 @@ +# gVisor container start experiment + +This is the first code experiment from +[the accepted runtime visibility proposal](https://github.com/kubescape/designs-and-proposals/pull/23). +It is a Linux-only, opt-in receiver for gVisor's SecCheck `container/start` +point. It does not change node-agent's normal startup, turn on tracing, or emit +existing eBPF event types. + +The receiver speaks the remote sink's Unix `SOCK_SEQPACKET` protocol. It reads +the version-1 handshake and the point header, accepts only message type 1 +(`MESSAGE_CONTAINER_START`), and compares the reported container ID with an +ID obtained independently from the local runtime. Other points and unresolved +IDs do not produce start events. The retained event contains the source, +container ID, observation time, and sender-reported cumulative drop count. + +**Privacy boundary:** gVisor's start message includes arguments and a working +directory even when the trace session selects no optional fields. The receiver +necessarily receives those bytes. Its decoder skips them without copying them +into an event, and clears the input buffer after processing. The experimental +command must not be used with real credentials until the Linux trial confirms +the protocol and the synthetic secret-in-argv/cwd test on the target `runsc` +build. This does not prevent the Sentry or transport from seeing the fields. + +## Controlled trial + +Use a self-managed Linux node with containerd and a pinned `runsc` release. +Record the kernel, containerd, `runsc`, and node-agent versions and runtime +root. Check the selected point with `runsc trace metadata` on that build. +Configure the `Default` trace session before the sandbox starts using +`--pod-init-config`; attaching with `runsc trace create` after startup cannot +demonstrate that `container/start` was captured. Do not use `--force` to replace +another monitor's session. + +The test input is: + +```json +{ + "trace_session": { + "name": "Default", + "points": [{ "name": "container/start" }], + "sinks": [{ + "name": "remote", + "config": { "endpoint": "/run/kubescape/gvisor-events.sock" } + }] + } +} +``` + +Create a private directory owned by the receiver, such as +`/run/kubescape` with mode `0700`. The receiver refuses an existing socket +path. Obtain the *exact* container ID from the local runtime before starting +that container, then run: + +```sh +go run ./cmd/gvisor-start-probe \ + --socket /run/kubescape/gvisor-events.sock \ + --container-id "$CONTAINER_ID" +``` + +The flag is a controlled-trial identity check, not a production runtime +inventory integration. Start the prepared sandbox only after the receiver is +listening. In the production integration, node-agent's runtime inventory must +perform this check. A start event from this probe is not evidence of an actor +or session identity. + +Test a normal start, twenty repeated starts, two concurrent sandboxes, an +absent or restarted receiver, and a root and child container with a harmless +synthetic canary in argv and cwd. Check stdout, stderr, errors, and any saved +artifacts for that canary before sharing results. Compare the received starts +and IDs with containerd or CRI records. A connection close is a disconnected +source, not a verified sandbox stop. No network meaning is inferred here. + +The package tests cover protocol parsing, identity mismatch, sensitive-field +discard, and a Linux socket exchange: + +```sh +go test ./pkg/gvisor -count=1 +``` + +Receiver callbacks receive the caller's cancellation context and must interrupt +blocking work when it is canceled. The probe uses a write deadline to interrupt +blocked pipe output on SIGINT or SIGTERM, allowing the receiver to finish and +remove its socket even if the output consumer stops reading. Tests in +`cmd/gvisor-start-probe` cover a full output pipe and ordinary JSON output; +receiver tests also cover draining a closed event queue in order. +The probe restores stdout's original file flags on both normal and error exits, +so a parent process sharing the inherited pipe or terminal keeps its prior mode. +If output fails, the probe stops collection, reports the first write error on +stderr, and exits with status 1. Signal cancellation of a blocked write remains +a normal exit. Actual-process tests cover a closed output reader as well as +flag restoration and signal cancellation. + +This document records the procedure, not results. Live `runsc` and node-agent +host eBPF observations must be added after the Linux trial; the existing +[gVisor proof of concept](https://github.com/yellow-forrest/gvisor-visibility-poc) +is useful prior work but is not validation of this receiver. diff --git a/pkg/gvisor/receiver_linux.go b/pkg/gvisor/receiver_linux.go new file mode 100644 index 0000000000..5ca02b440d --- /dev/null +++ b/pkg/gvisor/receiver_linux.go @@ -0,0 +1,187 @@ +//go:build linux + +package gvisor + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "syscall" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" +) + +const ( + maxConnections = 8 + queueSize = 128 +) + +// Receiver is an experimental, opt-in SecCheck remote sink listener. It does +// not configure runsc or acquire its single Default trace session. +type Receiver struct { + SocketPath string + Resolve Resolver + OnStart StartHandler + + queueDrops atomic.Uint64 +} + +// QueueDrops counts locally discarded events when the consumer cannot keep up. +func (r *Receiver) QueueDrops() uint64 { return r.queueDrops.Load() } + +// Run listens until ctx is canceled. The socket parent must already exist, +// belong to this process, and be private; no existing socket is unlinked. +func (r *Receiver) Run(ctx context.Context) error { + if r.Resolve == nil || r.OnStart == nil || r.SocketPath == "" { + return errors.New("gvisor receiver requires socket path, resolver, and handler") + } + if err := privateParent(filepath.Dir(r.SocketPath)); err != nil { + return err + } + if _, err := os.Lstat(r.SocketPath); err == nil { + return errors.New("gvisor socket path already exists") + } else if !os.IsNotExist(err) { + return fmt.Errorf("checking gvisor socket path: %w", err) + } + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("creating gvisor socket: %w", err) + } + defer unix.Close(fd) + if err := unix.Bind(fd, &unix.SockaddrUnix{Name: r.SocketPath}); err != nil { + return fmt.Errorf("binding gvisor socket: %w", err) + } + defer os.Remove(r.SocketPath) + if err := os.Chmod(r.SocketPath, 0600); err != nil { + return fmt.Errorf("setting gvisor socket permissions: %w", err) + } + if err := unix.Listen(fd, maxConnections); err != nil { + return fmt.Errorf("listening on gvisor socket: %w", err) + } + if err := unix.SetNonblock(fd, true); err != nil { + return fmt.Errorf("configuring gvisor listener: %w", err) + } + runCtx, cancel := context.WithCancel(ctx) + + starts := make(chan Start, queueSize) + workerDone := make(chan struct{}) + go func() { + defer close(workerDone) + r.deliverStarts(ctx, starts) + }() + + var clients sync.WaitGroup + slots := make(chan struct{}, maxConnections) + defer func() { + cancel() + clients.Wait() + close(starts) + <-workerDone + }() + + for { + if runCtx.Err() != nil { + return nil + } + client, _, acceptErr := unix.Accept4(fd, unix.SOCK_CLOEXEC) + if acceptErr != nil { + if acceptErr == unix.EINTR { + continue + } + if acceptErr == unix.EAGAIN || acceptErr == unix.EWOULDBLOCK { + select { + case <-runCtx.Done(): + return nil + case <-time.After(100 * time.Millisecond): + continue + } + } + return fmt.Errorf("accepting gvisor connection: %w", acceptErr) + } + select { + case slots <- struct{}{}: + clients.Add(1) + go func() { + defer clients.Done() + defer func() { <-slots }() + defer unix.Close(client) + r.receive(runCtx, client, starts) + }() + default: + unix.Close(client) + } + } +} + +func (r *Receiver) deliverStarts(ctx context.Context, starts <-chan Start) { + for start := range starts { + // Use the caller's context: an internal listener error still drains + // queued events, while caller cancellation interrupts the handler. + r.OnStart(ctx, start) + } +} + +func privateParent(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("checking gvisor socket directory: %w", err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.IsDir() || info.Mode().Perm()&0077 != 0 || stat.Uid != uint32(os.Geteuid()) { + return errors.New("gvisor socket directory must be private and owned by the receiver") + } + return nil +} + +func (r *Receiver) receive(ctx context.Context, fd int, starts chan<- Start) { + // The extra byte and MSG_TRUNC flag detect oversized seqpacket records. + buffer := make([]byte, maxFrameSize+1) + defer clear(buffer) + timeout := unix.NsecToTimeval((2 * time.Second).Nanoseconds()) + if unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &timeout) != nil { + return + } + if unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_SNDTIMEO, &timeout) != nil { + return + } + n, _, flags, _, err := unix.Recvmsg(fd, buffer, nil, 0) + if err != nil || n == 0 || flags&unix.MSG_TRUNC != 0 || n > maxFrameSize { + return + } + if _, err = handshakeVersion(buffer[:n]); err != nil { + return + } + response := protowire.AppendTag(nil, 1, protowire.VarintType) + response = protowire.AppendVarint(response, protocolVersion) + if _, err = unix.SendmsgN(fd, response, nil, nil, 0); err != nil { + return + } + for ctx.Err() == nil { + n, _, flags, _, err = unix.Recvmsg(fd, buffer, nil, 0) + if err == unix.EAGAIN || err == unix.EWOULDBLOCK { + continue + } + if err != nil || n == 0 { + return + } + if flags&unix.MSG_TRUNC != 0 || n > maxFrameSize { + return + } + start, ok, err := decodeStartFrame(buffer[:n], time.Now(), r.Resolve) + if err == nil && ok { + select { + case starts <- start: + default: + r.queueDrops.Add(1) + } + } + // Raw bytes may include argv, cwd, and env. Never retain them across reads. + clear(buffer[:n]) + } +} diff --git a/pkg/gvisor/receiver_linux_test.go b/pkg/gvisor/receiver_linux_test.go new file mode 100644 index 0000000000..5b85945db6 --- /dev/null +++ b/pkg/gvisor/receiver_linux_test.go @@ -0,0 +1,178 @@ +//go:build linux + +package gvisor + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" +) + +func TestReceiverVerifiesIdentityAndDropsSensitiveFields(t *testing.T) { + const canary = "synthetic-argv-secret" + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + starts := make(chan Start, 1) + receiver := &Receiver{ + SocketPath: path, + Resolve: func(id string) bool { return id == "known-container" }, + OnStart: func(_ context.Context, start Start) { starts <- start }, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- receiver.Run(ctx) }() + fd := connectReceiver(t, path) + defer unix.Close(fd) + timeout := unix.NsecToTimeval((3 * time.Second).Nanoseconds()) + if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &timeout); err != nil { + t.Fatal(err) + } + + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + sendPacket(t, fd, handshake) + buffer := make([]byte, 32) + if _, err := unix.Read(fd, buffer); err != nil { + t.Fatalf("handshake reply: %v", err) + } + sendPacket(t, fd, startFrame("unknown-container", "unknown-container", canary)) + sendPacket(t, fd, startFrame("known-container", "known-container", canary)) + select { + case start := <-starts: + if start.ContainerID != "known-container" || start.Source != "gvisor_trace" { + t.Fatalf("unexpected start: %+v", start) + } + encoded, err := json.Marshal(start) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), canary) { + t.Fatalf("secret retained: %s", encoded) + } + case <-time.After(3 * time.Second): + t.Fatal("verified start did not arrive") + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(4 * time.Second): + t.Fatal("receiver did not stop") + } +} + +func TestReceiverKeepsConcurrentConnectionsSeparate(t *testing.T) { + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + starts := make(chan Start, 40) + receiver := &Receiver{ + SocketPath: path, + Resolve: func(id string) bool { return id == "first" || id == "second" }, + OnStart: func(_ context.Context, start Start) { starts <- start }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- receiver.Run(ctx) }() + fds := make(map[string]int) + for _, id := range []string{"first", "second"} { + fd := connectReceiver(t, path) + fds[id] = fd + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + sendPacket(t, fd, handshake) + buffer := make([]byte, 32) + if _, err := unix.Read(fd, buffer); err != nil { + t.Fatal(err) + } + } + for range 20 { + for _, id := range []string{"first", "second"} { + sendPacket(t, fds[id], startFrame(id, id, "private")) + } + } + for _, fd := range fds { + unix.Close(fd) + } + counts := map[string]int{} + deadline := time.After(4 * time.Second) + for len(starts) > 0 || counts["first"]+counts["second"] < 40 { + select { + case event := <-starts: + counts[event.ContainerID]++ + case <-deadline: + t.Fatalf("missing events: %v", counts) + } + } + if counts["first"] != 20 || counts["second"] != 20 { + t.Fatalf("connections crossed or lost identity: %v", counts) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(4 * time.Second): + t.Fatal("receiver did not stop") + } +} + +func TestReceiverDrainsClosedQueue(t *testing.T) { + starts := make(chan Start, 3) + for _, id := range []string{"first", "second", "third"} { + starts <- Start{ContainerID: id} + } + close(starts) + var delivered []string + receiver := &Receiver{OnStart: func(ctx context.Context, start Start) { + if ctx.Err() != nil { + t.Fatal("normal queue drain received a canceled context") + } + delivered = append(delivered, start.ContainerID) + }} + receiver.deliverStarts(context.Background(), starts) + if strings.Join(delivered, ",") != "first,second,third" { + t.Fatalf("queue did not drain in order: %v", delivered) + } +} + +func connectReceiver(t *testing.T, path string) int { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + if err := unix.Connect(fd, &unix.SockaddrUnix{Name: path}); err == nil { + return fd + } + unix.Close(fd) + time.Sleep(20 * time.Millisecond) + } + t.Fatal("could not connect to receiver") + return -1 +} + +func sendPacket(t *testing.T, fd int, packet []byte) { + t.Helper() + if _, err := unix.SendmsgN(fd, packet, nil, nil, 0); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/gvisor/start.go b/pkg/gvisor/start.go new file mode 100644 index 0000000000..cb1582bd04 --- /dev/null +++ b/pkg/gvisor/start.go @@ -0,0 +1,167 @@ +// Package gvisor receives a narrow subset of gVisor's SecCheck remote sink. +// The first supported signal is container/start. The wire message contains +// command arguments and a working directory, so only allowlisted identity +// fields may leave the decoder. +package gvisor + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "time" + + "google.golang.org/protobuf/encoding/protowire" +) + +const ( + // These values come from gVisor's SecCheck common.proto and container.proto + // at 8a2c5049262ca84ea9c0981ac82eed02110d4ba7. The Linux trial must + // confirm them against its pinned runsc build before this is enabled. + protocolVersion = 1 + headerSize = 8 + startMessage = 1 // MESSAGE_CONTAINER_START in gVisor's common.proto + maxFrameSize = 1 << 20 + maxIDSize = 256 +) + +// Start is the entire retained event. In particular, Args, Cwd, Env, and raw +// protobuf bytes must never be added to this type or passed to the callback. +type Start struct { + Source string + ContainerID string + ObservedAt time.Time + Dropped uint32 // Cumulative sender-reported drop count on this connection. +} + +// Resolver checks a claimed container ID against the local runtime inventory. +// The receiver never attributes an unverified claim to a Kubernetes workload. +type Resolver func(containerID string) bool + +// StartHandler receives only runtime-verified start events. It must honor ctx +// cancellation, including interrupting any blocking I/O, so Run can finish. +type StartHandler func(context.Context, Start) + +func handshakeVersion(frame []byte) (uint64, error) { + var version uint64 + for len(frame) > 0 { + number, kind, n := protowire.ConsumeTag(frame) + if n < 0 { + return 0, errors.New("invalid handshake tag") + } + frame = frame[n:] + if number == 1 && kind == protowire.VarintType { + value, consumed := protowire.ConsumeVarint(frame) + if consumed < 0 { + return 0, errors.New("invalid handshake version") + } + version = value + frame = frame[consumed:] + continue + } + consumed := protowire.ConsumeFieldValue(number, kind, frame) + if consumed < 0 { + return 0, errors.New("invalid handshake field") + } + frame = frame[consumed:] + } + if version != protocolVersion { + return 0, fmt.Errorf("unsupported remote sink version %d", version) + } + return version, nil +} + +func decodeStartFrame(frame []byte, observedAt time.Time, resolve Resolver) (Start, bool, error) { + if len(frame) < headerSize { + return Start{}, false, errors.New("short remote sink header") + } + length := int(binary.LittleEndian.Uint16(frame[:2])) + if length < headerSize || length > len(frame) { + return Start{}, false, errors.New("invalid remote sink header size") + } + if binary.LittleEndian.Uint16(frame[2:4]) != startMessage { + return Start{}, false, nil // Forward-compatible: skip other point types. + } + id, err := startID(frame[length:]) + if err != nil { + return Start{}, false, err + } + if id == "" || resolve == nil || !resolve(id) { + return Start{}, false, nil + } + return Start{ + Source: "gvisor_trace", + ContainerID: id, + ObservedAt: observedAt, + Dropped: binary.LittleEndian.Uint32(frame[4:8]), + }, true, nil +} + +func startID(payload []byte) (string, error) { + var startID, contextID string + for len(payload) > 0 { + number, kind, n := protowire.ConsumeTag(payload) + if n < 0 { + return "", errors.New("invalid start tag") + } + payload = payload[n:] + if (number == 1 || number == 2) && kind == protowire.BytesType { + value, consumed := protowire.ConsumeBytes(payload) + if consumed < 0 { + return "", errors.New("invalid start field") + } + if number == 1 { + var err error + contextID, err = contextContainerID(value) + if err != nil { + return "", err + } + } else { + if len(value) > maxIDSize { + return "", errors.New("start ID exceeds limit") + } + startID = string(value) + } + payload = payload[consumed:] + continue + } + consumed := protowire.ConsumeFieldValue(number, kind, payload) + if consumed < 0 { + return "", errors.New("invalid start field") + } + payload = payload[consumed:] + } + if startID != "" && contextID != "" && startID != contextID { + return "", errors.New("start and context container IDs differ") + } + if startID != "" { + return startID, nil + } + return contextID, nil +} + +func contextContainerID(payload []byte) (string, error) { + var id string + for len(payload) > 0 { + number, kind, n := protowire.ConsumeTag(payload) + if n < 0 { + return "", errors.New("invalid context tag") + } + payload = payload[n:] + if number == 6 && kind == protowire.BytesType { + value, consumed := protowire.ConsumeBytes(payload) + if consumed < 0 || len(value) > maxIDSize { + return "", errors.New("invalid context container ID") + } + id = string(value) + payload = payload[consumed:] + continue + } + consumed := protowire.ConsumeFieldValue(number, kind, payload) + if consumed < 0 { + return "", errors.New("invalid context field") + } + payload = payload[consumed:] + } + return id, nil +} diff --git a/pkg/gvisor/start_test.go b/pkg/gvisor/start_test.go new file mode 100644 index 0000000000..893ba20cbb --- /dev/null +++ b/pkg/gvisor/start_test.go @@ -0,0 +1,95 @@ +package gvisor + +import ( + "encoding/binary" + "encoding/json" + "strings" + "testing" + "time" + + "google.golang.org/protobuf/encoding/protowire" +) + +func bytesField(number protowire.Number, value []byte) []byte { + field := protowire.AppendTag(nil, number, protowire.BytesType) + return protowire.AppendBytes(field, value) +} + +func startFrame(id, contextID, canary string) []byte { + context := bytesField(6, []byte(contextID)) + payload := bytesField(1, context) + payload = append(payload, bytesField(2, []byte(id))...) + payload = append(payload, bytesField(3, []byte("/work/"+canary))...) + payload = append(payload, bytesField(4, []byte("--token="+canary))...) + payload = append(payload, bytesField(5, []byte("KEY="+canary))...) + frame := make([]byte, headerSize) + binary.LittleEndian.PutUint16(frame[:2], headerSize) + binary.LittleEndian.PutUint16(frame[2:4], startMessage) + binary.LittleEndian.PutUint32(frame[4:8], 7) + return append(frame, payload...) +} + +func TestStartKeepsOnlyVerifiedIdentity(t *testing.T) { + const canary = "synthetic-secret-do-not-retain" + frame := startFrame("runtime-container-1", "runtime-container-1", canary) + when := time.Now() + event, ok, err := decodeStartFrame(frame, when, func(id string) bool { return id == "runtime-container-1" }) + if err != nil || !ok { + t.Fatalf("verified start: ok=%t err=%v", ok, err) + } + if event.Source != "gvisor_trace" || event.ContainerID != "runtime-container-1" || event.Dropped != 7 || !event.ObservedAt.Equal(when) { + t.Fatalf("unexpected retained event: %+v", event) + } + encoded, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), canary) || strings.Contains(string(encoded), "token") { + t.Fatalf("sensitive start fields escaped into retained event: %s", encoded) + } + _, ok, err = decodeStartFrame(frame, when, func(string) bool { return false }) + if err != nil || ok { + t.Fatalf("unverified ID was accepted: ok=%t err=%v", ok, err) + } +} + +func TestStartRejectsConflictingAndMalformedIdentity(t *testing.T) { + resolve := func(string) bool { return true } + frame := startFrame("one", "two", "canary") + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("conflicting IDs accepted: ok=%t err=%v", ok, err) + } + frame = startFrame(strings.Repeat("a", maxIDSize+1), "", "canary") + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("oversized ID accepted: ok=%t err=%v", ok, err) + } + frame = startFrame("one", "one", "canary") + frame[0] = 0xff + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("invalid header accepted: ok=%t err=%v", ok, err) + } + frame = append(startFrame("one", "one", "canary"), 0xff) + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("malformed trailing field accepted: ok=%t err=%v", ok, err) + } +} + +func TestUnknownPointIsSkipped(t *testing.T) { + frame := startFrame("one", "one", "canary") + binary.LittleEndian.PutUint16(frame[2:4], 999) + if _, ok, err := decodeStartFrame(frame, time.Now(), func(string) bool { return true }); err != nil || ok { + t.Fatalf("unknown point: ok=%t err=%v", ok, err) + } +} + +func TestHandshakeVersion(t *testing.T) { + frame := protowire.AppendTag(nil, 1, protowire.VarintType) + frame = protowire.AppendVarint(frame, 1) + if _, err := handshakeVersion(frame); err != nil { + t.Fatal(err) + } + frame[len(frame)-1] = 2 + if _, err := handshakeVersion(frame); err == nil { + t.Fatal("unsupported version accepted") + } +}