From f984879765a0b3c60f729d2200291f607923176b Mon Sep 17 00:00:00 2001 From: Daksh Pathak Date: Wed, 7 Oct 2026 21:13:02 +0530 Subject: [PATCH 1/4] feat(gvisor): prototype container start receiver Signed-off-by: Daksh Pathak --- cmd/gvisor-start-probe/main_linux.go | 43 +++++++ docs/gvisor-start-probe.md | 83 ++++++++++++ pkg/gvisor/receiver_linux.go | 181 +++++++++++++++++++++++++++ pkg/gvisor/receiver_linux_test.go | 159 +++++++++++++++++++++++ pkg/gvisor/start.go | 165 ++++++++++++++++++++++++ pkg/gvisor/start_test.go | 95 ++++++++++++++ 6 files changed, 726 insertions(+) create mode 100644 cmd/gvisor-start-probe/main_linux.go create mode 100644 docs/gvisor-start-probe.md create mode 100644 pkg/gvisor/receiver_linux.go create mode 100644 pkg/gvisor/receiver_linux_test.go create mode 100644 pkg/gvisor/start.go create mode 100644 pkg/gvisor/start_test.go diff --git a/cmd/gvisor-start-probe/main_linux.go b/cmd/gvisor-start-probe/main_linux.go new file mode 100644 index 0000000000..c2bbd1f43d --- /dev/null +++ b/cmd/gvisor-start-probe/main_linux.go @@ -0,0 +1,43 @@ +//go:build linux + +// Command gvisor-start-probe is a controlled experiment for the SecCheck +// container/start point. It does not enable gVisor collection in node-agent. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/kubescape/node-agent/pkg/gvisor" +) + +func main() { + socket := flag.String("socket", "/run/kubescape/gvisor-events.sock", "private Unix socket for the SecCheck remote sink") + containerID := flag.String("container-id", "", "exact container ID obtained independently from the local runtime before start") + flag.Parse() + if *containerID == "" { + fmt.Fprintln(os.Stderr, "container-id is required") + os.Exit(2) + } + encoder := json.NewEncoder(os.Stdout) + receiver := &gvisor.Receiver{ + SocketPath: *socket, + Resolve: func(id string) bool { + return id == *containerID + }, + OnStart: func(start gvisor.Start) { + _ = encoder.Encode(start) + }, + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := receiver.Run(ctx); err != nil { + fmt.Fprintf(os.Stderr, "gvisor start probe: %v\n", err) + os.Exit(1) + } +} diff --git a/docs/gvisor-start-probe.md b/docs/gvisor-start-probe.md new file mode 100644 index 0000000000..c159b0b205 --- /dev/null +++ b/docs/gvisor-start-probe.md @@ -0,0 +1,83 @@ +# gVisor container start experiment + +This is the first code experiment from +[the accepted runtime visibility proposal](https://github.com/kubescape/designs-and-proposals/pull/23). +It is a Linux-only, opt-in receiver for gVisor's SecCheck `container/start` +point. It does not change node-agent's normal startup, turn on tracing, or emit +existing eBPF event types. + +The receiver speaks the remote sink's Unix `SOCK_SEQPACKET` protocol. It reads +the version-1 handshake and the point header, accepts only message type 1 +(`MESSAGE_CONTAINER_START`), and compares the reported container ID with an +ID obtained independently from the local runtime. Other points and unresolved +IDs do not produce start events. The retained event contains the source, +container ID, observation time, and sender-reported cumulative drop count. + +**Privacy boundary:** gVisor's start message includes arguments and a working +directory even when the trace session selects no optional fields. The receiver +necessarily receives those bytes. Its decoder skips them without copying them +into an event, and clears the input buffer after processing. The experimental +command must not be used with real credentials until the Linux trial confirms +the protocol and the synthetic secret-in-argv/cwd test on the target `runsc` +build. This does not prevent the Sentry or transport from seeing the fields. + +## Controlled trial + +Use a self-managed Linux node with containerd and a pinned `runsc` release. +Record the kernel, containerd, `runsc`, and node-agent versions and runtime +root. Check the selected point with `runsc trace metadata` on that build. +Configure the `Default` trace session before the sandbox starts using +`--pod-init-config`; attaching with `runsc trace create` after startup cannot +demonstrate that `container/start` was captured. Do not use `--force` to replace +another monitor's session. + +The test input is: + +```json +{ + "trace_session": { + "name": "Default", + "points": [{ "name": "container/start" }], + "sinks": [{ + "name": "remote", + "config": { "endpoint": "/run/kubescape/gvisor-events.sock" } + }] + } +} +``` + +Create a private directory owned by the receiver, such as +`/run/kubescape` with mode `0700`. The receiver refuses an existing socket +path. Obtain the *exact* container ID from the local runtime before starting +that container, then run: + +```sh +go run ./cmd/gvisor-start-probe \ + --socket /run/kubescape/gvisor-events.sock \ + --container-id "$CONTAINER_ID" +``` + +The flag is a controlled-trial identity check, not a production runtime +inventory integration. Start the prepared sandbox only after the receiver is +listening. In the production integration, node-agent's runtime inventory must +perform this check. A start event from this probe is not evidence of an actor +or session identity. + +Test a normal start, twenty repeated starts, two concurrent sandboxes, an +absent or restarted receiver, and a root and child container with a harmless +synthetic canary in argv and cwd. Check stdout, stderr, errors, and any saved +artifacts for that canary before sharing results. Compare the received starts +and IDs with containerd or CRI records. A connection close is a disconnected +source, not a verified sandbox stop. No network meaning is inferred here. + +The package tests cover protocol parsing, identity mismatch, sensitive-field +discard, and a Linux socket exchange: + +```sh +go test ./pkg/gvisor -count=1 +``` + +This document records the procedure, not results. Live `runsc` and node-agent +host eBPF observations must be added after the Linux trial; the existing +[gVisor proof of concept](https://github.com/yellow-forrest/gvisor-visibility-poc) +is useful prior work but is not validation of this receiver. diff --git a/pkg/gvisor/receiver_linux.go b/pkg/gvisor/receiver_linux.go new file mode 100644 index 0000000000..6eb65bae98 --- /dev/null +++ b/pkg/gvisor/receiver_linux.go @@ -0,0 +1,181 @@ +//go:build linux + +package gvisor + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "syscall" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" +) + +const ( + maxConnections = 8 + queueSize = 128 +) + +// Receiver is an experimental, opt-in SecCheck remote sink listener. It does +// not configure runsc or acquire its single Default trace session. +type Receiver struct { + SocketPath string + Resolve Resolver + OnStart StartHandler + + queueDrops atomic.Uint64 +} + +// QueueDrops counts locally discarded events when the consumer cannot keep up. +func (r *Receiver) QueueDrops() uint64 { return r.queueDrops.Load() } + +// Run listens until ctx is canceled. The socket parent must already exist, +// belong to this process, and be private; no existing socket is unlinked. +func (r *Receiver) Run(ctx context.Context) error { + if r.Resolve == nil || r.OnStart == nil || r.SocketPath == "" { + return errors.New("gvisor receiver requires socket path, resolver, and handler") + } + if err := privateParent(filepath.Dir(r.SocketPath)); err != nil { + return err + } + if _, err := os.Lstat(r.SocketPath); err == nil { + return errors.New("gvisor socket path already exists") + } else if !os.IsNotExist(err) { + return fmt.Errorf("checking gvisor socket path: %w", err) + } + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("creating gvisor socket: %w", err) + } + defer unix.Close(fd) + if err := unix.Bind(fd, &unix.SockaddrUnix{Name: r.SocketPath}); err != nil { + return fmt.Errorf("binding gvisor socket: %w", err) + } + defer os.Remove(r.SocketPath) + if err := os.Chmod(r.SocketPath, 0600); err != nil { + return fmt.Errorf("setting gvisor socket permissions: %w", err) + } + if err := unix.Listen(fd, maxConnections); err != nil { + return fmt.Errorf("listening on gvisor socket: %w", err) + } + if err := unix.SetNonblock(fd, true); err != nil { + return fmt.Errorf("configuring gvisor listener: %w", err) + } + runCtx, cancel := context.WithCancel(ctx) + + starts := make(chan Start, queueSize) + workerDone := make(chan struct{}) + go func() { + defer close(workerDone) + for start := range starts { + r.OnStart(start) + } + }() + + var clients sync.WaitGroup + slots := make(chan struct{}, maxConnections) + defer func() { + cancel() + clients.Wait() + close(starts) + <-workerDone + }() + + for { + if runCtx.Err() != nil { + return nil + } + client, _, acceptErr := unix.Accept4(fd, unix.SOCK_CLOEXEC) + if acceptErr != nil { + if acceptErr == unix.EINTR { + continue + } + if acceptErr == unix.EAGAIN || acceptErr == unix.EWOULDBLOCK { + select { + case <-runCtx.Done(): + return nil + case <-time.After(100 * time.Millisecond): + continue + } + } + return fmt.Errorf("accepting gvisor connection: %w", acceptErr) + } + select { + case slots <- struct{}{}: + clients.Add(1) + go func() { + defer clients.Done() + defer func() { <-slots }() + defer unix.Close(client) + r.receive(runCtx, client, starts) + }() + default: + unix.Close(client) + } + } +} + +func privateParent(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("checking gvisor socket directory: %w", err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.IsDir() || info.Mode().Perm()&0077 != 0 || stat.Uid != uint32(os.Geteuid()) { + return errors.New("gvisor socket directory must be private and owned by the receiver") + } + return nil +} + +func (r *Receiver) receive(ctx context.Context, fd int, starts chan<- Start) { + // The extra byte and MSG_TRUNC flag detect oversized seqpacket records. + buffer := make([]byte, maxFrameSize+1) + defer clear(buffer) + timeout := unix.NsecToTimeval((2 * time.Second).Nanoseconds()) + if unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &timeout) != nil { + return + } + if unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_SNDTIMEO, &timeout) != nil { + return + } + n, _, flags, _, err := unix.Recvmsg(fd, buffer, nil, 0) + if err != nil || n == 0 || flags&unix.MSG_TRUNC != 0 || n > maxFrameSize { + return + } + if _, err = handshakeVersion(buffer[:n]); err != nil { + return + } + response := protowire.AppendTag(nil, 1, protowire.VarintType) + response = protowire.AppendVarint(response, protocolVersion) + if _, err = unix.SendmsgN(fd, response, nil, nil, 0); err != nil { + return + } + for ctx.Err() == nil { + n, _, flags, _, err = unix.Recvmsg(fd, buffer, nil, 0) + if err == unix.EAGAIN || err == unix.EWOULDBLOCK { + continue + } + if err != nil || n == 0 { + return + } + if flags&unix.MSG_TRUNC != 0 || n > maxFrameSize { + return + } + start, ok, err := decodeStartFrame(buffer[:n], time.Now(), r.Resolve) + if err == nil && ok { + select { + case starts <- start: + default: + r.queueDrops.Add(1) + } + } + // Raw bytes may include argv, cwd, and env. Never retain them across reads. + clear(buffer[:n]) + } +} diff --git a/pkg/gvisor/receiver_linux_test.go b/pkg/gvisor/receiver_linux_test.go new file mode 100644 index 0000000000..5b689f6777 --- /dev/null +++ b/pkg/gvisor/receiver_linux_test.go @@ -0,0 +1,159 @@ +//go:build linux + +package gvisor + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" +) + +func TestReceiverVerifiesIdentityAndDropsSensitiveFields(t *testing.T) { + const canary = "synthetic-argv-secret" + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + starts := make(chan Start, 1) + receiver := &Receiver{ + SocketPath: path, + Resolve: func(id string) bool { return id == "known-container" }, + OnStart: func(start Start) { starts <- start }, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- receiver.Run(ctx) }() + fd := connectReceiver(t, path) + defer unix.Close(fd) + timeout := unix.NsecToTimeval((3 * time.Second).Nanoseconds()) + if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &timeout); err != nil { + t.Fatal(err) + } + + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + sendPacket(t, fd, handshake) + buffer := make([]byte, 32) + if _, err := unix.Read(fd, buffer); err != nil { + t.Fatalf("handshake reply: %v", err) + } + sendPacket(t, fd, startFrame("unknown-container", "unknown-container", canary)) + sendPacket(t, fd, startFrame("known-container", "known-container", canary)) + select { + case start := <-starts: + if start.ContainerID != "known-container" || start.Source != "gvisor_trace" { + t.Fatalf("unexpected start: %+v", start) + } + encoded, err := json.Marshal(start) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), canary) { + t.Fatalf("secret retained: %s", encoded) + } + case <-time.After(3 * time.Second): + t.Fatal("verified start did not arrive") + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(4 * time.Second): + t.Fatal("receiver did not stop") + } +} + +func TestReceiverKeepsConcurrentConnectionsSeparate(t *testing.T) { + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + starts := make(chan Start, 40) + receiver := &Receiver{ + SocketPath: path, + Resolve: func(id string) bool { return id == "first" || id == "second" }, + OnStart: func(start Start) { starts <- start }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- receiver.Run(ctx) }() + fds := make(map[string]int) + for _, id := range []string{"first", "second"} { + fd := connectReceiver(t, path) + fds[id] = fd + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + sendPacket(t, fd, handshake) + buffer := make([]byte, 32) + if _, err := unix.Read(fd, buffer); err != nil { + t.Fatal(err) + } + } + for range 20 { + for _, id := range []string{"first", "second"} { + sendPacket(t, fds[id], startFrame(id, id, "private")) + } + } + for _, fd := range fds { + unix.Close(fd) + } + counts := map[string]int{} + deadline := time.After(4 * time.Second) + for len(starts) > 0 || counts["first"]+counts["second"] < 40 { + select { + case event := <-starts: + counts[event.ContainerID]++ + case <-deadline: + t.Fatalf("missing events: %v", counts) + } + } + if counts["first"] != 20 || counts["second"] != 20 { + t.Fatalf("connections crossed or lost identity: %v", counts) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(4 * time.Second): + t.Fatal("receiver did not stop") + } +} + +func connectReceiver(t *testing.T, path string) int { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + if err := unix.Connect(fd, &unix.SockaddrUnix{Name: path}); err == nil { + return fd + } + unix.Close(fd) + time.Sleep(20 * time.Millisecond) + } + t.Fatal("could not connect to receiver") + return -1 +} + +func sendPacket(t *testing.T, fd int, packet []byte) { + t.Helper() + if _, err := unix.SendmsgN(fd, packet, nil, nil, 0); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/gvisor/start.go b/pkg/gvisor/start.go new file mode 100644 index 0000000000..6c40f77442 --- /dev/null +++ b/pkg/gvisor/start.go @@ -0,0 +1,165 @@ +// Package gvisor receives a narrow subset of gVisor's SecCheck remote sink. +// The first supported signal is container/start. The wire message contains +// command arguments and a working directory, so only allowlisted identity +// fields may leave the decoder. +package gvisor + +import ( + "encoding/binary" + "errors" + "fmt" + "time" + + "google.golang.org/protobuf/encoding/protowire" +) + +const ( + // These values come from gVisor's SecCheck common.proto and container.proto + // at 8a2c5049262ca84ea9c0981ac82eed02110d4ba7. The Linux trial must + // confirm them against its pinned runsc build before this is enabled. + protocolVersion = 1 + headerSize = 8 + startMessage = 1 // MESSAGE_CONTAINER_START in gVisor's common.proto + maxFrameSize = 1 << 20 + maxIDSize = 256 +) + +// Start is the entire retained event. In particular, Args, Cwd, Env, and raw +// protobuf bytes must never be added to this type or passed to the callback. +type Start struct { + Source string + ContainerID string + ObservedAt time.Time + Dropped uint32 // Cumulative sender-reported drop count on this connection. +} + +// Resolver checks a claimed container ID against the local runtime inventory. +// The receiver never attributes an unverified claim to a Kubernetes workload. +type Resolver func(containerID string) bool + +// StartHandler receives only runtime-verified start events. +type StartHandler func(Start) + +func handshakeVersion(frame []byte) (uint64, error) { + var version uint64 + for len(frame) > 0 { + number, kind, n := protowire.ConsumeTag(frame) + if n < 0 { + return 0, errors.New("invalid handshake tag") + } + frame = frame[n:] + if number == 1 && kind == protowire.VarintType { + value, consumed := protowire.ConsumeVarint(frame) + if consumed < 0 { + return 0, errors.New("invalid handshake version") + } + version = value + frame = frame[consumed:] + continue + } + consumed := protowire.ConsumeFieldValue(number, kind, frame) + if consumed < 0 { + return 0, errors.New("invalid handshake field") + } + frame = frame[consumed:] + } + if version != protocolVersion { + return 0, fmt.Errorf("unsupported remote sink version %d", version) + } + return version, nil +} + +func decodeStartFrame(frame []byte, observedAt time.Time, resolve Resolver) (Start, bool, error) { + if len(frame) < headerSize { + return Start{}, false, errors.New("short remote sink header") + } + length := int(binary.LittleEndian.Uint16(frame[:2])) + if length < headerSize || length > len(frame) { + return Start{}, false, errors.New("invalid remote sink header size") + } + if binary.LittleEndian.Uint16(frame[2:4]) != startMessage { + return Start{}, false, nil // Forward-compatible: skip other point types. + } + id, err := startID(frame[length:]) + if err != nil { + return Start{}, false, err + } + if id == "" || resolve == nil || !resolve(id) { + return Start{}, false, nil + } + return Start{ + Source: "gvisor_trace", + ContainerID: id, + ObservedAt: observedAt, + Dropped: binary.LittleEndian.Uint32(frame[4:8]), + }, true, nil +} + +func startID(payload []byte) (string, error) { + var startID, contextID string + for len(payload) > 0 { + number, kind, n := protowire.ConsumeTag(payload) + if n < 0 { + return "", errors.New("invalid start tag") + } + payload = payload[n:] + if (number == 1 || number == 2) && kind == protowire.BytesType { + value, consumed := protowire.ConsumeBytes(payload) + if consumed < 0 { + return "", errors.New("invalid start field") + } + if number == 1 { + var err error + contextID, err = contextContainerID(value) + if err != nil { + return "", err + } + } else { + if len(value) > maxIDSize { + return "", errors.New("start ID exceeds limit") + } + startID = string(value) + } + payload = payload[consumed:] + continue + } + consumed := protowire.ConsumeFieldValue(number, kind, payload) + if consumed < 0 { + return "", errors.New("invalid start field") + } + payload = payload[consumed:] + } + if startID != "" && contextID != "" && startID != contextID { + return "", errors.New("start and context container IDs differ") + } + if startID != "" { + return startID, nil + } + return contextID, nil +} + +func contextContainerID(payload []byte) (string, error) { + var id string + for len(payload) > 0 { + number, kind, n := protowire.ConsumeTag(payload) + if n < 0 { + return "", errors.New("invalid context tag") + } + payload = payload[n:] + if number == 6 && kind == protowire.BytesType { + value, consumed := protowire.ConsumeBytes(payload) + if consumed < 0 || len(value) > maxIDSize { + return "", errors.New("invalid context container ID") + } + id = string(value) + payload = payload[consumed:] + continue + } + consumed := protowire.ConsumeFieldValue(number, kind, payload) + if consumed < 0 { + return "", errors.New("invalid context field") + } + payload = payload[consumed:] + } + return id, nil +} diff --git a/pkg/gvisor/start_test.go b/pkg/gvisor/start_test.go new file mode 100644 index 0000000000..893ba20cbb --- /dev/null +++ b/pkg/gvisor/start_test.go @@ -0,0 +1,95 @@ +package gvisor + +import ( + "encoding/binary" + "encoding/json" + "strings" + "testing" + "time" + + "google.golang.org/protobuf/encoding/protowire" +) + +func bytesField(number protowire.Number, value []byte) []byte { + field := protowire.AppendTag(nil, number, protowire.BytesType) + return protowire.AppendBytes(field, value) +} + +func startFrame(id, contextID, canary string) []byte { + context := bytesField(6, []byte(contextID)) + payload := bytesField(1, context) + payload = append(payload, bytesField(2, []byte(id))...) + payload = append(payload, bytesField(3, []byte("/work/"+canary))...) + payload = append(payload, bytesField(4, []byte("--token="+canary))...) + payload = append(payload, bytesField(5, []byte("KEY="+canary))...) + frame := make([]byte, headerSize) + binary.LittleEndian.PutUint16(frame[:2], headerSize) + binary.LittleEndian.PutUint16(frame[2:4], startMessage) + binary.LittleEndian.PutUint32(frame[4:8], 7) + return append(frame, payload...) +} + +func TestStartKeepsOnlyVerifiedIdentity(t *testing.T) { + const canary = "synthetic-secret-do-not-retain" + frame := startFrame("runtime-container-1", "runtime-container-1", canary) + when := time.Now() + event, ok, err := decodeStartFrame(frame, when, func(id string) bool { return id == "runtime-container-1" }) + if err != nil || !ok { + t.Fatalf("verified start: ok=%t err=%v", ok, err) + } + if event.Source != "gvisor_trace" || event.ContainerID != "runtime-container-1" || event.Dropped != 7 || !event.ObservedAt.Equal(when) { + t.Fatalf("unexpected retained event: %+v", event) + } + encoded, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), canary) || strings.Contains(string(encoded), "token") { + t.Fatalf("sensitive start fields escaped into retained event: %s", encoded) + } + _, ok, err = decodeStartFrame(frame, when, func(string) bool { return false }) + if err != nil || ok { + t.Fatalf("unverified ID was accepted: ok=%t err=%v", ok, err) + } +} + +func TestStartRejectsConflictingAndMalformedIdentity(t *testing.T) { + resolve := func(string) bool { return true } + frame := startFrame("one", "two", "canary") + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("conflicting IDs accepted: ok=%t err=%v", ok, err) + } + frame = startFrame(strings.Repeat("a", maxIDSize+1), "", "canary") + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("oversized ID accepted: ok=%t err=%v", ok, err) + } + frame = startFrame("one", "one", "canary") + frame[0] = 0xff + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("invalid header accepted: ok=%t err=%v", ok, err) + } + frame = append(startFrame("one", "one", "canary"), 0xff) + if _, ok, err := decodeStartFrame(frame, time.Now(), resolve); err == nil || ok { + t.Fatalf("malformed trailing field accepted: ok=%t err=%v", ok, err) + } +} + +func TestUnknownPointIsSkipped(t *testing.T) { + frame := startFrame("one", "one", "canary") + binary.LittleEndian.PutUint16(frame[2:4], 999) + if _, ok, err := decodeStartFrame(frame, time.Now(), func(string) bool { return true }); err != nil || ok { + t.Fatalf("unknown point: ok=%t err=%v", ok, err) + } +} + +func TestHandshakeVersion(t *testing.T) { + frame := protowire.AppendTag(nil, 1, protowire.VarintType) + frame = protowire.AppendVarint(frame, 1) + if _, err := handshakeVersion(frame); err != nil { + t.Fatal(err) + } + frame[len(frame)-1] = 2 + if _, err := handshakeVersion(frame); err == nil { + t.Fatal("unsupported version accepted") + } +} From 7853f55b2265ae834a2bedcf43b2ac8e737e9033 Mon Sep 17 00:00:00 2001 From: Daksh Pathak Date: Thu, 8 Oct 2026 13:24:25 +0530 Subject: [PATCH 2/4] fix(gvisor): interrupt blocked probe output on cancellation Signed-off-by: Daksh Pathak --- cmd/gvisor-start-probe/main_linux.go | 12 +- cmd/gvisor-start-probe/output_linux.go | 51 ++++++ cmd/gvisor-start-probe/output_linux_test.go | 165 ++++++++++++++++++++ docs/gvisor-start-probe.md | 7 + pkg/gvisor/receiver_linux.go | 12 +- pkg/gvisor/receiver_linux_test.go | 23 ++- pkg/gvisor/start.go | 6 +- 7 files changed, 265 insertions(+), 11 deletions(-) create mode 100644 cmd/gvisor-start-probe/output_linux.go create mode 100644 cmd/gvisor-start-probe/output_linux_test.go diff --git a/cmd/gvisor-start-probe/main_linux.go b/cmd/gvisor-start-probe/main_linux.go index c2bbd1f43d..a756a52058 100644 --- a/cmd/gvisor-start-probe/main_linux.go +++ b/cmd/gvisor-start-probe/main_linux.go @@ -6,7 +6,6 @@ package main import ( "context" - "encoding/json" "flag" "fmt" "os" @@ -24,14 +23,19 @@ func main() { fmt.Fprintln(os.Stderr, "container-id is required") os.Exit(2) } - encoder := json.NewEncoder(os.Stdout) + output, err := newJSONOutput(os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "gvisor start probe output: %v\n", err) + os.Exit(1) + } + defer output.file.Close() receiver := &gvisor.Receiver{ SocketPath: *socket, Resolve: func(id string) bool { return id == *containerID }, - OnStart: func(start gvisor.Start) { - _ = encoder.Encode(start) + OnStart: func(ctx context.Context, start gvisor.Start) { + _ = output.Encode(ctx, start) }, } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) diff --git a/cmd/gvisor-start-probe/output_linux.go b/cmd/gvisor-start-probe/output_linux.go new file mode 100644 index 0000000000..dcf9d8d956 --- /dev/null +++ b/cmd/gvisor-start-probe/output_linux.go @@ -0,0 +1,51 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/json" + "os" + "time" + + "github.com/kubescape/node-agent/pkg/gvisor" + "golang.org/x/sys/unix" +) + +type jsonOutput struct { + file *os.File + encoder *json.Encoder +} + +func newJSONOutput(output *os.File) (*jsonOutput, error) { + // Inherited stdout may be a blocking pipe. Register a nonblocking duplicate + // with Go's poller so a write deadline can interrupt a full pipe. The probe + // owns stdout; the duplicate shares its nonblocking status with the original. + fd, err := unix.FcntlInt(output.Fd(), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + return nil, err + } + if err := unix.SetNonblock(fd, true); err != nil { + unix.Close(fd) + return nil, err + } + file := os.NewFile(uintptr(fd), "probe-output") + return &jsonOutput{file: file, encoder: json.NewEncoder(file)}, nil +} + +func (o *jsonOutput) Encode(ctx context.Context, start gvisor.Start) error { + if err := ctx.Err(); err != nil { + return err + } + interrupted := make(chan struct{}) + stop := context.AfterFunc(ctx, func() { + // Regular files do not support deadlines; pipes and sockets do. + _ = o.file.SetWriteDeadline(time.Now()) + close(interrupted) + }) + err := o.encoder.Encode(start) + if !stop() { + <-interrupted + } + return err +} diff --git a/cmd/gvisor-start-probe/output_linux_test.go b/cmd/gvisor-start-probe/output_linux_test.go new file mode 100644 index 0000000000..14d6666b00 --- /dev/null +++ b/cmd/gvisor-start-probe/output_linux_test.go @@ -0,0 +1,165 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kubescape/node-agent/pkg/gvisor" + "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" +) + +func TestReceiverCancellationInterruptsFullOutputPipe(t *testing.T) { + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer reader.Close() // Keep the reader open throughout cancellation. + defer writer.Close() + fd := int(writer.Fd()) + if _, err := unix.FcntlInt(uintptr(fd), unix.F_SETPIPE_SZ, 4096); err != nil { + t.Fatal(err) + } + if err := unix.SetNonblock(fd, true); err != nil { + t.Fatal(err) + } + for { + _, err := unix.Write(fd, make([]byte, 4096)) + if err == unix.EAGAIN { + break + } + if err != nil { + t.Fatal(err) + } + } + // Mimic inherited stdout, which starts as a blocking descriptor. + if err := unix.SetNonblock(fd, false); err != nil { + t.Fatal(err) + } + output, err := newJSONOutput(writer) + if err != nil { + t.Fatal(err) + } + defer output.file.Close() + + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + entered := make(chan struct{}) + writeDone := make(chan error, 1) + receiver := &gvisor.Receiver{ + SocketPath: path, + Resolve: func(id string) bool { return id == "known" }, + OnStart: func(ctx context.Context, start gvisor.Start) { + close(entered) + writeDone <- output.Encode(ctx, start) + }, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- receiver.Run(ctx) }() + client := connectProbe(t, path) + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + if _, err := unix.Write(client, handshake); err != nil { + t.Fatal(err) + } + if _, err := unix.Read(client, make([]byte, 32)); err != nil { + t.Fatal(err) + } + frame := make([]byte, 8) + binary.LittleEndian.PutUint16(frame[:2], 8) + binary.LittleEndian.PutUint16(frame[2:4], 1) + frame = protowire.AppendTag(frame, 2, protowire.BytesType) + frame = protowire.AppendString(frame, "known") + if _, err := unix.Write(client, frame); err != nil { + t.Fatal(err) + } + unix.Close(client) + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("output callback did not start") + } + select { + case err := <-writeDone: + t.Fatalf("output did not block on the full pipe: %v", err) + case <-time.After(100 * time.Millisecond): + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("Run remained blocked after cancellation with pipe reader open") + } + if err := <-writeDone; !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("blocked write was not interrupted by deadline: %v", err) + } + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("receiver socket remains after shutdown: %v", err) + } +} + +func TestJSONOutputWritesNormally(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "events") + if err != nil { + t.Fatal(err) + } + defer file.Close() + output, err := newJSONOutput(file) + if err != nil { + t.Fatal(err) + } + defer output.file.Close() + for _, id := range []string{"first", "second", "third"} { + if err := output.Encode(context.Background(), gvisor.Start{ContainerID: id}); err != nil { + t.Fatal(err) + } + } + if _, err := file.Seek(0, 0); err != nil { + t.Fatal(err) + } + decoder := json.NewDecoder(file) + for _, id := range []string{"first", "second", "third"} { + var start gvisor.Start + if err := decoder.Decode(&start); err != nil || start.ContainerID != id { + t.Fatalf("normal output: got %q, want %q, err=%v", start.ContainerID, id, err) + } + } +} + +func connectProbe(t *testing.T, path string) int { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + if unix.Connect(fd, &unix.SockaddrUnix{Name: path}) == nil { + timeout := unix.NsecToTimeval((3 * time.Second).Nanoseconds()) + if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &timeout); err != nil { + t.Fatal(err) + } + return fd + } + unix.Close(fd) + time.Sleep(20 * time.Millisecond) + } + t.Fatal("could not connect to probe receiver") + return -1 +} diff --git a/docs/gvisor-start-probe.md b/docs/gvisor-start-probe.md index c159b0b205..33e38df538 100644 --- a/docs/gvisor-start-probe.md +++ b/docs/gvisor-start-probe.md @@ -77,6 +77,13 @@ discard, and a Linux socket exchange: go test ./pkg/gvisor -count=1 ``` +Receiver callbacks receive the caller's cancellation context and must interrupt +blocking work when it is canceled. The probe uses a write deadline to interrupt +blocked pipe output on SIGINT or SIGTERM, allowing the receiver to finish and +remove its socket even if the output consumer stops reading. Tests in +`cmd/gvisor-start-probe` cover a full output pipe and ordinary JSON output; +receiver tests also cover draining a closed event queue in order. + This document records the procedure, not results. Live `runsc` and node-agent host eBPF observations must be added after the Linux trial; the existing [gVisor proof of concept](https://github.com/yellow-forrest/gvisor-visibility-poc) diff --git a/pkg/gvisor/receiver_linux.go b/pkg/gvisor/receiver_linux.go index 6eb65bae98..5ca02b440d 100644 --- a/pkg/gvisor/receiver_linux.go +++ b/pkg/gvisor/receiver_linux.go @@ -73,9 +73,7 @@ func (r *Receiver) Run(ctx context.Context) error { workerDone := make(chan struct{}) go func() { defer close(workerDone) - for start := range starts { - r.OnStart(start) - } + r.deliverStarts(ctx, starts) }() var clients sync.WaitGroup @@ -121,6 +119,14 @@ func (r *Receiver) Run(ctx context.Context) error { } } +func (r *Receiver) deliverStarts(ctx context.Context, starts <-chan Start) { + for start := range starts { + // Use the caller's context: an internal listener error still drains + // queued events, while caller cancellation interrupts the handler. + r.OnStart(ctx, start) + } +} + func privateParent(path string) error { info, err := os.Stat(path) if err != nil { diff --git a/pkg/gvisor/receiver_linux_test.go b/pkg/gvisor/receiver_linux_test.go index 5b689f6777..5b85945db6 100644 --- a/pkg/gvisor/receiver_linux_test.go +++ b/pkg/gvisor/receiver_linux_test.go @@ -26,7 +26,7 @@ func TestReceiverVerifiesIdentityAndDropsSensitiveFields(t *testing.T) { receiver := &Receiver{ SocketPath: path, Resolve: func(id string) bool { return id == "known-container" }, - OnStart: func(start Start) { starts <- start }, + OnStart: func(_ context.Context, start Start) { starts <- start }, } ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -84,7 +84,7 @@ func TestReceiverKeepsConcurrentConnectionsSeparate(t *testing.T) { receiver := &Receiver{ SocketPath: path, Resolve: func(id string) bool { return id == "first" || id == "second" }, - OnStart: func(start Start) { starts <- start }, + OnStart: func(_ context.Context, start Start) { starts <- start }, } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) @@ -133,6 +133,25 @@ func TestReceiverKeepsConcurrentConnectionsSeparate(t *testing.T) { } } +func TestReceiverDrainsClosedQueue(t *testing.T) { + starts := make(chan Start, 3) + for _, id := range []string{"first", "second", "third"} { + starts <- Start{ContainerID: id} + } + close(starts) + var delivered []string + receiver := &Receiver{OnStart: func(ctx context.Context, start Start) { + if ctx.Err() != nil { + t.Fatal("normal queue drain received a canceled context") + } + delivered = append(delivered, start.ContainerID) + }} + receiver.deliverStarts(context.Background(), starts) + if strings.Join(delivered, ",") != "first,second,third" { + t.Fatalf("queue did not drain in order: %v", delivered) + } +} + func connectReceiver(t *testing.T, path string) int { t.Helper() deadline := time.Now().Add(3 * time.Second) diff --git a/pkg/gvisor/start.go b/pkg/gvisor/start.go index 6c40f77442..cb1582bd04 100644 --- a/pkg/gvisor/start.go +++ b/pkg/gvisor/start.go @@ -5,6 +5,7 @@ package gvisor import ( + "context" "encoding/binary" "errors" "fmt" @@ -37,8 +38,9 @@ type Start struct { // The receiver never attributes an unverified claim to a Kubernetes workload. type Resolver func(containerID string) bool -// StartHandler receives only runtime-verified start events. -type StartHandler func(Start) +// StartHandler receives only runtime-verified start events. It must honor ctx +// cancellation, including interrupting any blocking I/O, so Run can finish. +type StartHandler func(context.Context, Start) func handshakeVersion(frame []byte) (uint64, error) { var version uint64 From 1cd0e2a5fac9f6342aa24777ffb766fc89fa0f8a Mon Sep 17 00:00:00 2001 From: Daksh Pathak Date: Fri, 9 Oct 2026 17:19:30 +0530 Subject: [PATCH 3/4] fix(gvisor): restore inherited output flags before exit Signed-off-by: Daksh Pathak --- cmd/gvisor-start-probe/main_linux.go | 18 +++- cmd/gvisor-start-probe/main_linux_test.go | 106 ++++++++++++++++++++ cmd/gvisor-start-probe/output_linux.go | 39 ++++++- cmd/gvisor-start-probe/output_linux_test.go | 4 +- docs/gvisor-start-probe.md | 2 + 5 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 cmd/gvisor-start-probe/main_linux_test.go diff --git a/cmd/gvisor-start-probe/main_linux.go b/cmd/gvisor-start-probe/main_linux.go index a756a52058..ac675b4df6 100644 --- a/cmd/gvisor-start-probe/main_linux.go +++ b/cmd/gvisor-start-probe/main_linux.go @@ -16,19 +16,28 @@ import ( ) func main() { + os.Exit(run()) +} + +func run() (exitCode int) { socket := flag.String("socket", "/run/kubescape/gvisor-events.sock", "private Unix socket for the SecCheck remote sink") containerID := flag.String("container-id", "", "exact container ID obtained independently from the local runtime before start") flag.Parse() if *containerID == "" { fmt.Fprintln(os.Stderr, "container-id is required") - os.Exit(2) + return 2 } output, err := newJSONOutput(os.Stdout) if err != nil { fmt.Fprintf(os.Stderr, "gvisor start probe output: %v\n", err) - os.Exit(1) + return 1 } - defer output.file.Close() + defer func() { + if err := output.Close(); err != nil { + fmt.Fprintf(os.Stderr, "gvisor start probe output cleanup: %v\n", err) + exitCode = 1 + } + }() receiver := &gvisor.Receiver{ SocketPath: *socket, Resolve: func(id string) bool { @@ -42,6 +51,7 @@ func main() { defer stop() if err := receiver.Run(ctx); err != nil { fmt.Fprintf(os.Stderr, "gvisor start probe: %v\n", err) - os.Exit(1) + return 1 } + return 0 } diff --git a/cmd/gvisor-start-probe/main_linux_test.go b/cmd/gvisor-start-probe/main_linux_test.go new file mode 100644 index 0000000000..ff77d75c84 --- /dev/null +++ b/cmd/gvisor-start-probe/main_linux_test.go @@ -0,0 +1,106 @@ +//go:build linux + +package main + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func TestProbeRestoresInheritedOutputFlags(t *testing.T) { + binary := filepath.Join(t.TempDir(), "gvisor-start-probe") + build := exec.Command("go", "build", "-buildvcs=false", "-o", binary, ".") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("building probe: %v\n%s", err, output) + } + for _, mode := range []struct { + name string + flags int + }{{"blocking", 0}, {"nonblocking", unix.O_NONBLOCK}} { + for _, failure := range []bool{false, true} { + name := mode.name + "/signal_exit" + if failure { + name = mode.name + "/invalid_socket_directory" + } + t.Run(name, func(t *testing.T) { + fds := make([]int, 2) + if err := unix.Pipe2(fds, unix.O_CLOEXEC|mode.flags); err != nil { + t.Fatal(err) + } + reader := os.NewFile(uintptr(fds[0]), "reader") + writer := os.NewFile(uintptr(fds[1]), "writer") + defer reader.Close() + defer writer.Close() + original, err := unix.FcntlInt(uintptr(fds[1]), unix.F_GETFL, 0) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.sock") + if failure { + path = filepath.Join(dir, "missing", "events.sock") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, binary, "--socket", path, "--container-id", "known") + cmd.Stdout = writer + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + if !failure { + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Lstat(path); err == nil { + break + } + if time.Now().After(deadline) { + _ = cmd.Process.Kill() + _ = cmd.Wait() + t.Fatalf("probe did not create socket: %s", stderr.String()) + } + time.Sleep(20 * time.Millisecond) + } + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + } + err = cmd.Wait() + if ctx.Err() != nil { + t.Fatalf("probe did not exit: %v", ctx.Err()) + } + if failure { + if cmd.ProcessState.ExitCode() != 1 || stderr.Len() == 0 { + t.Fatalf("expected receiver setup error: exit=%d stderr=%s", cmd.ProcessState.ExitCode(), stderr.String()) + } + } else if err != nil { + t.Fatalf("normal probe exit: %v\n%s", err, stderr.String()) + } + after, err := unix.FcntlInt(uintptr(fds[1]), unix.F_GETFL, 0) + if err != nil { + t.Fatal(err) + } + if after != original { + t.Fatalf("probe changed parent's output flags: before=%#x after=%#x", original, after) + } + if !failure { + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("socket remains after exit: %v", err) + } + } + }) + } + } +} diff --git a/cmd/gvisor-start-probe/output_linux.go b/cmd/gvisor-start-probe/output_linux.go index dcf9d8d956..fb8f37377a 100644 --- a/cmd/gvisor-start-probe/output_linux.go +++ b/cmd/gvisor-start-probe/output_linux.go @@ -5,7 +5,9 @@ package main import ( "context" "encoding/json" + "errors" "os" + "sync" "time" "github.com/kubescape/node-agent/pkg/gvisor" @@ -13,24 +15,51 @@ import ( ) type jsonOutput struct { - file *os.File - encoder *json.Encoder + file *os.File + encoder *json.Encoder + fd uintptr + flags int + closeOnce sync.Once + closeErr error } func newJSONOutput(output *os.File) (*jsonOutput, error) { // Inherited stdout may be a blocking pipe. Register a nonblocking duplicate // with Go's poller so a write deadline can interrupt a full pipe. The probe - // owns stdout; the duplicate shares its nonblocking status with the original. - fd, err := unix.FcntlInt(output.Fd(), unix.F_DUPFD_CLOEXEC, 0) + // must restore the shared flags before exiting. SyscallConn avoids Fd(), + // which can itself switch a Go-managed pipe back to blocking mode. + raw, err := output.SyscallConn() if err != nil { return nil, err } + var flags, fd int + var setupErr error + if err := raw.Control(func(original uintptr) { + flags, setupErr = unix.FcntlInt(original, unix.F_GETFL, 0) + if setupErr == nil { + fd, setupErr = unix.FcntlInt(original, unix.F_DUPFD_CLOEXEC, 0) + } + }); err != nil { + return nil, err + } + if setupErr != nil { + return nil, setupErr + } if err := unix.SetNonblock(fd, true); err != nil { unix.Close(fd) return nil, err } file := os.NewFile(uintptr(fd), "probe-output") - return &jsonOutput{file: file, encoder: json.NewEncoder(file)}, nil + return &jsonOutput{file: file, encoder: json.NewEncoder(file), fd: uintptr(fd), flags: flags}, nil +} + +// Close restores the inherited file description after all output has finished. +func (o *jsonOutput) Close() error { + o.closeOnce.Do(func() { + _, restoreErr := unix.FcntlInt(o.fd, unix.F_SETFL, o.flags) + o.closeErr = errors.Join(restoreErr, o.file.Close()) + }) + return o.closeErr } func (o *jsonOutput) Encode(ctx context.Context, start gvisor.Start) error { diff --git a/cmd/gvisor-start-probe/output_linux_test.go b/cmd/gvisor-start-probe/output_linux_test.go index 14d6666b00..9f638b7263 100644 --- a/cmd/gvisor-start-probe/output_linux_test.go +++ b/cmd/gvisor-start-probe/output_linux_test.go @@ -48,7 +48,7 @@ func TestReceiverCancellationInterruptsFullOutputPipe(t *testing.T) { if err != nil { t.Fatal(err) } - defer output.file.Close() + defer output.Close() dir := t.TempDir() if err := os.Chmod(dir, 0700); err != nil { @@ -124,7 +124,7 @@ func TestJSONOutputWritesNormally(t *testing.T) { if err != nil { t.Fatal(err) } - defer output.file.Close() + defer output.Close() for _, id := range []string{"first", "second", "third"} { if err := output.Encode(context.Background(), gvisor.Start{ContainerID: id}); err != nil { t.Fatal(err) diff --git a/docs/gvisor-start-probe.md b/docs/gvisor-start-probe.md index 33e38df538..5c069fd60f 100644 --- a/docs/gvisor-start-probe.md +++ b/docs/gvisor-start-probe.md @@ -83,6 +83,8 @@ blocked pipe output on SIGINT or SIGTERM, allowing the receiver to finish and remove its socket even if the output consumer stops reading. Tests in `cmd/gvisor-start-probe` cover a full output pipe and ordinary JSON output; receiver tests also cover draining a closed event queue in order. +The probe restores stdout's original file flags on both normal and error exits, +so a parent process sharing the inherited pipe or terminal keeps its prior mode. This document records the procedure, not results. Live `runsc` and node-agent host eBPF observations must be added after the Linux trial; the existing From 858e3bc8077be21280da94aa267a582ebd515330 Mon Sep 17 00:00:00 2001 From: Daksh Pathak Date: Fri, 9 Oct 2026 18:29:34 +0530 Subject: [PATCH 4/4] fix(gvisor): stop the probe when output fails Signed-off-by: Daksh Pathak --- cmd/gvisor-start-probe/main_linux.go | 31 ++++++++++--- cmd/gvisor-start-probe/main_linux_test.go | 55 +++++++++++++++++------ docs/gvisor-start-probe.md | 4 ++ 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/cmd/gvisor-start-probe/main_linux.go b/cmd/gvisor-start-probe/main_linux.go index ac675b4df6..348e688420 100644 --- a/cmd/gvisor-start-probe/main_linux.go +++ b/cmd/gvisor-start-probe/main_linux.go @@ -6,6 +6,7 @@ package main import ( "context" + "errors" "flag" "fmt" "os" @@ -38,19 +39,39 @@ func run() (exitCode int) { exitCode = 1 } }() + signalCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithCancel(signalCtx) + defer cancel() + outputErrors := make(chan error, 1) receiver := &gvisor.Receiver{ SocketPath: *socket, Resolve: func(id string) bool { return id == *containerID }, OnStart: func(ctx context.Context, start gvisor.Start) { - _ = output.Encode(ctx, start) + if err := output.Encode(ctx, start); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || + (ctx.Err() != nil && errors.Is(err, os.ErrDeadlineExceeded)) { + return + } + select { + case outputErrors <- err: + default: + } + cancel() + } }, } - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - if err := receiver.Run(ctx); err != nil { - fmt.Fprintf(os.Stderr, "gvisor start probe: %v\n", err) + runErr := receiver.Run(ctx) + select { + case err := <-outputErrors: + fmt.Fprintf(os.Stderr, "gvisor start probe output: %v\n", err) + return 1 + default: + } + if runErr != nil { + fmt.Fprintf(os.Stderr, "gvisor start probe: %v\n", runErr) return 1 } return 0 diff --git a/cmd/gvisor-start-probe/main_linux_test.go b/cmd/gvisor-start-probe/main_linux_test.go index ff77d75c84..4854300265 100644 --- a/cmd/gvisor-start-probe/main_linux_test.go +++ b/cmd/gvisor-start-probe/main_linux_test.go @@ -5,19 +5,22 @@ package main import ( "bytes" "context" + "encoding/binary" "os" "os/exec" "path/filepath" + "strings" "syscall" "testing" "time" "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protowire" ) func TestProbeRestoresInheritedOutputFlags(t *testing.T) { - binary := filepath.Join(t.TempDir(), "gvisor-start-probe") - build := exec.Command("go", "build", "-buildvcs=false", "-o", binary, ".") + probeBinary := filepath.Join(t.TempDir(), "gvisor-start-probe") + build := exec.Command("go", "build", "-buildvcs=false", "-o", probeBinary, ".") if output, err := build.CombinedOutput(); err != nil { t.Fatalf("building probe: %v\n%s", err, output) } @@ -25,11 +28,8 @@ func TestProbeRestoresInheritedOutputFlags(t *testing.T) { name string flags int }{{"blocking", 0}, {"nonblocking", unix.O_NONBLOCK}} { - for _, failure := range []bool{false, true} { - name := mode.name + "/signal_exit" - if failure { - name = mode.name + "/invalid_socket_directory" - } + for _, scenario := range []string{"signal_exit", "invalid_socket_directory", "closed_output"} { + name := mode.name + "/" + scenario t.Run(name, func(t *testing.T) { fds := make([]int, 2) if err := unix.Pipe2(fds, unix.O_CLOEXEC|mode.flags); err != nil { @@ -39,6 +39,11 @@ func TestProbeRestoresInheritedOutputFlags(t *testing.T) { writer := os.NewFile(uintptr(fds[1]), "writer") defer reader.Close() defer writer.Close() + if scenario == "closed_output" { + if err := reader.Close(); err != nil { + t.Fatal(err) + } + } original, err := unix.FcntlInt(uintptr(fds[1]), unix.F_GETFL, 0) if err != nil { t.Fatal(err) @@ -48,19 +53,19 @@ func TestProbeRestoresInheritedOutputFlags(t *testing.T) { t.Fatal(err) } path := filepath.Join(dir, "events.sock") - if failure { + if scenario == "invalid_socket_directory" { path = filepath.Join(dir, "missing", "events.sock") } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - cmd := exec.CommandContext(ctx, binary, "--socket", path, "--container-id", "known") + cmd := exec.CommandContext(ctx, probeBinary, "--socket", path, "--container-id", "known") cmd.Stdout = writer var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Start(); err != nil { t.Fatal(err) } - if !failure { + if scenario != "invalid_socket_directory" { deadline := time.Now().Add(3 * time.Second) for { if _, err := os.Lstat(path); err == nil { @@ -73,7 +78,26 @@ func TestProbeRestoresInheritedOutputFlags(t *testing.T) { } time.Sleep(20 * time.Millisecond) } - if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + if scenario == "closed_output" { + client := connectProbe(t, path) + handshake := protowire.AppendTag(nil, 1, protowire.VarintType) + handshake = protowire.AppendVarint(handshake, 1) + if _, err := unix.Write(client, handshake); err != nil { + t.Fatal(err) + } + if _, err := unix.Read(client, make([]byte, 32)); err != nil { + t.Fatal(err) + } + frame := make([]byte, 8) + binary.LittleEndian.PutUint16(frame[:2], 8) + binary.LittleEndian.PutUint16(frame[2:4], 1) + frame = protowire.AppendTag(frame, 2, protowire.BytesType) + frame = protowire.AppendString(frame, "known") + if _, err := unix.Write(client, frame); err != nil { + t.Fatal(err) + } + unix.Close(client) + } else if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { t.Fatal(err) } } @@ -81,9 +105,12 @@ func TestProbeRestoresInheritedOutputFlags(t *testing.T) { if ctx.Err() != nil { t.Fatalf("probe did not exit: %v", ctx.Err()) } - if failure { + if scenario != "signal_exit" { if cmd.ProcessState.ExitCode() != 1 || stderr.Len() == 0 { - t.Fatalf("expected receiver setup error: exit=%d stderr=%s", cmd.ProcessState.ExitCode(), stderr.String()) + t.Fatalf("expected probe failure: exit=%d stderr=%s", cmd.ProcessState.ExitCode(), stderr.String()) + } + if scenario == "closed_output" && !strings.Contains(stderr.String(), "gvisor start probe output:") { + t.Fatalf("missing output failure diagnostic: %s", stderr.String()) } } else if err != nil { t.Fatalf("normal probe exit: %v\n%s", err, stderr.String()) @@ -95,7 +122,7 @@ func TestProbeRestoresInheritedOutputFlags(t *testing.T) { if after != original { t.Fatalf("probe changed parent's output flags: before=%#x after=%#x", original, after) } - if !failure { + if scenario != "invalid_socket_directory" { if _, err := os.Lstat(path); !os.IsNotExist(err) { t.Fatalf("socket remains after exit: %v", err) } diff --git a/docs/gvisor-start-probe.md b/docs/gvisor-start-probe.md index 5c069fd60f..eaf4a60c39 100644 --- a/docs/gvisor-start-probe.md +++ b/docs/gvisor-start-probe.md @@ -85,6 +85,10 @@ remove its socket even if the output consumer stops reading. Tests in receiver tests also cover draining a closed event queue in order. The probe restores stdout's original file flags on both normal and error exits, so a parent process sharing the inherited pipe or terminal keeps its prior mode. +If output fails, the probe stops collection, reports the first write error on +stderr, and exits with status 1. Signal cancellation of a blocked write remains +a normal exit. Actual-process tests cover a closed output reader as well as +flag restoration and signal cancellation. This document records the procedure, not results. Live `runsc` and node-agent host eBPF observations must be added after the Linux trial; the existing