feat(server): Add configurable network rate limiting for MicroVMs

docs/reference/configuration.md

@@ -147,6 +147,67 @@ pools:
       #
       vcpu_count: 2
     #
+    # Firecracker network interface configuration.
+    #
+    # Default: {} (no rate limiting)
+    #
+    network_interface:
+      #
+      # Rate limiter for incoming (ingress) traffic. Maps to the Firecracker
+      # network interface `rx_rate_limiter`. Both the `bandwidth` and `ops`
+      # token buckets are optional, an omitted bucket means unlimited.
+      #
+      # The resulting rate is `size` / `refill_time`.
+      #
+      # Default: {} (unlimited)
+      #
+      in_rate_limiter:
+        #
+        # Token bucket with bytes as tokens.
+        #
+        # Default: {} (unlimited)
+        #
+        bandwidth:
+          #
+          # The total number of tokens (bytes) the bucket can hold.
+          #
+          # Required: true
+          #
+          size: 131072000
+          #
+          # The amount of milliseconds it takes for the bucket to refill.
+          # 131072000 bytes per 1000 ms is ~125 MiB/s.
+          #
+          # Required: true
+          #
+          refill_time: 1000
+          #
+          # The initial burst size (bytes). Consumed before the refill process
+          # starts happening.
+          #
+          # Default: 0
+          #
+          one_time_burst: 262144000
+        #
+        # Token bucket with operations (packets) as tokens.
+        #
+        # Default: {} (unlimited)
+        #
+        ops:
+          size: 10000
+          refill_time: 1000
+      #
+      # Rate limiter for outgoing (egress) traffic. Maps to the Firecracker
+      # network interface `tx_rate_limiter`. Same structure as
+      # `in_rate_limiter`.
+      #
+      # Default: {} (unlimited)
+      #
+      out_rate_limiter:
+        bandwidth:
+          size: 26214400
+          refill_time: 1000
+    #
     # Metadata to pass to the Firecracker VM via MMDS.
     #
     # Default: {}

server/config.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/firecracker-microvm/firecracker-go-sdk"
+	"github.com/firecracker-microvm/firecracker-go-sdk/client/models"
 	"github.com/go-playground/validator/v10"
 	"gopkg.in/yaml.v3"
 )
@@ -16,7 +18,7 @@ type Config struct {
 	BasicAuthEnabled bool              `yaml:"basic_auth_enabled" validate:""`
 	BasicAuthUsers   map[string]string `yaml:"basic_auth_users" validate:"required_if=basic_auth_enabled true"`
 	GitHub           *GitHubConfig     `yaml:"github" validate:"required"`
-	Pools            []*PoolConfig     `yaml:"pools" validate:"required,min=1"`
+	Pools            []*PoolConfig     `yaml:"pools" validate:"required,min=1,dive,required"`
 	LogLevel         string            `yaml:"log_level" validate:"required,oneof=debug info warn error fatal panic trace"`
 
 	path string
@@ -47,18 +49,67 @@ type RunnerConfig struct {
 }
 
 type FirecrackerConfig struct {
-	BinaryPath      string                   `yaml:"binary_path" `
-	KernelImagePath string                   `yaml:"kernel_image_path"`
-	KernelArgs      string                   `yaml:"kernel_args"`
-	MachineConfig   FirecrackerMachineConfig `yaml:"machine_config"`
-	Metadata        map[string]interface{}   `yaml:"metadata"`
+	BinaryPath       string                             `yaml:"binary_path" `
+	KernelImagePath  string                             `yaml:"kernel_image_path"`
+	KernelArgs       string                             `yaml:"kernel_args"`
+	MachineConfig    FirecrackerMachineConfig           `yaml:"machine_config"`
+	NetworkInterface *FirecrackerNetworkInterfaceConfig `yaml:"network_interface"`
+	Metadata         map[string]interface{}             `yaml:"metadata"`
 }
 
 type FirecrackerMachineConfig struct {
 	VcpuCount  int64 `yaml:"vcpu_count"`
 	MemSizeMib int64 `yaml:"mem_size_mib"`
 }
 
+// FirecrackerNetworkInterfaceConfig configures the MicroVM's network interface.
+// Rate limiters are optional, a nil limiter leaves that direction unlimited.
+type FirecrackerNetworkInterfaceConfig struct {
+	InRateLimiter  *FirecrackerRateLimiterConfig `yaml:"in_rate_limiter"`
+	OutRateLimiter *FirecrackerRateLimiterConfig `yaml:"out_rate_limiter"`
+}
+
+// FirecrackerRateLimiterConfig defines an IO rate limiter with independent
+// bytes/s and ops/s limits. A nil token bucket leaves that limit unlimited.
+type FirecrackerRateLimiterConfig struct {
+	Bandwidth *FirecrackerTokenBucketConfig `yaml:"bandwidth"`
+	Ops       *FirecrackerTokenBucketConfig `yaml:"ops"`
+}
+
+// FirecrackerTokenBucketConfig defines a token bucket with a maximum capacity
+// (Size), an optional initial burst size (OneTimeBurst) and the interval in
+// milliseconds it takes to refill the bucket (RefillTime). The resulting rate
+// is Size / RefillTime.
+type FirecrackerTokenBucketConfig struct {
+	Size         int64  `yaml:"size" validate:"required,gt=0"`
+	OneTimeBurst *int64 `yaml:"one_time_burst" validate:"omitempty,gte=0"`
+	RefillTime   int64  `yaml:"refill_time" validate:"required,gt=0"`
+}
+
+// toSDK converts the rate limiter configuration into its Firecracker SDK
+// representation. Returns nil if the rate limiter isn't configured.
+func (c *FirecrackerRateLimiterConfig) toSDK() *models.RateLimiter {
+	if c == nil {
+		return nil
+	}
+
+	return &models.RateLimiter{Bandwidth: c.Bandwidth.toSDK(), Ops: c.Ops.toSDK()}
+}
+
+// toSDK converts the token bucket configuration into its Firecracker SDK
+// representation. Returns nil if the token bucket isn't configured.
+func (c *FirecrackerTokenBucketConfig) toSDK() *models.TokenBucket {
+	if c == nil {
+		return nil
+	}
+
+	return &models.TokenBucket{
+		Size:         firecracker.Int64(c.Size),
+		RefillTime:   firecracker.Int64(c.RefillTime),
+		OneTimeBurst: c.OneTimeBurst,
+	}
+}
+
 // DefaultConfig creates a new Config with default values.
 func DefaultConfig() *Config {
 	c := &Config{

server/config_test.go

@@ -3,6 +3,8 @@ package server
 import (
 	"testing"
 
+	"github.com/firecracker-microvm/firecracker-go-sdk"
+	"github.com/firecracker-microvm/firecracker-go-sdk/client/models"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -14,3 +16,52 @@ func TestNewConfig(t *testing.T) {
 
 	assert.Equal(t, "testdata/config1.yaml", config.path)
 }
+
+func TestNewConfigNetworkInterface(t *testing.T) {
+	config, err := NewConfig("testdata/config1.yaml")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	networkInterface := config.Pools[0].Firecracker.NetworkInterface
+	if networkInterface == nil {
+		t.Fatal("expected network interface configuration to be set")
+	}
+
+	assert.Equal(t, &FirecrackerTokenBucketConfig{
+		Size: 131072000, RefillTime: 1000, OneTimeBurst: firecracker.Int64(262144000),
+	}, networkInterface.InRateLimiter.Bandwidth)
+	assert.Equal(t, &FirecrackerTokenBucketConfig{
+		Size: 10000, RefillTime: 1000,
+	}, networkInterface.InRateLimiter.Ops)
+	assert.Equal(t, &FirecrackerTokenBucketConfig{
+		Size: 26214400, RefillTime: 1000,
+	}, networkInterface.OutRateLimiter.Bandwidth)
+	assert.Nil(t, networkInterface.OutRateLimiter.Ops)
+
+	// A pool without a network_interface block leaves the interface unlimited.
+	assert.Nil(t, config.Pools[1].Firecracker.NetworkInterface)
+}
+
+func TestNewConfigNetworkInterfaceInvalid(t *testing.T) {
+	// A token bucket without a size is rejected.
+	_, err := NewConfig("testdata/config2.yaml")
+	assert.ErrorContains(t, err, "Config.Pools[0].Firecracker.NetworkInterface.InRateLimiter.Bandwidth.Size")
+}
+
+func TestFirecrackerRateLimiterConfigToSDK(t *testing.T) {
+	var nilRateLimiter *FirecrackerRateLimiterConfig
+	assert.Nil(t, nilRateLimiter.toSDK())
+
+	rateLimiter := &FirecrackerRateLimiterConfig{
+		Bandwidth: &FirecrackerTokenBucketConfig{Size: 131072000, RefillTime: 1000, OneTimeBurst: firecracker.Int64(262144000)},
+	}
+
+	assert.Equal(t, &models.RateLimiter{
+		Bandwidth: &models.TokenBucket{
+			Size:         firecracker.Int64(131072000),
+			RefillTime:   firecracker.Int64(1000),
+			OneTimeBurst: firecracker.Int64(262144000),
+		},
+	}, rateLimiter.toSDK())
+}

server/pool.go

@@ -461,6 +461,16 @@ func (p *Pool) createMachine(ctx context.Context) error {
 	vsockPath := filepath.Join(p.GetDir(), fmt.Sprintf("%s.vsock", runnerName))
 	vsockCID := p.nextCID.Add(1)
 
+	networkInterface := firecracker.NetworkInterface{
+		AllowMMDS:        true,
+		CNIConfiguration: &firecracker.CNIConfiguration{NetworkName: "fireactions", IfName: "eth0", ConfDir: "/etc/cni/net.d", BinPath: []string{"/opt/cni/bin"}},
+	}
+
+	if networkInterfaceConfig := p.config.Firecracker.NetworkInterface; networkInterfaceConfig != nil {
+		networkInterface.InRateLimiter = networkInterfaceConfig.InRateLimiter.toSDK()
+		networkInterface.OutRateLimiter = networkInterfaceConfig.OutRateLimiter.toSDK()
+	}
+
 	fcMachine, err := firecracker.NewMachine(ctx, firecracker.Config{
 		VMID:            runnerName,
 		SocketPath:      filepath.Join(p.GetDir(), fmt.Sprintf("%s.sock", runnerName)),
@@ -476,16 +486,13 @@ func (p *Pool) createMachine(ctx context.Context) error {
 			IsRootDevice: firecracker.Bool(true),
 			IsReadOnly:   firecracker.Bool(false),
 		}},
-		NetworkInterfaces: []firecracker.NetworkInterface{{
-			AllowMMDS:        true,
-			CNIConfiguration: &firecracker.CNIConfiguration{NetworkName: "fireactions", IfName: "eth0", ConfDir: "/etc/cni/net.d", BinPath: []string{"/opt/cni/bin"}},
-		}},
-		VsockDevices:   []firecracker.VsockDevice{{Path: vsockPath, CID: vsockCID}},
-		MmdsAddress:    net.IPv4(169, 254, 169, 254),
-		MmdsVersion:    firecracker.MMDSv2,
-		ForwardSignals: []os.Signal{},
-		LogPath:        filepath.Join(p.GetDir(), fmt.Sprintf("%s.firecracker.log", runnerName)),
-		LogLevel:       "Debug",
+		NetworkInterfaces: []firecracker.NetworkInterface{networkInterface},
+		VsockDevices:      []firecracker.VsockDevice{{Path: vsockPath, CID: vsockCID}},
+		MmdsAddress:       net.IPv4(169, 254, 169, 254),
+		MmdsVersion:       firecracker.MMDSv2,
+		ForwardSignals:    []os.Signal{},
+		LogPath:           filepath.Join(p.GetDir(), fmt.Sprintf("%s.firecracker.log", runnerName)),
+		LogLevel:          "Debug",
 	}, firecracker.WithProcessRunner(machineCmd), firecracker.WithLogger(logrus.NewEntry(logger)))
 	if err != nil {
 		return fmt.Errorf("firecracker: creating machine: %w", err)

server/testdata/config1.yaml

@@ -37,6 +37,19 @@ pools:
     machine_config:
       mem_size_mib: 2048
       vcpu_count: 2
+    network_interface:
+      in_rate_limiter:
+        bandwidth:
+          size: 131072000
+          refill_time: 1000
+          one_time_burst: 262144000
+        ops:
+          size: 10000
+          refill_time: 1000
+      out_rate_limiter:
+        bandwidth:
+          size: 26214400
+          refill_time: 1000
     metadata:
       example1: value1
 - name: fireactions-2vcpu-4gb

server/testdata/config2.yaml

@@ -0,0 +1,32 @@
+---
+bind_address: 0.0.0.0:8080
+
+github:
+  app_private_key: |
+    -----BEGIN RSA PRIVATE KEY-----
+  app_id: 12345
+
+pools:
+- name: fireactions-2vcpu-2gb
+  replicas: 1
+  runner:
+    name: fireactions-2vcpu-2gb
+    image: ghcr.io/hostinger/fireactions/runner:ubuntu-20.04-x64-2.310.2
+    image_pull_policy: IfNotPresent
+    group_id: 1
+    organization: hostinger
+    labels:
+    - self-hosted
+  firecracker:
+    binary_path: firecracker
+    kernel_image_path: /var/lib/fireactions/vmlinux
+    kernel_args: "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules rw"
+    machine_config:
+      mem_size_mib: 2048
+      vcpu_count: 2
+    network_interface:
+      in_rate_limiter:
+        bandwidth:
+          refill_time: 1000
+
+log_level: debug