diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 6b8a76d..bb84031 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -147,6 +147,67 @@ pools: # vcpu_count: 2 # + # Firecracker network interface configuration. + # + # Default: {} (no rate limiting) + # + network_interface: + # + # Rate limiter for incoming (ingress) traffic. Maps to the Firecracker + # network interface `rx_rate_limiter`. Both the `bandwidth` and `ops` + # token buckets are optional, an omitted bucket means unlimited. + # + # The resulting rate is `size` / `refill_time`. + # + # Default: {} (unlimited) + # + in_rate_limiter: + # + # Token bucket with bytes as tokens. + # + # Default: {} (unlimited) + # + bandwidth: + # + # The total number of tokens (bytes) the bucket can hold. + # + # Required: true + # + size: 131072000 + # + # The amount of milliseconds it takes for the bucket to refill. + # 131072000 bytes per 1000 ms is ~125 MiB/s. + # + # Required: true + # + refill_time: 1000 + # + # The initial burst size (bytes). Consumed before the refill process + # starts happening. + # + # Default: 0 + # + one_time_burst: 262144000 + # + # Token bucket with operations (packets) as tokens. + # + # Default: {} (unlimited) + # + ops: + size: 10000 + refill_time: 1000 + # + # Rate limiter for outgoing (egress) traffic. Maps to the Firecracker + # network interface `tx_rate_limiter`. Same structure as + # `in_rate_limiter`. + # + # Default: {} (unlimited) + # + out_rate_limiter: + bandwidth: + size: 26214400 + refill_time: 1000 + # # Metadata to pass to the Firecracker VM via MMDS. # # Default: {} diff --git a/server/config.go b/server/config.go index b267348..a50fd50 100644 --- a/server/config.go +++ b/server/config.go @@ -4,6 +4,8 @@ import ( "fmt" "os" + "github.com/firecracker-microvm/firecracker-go-sdk" + "github.com/firecracker-microvm/firecracker-go-sdk/client/models" "github.com/go-playground/validator/v10" "gopkg.in/yaml.v3" ) @@ -16,7 +18,7 @@ type Config struct { BasicAuthEnabled bool `yaml:"basic_auth_enabled" validate:""` BasicAuthUsers map[string]string `yaml:"basic_auth_users" validate:"required_if=basic_auth_enabled true"` GitHub *GitHubConfig `yaml:"github" validate:"required"` - Pools []*PoolConfig `yaml:"pools" validate:"required,min=1"` + Pools []*PoolConfig `yaml:"pools" validate:"required,min=1,dive,required"` LogLevel string `yaml:"log_level" validate:"required,oneof=debug info warn error fatal panic trace"` path string @@ -47,11 +49,12 @@ type RunnerConfig struct { } type FirecrackerConfig struct { - BinaryPath string `yaml:"binary_path" ` - KernelImagePath string `yaml:"kernel_image_path"` - KernelArgs string `yaml:"kernel_args"` - MachineConfig FirecrackerMachineConfig `yaml:"machine_config"` - Metadata map[string]interface{} `yaml:"metadata"` + BinaryPath string `yaml:"binary_path" ` + KernelImagePath string `yaml:"kernel_image_path"` + KernelArgs string `yaml:"kernel_args"` + MachineConfig FirecrackerMachineConfig `yaml:"machine_config"` + NetworkInterface *FirecrackerNetworkInterfaceConfig `yaml:"network_interface"` + Metadata map[string]interface{} `yaml:"metadata"` } type FirecrackerMachineConfig struct { @@ -59,6 +62,54 @@ type FirecrackerMachineConfig struct { MemSizeMib int64 `yaml:"mem_size_mib"` } +// FirecrackerNetworkInterfaceConfig configures the MicroVM's network interface. +// Rate limiters are optional, a nil limiter leaves that direction unlimited. +type FirecrackerNetworkInterfaceConfig struct { + InRateLimiter *FirecrackerRateLimiterConfig `yaml:"in_rate_limiter"` + OutRateLimiter *FirecrackerRateLimiterConfig `yaml:"out_rate_limiter"` +} + +// FirecrackerRateLimiterConfig defines an IO rate limiter with independent +// bytes/s and ops/s limits. A nil token bucket leaves that limit unlimited. +type FirecrackerRateLimiterConfig struct { + Bandwidth *FirecrackerTokenBucketConfig `yaml:"bandwidth"` + Ops *FirecrackerTokenBucketConfig `yaml:"ops"` +} + +// FirecrackerTokenBucketConfig defines a token bucket with a maximum capacity +// (Size), an optional initial burst size (OneTimeBurst) and the interval in +// milliseconds it takes to refill the bucket (RefillTime). The resulting rate +// is Size / RefillTime. +type FirecrackerTokenBucketConfig struct { + Size int64 `yaml:"size" validate:"required,gt=0"` + OneTimeBurst *int64 `yaml:"one_time_burst" validate:"omitempty,gte=0"` + RefillTime int64 `yaml:"refill_time" validate:"required,gt=0"` +} + +// toSDK converts the rate limiter configuration into its Firecracker SDK +// representation. Returns nil if the rate limiter isn't configured. +func (c *FirecrackerRateLimiterConfig) toSDK() *models.RateLimiter { + if c == nil { + return nil + } + + return &models.RateLimiter{Bandwidth: c.Bandwidth.toSDK(), Ops: c.Ops.toSDK()} +} + +// toSDK converts the token bucket configuration into its Firecracker SDK +// representation. Returns nil if the token bucket isn't configured. +func (c *FirecrackerTokenBucketConfig) toSDK() *models.TokenBucket { + if c == nil { + return nil + } + + return &models.TokenBucket{ + Size: firecracker.Int64(c.Size), + RefillTime: firecracker.Int64(c.RefillTime), + OneTimeBurst: c.OneTimeBurst, + } +} + // DefaultConfig creates a new Config with default values. func DefaultConfig() *Config { c := &Config{ diff --git a/server/config_test.go b/server/config_test.go index f2b3ae7..0c52a3f 100644 --- a/server/config_test.go +++ b/server/config_test.go @@ -3,6 +3,8 @@ package server import ( "testing" + "github.com/firecracker-microvm/firecracker-go-sdk" + "github.com/firecracker-microvm/firecracker-go-sdk/client/models" "github.com/stretchr/testify/assert" ) @@ -14,3 +16,52 @@ func TestNewConfig(t *testing.T) { assert.Equal(t, "testdata/config1.yaml", config.path) } + +func TestNewConfigNetworkInterface(t *testing.T) { + config, err := NewConfig("testdata/config1.yaml") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + networkInterface := config.Pools[0].Firecracker.NetworkInterface + if networkInterface == nil { + t.Fatal("expected network interface configuration to be set") + } + + assert.Equal(t, &FirecrackerTokenBucketConfig{ + Size: 131072000, RefillTime: 1000, OneTimeBurst: firecracker.Int64(262144000), + }, networkInterface.InRateLimiter.Bandwidth) + assert.Equal(t, &FirecrackerTokenBucketConfig{ + Size: 10000, RefillTime: 1000, + }, networkInterface.InRateLimiter.Ops) + assert.Equal(t, &FirecrackerTokenBucketConfig{ + Size: 26214400, RefillTime: 1000, + }, networkInterface.OutRateLimiter.Bandwidth) + assert.Nil(t, networkInterface.OutRateLimiter.Ops) + + // A pool without a network_interface block leaves the interface unlimited. + assert.Nil(t, config.Pools[1].Firecracker.NetworkInterface) +} + +func TestNewConfigNetworkInterfaceInvalid(t *testing.T) { + // A token bucket without a size is rejected. + _, err := NewConfig("testdata/config2.yaml") + assert.ErrorContains(t, err, "Config.Pools[0].Firecracker.NetworkInterface.InRateLimiter.Bandwidth.Size") +} + +func TestFirecrackerRateLimiterConfigToSDK(t *testing.T) { + var nilRateLimiter *FirecrackerRateLimiterConfig + assert.Nil(t, nilRateLimiter.toSDK()) + + rateLimiter := &FirecrackerRateLimiterConfig{ + Bandwidth: &FirecrackerTokenBucketConfig{Size: 131072000, RefillTime: 1000, OneTimeBurst: firecracker.Int64(262144000)}, + } + + assert.Equal(t, &models.RateLimiter{ + Bandwidth: &models.TokenBucket{ + Size: firecracker.Int64(131072000), + RefillTime: firecracker.Int64(1000), + OneTimeBurst: firecracker.Int64(262144000), + }, + }, rateLimiter.toSDK()) +} diff --git a/server/pool.go b/server/pool.go index fdd283b..4567fd9 100644 --- a/server/pool.go +++ b/server/pool.go @@ -461,6 +461,16 @@ func (p *Pool) createMachine(ctx context.Context) error { vsockPath := filepath.Join(p.GetDir(), fmt.Sprintf("%s.vsock", runnerName)) vsockCID := p.nextCID.Add(1) + networkInterface := firecracker.NetworkInterface{ + AllowMMDS: true, + CNIConfiguration: &firecracker.CNIConfiguration{NetworkName: "fireactions", IfName: "eth0", ConfDir: "/etc/cni/net.d", BinPath: []string{"/opt/cni/bin"}}, + } + + if networkInterfaceConfig := p.config.Firecracker.NetworkInterface; networkInterfaceConfig != nil { + networkInterface.InRateLimiter = networkInterfaceConfig.InRateLimiter.toSDK() + networkInterface.OutRateLimiter = networkInterfaceConfig.OutRateLimiter.toSDK() + } + fcMachine, err := firecracker.NewMachine(ctx, firecracker.Config{ VMID: runnerName, SocketPath: filepath.Join(p.GetDir(), fmt.Sprintf("%s.sock", runnerName)), @@ -476,16 +486,13 @@ func (p *Pool) createMachine(ctx context.Context) error { IsRootDevice: firecracker.Bool(true), IsReadOnly: firecracker.Bool(false), }}, - NetworkInterfaces: []firecracker.NetworkInterface{{ - AllowMMDS: true, - CNIConfiguration: &firecracker.CNIConfiguration{NetworkName: "fireactions", IfName: "eth0", ConfDir: "/etc/cni/net.d", BinPath: []string{"/opt/cni/bin"}}, - }}, - VsockDevices: []firecracker.VsockDevice{{Path: vsockPath, CID: vsockCID}}, - MmdsAddress: net.IPv4(169, 254, 169, 254), - MmdsVersion: firecracker.MMDSv2, - ForwardSignals: []os.Signal{}, - LogPath: filepath.Join(p.GetDir(), fmt.Sprintf("%s.firecracker.log", runnerName)), - LogLevel: "Debug", + NetworkInterfaces: []firecracker.NetworkInterface{networkInterface}, + VsockDevices: []firecracker.VsockDevice{{Path: vsockPath, CID: vsockCID}}, + MmdsAddress: net.IPv4(169, 254, 169, 254), + MmdsVersion: firecracker.MMDSv2, + ForwardSignals: []os.Signal{}, + LogPath: filepath.Join(p.GetDir(), fmt.Sprintf("%s.firecracker.log", runnerName)), + LogLevel: "Debug", }, firecracker.WithProcessRunner(machineCmd), firecracker.WithLogger(logrus.NewEntry(logger))) if err != nil { return fmt.Errorf("firecracker: creating machine: %w", err) diff --git a/server/testdata/config1.yaml b/server/testdata/config1.yaml index f598a86..43f5990 100644 --- a/server/testdata/config1.yaml +++ b/server/testdata/config1.yaml @@ -37,6 +37,19 @@ pools: machine_config: mem_size_mib: 2048 vcpu_count: 2 + network_interface: + in_rate_limiter: + bandwidth: + size: 131072000 + refill_time: 1000 + one_time_burst: 262144000 + ops: + size: 10000 + refill_time: 1000 + out_rate_limiter: + bandwidth: + size: 26214400 + refill_time: 1000 metadata: example1: value1 - name: fireactions-2vcpu-4gb diff --git a/server/testdata/config2.yaml b/server/testdata/config2.yaml new file mode 100644 index 0000000..a278667 --- /dev/null +++ b/server/testdata/config2.yaml @@ -0,0 +1,32 @@ +--- +bind_address: 0.0.0.0:8080 + +github: + app_private_key: | + -----BEGIN RSA PRIVATE KEY----- + app_id: 12345 + +pools: +- name: fireactions-2vcpu-2gb + replicas: 1 + runner: + name: fireactions-2vcpu-2gb + image: ghcr.io/hostinger/fireactions/runner:ubuntu-20.04-x64-2.310.2 + image_pull_policy: IfNotPresent + group_id: 1 + organization: hostinger + labels: + - self-hosted + firecracker: + binary_path: firecracker + kernel_image_path: /var/lib/fireactions/vmlinux + kernel_args: "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules rw" + machine_config: + mem_size_mib: 2048 + vcpu_count: 2 + network_interface: + in_rate_limiter: + bandwidth: + refill_time: 1000 + +log_level: debug