From 84855bb219a66d7f97d3b48f193ff7001eef8dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Sch=C3=A4fer?= Date: Mon, 19 Oct 2026 11:28:42 +0200 Subject: [PATCH] Add optional egress policy to firecracker network flake-ctl firecracker network init now supports the new option --egress-deny. The implementation adds a new chain FLAKE_PILOT_EGRESS. VM traffic to the outside world now jumps from FLAKE_PILOT_FORWARD into the new chain. The new chain holds the egress drop rules first. For two prominent restrictions the --egress-deny option was simplified: * --egress-deny link-local * --egress-deny private link-local blocks 169.254.0.0/16, which includes the cloud metadata service at 169.254.169.254. Without it, a VM reaches that service as the host and can read the instance's credentials. private blocks the RFC 1918 ranges: 10/8, 172.16/12 and 192.168/16. The policy is IPv4 only like the rest of the setup --- common/src/defaults.rs | 19 ++ common/src/network.rs | 94 ++++++++- completion/flake-ctl | 2 +- doc/flake-ctl-firecracker-network-init.rst | 60 +++++- doc/flake-pilot-helper.rst | 5 +- doc/user_guide/firecracker_networking.rst | 21 ++ doc/user_guide/spelling_wordlist.txt | 1 + flake-ctl/src/cli.rs | 11 ++ flake-ctl/src/main.rs | 4 +- flake-ctl/src/network.rs | 34 +++- flake-pilot-helper/src/cli.rs | 6 + flake-pilot-helper/src/firewall.rs | 212 ++++++++++++++++++--- flake-pilot-helper/src/main.rs | 5 +- flake-pilot-helper/src/network.rs | 42 +++- 14 files changed, 458 insertions(+), 58 deletions(-) diff --git a/common/src/defaults.rs b/common/src/defaults.rs index 529be5c..3a20924 100644 --- a/common/src/defaults.rs +++ b/common/src/defaults.rs @@ -88,6 +88,10 @@ pub const TAP_DEVICE_PREFIX: &str = "tap-"; pub const FIREWALL_NAT_CHAIN: &str = "FLAKE_PILOT_NAT"; pub const FIREWALL_FORWARD_CHAIN: &str = "FLAKE_PILOT_FORWARD"; pub const FIREWALL_INPUT_CHAIN: &str = "FLAKE_PILOT_INPUT"; +// Chain of the egress policy of the VM network. The traffic of the +// VMs to the outside world passes it before it is forwarded, see +// NETWORK_EGRESS_PRESETS +pub const FIREWALL_EGRESS_CHAIN: &str = "FLAKE_PILOT_EGRESS"; // Max. size of a network interface name including the // terminating zero byte as defined by IFNAMSIZ in linux/if.h pub const IFNAMSIZ: usize = 16; @@ -209,6 +213,21 @@ pub const NETWORK_HOST_PREFIX_LEN: u8 = // a resolver on the host, which the VMs use through their gateway pub const NETWORK_DNS: &str = "8.8.8.8"; +// Named sets of networks the egress policy of the VM network can +// deny, see 'flake-ctl firecracker network init --egress-deny'. +// The traffic of the VMs to them is dropped instead of being +// forwarded to the outgoing interface: +// +// link-local: the link local network, which provides the metadata +// service of the cloud the host runs in at 169.254.169.254. +// Without the policy a VM reaches it with the identity of the +// host +// private: the private networks of RFC 1918, e.g the local +// network of the host and the services in it +pub const NETWORK_EGRESS_PRESETS: &[(&str, &[&str])] = &[ + ("link-local", &["169.254.0.0/16"]), + ("private", &["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]) +]; // Directory the overlay setups of the bubblewrap sandboxes are // created in. The directories are created by bubblewrap-pilot and // mounted by the helper. It is shared diff --git a/common/src/network.rs b/common/src/network.rs index b9a0662..3c13a2e 100644 --- a/common/src/network.rs +++ b/common/src/network.rs @@ -683,6 +683,12 @@ pub struct NetworkState { /// The host runs a DHCP server for the VMs #[serde(default)] pub host_dhcp: bool, + /// Networks in their ADDRESS/PREFIX_LEN notation the traffic + /// of the VMs to the outside world must not go to, see + /// get_egress_networks(). A state written before the egress + /// policy became selectable provides none + #[serde(default)] + pub egress_deny: Vec, /// The addresses handed out to the VM instances of all users #[serde(default)] pub allocations: Vec @@ -702,6 +708,52 @@ impl NetworkState { !*/ HostServices { resolver: self.host_resolver, dhcp: self.host_dhcp } } + + pub fn get_egress_deny(&self) -> Option> { + /*! + Provide the recorded networks of the egress policy, None + if one of them is invalid + !*/ + self.egress_deny.iter() + .map(|network| Ipv4Network::parse(network)) + .collect() + } +} + +pub fn get_egress_networks( + egress_deny: &[String] +) -> Result, String> { + /*! + Provide the networks of the egress policy of the VM network + + Each entry is the name of one of the sets of networks in + defaults::NETWORK_EGRESS_PRESETS or a network in its + ADDRESS/PREFIX_LEN notation. An address without a prefix length + is a single host. Networks which are given more than once are + only listed once, in the order they are given first + !*/ + let mut networks: Vec = Vec::new(); + for entry in egress_deny { + let preset = defaults::NETWORK_EGRESS_PRESETS.iter() + .find(|(name, _)| name == entry); + let entry_networks = match preset { + Some((_, preset_networks)) => preset_networks.iter() + .filter_map(|network| Ipv4Network::parse(network)) + .collect(), + None => vec![Ipv4Network::parse(entry).ok_or(format!( + "Invalid egress policy entry {entry:?}: expected one of {} \ + or ADDRESS[/PREFIX_LEN]", + defaults::NETWORK_EGRESS_PRESETS.iter() + .map(|(name, _)| *name).collect::>().join(", ") + ))?] + }; + for network in entry_networks { + if ! networks.contains(&network) { + networks.push(network) + } + } + } + Ok(networks) } // Allocation is an address of the private network handed out to @@ -956,8 +1008,8 @@ mod tests { use super::{ get_jump_rules, get_masquerade_rule, get_network_config_file, get_tap_name, NetworkConfig, get_address_list_networks, - get_flake_allocations, get_free_address, get_nameserver, - get_network_candidates, get_preferred_network, + get_egress_networks, get_flake_allocations, get_free_address, + get_nameserver, get_network_candidates, get_preferred_network, get_route_list_default_device, get_route_list_networks, is_vm_address, lock_file_of, select_free_network, Allocation, HostServices, Ipv4Network, NetworkState @@ -1180,6 +1232,7 @@ mod tests { outgoing_interface: Some("eth0".to_string()), host_resolver: true, host_dhcp: false, + egress_deny: vec!["169.254.0.0/16".to_string()], allocations: vec![ allocation( "172.16.0.2", 1000, "tap-1000_myapp", @@ -1193,9 +1246,46 @@ mod tests { assert_eq!( HostServices { resolver: true, dhcp: false }, state.get_services() ); + assert_eq!( + Some(vec![network("169.254.0.0/16")]), state.get_egress_deny() + ); assert_eq!( state, serde_yaml::from_str::(&yaml_data).unwrap() ); + // an invalid network of the egress policy is not ignored + let state = NetworkState { + egress_deny: vec!["10.0.0.0/8".to_string(), "bogus".to_string()], + ..Default::default() + }; + assert_eq!(None, state.get_egress_deny()); + } + + #[test] + fn test_get_egress_networks() { + let egress_networks = |entries: &[&str]| get_egress_networks( + &entries.iter().map(|entry| entry.to_string()).collect::>() + ); + assert_eq!(Ok(vec![]), egress_networks(&[])); + assert_eq!( + Ok(vec![network("169.254.0.0/16")]), + egress_networks(&["link-local"]) + ); + // presets and networks are combined, duplicates listed once + assert_eq!( + Ok(vec![ + network("10.0.0.0/8"), network("172.16.0.0/12"), + network("192.168.0.0/16"), network("100.100.100.200/32") + ]), + egress_networks(&["private", "10.0.0.0/8", "100.100.100.200"]) + ); + // a network is taken by its network address + assert_eq!( + Ok(vec![network("10.1.0.0/16")]), + egress_networks(&["10.1.2.3/16"]) + ); + assert!(egress_networks(&["metadata"]).is_err()); + assert!(egress_networks(&["10.0.0.0/33"]).is_err()); + assert!(egress_networks(&["-j ACCEPT"]).is_err()); } #[test] diff --git a/completion/flake-ctl b/completion/flake-ctl index a18e3b5..f1af990 100755 --- a/completion/flake-ctl +++ b/completion/flake-ctl @@ -40,7 +40,7 @@ __flake_ctl_firecracker_show() { __flake_ctl_firecracker_network() { if [ "${COMP_WORDS[3]}" = "init" ];then - __comp_reply_unused "--outgoing-interface --host-resolver --host-dhcp --help" + __comp_reply_unused "--outgoing-interface --host-resolver --host-dhcp --egress-deny --help" return 0 fi if [ "${COMP_WORDS[3]}" = "add" ] || [ "${COMP_WORDS[3]}" = "remove" ];then diff --git a/doc/flake-ctl-firecracker-network-init.rst b/doc/flake-ctl-firecracker-network-init.rst index a72eda2..6b6c56b 100644 --- a/doc/flake-ctl-firecracker-network-init.rst +++ b/doc/flake-ctl-firecracker-network-init.rst @@ -12,12 +12,13 @@ SYNOPSIS .. code:: bash USAGE: - flake-ctl firecracker network init [--outgoing-interface ] [--host-resolver] [--host-dhcp] + flake-ctl firecracker network init [--outgoing-interface ] [--host-resolver] [--host-dhcp] [--egress-deny ]... OPTIONS: --outgoing-interface --host-resolver --host-dhcp + --egress-deny --help DESCRIPTION @@ -62,7 +63,8 @@ again: the network has to be one of the private networks the command selects from, see NETWORK below, and must not collide with a network of the host, the interface has to exist. Both are recorded in the network state file along with the services of the host given by -``--host-resolver`` and ``--host-dhcp``. Connecting an application to +``--host-resolver`` and ``--host-dhcp`` and the networks of the egress +policy given by ``--egress-deny``. Connecting an application to the host network with **flake-ctl-firecracker-network-add**(8) takes them from there: the address of the application, the route to it and its firewall rules are only ever created within the recorded network. @@ -106,10 +108,13 @@ follows, shown in the notation of ``iptables -S``: # table filter -A FORWARD -j FLAKE_PILOT_FORWARD -A FLAKE_PILOT_FORWARD -o tap-+ -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - -A FLAKE_PILOT_FORWARD -i tap-+ -s -o -j ACCEPT + -A FLAKE_PILOT_FORWARD -i tap-+ -s -o -j FLAKE_PILOT_EGRESS -A FLAKE_PILOT_FORWARD -i tap-+ -j DROP -A FLAKE_PILOT_FORWARD -o tap-+ -j DROP + -A FLAKE_PILOT_EGRESS -d -j DROP + -A FLAKE_PILOT_EGRESS -j ACCEPT + -A INPUT -i tap-+ -j FLAKE_PILOT_INPUT -A FLAKE_PILOT_INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A FLAKE_PILOT_INPUT -j DROP @@ -120,6 +125,11 @@ The effect of the setup is: the outgoing interface is masqueraded and forwarded. The answers to it pass back to the VMs. +* The traffic of the VMs to the outside world passes the egress + policy before it is forwarded. Its traffic to a network given by + ``--egress-deny`` is dropped, there is one ``DROP`` rule for each + of them. Without the option the egress chain accepts all traffic. + * Traffic between two VMs, traffic from the outside world to a VM and traffic from a VM to any other interface of the host is dropped. @@ -134,7 +144,10 @@ The chains are deleted as a whole by Former versions of flake-pilot created their rules in the builtin chains. Their rules for single TAP devices are deleted by this -command. A MASQUERADE rule for all traffic of the outgoing interface +command, as well as a rule of the FLAKE_PILOT_FORWARD chain which +accepts the traffic of the VMs without passing it to the egress +chain. Until the command is called again, a setup created by a former +version keeps working without an egress policy. A MASQUERADE rule for all traffic of the outgoing interface and a FORWARD rule accepting all answers, which former versions created as well, cannot be told apart from rules created by other means. If one of them is found, it is reported along with the command @@ -221,6 +234,43 @@ OPTIONS Applications which were connected before the setting changed keep their firewall rules until they are connected again +--egress-deny + + Network the traffic of the VMs to the outside world must not go + to. Can be specified multiple times, the traffic to all of them is + dropped. Either the name of a set of networks or a network in its + ``ADDRESS[/PREFIX_LEN]`` notation, an address without a prefix + length is a single host: + + link-local + ``169.254.0.0/16``. It covers the metadata service a cloud + provides to its instances at ``169.254.169.254``, e.g on AWS, + Azure, Google Cloud or OpenStack. Without the policy a VM on a + cloud instance reaches the metadata service with the identity + of the host and may e.g read the credentials of the instance. + Clouds which provide their metadata service at another address + have to be covered by that address, e.g ``100.100.100.200`` + + private + ``10.0.0.0/8``, ``172.16.0.0/12`` and ``192.168.0.0/16``, the + private networks of RFC 1918. It covers the local network of + the host and the services in it. A VM which needs one of them, + e.g a proxy or a name server of the local network, cannot reach + it anymore + + The policy only applies to the traffic which is forwarded to the + outgoing interface. The services of the host a VM may use through + its gateway, see FIREWALL above, are not affected. The networks + are recorded in the network state file and take effect for all + applications at once, including the ones which are already + connected. Calling the command again replaces the policy, without + the option the policy is removed. + **flake-ctl-firecracker-network-add**\(8) refuses to connect an + application if the recorded policy is not active + + The policy is IPv4 only, as is the whole setup. IPv6 is switched + off on the TAP devices of the VMs + FILES ----- @@ -239,6 +289,8 @@ EXAMPLE $ flake-ctl firecracker network init --outgoing-interface eth0 + $ flake-ctl firecracker network init --egress-deny link-local --egress-deny private + SEE ALSO -------- diff --git a/doc/flake-pilot-helper.rst b/doc/flake-pilot-helper.rst index d332068..fe90289 100644 --- a/doc/flake-pilot-helper.rst +++ b/doc/flake-pilot-helper.rst @@ -15,7 +15,7 @@ SYNOPSIS sudo /usr/libexec/flake-pilot/flake-pilot-helper [--debug] SUBCOMMANDS: - network-init --network --outgoing-interface [--host-resolver] [--host-dhcp] + network-init --network --outgoing-interface [--host-resolver] [--host-dhcp] [--egress-deny ]... network-cleanup network-connect --config-file [--instance ] [--uid ] [--prefer ] [--exclude ]... network-disconnect --config-file [--instance ] [--uid ] @@ -68,7 +68,8 @@ network-init network has to be one of the private networks flake-pilot selects from and must not collide with a network of the host, the interface has to exist. Both are recorded in the network state - along with the services of the host the VMs use. Called by + along with the services of the host the VMs use and the networks + of the egress policy, which only adds drop rules. Called by **flake-ctl-firecracker-network-init**\(8) network-cleanup diff --git a/doc/user_guide/firecracker_networking.rst b/doc/user_guide/firecracker_networking.rst index cd6065c..1b75435 100644 --- a/doc/user_guide/firecracker_networking.rst +++ b/doc/user_guide/firecracker_networking.rst @@ -192,6 +192,27 @@ the instance are bound to it. A change of these settings takes effect for an application once it is connected again. +By default the VMs may reach every network the host reaches through +the outgoing interface. An egress policy denies them networks with +``--egress-deny``, which can be given multiple times: + +.. code-block:: bash + + flake-ctl firecracker network init --egress-deny link-local --egress-deny private + +``link-local`` denies ``169.254.0.0/16``, which covers the metadata +service of a cloud at ``169.254.169.254``. On a cloud instance a VM +reaches that service with the identity of the host otherwise, and may +e.g read the credentials of the instance. ``private`` denies the +private networks of RFC 1918, ``10.0.0.0/8``, ``172.16.0.0/12`` and +``192.168.0.0/16``, which keeps the VMs away from the local network of +the host. Any other network can be given as ``ADDRESS[/PREFIX_LEN]``, +e.g ``--egress-deny 100.100.100.200`` for the metadata service of a +cloud which uses another address. The services of the host the VMs use +through their gateway are not affected. Unlike the settings above, the +policy takes effect for all applications at once. Calling ``init`` +again replaces it. + Rules of former versions of Flake Pilot for single TAP devices are deleted. The rules for all traffic former versions created, a ``MASQUERADE`` rule for the outgoing interface and a ``FORWARD`` rule diff --git a/doc/user_guide/spelling_wordlist.txt b/doc/user_guide/spelling_wordlist.txt index cde46ea..61770f9 100644 --- a/doc/user_guide/spelling_wordlist.txt +++ b/doc/user_guide/spelling_wordlist.txt @@ -42,3 +42,4 @@ toolchain vsock appstore boxbuild +RFC diff --git a/flake-ctl/src/cli.rs b/flake-ctl/src/cli.rs index 71c0f26..d2f70e4 100644 --- a/flake-ctl/src/cli.rs +++ b/flake-ctl/src/cli.rs @@ -261,6 +261,17 @@ pub enum Network { /// closed to them #[clap(long)] host_dhcp: bool, + + /// Network the traffic of the VMs to the outside world + /// must not go to. Either link-local, which covers the + /// metadata service of a cloud at 169.254.169.254, private, + /// which covers the private networks of RFC 1918, or a + /// network as ADDRESS[/PREFIX_LEN]. Can be specified + /// multiple times. Without this option the VMs may reach + /// every network the host reaches through the outgoing + /// interface + #[clap(long, multiple_occurrences = true)] + egress_deny: Vec, }, /// Connect a VM application to the host network Add { diff --git a/flake-ctl/src/main.rs b/flake-ctl/src/main.rs index 99c3b32..5fa7ad7 100644 --- a/flake-ctl/src/main.rs +++ b/flake-ctl/src/main.rs @@ -160,7 +160,8 @@ async fn main() -> Result> { match &command { // init cli::Network::Init { - outgoing_interface, host_resolver, host_dhcp + outgoing_interface, host_resolver, host_dhcp, + egress_deny } => { if ! network::init( outgoing_interface.as_deref(), @@ -168,6 +169,7 @@ async fn main() -> Result> { resolver: *host_resolver, dhcp: *host_dhcp }, + egress_deny, user ) { return Ok(ExitCode::FAILURE) diff --git a/flake-ctl/src/network.rs b/flake-ctl/src/network.rs index cc0eaca..8401155 100644 --- a/flake-ctl/src/network.rs +++ b/flake-ctl/src/network.rs @@ -23,8 +23,9 @@ // use flakes::helper::helper_call; use flakes::network::{ - get_default_route_interface, get_free_network, get_nameserver, - get_network_config_file, get_preferred_network, get_tap_name, + get_default_route_interface, get_egress_networks, get_free_network, + get_nameserver, get_network_config_file, get_preferred_network, + get_tap_name, lock_network_state, network_collides, read_network_config, read_network_state, tap_exists, HostServices, Ipv4Network, NetworkConfig, NetworkState @@ -44,7 +45,8 @@ use crate::app_config::{ use crate::defaults; pub fn init( - outgoing_interface: Option<&str>, services: HostServices, usermode: bool + outgoing_interface: Option<&str>, services: HostServices, + egress_deny: &[String], usermode: bool ) -> bool { /*! Prepare the host for NAT based VM networking @@ -60,9 +62,10 @@ pub fn init( The private network of the VMs is selected here, the setup itself is created by the helper, see its network-init command. The helper checks the network and the interface again and - records them along with the given services of the host in the - network state. The setup of the single flakes, done by add(), - uses what is recorded there. + records them along with the given services of the host and the + networks of the given egress policy in the network state. The + setup of the single flakes, done by add(), uses what is + recorded there. The setup is also recorded in the network configuration file, which the pilot reads to check the firewall setup. @@ -73,6 +76,13 @@ pub fn init( firewall is managed by another tool, the rules have to be created with that tool instead !*/ + let egress_networks = match get_egress_networks(egress_deny) { + Ok(egress_networks) => egress_networks, + Err(error) => { + error!("{error}"); + return false + } + }; let outgoing_interface = match select_outgoing_interface( outgoing_interface ) { @@ -92,6 +102,9 @@ pub fn init( if services.dhcp { call.arg("--host-dhcp"); } + for egress_network in &egress_networks { + call.arg("--egress-deny").arg(egress_network.to_string()); + } if ! run_helper(&mut call, "prepare the host network") { return false } @@ -109,6 +122,15 @@ pub fn init( " DHCP server of the host: {}", if services.dhcp { "yes" } else { "no" } ); + if egress_networks.is_empty() { + info!(" egress denied: none"); + } else { + info!( + " egress denied: {}", + egress_networks.iter() + .map(ToString::to_string).collect::>().join(", ") + ); + } true } diff --git a/flake-pilot-helper/src/cli.rs b/flake-pilot-helper/src/cli.rs index f9cb402..e48420c 100644 --- a/flake-pilot-helper/src/cli.rs +++ b/flake-pilot-helper/src/cli.rs @@ -64,6 +64,12 @@ pub enum Commands { /// The host runs a DHCP server for the VMs #[clap(long)] host_dhcp: bool, + + /// Network the traffic of the VMs to the outside world must + /// not go to: link-local, private or ADDRESS[/PREFIX_LEN]. + /// Given more than once, all of them are denied + #[clap(long, multiple_occurrences = true)] + egress_deny: Vec, }, /// Delete the network setup of the host NetworkCleanup, diff --git a/flake-pilot-helper/src/firewall.rs b/flake-pilot-helper/src/firewall.rs index 40f3f17..aa43cb0 100644 --- a/flake-pilot-helper/src/firewall.rs +++ b/flake-pilot-helper/src/firewall.rs @@ -34,7 +34,7 @@ use std::process::{Command, Stdio}; use flakes::defaults::{ self, TAP_DEVICE_PREFIX, FIREWALL_NAT_CHAIN, FIREWALL_FORWARD_CHAIN, - FIREWALL_INPUT_CHAIN + FIREWALL_INPUT_CHAIN, FIREWALL_EGRESS_CHAIN }; use flakes::network::{ FirewallRule, HostServices, Ipv4Network, get_tap_match, get_jump_rules, @@ -46,50 +46,64 @@ use crate::run::run_ok; // Position a rule is created at in its chain. Accepting rules are // inserted at the top, dropping rules are appended. This keeps // every accepting rule in front of the drop rules, no matter in -// which order the rules are created +// which order the rules are created. The egress chain is the other +// way round: the drop rules of the egress policy are inserted at +// the top, in front of the rule which accepts everything else pub enum RulePosition { Top, Bottom } -pub fn get_firewall_chains() -> [(&'static str, &'static str); 3] { +pub fn get_firewall_chains() -> [(&'static str, &'static str); 4] { /*! Provide the table and the name of the chains of the host firewall setup + + The egress chain is entered from the forward chain and is + therefore listed after it, the forward chain has to be deleted + first !*/ [ ("nat", FIREWALL_NAT_CHAIN), ("filter", FIREWALL_FORWARD_CHAIN), - ("filter", FIREWALL_INPUT_CHAIN) + ("filter", FIREWALL_INPUT_CHAIN), + ("filter", FIREWALL_EGRESS_CHAIN) ] } pub fn get_firewall_rules( - network: &str, outgoing_interface: &str + network: &str, outgoing_interface: &str, egress_deny: &[Ipv4Network] ) -> Vec<(FirewallRule, RulePosition)> { /*! Provide the rules of the host firewall setup and the position they are created at The traffic of the private network of the VMs is masqueraded - and forwarded if it leaves through the outgoing interface. The - answers to it may pass back. All other traffic which comes from - or goes to a TAP device is dropped: traffic between two VMs, - traffic from the outside world to a VM and traffic from a VM to - another interface of the host. Traffic from a VM to the host - itself is dropped as well, except for answers to traffic of the - host and the services connect() allows for each instance + and forwarded if it leaves through the outgoing interface and + passes the egress policy, see get_egress_rules(). The answers + to it may pass back. All other traffic which comes from or goes + to a TAP device is dropped: traffic between two VMs, traffic + from the outside world to a VM and traffic from a VM to another + interface of the host. Traffic from a VM to the host itself is + dropped as well, except for answers to traffic of the host and + the services connect() allows for each instance + + The rules of the egress chain are listed first, the chain is + complete before traffic is passed to it !*/ let tap_match = get_tap_match(); - vec![ - (get_masquerade_rule(network, outgoing_interface), RulePosition::Bottom), + let mut rules: Vec<(FirewallRule, RulePosition)> = get_egress_rules( + egress_deny + ).into_iter().map(|rule| (rule, RulePosition::Top)).collect(); + rules.extend([ ( - FirewallRule::new("filter", FIREWALL_FORWARD_CHAIN, &[ - "-i", &tap_match, "-s", network, "-o", outgoing_interface, + FirewallRule::new("filter", FIREWALL_EGRESS_CHAIN, &[ "-j", "ACCEPT" ]), - RulePosition::Top + RulePosition::Bottom ), + (get_masquerade_rule(network, outgoing_interface), RulePosition::Bottom), + (get_egress_jump_rule(network, outgoing_interface), RulePosition::Top), ( FirewallRule::new("filter", FIREWALL_FORWARD_CHAIN, &[ "-o", &tap_match, "-m", "conntrack", @@ -120,7 +134,38 @@ pub fn get_firewall_rules( FirewallRule::new("filter", FIREWALL_INPUT_CHAIN, &["-j", "DROP"]), RulePosition::Bottom ) - ] + ]); + rules +} + +pub fn get_egress_jump_rule( + network: &str, outgoing_interface: &str +) -> FirewallRule { + /*! + Provide the rule which passes the traffic of the VMs to the + outside world to the egress chain. It is the only way traffic + of a TAP device gets forwarded + !*/ + FirewallRule::new("filter", FIREWALL_FORWARD_CHAIN, &[ + "-i", &get_tap_match(), "-s", network, "-o", outgoing_interface, + "-j", FIREWALL_EGRESS_CHAIN + ]) +} + +pub fn get_egress_rules(egress_deny: &[Ipv4Network]) -> Vec { + /*! + Provide the rules of the egress policy + + The traffic of the VMs to the given networks is dropped, all + other traffic which enters the egress chain is accepted by its + last rule. The rules are given in the notation iptables lists + them with, see delete_stale_egress_rules() + !*/ + egress_deny.iter() + .map(|network| FirewallRule::new("filter", FIREWALL_EGRESS_CHAIN, &[ + "-d", &network.to_string(), "-j", "DROP" + ])) + .collect() } pub fn get_guest_service_rules( @@ -170,14 +215,19 @@ pub fn get_guest_service_rules( rules } -pub fn setup_firewall(network: &Ipv4Network, outgoing_interface: &str) -> bool { +pub fn setup_firewall( + network: &Ipv4Network, outgoing_interface: &str, + egress_deny: &[Ipv4Network] +) -> bool { /*! Create the chains and rules of the host firewall setup The chains are created first and are entered from the builtin chains last, traffic is therefore only passed to them once they are complete. Chains and rules which already exist are kept, - this allows to call the setup more than once + this allows to call the setup more than once. The drop rules of + a former egress policy are deleted once the rules of the given + one are in place !*/ for (table, chain) in get_firewall_chains() { if ! create_chain(table, chain) { @@ -185,12 +235,17 @@ pub fn setup_firewall(network: &Ipv4Network, outgoing_interface: &str) -> bool { } } for (rule, position) in get_firewall_rules( - &network.to_string(), outgoing_interface + &network.to_string(), outgoing_interface, egress_deny ) { if ! add_rule(&rule, position) { return false } } + if ! delete_stale_egress_rules(egress_deny) + || ! delete_unfiltered_forward_rules() + { + return false + } for rule in get_jump_rules() { if ! add_rule(&rule, RulePosition::Top) { return false @@ -216,14 +271,62 @@ pub fn delete_firewall() -> bool { ok } -pub fn firewall_is_active() -> bool { +pub fn firewall_is_active( + network: &Ipv4Network, outgoing_interface: &str, + egress_deny: &[Ipv4Network] +) -> bool { /*! Check if the host firewall setup is complete, the chains are present and are entered from the builtin chains + + With an egress policy, the traffic of the VMs has to be passed + to the egress chain and the drop rules of the policy have to be + present. A setup created before the egress policy existed has + no egress chain, it is complete without a policy + !*/ + let chains_exist = get_firewall_chains().iter() + .filter(|(_, chain)| *chain != FIREWALL_EGRESS_CHAIN) + .all(|(table, chain)| chain_exists(table, chain)); + if ! chains_exist || ! get_jump_rules().iter().all(rule_exists) { + return false + } + egress_deny.is_empty() || ( + rule_exists( + &get_egress_jump_rule(&network.to_string(), outgoing_interface) + ) && get_egress_rules(egress_deny).iter().all(rule_exists) + ) +} + +pub fn delete_stale_egress_rules(egress_deny: &[Ipv4Network]) -> bool { + /*! + Delete the drop rules of the egress chain which are not part of + the given egress policy !*/ - get_firewall_chains().iter().all( - |(table, chain)| chain_exists(table, chain) - ) && get_jump_rules().iter().all(rule_exists) + let wanted: Vec> = get_egress_rules(egress_deny).into_iter() + .map(|rule| rule.spec) + .collect(); + delete_listed_rules("filter", FIREWALL_EGRESS_CHAIN, |spec| { + spec.last().is_some_and(|target| target == "DROP") + && ! wanted.iter().any(|wanted_spec| wanted_spec == spec) + }) +} + +pub fn delete_unfiltered_forward_rules() -> bool { + /*! + Delete the rules which forward the traffic of the VMs without + passing it to the egress chain + + Former versions accepted the traffic of the TAP devices in the + forward chain directly. Such a rule is recognized by its match + of the TAP devices as the incoming interface and its target + !*/ + let tap_match = get_tap_match(); + delete_listed_rules("filter", FIREWALL_FORWARD_CHAIN, |spec| { + spec.ends_with(&["-j".to_string(), "ACCEPT".to_string()]) + && spec.windows(2).any( + |pair| pair[0] == "-i" && pair[1] == tap_match + ) + }) } pub fn allow_guest_services( @@ -469,8 +572,8 @@ mod tests { use flakes::network::{HostServices, Ipv4Network}; use super::{ - get_firewall_rules, get_guest_service_rules, get_listed_rules, - RulePosition + get_egress_rules, get_firewall_rules, get_guest_service_rules, + get_listed_rules, RulePosition }; fn network(network: &str) -> Ipv4Network { @@ -483,7 +586,10 @@ mod tests { #[test] fn test_firewall_rules() { - let rules = get_firewall_rules("172.16.0.0/24", "eth0"); + let rules = get_firewall_rules( + "172.16.0.0/24", "eth0", + &[network("169.254.0.0/16"), network("10.0.0.0/8")] + ); let rule_args = |chain: &str| -> Vec<(String, bool)> { rules.iter() .filter(|(rule, _)| rule.chain == chain) @@ -501,12 +607,33 @@ mod tests { )], rule_args("FLAKE_PILOT_NAT") ); + // the egress chain drops the traffic to the denied networks + // in front of the rule which accepts everything else + assert_eq!( + vec![ + ("-d 169.254.0.0/16 -j DROP".to_string(), true), + ("-d 10.0.0.0/8 -j DROP".to_string(), true), + ("-j ACCEPT".to_string(), false) + ], + rule_args("FLAKE_PILOT_EGRESS") + ); + // the egress chain is complete before traffic is passed to it + let last_egress_rule = rules.iter() + .rposition(|(rule, _)| rule.chain == "FLAKE_PILOT_EGRESS"); + let egress_jump_rule = rules.iter() + .position(|(rule, _)| rule.spec.ends_with(&[ + "-j".to_string(), "FLAKE_PILOT_EGRESS".to_string() + ])); + assert!(last_egress_rule.unwrap() < egress_jump_rule.unwrap()); // accepting rules are created at the top, the drop rules - // for everything else to and from a VM at the bottom + // for everything else to and from a VM at the bottom. The + // traffic to the outside world is only accepted after it + // passed the egress chain assert_eq!( vec![ ( - "-i tap-+ -s 172.16.0.0/24 -o eth0 -j ACCEPT".to_string(), + "-i tap-+ -s 172.16.0.0/24 -o eth0 -j FLAKE_PILOT_EGRESS" + .to_string(), true ), ( @@ -530,6 +657,31 @@ mod tests { ], rule_args("FLAKE_PILOT_INPUT") ); + // without an egress policy everything passes the egress chain + let rules = get_firewall_rules("172.16.0.0/24", "eth0", &[]); + assert_eq!( + vec!["-j ACCEPT"], + rules.iter() + .filter(|(rule, _)| rule.chain == "FLAKE_PILOT_EGRESS") + .map(|(rule, _)| rule.spec.join(" ")) + .collect::>() + ); + } + + #[test] + fn test_egress_rules_are_listed_as_created() { + // stale drop rules are recognized by comparing the listed + // rules of the egress chain with the created ones + let rule_list = "-N FLAKE_PILOT_EGRESS\n\ + -A FLAKE_PILOT_EGRESS -d 169.254.0.0/16 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -d 100.100.100.200/32 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -j ACCEPT\n"; + assert_eq!( + get_listed_rules(rule_list, "FLAKE_PILOT_EGRESS")[..2].to_vec(), + get_egress_rules(&[ + network("169.254.0.0/16"), network("100.100.100.200") + ]).into_iter().map(|rule| rule.spec).collect::>() + ); } #[test] diff --git a/flake-pilot-helper/src/main.rs b/flake-pilot-helper/src/main.rs index 477f726..22f5edc 100644 --- a/flake-pilot-helper/src/main.rs +++ b/flake-pilot-helper/src/main.rs @@ -79,9 +79,10 @@ fn main() -> ExitCode { setup_logger(debug); let ok = match &args.command { cli::Commands::NetworkInit { - network, outgoing_interface, host_resolver, host_dhcp + network, outgoing_interface, host_resolver, host_dhcp, egress_deny } => network::init( - &caller, network, outgoing_interface, *host_resolver, *host_dhcp + &caller, network, outgoing_interface, *host_resolver, *host_dhcp, + egress_deny ), cli::Commands::NetworkCleanup => network::cleanup(&caller), cli::Commands::NetworkConnect { diff --git a/flake-pilot-helper/src/network.rs b/flake-pilot-helper/src/network.rs index 2639356..d7070b4 100644 --- a/flake-pilot-helper/src/network.rs +++ b/flake-pilot-helper/src/network.rs @@ -38,10 +38,10 @@ use tempfile::NamedTempFile; use flakes::defaults; use flakes::network::{ - get_free_address, get_network_candidates, get_tap_name, - is_flake_tap_device, is_vm_address, lock_network_state, network_collides, - read_network_config, read_network_state, tap_exists, Allocation, - Ipv4Network, NetworkState + get_egress_networks, get_free_address, get_network_candidates, + get_tap_name, is_flake_tap_device, is_vm_address, lock_network_state, + network_collides, read_network_config, read_network_state, tap_exists, + Allocation, Ipv4Network, NetworkState }; use crate::caller::Caller; @@ -69,7 +69,7 @@ pub struct VmInstance<'a> { pub fn init( caller: &Caller, network: &str, outgoing_interface: &str, - host_resolver: bool, host_dhcp: bool + host_resolver: bool, host_dhcp: bool, egress_deny: &[String] ) -> bool { /*! Prepare the host for NAT based VM networking @@ -85,9 +85,11 @@ pub fn init( selects from, and it must not collide with a network the host is connected to. The outgoing interface has to exist. Both are recorded in the network state along with the services of the - host the VMs use. connect() takes them from there and nowhere - else: the address of a VM, the route to it and its firewall - rules are only ever created within the recorded network. + host the VMs use and the networks of the egress policy, see + flakes::network::get_egress_networks(). connect() takes them + from there and nowhere else: the address of a VM, the route to + it and its firewall rules are only ever created within the + recorded network. If IP forwarding was switched off, the state records that flake-pilot switched it on, cleanup() switches it off again @@ -103,6 +105,13 @@ pub fn init( error!("{error}"); return false } + let egress_deny = match get_egress_networks(egress_deny) { + Ok(egress_deny) => egress_deny, + Err(error) => { + error!("{error}"); + return false + } + }; let _lock = match lock_network_state(true) { Some(lock) => lock, None => return false @@ -132,10 +141,13 @@ pub fn init( state.outgoing_interface = Some(outgoing_interface.to_string()); state.host_resolver = host_resolver; state.host_dhcp = host_dhcp; + state.egress_deny = egress_deny.iter() + .map(ToString::to_string) + .collect(); if ! write_network_state(&state) { return false } - if ! setup_firewall(&network, outgoing_interface) { + if ! setup_firewall(&network, outgoing_interface, &egress_deny) { return false } delete_legacy_rules(outgoing_interface); @@ -238,7 +250,17 @@ pub fn connect( } }; let services = state.get_services(); - if ! firewall_is_active() { + let egress_deny = match state.get_egress_deny() { + Some(egress_deny) => egress_deny, + None => { + error!("The recorded egress policy is invalid"); + error!("Please run 'flake-ctl firecracker network init' again"); + return false + } + }; + let outgoing_interface = state.outgoing_interface.as_deref() + .unwrap_or_default(); + if ! firewall_is_active(&network, outgoing_interface, &egress_deny) { error!("The firewall setup of the host is not active"); error!( "Please run 'flake-ctl firecracker network init' first, \