Add optional egress policy to firecracker network

common/src/defaults.rs

@@ -88,6 +88,10 @@ pub const TAP_DEVICE_PREFIX: &str = "tap-";
 pub const FIREWALL_NAT_CHAIN: &str = "FLAKE_PILOT_NAT";
 pub const FIREWALL_FORWARD_CHAIN: &str = "FLAKE_PILOT_FORWARD";
 pub const FIREWALL_INPUT_CHAIN: &str = "FLAKE_PILOT_INPUT";
+// Chain of the egress policy of the VM network. The traffic of the
+// VMs to the outside world passes it before it is forwarded, see
+// NETWORK_EGRESS_PRESETS
+pub const FIREWALL_EGRESS_CHAIN: &str = "FLAKE_PILOT_EGRESS";
 // Max. size of a network interface name including the
 // terminating zero byte as defined by IFNAMSIZ in linux/if.h
 pub const IFNAMSIZ: usize = 16;
@@ -209,6 +213,21 @@ pub const NETWORK_HOST_PREFIX_LEN: u8 =
 // a resolver on the host, which the VMs use through their gateway
 pub const NETWORK_DNS: &str =
     "8.8.8.8";
+// Named sets of networks the egress policy of the VM network can
+// deny, see 'flake-ctl firecracker network init --egress-deny'.
+// The traffic of the VMs to them is dropped instead of being
+// forwarded to the outgoing interface:
+//
+// link-local: the link local network, which provides the metadata
+//   service of the cloud the host runs in at 169.254.169.254.
+//   Without the policy a VM reaches it with the identity of the
+//   host
+// private: the private networks of RFC 1918, e.g the local
+//   network of the host and the services in it
+pub const NETWORK_EGRESS_PRESETS: &[(&str, &[&str])] = &[
+    ("link-local", &["169.254.0.0/16"]),
+    ("private", &["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"])
+];
 // Directory the overlay setups of the bubblewrap sandboxes are
 // created in. The directories are created by bubblewrap-pilot and
 // mounted by the helper. It is shared

common/src/network.rs

@@ -683,6 +683,12 @@ pub struct NetworkState {
     /// The host runs a DHCP server for the VMs
     #[serde(default)]
     pub host_dhcp: bool,
+    /// Networks in their ADDRESS/PREFIX_LEN notation the traffic
+    /// of the VMs to the outside world must not go to, see
+    /// get_egress_networks(). A state written before the egress
+    /// policy became selectable provides none
+    #[serde(default)]
+    pub egress_deny: Vec<String>,
     /// The addresses handed out to the VM instances of all users
     #[serde(default)]
     pub allocations: Vec<Allocation>
@@ -702,6 +708,52 @@ impl NetworkState {
         !*/
         HostServices { resolver: self.host_resolver, dhcp: self.host_dhcp }
     }
+
+    pub fn get_egress_deny(&self) -> Option<Vec<Ipv4Network>> {
+        /*!
+        Provide the recorded networks of the egress policy, None
+        if one of them is invalid
+        !*/
+        self.egress_deny.iter()
+            .map(|network| Ipv4Network::parse(network))
+            .collect()
+    }
+}
+
+pub fn get_egress_networks(
+    egress_deny: &[String]
+) -> Result<Vec<Ipv4Network>, String> {
+    /*!
+    Provide the networks of the egress policy of the VM network
+
+    Each entry is the name of one of the sets of networks in
+    defaults::NETWORK_EGRESS_PRESETS or a network in its
+    ADDRESS/PREFIX_LEN notation. An address without a prefix length
+    is a single host. Networks which are given more than once are
+    only listed once, in the order they are given first
+    !*/
+    let mut networks: Vec<Ipv4Network> = Vec::new();
+    for entry in egress_deny {
+        let preset = defaults::NETWORK_EGRESS_PRESETS.iter()
+            .find(|(name, _)| name == entry);
+        let entry_networks = match preset {
+            Some((_, preset_networks)) => preset_networks.iter()
+                .filter_map(|network| Ipv4Network::parse(network))
+                .collect(),
+            None => vec![Ipv4Network::parse(entry).ok_or(format!(
+                "Invalid egress policy entry {entry:?}: expected one of {} \
+                or ADDRESS[/PREFIX_LEN]",
+                defaults::NETWORK_EGRESS_PRESETS.iter()
+                    .map(|(name, _)| *name).collect::<Vec<_>>().join(", ")
+            ))?]
+        };
+        for network in entry_networks {
+            if ! networks.contains(&network) {
+                networks.push(network)
+            }
+        }
+    }
+    Ok(networks)
 }
 
 // Allocation is an address of the private network handed out to
@@ -956,8 +1008,8 @@ mod tests {
     use super::{
         get_jump_rules, get_masquerade_rule, get_network_config_file,
         get_tap_name, NetworkConfig, get_address_list_networks,
-        get_flake_allocations, get_free_address, get_nameserver,
-        get_network_candidates, get_preferred_network,
+        get_egress_networks, get_flake_allocations, get_free_address,
+        get_nameserver, get_network_candidates, get_preferred_network,
         get_route_list_default_device, get_route_list_networks,
         is_vm_address, lock_file_of, select_free_network, Allocation,
         HostServices, Ipv4Network, NetworkState
@@ -1180,6 +1232,7 @@ mod tests {
             outgoing_interface: Some("eth0".to_string()),
             host_resolver: true,
             host_dhcp: false,
+            egress_deny: vec!["169.254.0.0/16".to_string()],
             allocations: vec![
                 allocation(
                     "172.16.0.2", 1000, "tap-1000_myapp",
@@ -1193,9 +1246,46 @@ mod tests {
         assert_eq!(
             HostServices { resolver: true, dhcp: false }, state.get_services()
         );
+        assert_eq!(
+            Some(vec![network("169.254.0.0/16")]), state.get_egress_deny()
+        );
         assert_eq!(
             state, serde_yaml::from_str::<NetworkState>(&yaml_data).unwrap()
         );
+        // an invalid network of the egress policy is not ignored
+        let state = NetworkState {
+            egress_deny: vec!["10.0.0.0/8".to_string(), "bogus".to_string()],
+            ..Default::default()
+        };
+        assert_eq!(None, state.get_egress_deny());
+    }
+
+    #[test]
+    fn test_get_egress_networks() {
+        let egress_networks = |entries: &[&str]| get_egress_networks(
+            &entries.iter().map(|entry| entry.to_string()).collect::<Vec<_>>()
+        );
+        assert_eq!(Ok(vec![]), egress_networks(&[]));
+        assert_eq!(
+            Ok(vec![network("169.254.0.0/16")]),
+            egress_networks(&["link-local"])
+        );
+        // presets and networks are combined, duplicates listed once
+        assert_eq!(
+            Ok(vec![
+                network("10.0.0.0/8"), network("172.16.0.0/12"),
+                network("192.168.0.0/16"), network("100.100.100.200/32")
+            ]),
+            egress_networks(&["private", "10.0.0.0/8", "100.100.100.200"])
+        );
+        // a network is taken by its network address
+        assert_eq!(
+            Ok(vec![network("10.1.0.0/16")]),
+            egress_networks(&["10.1.2.3/16"])
+        );
+        assert!(egress_networks(&["metadata"]).is_err());
+        assert!(egress_networks(&["10.0.0.0/33"]).is_err());
+        assert!(egress_networks(&["-j ACCEPT"]).is_err());
     }
 
     #[test]

completion/flake-ctl

@@ -40,7 +40,7 @@ __flake_ctl_firecracker_show() {
 
 __flake_ctl_firecracker_network() {
     if [ "${COMP_WORDS[3]}" = "init" ];then
-        __comp_reply_unused "--outgoing-interface --host-resolver --host-dhcp --help"
+        __comp_reply_unused "--outgoing-interface --host-resolver --host-dhcp --egress-deny --help"
         return 0
     fi
     if [ "${COMP_WORDS[3]}" = "add" ] || [ "${COMP_WORDS[3]}" = "remove" ];then

doc/flake-ctl-firecracker-network-init.rst

@@ -12,12 +12,13 @@ SYNOPSIS
 .. code:: bash
 
    USAGE:
-       flake-ctl firecracker network init [--outgoing-interface <OUTGOING_INTERFACE>] [--host-resolver] [--host-dhcp]
+       flake-ctl firecracker network init [--outgoing-interface <OUTGOING_INTERFACE>] [--host-resolver] [--host-dhcp] [--egress-deny <EGRESS_DENY>]...
 
    OPTIONS:
        --outgoing-interface <OUTGOING_INTERFACE>
        --host-resolver
        --host-dhcp
+       --egress-deny <EGRESS_DENY>
        --help
 
 DESCRIPTION
@@ -62,7 +63,8 @@ again: the network has to be one of the private networks the command
 selects from, see NETWORK below, and must not collide with a network
 of the host, the interface has to exist. Both are recorded in the
 network state file along with the services of the host given by
-``--host-resolver`` and ``--host-dhcp``. Connecting an application to
+``--host-resolver`` and ``--host-dhcp`` and the networks of the egress
+policy given by ``--egress-deny``. Connecting an application to
 the host network with **flake-ctl-firecracker-network-add**(8) takes
 them from there: the address of the application, the route to it and
 its firewall rules are only ever created within the recorded network.
@@ -106,10 +108,13 @@ follows, shown in the notation of ``iptables -S``:
    # table filter
    -A FORWARD -j FLAKE_PILOT_FORWARD
    -A FLAKE_PILOT_FORWARD -o tap-+ -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-   -A FLAKE_PILOT_FORWARD -i tap-+ -s <NETWORK> -o <OUTGOING_INTERFACE> -j ACCEPT
+   -A FLAKE_PILOT_FORWARD -i tap-+ -s <NETWORK> -o <OUTGOING_INTERFACE> -j FLAKE_PILOT_EGRESS
    -A FLAKE_PILOT_FORWARD -i tap-+ -j DROP
    -A FLAKE_PILOT_FORWARD -o tap-+ -j DROP
 
+   -A FLAKE_PILOT_EGRESS -d <EGRESS_DENY> -j DROP
+   -A FLAKE_PILOT_EGRESS -j ACCEPT
+
    -A INPUT -i tap-+ -j FLAKE_PILOT_INPUT
    -A FLAKE_PILOT_INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
    -A FLAKE_PILOT_INPUT -j DROP
@@ -120,6 +125,11 @@ The effect of the setup is:
   the outgoing interface is masqueraded and forwarded. The answers to
   it pass back to the VMs.
 
+* The traffic of the VMs to the outside world passes the egress
+  policy before it is forwarded. Its traffic to a network given by
+  ``--egress-deny`` is dropped, there is one ``DROP`` rule for each
+  of them. Without the option the egress chain accepts all traffic.
+
 * Traffic between two VMs, traffic from the outside world to a VM and
   traffic from a VM to any other interface of the host is dropped.
 
@@ -134,7 +144,10 @@ The chains are deleted as a whole by
 
 Former versions of flake-pilot created their rules in the builtin
 chains. Their rules for single TAP devices are deleted by this
-command. A MASQUERADE rule for all traffic of the outgoing interface
+command, as well as a rule of the FLAKE_PILOT_FORWARD chain which
+accepts the traffic of the VMs without passing it to the egress
+chain. Until the command is called again, a setup created by a former
+version keeps working without an egress policy. A MASQUERADE rule for all traffic of the outgoing interface
 and a FORWARD rule accepting all answers, which former versions
 created as well, cannot be told apart from rules created by other
 means. If one of them is found, it is reported along with the command
@@ -221,6 +234,43 @@ OPTIONS
   Applications which were connected before the setting changed keep
   their firewall rules until they are connected again
 
+--egress-deny <EGRESS_DENY>
+
+  Network the traffic of the VMs to the outside world must not go
+  to. Can be specified multiple times, the traffic to all of them is
+  dropped. Either the name of a set of networks or a network in its
+  ``ADDRESS[/PREFIX_LEN]`` notation, an address without a prefix
+  length is a single host:
+
+  link-local
+    ``169.254.0.0/16``. It covers the metadata service a cloud
+    provides to its instances at ``169.254.169.254``, e.g on AWS,
+    Azure, Google Cloud or OpenStack. Without the policy a VM on a
+    cloud instance reaches the metadata service with the identity
+    of the host and may e.g read the credentials of the instance.
+    Clouds which provide their metadata service at another address
+    have to be covered by that address, e.g ``100.100.100.200``
+
+  private
+    ``10.0.0.0/8``, ``172.16.0.0/12`` and ``192.168.0.0/16``, the
+    private networks of RFC 1918. It covers the local network of
+    the host and the services in it. A VM which needs one of them,
+    e.g a proxy or a name server of the local network, cannot reach
+    it anymore
+
+  The policy only applies to the traffic which is forwarded to the
+  outgoing interface. The services of the host a VM may use through
+  its gateway, see FIREWALL above, are not affected. The networks
+  are recorded in the network state file and take effect for all
+  applications at once, including the ones which are already
+  connected. Calling the command again replaces the policy, without
+  the option the policy is removed.
+  **flake-ctl-firecracker-network-add**\(8) refuses to connect an
+  application if the recorded policy is not active
+
+  The policy is IPv4 only, as is the whole setup. IPv6 is switched
+  off on the TAP devices of the VMs
+
 FILES
 -----
 
@@ -239,6 +289,8 @@ EXAMPLE
 
    $ flake-ctl firecracker network init --outgoing-interface eth0
 
+   $ flake-ctl firecracker network init --egress-deny link-local --egress-deny private
+
 SEE ALSO
 --------
 

doc/flake-pilot-helper.rst

@@ -15,7 +15,7 @@ SYNOPSIS
        sudo /usr/libexec/flake-pilot/flake-pilot-helper <SUBCOMMAND> [--debug]
 
    SUBCOMMANDS:
-       network-init --network <NETWORK> --outgoing-interface <IF> [--host-resolver] [--host-dhcp]
+       network-init --network <NETWORK> --outgoing-interface <IF> [--host-resolver] [--host-dhcp] [--egress-deny <NET>]...
        network-cleanup
        network-connect --config-file <FILE> [--instance <NAME>] [--uid <UID>] [--prefer <ADDR>] [--exclude <ADDR>]...
        network-disconnect --config-file <FILE> [--instance <NAME>] [--uid <UID>]
@@ -68,7 +68,8 @@ network-init
   network has to be one of the private networks flake-pilot selects
   from and must not collide with a network of the host, the
   interface has to exist. Both are recorded in the network state
-  along with the services of the host the VMs use. Called by
+  along with the services of the host the VMs use and the networks
+  of the egress policy, which only adds drop rules. Called by
   **flake-ctl-firecracker-network-init**\(8)
 
 network-cleanup

doc/user_guide/firecracker_networking.rst

@@ -192,6 +192,27 @@ the instance are bound to it.
 A change of these settings takes effect for an application once it
 is connected again.
 
+By default the VMs may reach every network the host reaches through
+the outgoing interface. An egress policy denies them networks with
+``--egress-deny``, which can be given multiple times:
+
+.. code-block:: bash
+
+   flake-ctl firecracker network init --egress-deny link-local --egress-deny private
+
+``link-local`` denies ``169.254.0.0/16``, which covers the metadata
+service of a cloud at ``169.254.169.254``. On a cloud instance a VM
+reaches that service with the identity of the host otherwise, and may
+e.g read the credentials of the instance. ``private`` denies the
+private networks of RFC 1918, ``10.0.0.0/8``, ``172.16.0.0/12`` and
+``192.168.0.0/16``, which keeps the VMs away from the local network of
+the host. Any other network can be given as ``ADDRESS[/PREFIX_LEN]``,
+e.g ``--egress-deny 100.100.100.200`` for the metadata service of a
+cloud which uses another address. The services of the host the VMs use
+through their gateway are not affected. Unlike the settings above, the
+policy takes effect for all applications at once. Calling ``init``
+again replaces it.
+
 Rules of former versions of Flake Pilot for single TAP devices are
 deleted. The rules for all traffic former versions created, a
 ``MASQUERADE`` rule for the outgoing interface and a ``FORWARD`` rule

doc/user_guide/spelling_wordlist.txt

@@ -42,3 +42,4 @@ toolchain
 vsock
 appstore
 boxbuild
+RFC

flake-ctl/src/cli.rs

@@ -261,6 +261,17 @@ pub enum Network {
         /// closed to them
         #[clap(long)]
         host_dhcp: bool,
+
+        /// Network the traffic of the VMs to the outside world
+        /// must not go to. Either link-local, which covers the
+        /// metadata service of a cloud at 169.254.169.254, private,
+        /// which covers the private networks of RFC 1918, or a
+        /// network as ADDRESS[/PREFIX_LEN]. Can be specified
+        /// multiple times. Without this option the VMs may reach
+        /// every network the host reaches through the outgoing
+        /// interface
+        #[clap(long, multiple_occurrences = true)]
+        egress_deny: Vec<String>,
     },
     /// Connect a VM application to the host network
     Add {

flake-ctl/src/main.rs

@@ -160,14 +160,16 @@ async fn main() -> Result<ExitCode, Box<dyn std::error::Error>> {
                     match &command {
                         // init
                         cli::Network::Init {
-                            outgoing_interface, host_resolver, host_dhcp
+                            outgoing_interface, host_resolver, host_dhcp,
+                            egress_deny
                         } => {
                             if ! network::init(
                                 outgoing_interface.as_deref(),
                                 flakes::network::HostServices {
                                     resolver: *host_resolver,
                                     dhcp: *host_dhcp
                                 },
+                                egress_deny,
                                 user
                             ) {
                                 return Ok(ExitCode::FAILURE)