[PRODENG-3929] Admit the NodePort range the aws-simple example installs

examples/terraform/aws-simple/README.md

@@ -25,6 +25,39 @@ admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"]
 Node-to-node traffic does not pass through these rules, so narrowing this does
 not affect the cluster talking to itself.
 
+#### NodePort exposure — `nodeport_cidrs`
+
+This example installs MKE with `--nodeport-range=32768-35535`, so Kubernetes
+NodePort services land in that range on every node. `nodeport_cidrs` lists the
+CIDRs allowed to reach it. The range itself is stated once, as
+`local.nodeport_range` in `launchpad.tf`, and feeds both the install flag and
+the security group rule so the two cannot disagree.
+
+Leaving it unset means "use `admin_cidrs`", which keeps an existing
+`terraform.tfvars` working and carries `admin_cidrs`'s exposure with it: at
+both defaults the NodePort range is reachable from the whole internet. Set it
+when the people who reach your applications are not the people who administer
+the cluster — widening `admin_cidrs` to let application clients in would also
+hand them SSH and the MKE and Kubernetes APIs:
+
+```hcl
+nodeport_cidrs = ["203.0.113.0/24"]
+```
+
+The rule covers TCP and UDP, and does not include the VPC CIDR: nothing in
+this example load-balances the NodePort range, and node-to-node traffic is
+already covered by the cluster-internal rule.
+
+On `rhel_9` and `sles_15` nodes the host firewall is a second gate. The
+`user_data` in `terraform.tfvars.template` opens 2377, 7946, 4789 and 10250
+but not the NodePort range, so `firewall-cmd` still blocks it there whatever
+the security group says. Add the range to that `user_data` if you need
+NodePort services on those platforms.
+The `ubuntu_22.04` `user_data` omits the range in exactly the same way — it
+opens 2377, 7946, 10250/tcp and 7946, 4789/udp through `ufw` and nothing else —
+so the same gate applies on Ubuntu whenever `ufw` is active in that AMI, which
+is unverified here because confirming it needs a running AWS cluster.
+
 #### Credentials
 
 The MKE admin password (`launchpad.mke_connect.password`) and

examples/terraform/aws-simple/launchpad.tf

@@ -62,6 +62,21 @@ locals {
     }
   }
 
+  // The Kubernetes NodePort range.  Stated once here and used twice: for the
+  // ingress rule below and for the --nodeport-range install flag in the
+  // generated launchpad.yaml.  Declaring a range the security group does not
+  // admit produces a cluster whose NodePort services are unreachable, so the
+  // two must not be able to drift apart.
+  nodeport_range = {
+    from = 32768
+    to   = 35535
+  }
+
+  // NodePort services are reached by application clients from outside the
+  // cluster, who are not necessarily the administrators var.admin_cidrs
+  // names.  Unset means inherit admin_cidrs; see variables.tf.
+  nodeport_cidrs = var.nodeport_cidrs != null ? var.nodeport_cidrs : var.admin_cidrs
+
   // MCR/MKE/MSR firewall rules.
   //
   // Node-to-node traffic inside the group stays unrestricted: MKE, swarm and
@@ -75,6 +90,11 @@ locals {
   // local.launchpad_ingresses above reach the nodes from inside the VPC
   // rather than from an administrator's address, and their health checks
   // would fail if only var.admin_cidrs were allowed.
+  //
+  // The NodePort range takes its own CIDR list and no VPC CIDR: nothing in
+  // local.launchpad_ingresses load-balances it, so there is no in-VPC health
+  // check to admit, and node-to-node NodePort traffic is already covered by
+  // the self rule.
   launchpad_securitygroups = {
     "permissive" = {
       description = "Common SG for all cluster machines"
@@ -119,6 +139,22 @@ locals {
           protocol : "tcp"
           self : false
           cidr_blocks : concat(var.admin_cidrs, [var.network.cidr])
+        },
+        {
+          description : "Kubernetes NodePort services (TCP), from application clients"
+          from_port : local.nodeport_range.from
+          to_port : local.nodeport_range.to
+          protocol : "tcp"
+          self : false
+          cidr_blocks : local.nodeport_cidrs
+        },
+        {
+          description : "Kubernetes NodePort services (UDP), from application clients"
+          from_port : local.nodeport_range.from
+          to_port : local.nodeport_range.to
+          protocol : "udp"
+          self : false
+          cidr_blocks : local.nodeport_cidrs
         }
       ]
       egress_ipv4 = [
@@ -230,7 +266,7 @@ spec:
     installFlags: 
     - "--san=${local.MKE_URL}"
     - "--default-node-orchestrator=kubernetes"
-    - "--nodeport-range=32768-35535"
+    - "--nodeport-range=${local.nodeport_range.from}-${local.nodeport_range.to}"
     upgradeFlags:
     - "--force-recent-backup"
     - "--force-minimums"

examples/terraform/aws-simple/terraform.tfvars.template

@@ -12,6 +12,19 @@
 //
 // admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"]
 
+// CIDRs allowed to reach the Kubernetes NodePort range (32768-35535, the
+// range this example installs MKE with).
+//
+// Unset means "the same as admin_cidrs", so with both left alone the
+// NodePort range is reachable from the whole internet -- the same real risk
+// as above, and the exposure this example had before its ingress rules were
+// scoped. Set this when the people who reach your applications are not the
+// people who administer the cluster: widening admin_cidrs to admit
+// application clients would also hand them SSH and the MKE and Kubernetes
+// APIs. Uncomment and edit:
+//
+// nodeport_cidrs = ["203.0.113.0/24"]
+
 aws = {
   region = "us-east-2"
 }

examples/terraform/aws-simple/variables.tf

@@ -96,6 +96,39 @@ variable "admin_cidrs" {
   }
 }
 
+variable "nodeport_cidrs" {
+  description = <<-EOT
+    CIDRs allowed to reach the Kubernetes NodePort range on the cluster
+    nodes.  The range itself is local.nodeport_range in launchpad.tf, which
+    also builds the --nodeport-range install flag, so the rule and the flag
+    cannot drift apart.
+
+    Unset (null) means "use admin_cidrs", which is the default and keeps an
+    existing tfvars working unchanged.  Set it when the people who reach
+    your applications are not the people who administer the cluster:
+    admin_cidrs admits SSH, WinRM and the MKE and Kubernetes APIs, and
+    widening it to admit application clients would hand them those too.
+
+      nodeport_cidrs = ["203.0.113.0/24"]
+
+    Inheriting admin_cidrs inherits its exposure as well: with both left at
+    their defaults the NodePort range is reachable from the whole internet.
+    That is a real risk and not a recommendation -- it is the exposure this
+    example had before the ingress rules were scoped.
+
+    An empty list is rejected rather than treated as "no rule", because a
+    cluster installed with --nodeport-range and no way in is the failure
+    this variable exists to prevent.  Node-to-node traffic does not come
+    through here.
+  EOT
+  type        = list(string)
+  default     = null
+  validation {
+    condition     = var.nodeport_cidrs == null || length(coalesce(var.nodeport_cidrs, [])) > 0
+    error_message = "nodeport_cidrs must list at least one CIDR, or be left unset to inherit admin_cidrs."
+  }
+}
+
 variable "windows_password" {
   description = <<-EOT
     Password set on the local Administrator account of Windows nodes and used