diff --git a/examples/terraform/aws-simple/README.md b/examples/terraform/aws-simple/README.md index 42c70103..4eb4b75c 100644 --- a/examples/terraform/aws-simple/README.md +++ b/examples/terraform/aws-simple/README.md @@ -25,6 +25,39 @@ admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] Node-to-node traffic does not pass through these rules, so narrowing this does not affect the cluster talking to itself. +#### NodePort exposure — `nodeport_cidrs` + +This example installs MKE with `--nodeport-range=32768-35535`, so Kubernetes +NodePort services land in that range on every node. `nodeport_cidrs` lists the +CIDRs allowed to reach it. The range itself is stated once, as +`local.nodeport_range` in `launchpad.tf`, and feeds both the install flag and +the security group rule so the two cannot disagree. + +Leaving it unset means "use `admin_cidrs`", which keeps an existing +`terraform.tfvars` working and carries `admin_cidrs`'s exposure with it: at +both defaults the NodePort range is reachable from the whole internet. Set it +when the people who reach your applications are not the people who administer +the cluster — widening `admin_cidrs` to let application clients in would also +hand them SSH and the MKE and Kubernetes APIs: + +```hcl +nodeport_cidrs = ["203.0.113.0/24"] +``` + +The rule covers TCP and UDP, and does not include the VPC CIDR: nothing in +this example load-balances the NodePort range, and node-to-node traffic is +already covered by the cluster-internal rule. + +On `rhel_9` and `sles_15` nodes the host firewall is a second gate. The +`user_data` in `terraform.tfvars.template` opens 2377, 7946, 4789 and 10250 +but not the NodePort range, so `firewall-cmd` still blocks it there whatever +the security group says. Add the range to that `user_data` if you need +NodePort services on those platforms. +The `ubuntu_22.04` `user_data` omits the range in exactly the same way — it +opens 2377, 7946, 10250/tcp and 7946, 4789/udp through `ufw` and nothing else — +so the same gate applies on Ubuntu whenever `ufw` is active in that AMI, which +is unverified here because confirming it needs a running AWS cluster. + #### Credentials The MKE admin password (`launchpad.mke_connect.password`) and diff --git a/examples/terraform/aws-simple/launchpad.tf b/examples/terraform/aws-simple/launchpad.tf index 2e9a671b..93739644 100644 --- a/examples/terraform/aws-simple/launchpad.tf +++ b/examples/terraform/aws-simple/launchpad.tf @@ -62,6 +62,21 @@ locals { } } + // The Kubernetes NodePort range. Stated once here and used twice: for the + // ingress rule below and for the --nodeport-range install flag in the + // generated launchpad.yaml. Declaring a range the security group does not + // admit produces a cluster whose NodePort services are unreachable, so the + // two must not be able to drift apart. + nodeport_range = { + from = 32768 + to = 35535 + } + + // NodePort services are reached by application clients from outside the + // cluster, who are not necessarily the administrators var.admin_cidrs + // names. Unset means inherit admin_cidrs; see variables.tf. + nodeport_cidrs = var.nodeport_cidrs != null ? var.nodeport_cidrs : var.admin_cidrs + // MCR/MKE/MSR firewall rules. // // Node-to-node traffic inside the group stays unrestricted: MKE, swarm and @@ -75,6 +90,11 @@ locals { // local.launchpad_ingresses above reach the nodes from inside the VPC // rather than from an administrator's address, and their health checks // would fail if only var.admin_cidrs were allowed. + // + // The NodePort range takes its own CIDR list and no VPC CIDR: nothing in + // local.launchpad_ingresses load-balances it, so there is no in-VPC health + // check to admit, and node-to-node NodePort traffic is already covered by + // the self rule. launchpad_securitygroups = { "permissive" = { description = "Common SG for all cluster machines" @@ -119,6 +139,22 @@ locals { protocol : "tcp" self : false cidr_blocks : concat(var.admin_cidrs, [var.network.cidr]) + }, + { + description : "Kubernetes NodePort services (TCP), from application clients" + from_port : local.nodeport_range.from + to_port : local.nodeport_range.to + protocol : "tcp" + self : false + cidr_blocks : local.nodeport_cidrs + }, + { + description : "Kubernetes NodePort services (UDP), from application clients" + from_port : local.nodeport_range.from + to_port : local.nodeport_range.to + protocol : "udp" + self : false + cidr_blocks : local.nodeport_cidrs } ] egress_ipv4 = [ @@ -230,7 +266,7 @@ spec: installFlags: - "--san=${local.MKE_URL}" - "--default-node-orchestrator=kubernetes" - - "--nodeport-range=32768-35535" + - "--nodeport-range=${local.nodeport_range.from}-${local.nodeport_range.to}" upgradeFlags: - "--force-recent-backup" - "--force-minimums" diff --git a/examples/terraform/aws-simple/terraform.tfvars.template b/examples/terraform/aws-simple/terraform.tfvars.template index 70163550..3699d465 100644 --- a/examples/terraform/aws-simple/terraform.tfvars.template +++ b/examples/terraform/aws-simple/terraform.tfvars.template @@ -12,6 +12,19 @@ // // admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] +// CIDRs allowed to reach the Kubernetes NodePort range (32768-35535, the +// range this example installs MKE with). +// +// Unset means "the same as admin_cidrs", so with both left alone the +// NodePort range is reachable from the whole internet -- the same real risk +// as above, and the exposure this example had before its ingress rules were +// scoped. Set this when the people who reach your applications are not the +// people who administer the cluster: widening admin_cidrs to admit +// application clients would also hand them SSH and the MKE and Kubernetes +// APIs. Uncomment and edit: +// +// nodeport_cidrs = ["203.0.113.0/24"] + aws = { region = "us-east-2" } diff --git a/examples/terraform/aws-simple/variables.tf b/examples/terraform/aws-simple/variables.tf index 4c86f486..f1c981eb 100644 --- a/examples/terraform/aws-simple/variables.tf +++ b/examples/terraform/aws-simple/variables.tf @@ -96,6 +96,39 @@ variable "admin_cidrs" { } } +variable "nodeport_cidrs" { + description = <<-EOT + CIDRs allowed to reach the Kubernetes NodePort range on the cluster + nodes. The range itself is local.nodeport_range in launchpad.tf, which + also builds the --nodeport-range install flag, so the rule and the flag + cannot drift apart. + + Unset (null) means "use admin_cidrs", which is the default and keeps an + existing tfvars working unchanged. Set it when the people who reach + your applications are not the people who administer the cluster: + admin_cidrs admits SSH, WinRM and the MKE and Kubernetes APIs, and + widening it to admit application clients would hand them those too. + + nodeport_cidrs = ["203.0.113.0/24"] + + Inheriting admin_cidrs inherits its exposure as well: with both left at + their defaults the NodePort range is reachable from the whole internet. + That is a real risk and not a recommendation -- it is the exposure this + example had before the ingress rules were scoped. + + An empty list is rejected rather than treated as "no rule", because a + cluster installed with --nodeport-range and no way in is the failure + this variable exists to prevent. Node-to-node traffic does not come + through here. + EOT + type = list(string) + default = null + validation { + condition = var.nodeport_cidrs == null || length(coalesce(var.nodeport_cidrs, [])) > 0 + error_message = "nodeport_cidrs must list at least one CIDR, or be left unset to inherit admin_cidrs." + } +} + variable "windows_password" { description = <<-EOT Password set on the local Administrator account of Windows nodes and used