diff --git a/examples/terraform/aws-complete/README.md b/examples/terraform/aws-complete/README.md index 7877229d8..83bd829e6 100644 --- a/examples/terraform/aws-complete/README.md +++ b/examples/terraform/aws-complete/README.md @@ -9,4 +9,43 @@ - This configuration uses Launchpad terraform provider for installing MCR, MKE and MSR 2.9x on the provisioned infrstructure - After installing MKE, the configuration uses MKE terraform provider to create a kubernetes context to install kubernetes workloads - After creating a kubernetes context the same is used for installing AWS cloud controller manager and AWS EBS CSI driver -- The content of test.tf file is commented out and the resources in that file can be used testing the functionality of AWS cloud controller manager and AWS EBS CSI driver \ No newline at end of file +- The content of test.tf file is commented out and the resources in that file can be used testing the functionality of AWS cloud controller manager and AWS EBS CSI driver + +## Inputs worth knowing about + +### Network exposure — `admin_cidrs` + +`admin_cidrs` lists the CIDRs allowed to reach the cluster from outside it: +SSH, WinRM over HTTPS, and the MKE and Kubernetes API ports. + +It defaults to `["0.0.0.0/0"]` — open to the whole internet. That is a real +risk, and it is the exposure this example has always had; it is the default so +that an existing `terraform.tfvars` keeps working and a first run needs no +extra arguments. Narrow it whenever you can: + +```hcl +admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] +``` + +Node-to-node traffic does not pass through these rules, so narrowing this does +not affect the cluster talking to itself. + +### Credentials + +The MKE admin password (`launchpad.mke_connect.password`) and +`windows_password` are ordinary terraform variables. Setting them in +`terraform.tfvars` is all that is required, and the generated +`launchpad_yaml` output is usable as-is. + +To keep a password off disk, use terraform's own environment mechanism — +terraform reads any input variable from `TF_VAR_`: + +```sh +export TF_VAR_launchpad='{"drain":false,"mcr_version":"25.0.13","mke_version":"3.8.8","msr_version":"","mke_connect":{"username":"admin","password":"...","insecure":false},"skip_create":false,"skip_destroy":true}' +export TF_VAR_windows_password='...' +``` + +The MKE password is nested inside the `launchpad` object, so `TF_VAR_launchpad` +carries that whole object as JSON. Either route works; the environment is an +option for keeping secrets out of a file, not a requirement for running the +example. \ No newline at end of file diff --git a/examples/terraform/aws-complete/iam.tf b/examples/terraform/aws-complete/iam.tf index f771126d2..ba58f847e 100644 --- a/examples/terraform/aws-complete/iam.tf +++ b/examples/terraform/aws-complete/iam.tf @@ -1,5 +1,24 @@ -resource "aws_iam_role" "common_role" { - name = "common-iam-role-${var.name}" +// Manager and worker nodes carry separate roles so that the cloud-controller +// permissions, which only the control plane uses, are not held by every +// worker in the cluster. Both roles share the same trust policy. +// +// APPLYING THIS TO AN EXISTING STACK REPLACES ITS IAM RESOURCES. The previous +// common_role, common_profile and their policy attachments are destroyed and +// the new profiles are attached to the launch templates, not to instances +// that are already running: provision-aws does not refresh instances on a +// launch-template change, so nodes provisioned before this change keep +// running with an instance profile whose role no longer exists, and the +// cloud controller manager and EBS CSI driver on them lose their permissions +// until the nodes are replaced. Replace the nodes, or recreate the stack, +// rather than re-applying in place against a live cluster. +resource "aws_iam_role" "manager_role" { + name = "manager-iam-role-${var.name}" + path = "/" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role" "worker_role" { + name = "worker-iam-role-${var.name}" path = "/" assume_role_policy = data.aws_iam_policy_document.assume_role.json } @@ -17,7 +36,12 @@ data "aws_iam_policy_document" "assume_role" { } } -resource "aws_iam_instance_profile" "common_profile" { - name = "common-instance-profile-${var.name}" - role = aws_iam_role.common_role.name +resource "aws_iam_instance_profile" "manager_profile" { + name = "manager-instance-profile-${var.name}" + role = aws_iam_role.manager_role.name +} + +resource "aws_iam_instance_profile" "worker_profile" { + name = "worker-instance-profile-${var.name}" + role = aws_iam_role.worker_role.name } diff --git a/examples/terraform/aws-complete/launchpad.tf b/examples/terraform/aws-complete/launchpad.tf index 78fa8b91d..182e4673b 100644 --- a/examples/terraform/aws-complete/launchpad.tf +++ b/examples/terraform/aws-complete/launchpad.tf @@ -52,7 +52,10 @@ locals { } } - // standard MCR/MKE/MSR firewall rules [here we just leave it open until we can figure this out] + // MCR/MKE/MSR firewall rules. The load-balanced ports accept the VPC CIDR + // as well as var.admin_cidrs, because the ingress load balancers reach the + // manager nodes from inside the VPC rather than from an administrator's + // address, and their health checks would fail without it. launchpad_securitygroups = { "manager" = { description = "Common security group for manager nodes" @@ -60,20 +63,20 @@ locals { tags = local.common_tags ingress_ipv4 = [ { - description : "Allow https traffic from anywhere" + description : "Allow https traffic from administrators and the load balancers" from_port : 443 to_port : 443 protocol : "tcp" self : false - cidr_blocks : ["0.0.0.0/0"] + cidr_blocks : concat(var.admin_cidrs, [var.network.cidr]) }, { - description : "Allow https traffic from anywhere for kube api server" + description : "Allow kube api server traffic from administrators and the load balancer" from_port : 6443 to_port : 6443 protocol : "tcp" self : false - cidr_blocks : ["0.0.0.0/0"] + cidr_blocks : concat(var.admin_cidrs, [var.network.cidr]) }, ] } @@ -107,7 +110,7 @@ locals { winrm_address : ngn.public_ip winrm_user : ng.winrm_user - winrm_password : var.windows_password + winrm_password : local.windows_password winrm_useHTTPS : ng.winrm_useHTTPS winrm_insecure : ng.winrm_insecure } if contains(local.launchpad_roles, ng.role) && ng.connection == "winrm" }]...) @@ -182,6 +185,23 @@ resource "launchpad_config" "cluster" { } // ------- Ye old launchpad yaml (just for debugging) +// +// The credentials are written from the terraform variables that carry them: +// var.launchpad.mke_connect.password and var.windows_password. The document +// this produces is usable as-is, with nothing exported into the environment +// first. +// +// To keep a password out of a tfvars file on disk, supply the variable +// through terraform's own environment mechanism instead -- TF_VAR_, +// which terraform reads for any input variable: +// +// export TF_VAR_windows_password='...' +// export TF_VAR_launchpad='{"drain":false,...,"mke_connect":{"username":"admin","password":"...","insecure":true}}' +// +// TF_VAR_launchpad takes the whole object as JSON because the MKE password +// is nested inside it; test/terraform.go does exactly this. Either route +// produces the same configuration -- the environment is an option for +// keeping secrets off disk, not a requirement for running the example. output "launchpad_yaml" { description = "launchpad config file yaml (for debugging)" sensitive = true diff --git a/examples/terraform/aws-complete/main.tf b/examples/terraform/aws-complete/main.tf index 656e3b610..337bbc10f 100644 --- a/examples/terraform/aws-complete/main.tf +++ b/examples/terraform/aws-complete/main.tf @@ -24,12 +24,17 @@ locals { module "aws_ccm" { - source = "./modules/aws-ccm" - iam_role_name = aws_iam_role.common_role.name - cluster_name = var.name + source = "./modules/aws-ccm" + // The control-plane policy goes to managers only; the worker policy, which + // is ECR pull access, is needed by every node that runs containers. + control_plane_role_name = aws_iam_role.manager_role.name + worker_role_names = [aws_iam_role.manager_role.name, aws_iam_role.worker_role.name] + cluster_name = var.name } module "aws_ebs_csi_driver" { - source = "./modules/ebs-csi-driver" - iam_role_name = aws_iam_role.common_role.name + source = "./modules/ebs-csi-driver" + // Attached to both roles, as it was to the single role before: the CSI + // controller runs on a manager and the node plugin runs everywhere. + iam_role_names = [aws_iam_role.manager_role.name, aws_iam_role.worker_role.name] } diff --git a/examples/terraform/aws-complete/modules/aws-ccm/iam.tf b/examples/terraform/aws-complete/modules/aws-ccm/iam.tf index 58a4360f4..06a63eb49 100644 --- a/examples/terraform/aws-complete/modules/aws-ccm/iam.tf +++ b/examples/terraform/aws-complete/modules/aws-ccm/iam.tf @@ -1,117 +1,212 @@ -resource "aws_iam_policy" "control_plane_policy" { - name = "control-plane-policy-${var.cluster_name}" - description = "Iam policy for attaching to the IAM instance profile" +// The AWS cloud controller manager's permissions are split into three +// statements, following the shape of the policy published by the upstream +// cloud-provider-aws project: +// +// - discovery calls, which AWS does not support resource-level permissions +// for at all, so they stay at Resource "*"; +// - calls that create new resources, which cannot be constrained by the tags +// of a resource that does not exist yet; +// - calls that mutate or delete existing resources, which are constrained to +// resources carrying this cluster's ownership tag. +// +// The cluster tag is the one the stack applies to everything it provisions +// (see local.kube_tags in the example root), which is also the tag the cloud +// controller manager puts on the resources it creates. +locals { + cluster_tag_key = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" +} + +data "aws_iam_policy_document" "control_plane" { + // Discovery. No resource-level permissions exist for these actions. + statement { + sid = "Discovery" + effect = "Allow" + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:DescribeVpcs", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "kms:DescribeKey", + ] + resources = ["*"] + } + + // Creation. The resource does not exist when the call is made, so it cannot + // carry the cluster tag yet. + // + // CreateLoadBalancerListeners and CreateLoadBalancerPolicy are NOT here: + // despite the names they operate on an existing Classic load balancer, so + // they belong with the mutations below and are constrained by the cluster + // tag like the rest of them. + statement { + sid = "CreateClusterResources" + effect = "Allow" + actions = [ + "ec2:CreateSecurityGroup", + "ec2:CreateVolume", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", + "iam:CreateServiceLinkedRole", + ] + resources = ["*"] + } + + // Tagging, restricted to tags applied as part of a create call. + // + // An unrestricted ec2:CreateTags or elasticloadbalancing:AddTags would + // defeat the MutateClusterResources statement below: a holder of this role + // could tag an unrelated security group, volume or load balancer as owned + // by this cluster and then mutate or delete it under a condition that reads + // as constrained. ec2:CreateAction and elasticloadbalancing:CreateAction + // restrict tagging to the resource-creating calls granted above, which is + // the same construction the AWS-managed AmazonEBSCSIDriverPolicy uses. + statement { + sid = "TagOnCreateOnly" + effect = "Allow" + actions = ["ec2:CreateTags"] + resources = ["*"] - policy = < merge(ngd, local.platforms_with_ami[ngd.platform]) } + // Combine the nodegroup definition with the platform data. The platform + // data wins on every key except user_data: a caller-supplied + // ngd.user_data (e.g. the firewall-cmd/ufw rules in + // terraform.tfvars.template) takes precedence over the platform's coded + // default user_data, EXCEPT for winrm platforms, where the platform's + // default is not merely a convenience default but a fixed requirement -- + // it resets the local Administrator password and stands up the WinRM + // HTTPS listener launchpad needs in order to connect at all. For those + // platforms the caller's user_data is appended after the required setup + // instead of replacing it, so a custom user_data on a Windows nodegroup + // can't accidentally disable WinRM connectivity. + nodegroups_wplatform = { for k, ngd in var.nodegroups : k => merge(ngd, local.platforms_with_ami[ngd.platform], { + user_data = ( + local.platforms_with_ami[ngd.platform].connection == "winrm" + ? join("\n", compact([local.platforms_with_ami[ngd.platform].user_data, ngd.user_data])) + : (ngd.user_data != "" ? ngd.user_data : local.platforms_with_ami[ngd.platform].user_data) + ) + }) } } # PROVISION MACHINES/NETWORK @@ -26,7 +42,7 @@ module "provision" { role : ngd.role public : ngd.public user_data : ngd.user_data - instance_profile_name : aws_iam_instance_profile.common_profile.name + instance_profile_name : ngd.role == "manager" ? aws_iam_instance_profile.manager_profile.name : aws_iam_instance_profile.worker_profile.name tags : local.tags } } diff --git a/examples/terraform/aws-complete/sg.tf b/examples/terraform/aws-complete/sg.tf index cf8c01784..70c23e474 100644 --- a/examples/terraform/aws-complete/sg.tf +++ b/examples/terraform/aws-complete/sg.tf @@ -32,12 +32,12 @@ locals { tags = local.common_tags ingress_ipv4 = [ { - description : "Allow ssh traffic from anywhere" + description : "Allow ssh traffic from the administrative CIDRs" from_port : 22 to_port : 22 protocol : "tcp" self : false - cidr_blocks : ["0.0.0.0/0"] + cidr_blocks : var.admin_cidrs }, ] } @@ -48,12 +48,14 @@ locals { tags = local.common_tags ingress_ipv4 = [ { - description : "Allow winrm traffic from anywhere" - from_port : 5985 + // 5985 is WinRM over cleartext HTTP and is deliberately absent: + // these nodes serve WinRM over HTTPS on 5986. + description : "Allow winrm HTTPS traffic from the administrative CIDRs" + from_port : 5986 to_port : 5986 protocol : "tcp" self : false - cidr_blocks : ["0.0.0.0/0"] + cidr_blocks : var.admin_cidrs }, ] } diff --git a/examples/terraform/aws-complete/terraform.tfvars.template b/examples/terraform/aws-complete/terraform.tfvars.template index 46db5abb2..36d5fc113 100644 --- a/examples/terraform/aws-complete/terraform.tfvars.template +++ b/examples/terraform/aws-complete/terraform.tfvars.template @@ -1,10 +1,35 @@ // used to name infrastructure (CHANGE THIS) //name = "test" +// CIDRs allowed to reach the cluster from outside it (SSH, WinRM over HTTPS, +// MKE and the Kubernetes API). +// +// Defaults to ["0.0.0.0/0"] -- open to the whole internet. That is a real +// risk, and it is the exposure this example has always had; it is left as the +// default so an existing tfvars keeps working and a first run needs no extra +// arguments. Narrow it whenever you can: list the machine running launchpad +// and the operators who need the MKE UI. Uncomment and edit: +// +// admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] + aws = { region = "us-east-2" } +// Credentials: both the MKE admin password below and windows_password at the +// bottom of this file are ordinary terraform variables, and setting them here +// is all that is needed. +// +// If you would rather not keep a password in this file, terraform reads any +// input variable from TF_VAR_ in the environment instead: +// +// export TF_VAR_launchpad='{"drain":false,"mcr_version":"25.0.13","mke_version":"3.8.8","msr_version":"","mke_connect":{"username":"admin","password":"...","insecure":false},"skip_create":false,"skip_destroy":true}' +// export TF_VAR_windows_password='...' +// +// The MKE password is nested inside the launchpad object, so TF_VAR_launchpad +// carries that whole object as JSON. Omit the launchpad block below if you +// use it. Either route works; the environment is optional. + launchpad = { drain = false @@ -54,4 +79,6 @@ nodegroups = { // set a windows password, if you have windows nodes // windows passwords must match a pattern, or connections will fail. // use something like: `testp@ss!` +// Required only when a nodegroup uses a windows_ platform; may also be +// supplied as TF_VAR_windows_password instead of being written here. // windows_password = "" diff --git a/examples/terraform/aws-complete/variables.tf b/examples/terraform/aws-complete/variables.tf index e90cac629..a66a19c1c 100644 --- a/examples/terraform/aws-complete/variables.tf +++ b/examples/terraform/aws-complete/variables.tf @@ -62,8 +62,60 @@ variable "extra_tags" { default = {} } +variable "admin_cidrs" { + description = <<-EOT + CIDRs allowed to reach the cluster from outside it: SSH, WinRM over + HTTPS, and the MKE and Kubernetes API ports. + + Open by default. ["0.0.0.0/0"] means every one of those ports is + reachable from the whole internet, which is a real risk and not a + recommendation -- it is the same exposure this example has always had, + kept so that an existing tfvars still works and a first run still comes + up without extra arguments. + + Narrowing it is one line: set admin_cidrs to the addresses that actually + need in -- the machine running launchpad, and the operators who need the + MKE UI. For example: + + admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] + + Node-to-node traffic does not come through here, so narrowing this does + not affect the cluster talking to itself. + EOT + type = list(string) + default = ["0.0.0.0/0"] + validation { + condition = length(var.admin_cidrs) > 0 + error_message = "admin_cidrs must list at least one CIDR." + } +} + variable "windows_password" { - description = "Password to use with windows & winrm" + description = <<-EOT + Password set on the local Administrator account of Windows nodes and used + by launchpad for WinRM. Required when any nodegroup runs a Windows + platform; leave unset for Linux-only stacks. + + The validation below references var.nodegroups, which Terraform supports + from 1.9 onwards. + EOT type = string - default = "" + sensitive = true + default = null + + // Without this, a stack with Windows nodegroups and no windows_password + // generated a launchpad.yaml carrying "password:" with an empty value, and + // launchpad then authenticated as Administrator with no password. Failing + // here names the variable at plan time instead. + // + // The whole requirement is gated on a Windows nodegroup being present, so a + // Linux-only stack may leave the variable unset or pass an empty string -- + // neither reaches a host that would authenticate with it. + validation { + condition = ( + !anytrue([for ng in var.nodegroups : startswith(ng.platform, "windows_")]) + || (var.windows_password != null && length(var.windows_password) > 0) + ) + error_message = "windows_password must be set to a non-empty value when any nodegroup uses a windows_ platform: it is the local Administrator password the nodes are configured with and the credential launchpad uses for WinRM. Leave it unset only for Linux-only stacks." + } } diff --git a/examples/terraform/aws-simple/README.md b/examples/terraform/aws-simple/README.md index 2677a3421..42c701034 100644 --- a/examples/terraform/aws-simple/README.md +++ b/examples/terraform/aws-simple/README.md @@ -8,3 +8,40 @@ ### Launchpad +#### Network exposure — `admin_cidrs` + +`admin_cidrs` lists the CIDRs allowed to reach the cluster from outside it: +SSH, WinRM over HTTPS, and the MKE/MSR and Kubernetes API ports. + +It defaults to `["0.0.0.0/0"]` — open to the whole internet. That is a real +risk, and it is the exposure this example has always had; it is the default so +that an existing `terraform.tfvars` keeps working and a first run needs no +extra arguments. Narrow it whenever you can: + +```hcl +admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] +``` + +Node-to-node traffic does not pass through these rules, so narrowing this does +not affect the cluster talking to itself. + +#### Credentials + +The MKE admin password (`launchpad.mke_connect.password`) and +`windows_password` are ordinary terraform variables. Setting them in +`terraform.tfvars` is all that is required, and the generated +`launchpad_yaml` output is usable as-is. + +To keep a password off disk, use terraform's own environment mechanism — +terraform reads any input variable from `TF_VAR_`: + +```sh +export TF_VAR_launchpad='{"drain":false,"mcr_channel":"stable-25.0.14","mke_version":"3.8.8","msr_version":"2.9.28","mke_connect":{"username":"admin","password":"...","insecure":true},"skip_create":false,"skip_destroy":true}' +export TF_VAR_windows_password='...' +``` + +The MKE password is nested inside the `launchpad` object, so `TF_VAR_launchpad` +carries that whole object as JSON. Either route works; the environment is an +option for keeping secrets out of a file, not a requirement for running the +example. + diff --git a/examples/terraform/aws-simple/launchpad.tf b/examples/terraform/aws-simple/launchpad.tf index ad4c5416a..2e9a671bc 100644 --- a/examples/terraform/aws-simple/launchpad.tf +++ b/examples/terraform/aws-simple/launchpad.tf @@ -62,14 +62,26 @@ locals { } } - // standard MCR/MKE/MSR firewall rules [here we just leave it open until we can figure this out] + // MCR/MKE/MSR firewall rules. + // + // Node-to-node traffic inside the group stays unrestricted: MKE, swarm and + // the kubelet use a wide and version-dependent port range between cluster + // members, and every member is in this security group. + // + // Everything arriving from outside the group is restricted to the ports the + // cluster actually serves, from the CIDRs the operator names in + // var.admin_cidrs. The two load-balanced ports additionally accept the VPC + // CIDR, because the MKE and MSR load balancers defined in + // local.launchpad_ingresses above reach the nodes from inside the VPC + // rather than from an administrator's address, and their health checks + // would fail if only var.admin_cidrs were allowed. launchpad_securitygroups = { "permissive" = { description = "Common SG for all cluster machines" nodegroups = [for n, ng in var.nodegroups : n] ingress_ipv4 = [ { - description : "Permissive internal traffic [BAD RULE]" + description : "Cluster-internal traffic between members of this group" from_port : 0 to_port : 0 protocol : "-1" @@ -77,12 +89,36 @@ locals { cidr_blocks : [] }, { - description : "Permissive external traffic [BAD RULE]" - from_port : 0 - to_port : 0 - protocol : "-1" + description : "SSH for launchpad host configuration" + from_port : 22 + to_port : 22 + protocol : "tcp" self : false - cidr_blocks : ["0.0.0.0/0"] + cidr_blocks : var.admin_cidrs + }, + { + description : "WinRM over HTTPS for launchpad host configuration" + from_port : 5986 + to_port : 5986 + protocol : "tcp" + self : false + cidr_blocks : var.admin_cidrs + }, + { + description : "MKE and MSR web/API, from administrators and from the load balancers" + from_port : 443 + to_port : 443 + protocol : "tcp" + self : false + cidr_blocks : concat(var.admin_cidrs, [var.network.cidr]) + }, + { + description : "Kubernetes API, from administrators and from the load balancer" + from_port : 6443 + to_port : 6443 + protocol : "tcp" + self : false + cidr_blocks : concat(var.admin_cidrs, [var.network.cidr]) } ] egress_ipv4 = [ @@ -126,7 +162,7 @@ locals { winrm_address : ngn.public_ip winrm_user : ng.winrm_user - winrm_password : var.windows_password + winrm_password : local.windows_password winrm_useHTTPS : ng.winrm_useHTTPS winrm_insecure : ng.winrm_insecure } if contains(local.launchpad_roles, ng.role) && ng.connection == "winrm" }]...) @@ -136,7 +172,23 @@ locals { } // ------- Ye old launchpad yaml (just for debugging) - +// +// The credentials are written from the terraform variables that carry them: +// var.launchpad.mke_connect.password and var.windows_password. The document +// this produces is usable as-is, with nothing exported into the environment +// first. +// +// To keep a password out of a tfvars file on disk, supply the variable +// through terraform's own environment mechanism instead -- TF_VAR_, +// which terraform reads for any input variable: +// +// export TF_VAR_windows_password='...' +// export TF_VAR_launchpad='{"drain":false,...,"mke_connect":{"username":"admin","password":"...","insecure":true}}' +// +// TF_VAR_launchpad takes the whole object as JSON because the MKE password +// is nested inside it; test/terraform.go does exactly this. Either route +// produces the same configuration -- the environment is an option for +// keeping secrets off disk, not a requirement for running the example. locals { launchpad_yaml = <<-EOT apiVersion: launchpad.mirantis.com/mke/v1.6 diff --git a/examples/terraform/aws-simple/platform.tf b/examples/terraform/aws-simple/platform.tf index 0f116d95c..a4185db51 100644 --- a/examples/terraform/aws-simple/platform.tf +++ b/examples/terraform/aws-simple/platform.tf @@ -1,5 +1,11 @@ // variables calculated before ami data is retrieved locals { + // var.windows_password is null on Linux-only stacks, where no Windows node + // exists to carry it; the consumers below take a string. Its validation + // rejects null when a windows_ platform is in use, so reaching here with + // null means no Windows node is being provisioned. + windows_password = var.windows_password == null ? "" : var.windows_password + // find the unique platforms actually used in the node_group_definitions unique_used_platforms = distinct([for ngd in var.nodegroups : ngd.platform]) @@ -31,7 +37,7 @@ module "platform" { version = "0.3.0" platform_key = local.upstream_platform_keys[count.index] - windows_password = var.windows_password + windows_password = local.windows_password } data "aws_ami" "local" { diff --git a/examples/terraform/aws-simple/terraform.tfvars.template b/examples/terraform/aws-simple/terraform.tfvars.template index 66fc4a590..701635508 100644 --- a/examples/terraform/aws-simple/terraform.tfvars.template +++ b/examples/terraform/aws-simple/terraform.tfvars.template @@ -1,10 +1,35 @@ // used to name infrastructure (CHANGE THIS) //name = "" +// CIDRs allowed to reach the cluster from outside it (SSH, WinRM over HTTPS, +// MKE/MSR and the Kubernetes API). +// +// Defaults to ["0.0.0.0/0"] -- open to the whole internet. That is a real +// risk, and it is the exposure this example has always had; it is left as the +// default so an existing tfvars keeps working and a first run needs no extra +// arguments. Narrow it whenever you can: list the machine running launchpad +// and the operators who need the MKE UI. Uncomment and edit: +// +// admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] + aws = { region = "us-east-2" } +// Credentials: both the MKE admin password below and windows_password at the +// bottom of this file are ordinary terraform variables, and setting them here +// is all that is needed. +// +// If you would rather not keep a password in this file, terraform reads any +// input variable from TF_VAR_ in the environment instead: +// +// export TF_VAR_launchpad='{"drain":false,"mcr_channel":"stable-25.0.14","mke_version":"3.8.8","msr_version":"2.9.28","mke_connect":{"username":"admin","password":"...","insecure":true},"skip_create":false,"skip_destroy":true}' +// export TF_VAR_windows_password='...' +// +// The MKE password is nested inside the launchpad object, so TF_VAR_launchpad +// carries that whole object as JSON. Omit the launchpad block below if you +// use it. Either route works; the environment is optional. + launchpad = { drain = false @@ -75,4 +100,6 @@ nodegroups = { // set a windows password, if you have windows nodes // windows passwords must match a pattern, or connections will fail. // use something like: `testp@ss!` +// Required only when a nodegroup uses a windows_ platform; may also be +// supplied as TF_VAR_windows_password instead of being written here. // windows_password = "" diff --git a/examples/terraform/aws-simple/variables.tf b/examples/terraform/aws-simple/variables.tf index c0e2d2160..4c86f4867 100644 --- a/examples/terraform/aws-simple/variables.tf +++ b/examples/terraform/aws-simple/variables.tf @@ -68,10 +68,62 @@ variable "extra_tags" { default = {} } +variable "admin_cidrs" { + description = <<-EOT + CIDRs allowed to reach the cluster from outside it: SSH, WinRM, and the + MKE/MSR and Kubernetes API ports. + + Open by default. ["0.0.0.0/0"] means every one of those ports is + reachable from the whole internet, which is a real risk and not a + recommendation -- it is the same exposure this example has always had, + kept so that an existing tfvars still works and a first run still comes + up without extra arguments. + + Narrowing it is one line: set admin_cidrs to the addresses that actually + need in -- the machine running launchpad, and the operators who need the + MKE UI. For example: + + admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"] + + Node-to-node traffic does not come through here, so narrowing this does + not affect the cluster talking to itself. + EOT + type = list(string) + default = ["0.0.0.0/0"] + validation { + condition = length(var.admin_cidrs) > 0 + error_message = "admin_cidrs must list at least one CIDR." + } +} + variable "windows_password" { - description = "Password to use with windows & winrm" + description = <<-EOT + Password set on the local Administrator account of Windows nodes and used + by launchpad for WinRM. Required when any nodegroup runs a Windows + platform; leave unset for Linux-only stacks. + + The validation below references var.nodegroups, which Terraform supports + from 1.9 onwards. + EOT type = string - default = "" + sensitive = true + default = null + + // Without this, a stack with Windows nodegroups and no windows_password + // generated a launchpad.yaml carrying "password:" with an empty value, and + // launchpad then authenticated as Administrator with no password. Failing + // here names the variable at plan time instead. + // + // The whole requirement is gated on a Windows nodegroup being present, so a + // Linux-only stack may leave the variable unset or pass an empty string -- + // neither reaches a host that would authenticate with it. + validation { + condition = ( + !anytrue([for ng in var.nodegroups : startswith(ng.platform, "windows_")]) + || (var.windows_password != null && length(var.windows_password) > 0) + ) + error_message = "windows_password must be set to a non-empty value when any nodegroup uses a windows_ platform: it is the local Administrator password the nodes are configured with and the credential launchpad uses for WinRM. Leave it unset only for Linux-only stacks." + } } variable "ssh_pk_location" { diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 022323831..f16bd8c84 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -79,7 +79,8 @@ func TestMain(m *testing.M) { name := fmt.Sprintf("smoke-%s", uTestId) - MKE_CONNECT["password"] = test.GenerateRandomAlphaNumericString(12) + mkePassword := test.GenerateRandomAlphaNumericString(12) + MKE_CONNECT["password"] = mkePassword options := test.TerraformOptions( t, @@ -88,6 +89,7 @@ func TestMain(m *testing.M) { map[string]any{ "name": name, "aws": AWS, + "admin_cidrs": test.AdminCIDRs(t), "network": NETWORK, "subnets": SUBNETS, "ssh_pk_location": tempSSHKeyPathDir, diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index b455a0b80..c542f0538 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -154,6 +154,7 @@ func runSmokeTest(t *testing.T, cfg smokeConfig) { vars := map[string]interface{}{ "name": name, "aws": awsConfig, + "admin_cidrs": test.AdminCIDRs(t), "network": networkConfig, "subnets": subnets, "ssh_pk_location": tempSSHKeyPathDir, @@ -184,8 +185,10 @@ func runSmokeTest(t *testing.T, cfg smokeConfig) { break } } + windowsPassword := "" if hasWindows { - secretVars["windows_password"] = generateWindowsPassword(t) + windowsPassword = generateWindowsPassword(t) + secretVars["windows_password"] = windowsPassword } options := test.TerraformOptions(t, "../../examples/terraform/aws-simple", vars, secretVars) diff --git a/test/smoke/swarm_addr_pool_test.go b/test/smoke/swarm_addr_pool_test.go index 15ae8e50b..c0e398052 100644 --- a/test/smoke/swarm_addr_pool_test.go +++ b/test/smoke/swarm_addr_pool_test.go @@ -158,9 +158,10 @@ func provisionSwarmPoolStack(t *testing.T, cfg smokeConfig) string { } vars := map[string]interface{}{ - "name": name, - "aws": awsConfig, - "network": networkConfig, + "name": name, + "aws": awsConfig, + "admin_cidrs": test.AdminCIDRs(t), + "network": networkConfig, "subnets": map[string]interface{}{ "main": map[string]interface{}{ "cidr": "172.31.0.0/17", @@ -177,6 +178,8 @@ func provisionSwarmPoolStack(t *testing.T, cfg smokeConfig) string { }, } + mkePassword := test.GenerateRandomAlphaNumericString(12) + // launchpad nests the MKE admin password, so it travels by TF_VAR_* rather // than as a "-var" argument. See test.TerraformOptions. secretVars := map[string]interface{}{ @@ -187,7 +190,7 @@ func provisionSwarmPoolStack(t *testing.T, cfg smokeConfig) string { "msr_version": cfg.MSRVersion, "mke_connect": map[string]interface{}{ "username": "admin", - "password": test.GenerateRandomAlphaNumericString(12), + "password": mkePassword, "insecure": true, }, }, diff --git a/test/smoke/upgrade_test.go b/test/smoke/upgrade_test.go index 4d7c0e922..0aca50040 100644 --- a/test/smoke/upgrade_test.go +++ b/test/smoke/upgrade_test.go @@ -89,6 +89,7 @@ func runUpgradeTest(t *testing.T, cfg upgradeConfig) { vars := map[string]interface{}{ "name": name, "aws": awsConfig, + "admin_cidrs": test.AdminCIDRs(t), "network": networkConfig, "subnets": subnets, "ssh_pk_location": tempSSHKeyPathDir, diff --git a/test/terraform.go b/test/terraform.go index 13741fa4f..e4c4a109f 100644 --- a/test/terraform.go +++ b/test/terraform.go @@ -2,12 +2,40 @@ package test import ( "encoding/json" + "os" + "strings" "testing" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/terraform" ) +// AdminCIDRs returns the value for the examples' admin_cidrs variable, which +// scopes SSH, WinRM and the MKE/Kubernetes API to named networks. +// +// The variable defaults to 0.0.0.0/0 in the examples. These suites provision +// throwaway stacks from runners whose egress address is not known ahead of +// time, so they pass that same open range explicitly rather than inheriting +// it, which keeps the exposure stated at the caller where it can be found and +// changed. Set SMOKE_ADMIN_CIDRS (comma separated) to narrow it. +func AdminCIDRs(t *testing.T) []string { + t.Helper() + + configured := os.Getenv("SMOKE_ADMIN_CIDRS") + if configured == "" { + t.Logf("SMOKE_ADMIN_CIDRS is unset: provisioning with admin_cidrs = [0.0.0.0/0]") + + return []string{"0.0.0.0/0"} + } + + cidrs := strings.Split(configured, ",") + for i, cidr := range cidrs { + cidrs[i] = strings.TrimSpace(cidr) + } + + return cidrs +} + // TerraformOptions builds terraform.Options that keep credentials out of the // terraform command line. // diff --git a/test/terraform_test.go b/test/terraform_test.go index 0baeb3a0e..cab9a03ff 100644 --- a/test/terraform_test.go +++ b/test/terraform_test.go @@ -196,3 +196,17 @@ func copyFixture(t *testing.T, fixtureDir string) string { return dest } + +func TestAdminCIDRs(t *testing.T) { + t.Run("falls back to the open range when unset", func(t *testing.T) { + t.Setenv("SMOKE_ADMIN_CIDRS", "") + + assert.Equal(t, []string{"0.0.0.0/0"}, AdminCIDRs(t)) + }) + + t.Run("honours a configured list", func(t *testing.T) { + t.Setenv("SMOKE_ADMIN_CIDRS", "203.0.113.4/32, 198.51.100.0/24") + + assert.Equal(t, []string{"203.0.113.4/32", "198.51.100.0/24"}, AdminCIDRs(t)) + }) +}