Add optional subnet_names to EKS node group configuration (#447)

This allows to define, for example, that all nodes in a group would be in the availability zones of a specific region. --------- Co-authored-by: Rafael Mendes Pereira <[email protected]>
Azure · Dec 23, 2024 · c2b5e60 · c2b5e60
1 parent 5952635
commit c2b5e60
Show file tree

Hide file tree

Showing 6 changed files with 185 additions and 2 deletions.
diff --git a/.github/workflows/terraform-validation.yml b/.github/workflows/terraform-validation.yml
@@ -23,7 +23,7 @@ jobs:
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v2
         with:
-          terraform_version: 1.6.0
+          terraform_version: 1.10.0
 
       - name: Terraform Format Check
         if: always()

diff --git a/modules/terraform/aws/eks/main.tf b/modules/terraform/aws/eks/main.tf
@@ -5,6 +5,8 @@ locals {
 
   eks_config_addons_map = { for addon in var.eks_config.eks_addons : addon.name => addon }
 
+  eks_nodes_subnets_list = flatten([for node_group in var.eks_config.eks_managed_node_groups : node_group.subnet_names if node_group.subnet_names != null])
+
   karpenter_addons_map = {
     for addon in [
       { name = "vpc-cni", vpc_cni_warm_prefix_target = 1 },
@@ -59,6 +61,21 @@ data "aws_subnets" "subnets" {
   }
 }
 
+data "aws_subnet" "subnet_details" {
+  for_each = toset(local.eks_nodes_subnets_list)
+
+  filter {
+    name   = "tag:run_id"
+    values = [var.run_id]
+  }
+
+  filter {
+    name   = "tag:Name"
+    values = [each.value]
+  }
+}
+
+
 data "aws_iam_policy_document" "assume_role" {
   statement {
     effect = "Allow"
@@ -175,7 +192,7 @@ resource "aws_eks_node_group" "eks_managed_node_groups" {
   node_group_name = each.value.name
   cluster_name    = aws_eks_cluster.eks.name
   node_role_arn   = aws_iam_role.eks_cluster_role.arn
-  subnet_ids      = toset(data.aws_subnets.subnets.ids)
+  subnet_ids      = each.value.subnet_names != null ? toset([for subnet_name in each.value.subnet_names : data.aws_subnet.subnet_details[subnet_name].id]) : toset(data.aws_subnets.subnets.ids)
 
   scaling_config {
     min_size     = each.value.min_size

diff --git a/modules/terraform/aws/eks/variables.tf b/modules/terraform/aws/eks/variables.tf
@@ -43,6 +43,7 @@ variable "eks_config" {
       desired_size   = number
       capacity_type  = optional(string, "ON_DEMAND")
       labels         = optional(map(string), {})
+      subnet_names   = optional(list(string), null)
       taints = optional(list(object({
         key    = string
         value  = string

diff --git a/modules/terraform/aws/tests/data.tfmock.hcl b/modules/terraform/aws/tests/data.tfmock.hcl
@@ -0,0 +1,46 @@
+override_data {
+  target = module.eks["eks_name"].data.aws_iam_policy_document.assume_role
+  values = {
+    json = <<EOT
+    {
+      "Statement": [{
+        "Effect": "Allow",
+        "Principal": {
+          "Service": ["eks.amazonaws.com", "ec2.amazonaws.com"]
+        },
+        "Action": ["sts:AssumeRole"]
+      }]
+    } 
+    EOT
+  }
+}
+
+override_resource {
+  target = module.eks["eks_name"].aws_iam_role.eks_cluster_role
+  values = { arn = "arn:aws:iam::123456789012:role/eks_cluster_role" }
+}
+
+override_resource {
+  target = module.eks["eks_name"].aws_eks_cluster.eks
+  values = {
+    identity = [{
+      oidc = [{
+        issuer = "https://oidc.eks.us-west-2.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E"
+      }]
+    }]
+  }
+}
+
+override_resource {
+  target = module.eks["eks_name"].aws_launch_template.launch_template["default"]
+  values = {
+    id = "lt-0abcd1234efgh5678"
+  }
+}
+
+override_resource {
+  target = module.eks["eks_name"].aws_launch_template.launch_template["userpool"]
+  values = {
+    id = "lt-0abcd1234efgh5679"
+  }
+}
diff --git a/modules/terraform/aws/tests/test_eks_node_groups_subnets.tftest.hcl b/modules/terraform/aws/tests/test_eks_node_groups_subnets.tftest.hcl
@@ -0,0 +1,118 @@
+variables {
+  scenario_type  = "perf-eval"
+  scenario_name  = "my_scenario"
+  deletion_delay = "2h"
+  owner          = "aks"
+  json_input = {
+    "run_id" : "123456789",
+    "region" : "us-east-1",
+    "creation_time" : "2024-11-12T16:39:54Z"
+  }
+
+  network_config_list = [
+    {
+      role           = "nap"
+      vpc_name       = "nap-vpc"
+      vpc_cidr_block = "10.0.0.0/16"
+      subnet = [
+        {
+          name                    = "nap-subnet-1"
+          cidr_block              = "10.0.32.0/19"
+          zone_suffix             = "a"
+          map_public_ip_on_launch = true
+        },
+        {
+          name                    = "nap-subnet-2"
+          cidr_block              = "10.0.64.0/19"
+          zone_suffix             = "b"
+          map_public_ip_on_launch = true
+        }
+      ]
+      security_group_name = "nap-sg"
+      route_tables = [
+        {
+          name       = "internet-rt"
+          cidr_block = "0.0.0.0/0"
+        }
+      ],
+      route_table_associations = [
+        {
+          name             = "nap-subnet-rt-assoc"
+          subnet_name      = "nap-subnet-1"
+          route_table_name = "internet-rt"
+        }
+      ]
+      sg_rules = {
+        ingress = []
+        egress = [
+          {
+            from_port  = 0
+            to_port    = 0
+            protocol   = "-1"
+            cidr_block = "0.0.0.0/0"
+          }
+        ]
+      }
+    }
+  ]
+
+  eks_config_list = [{
+    role        = "nap"
+    eks_name    = "eks_name"
+    vpc_name    = "nap-vpc"
+    policy_arns = ["AmazonEKS_CNI_Policy"]
+    eks_managed_node_groups = [
+      {
+        name           = "default"
+        ami_type       = "AL2_x86_64"
+        instance_types = ["m4.large"]
+        min_size       = 1
+        max_size       = 2
+        desired_size   = 1
+      },
+      {
+        name           = "userpool"
+        ami_type       = "AL2_x86_64"
+        instance_types = ["m4.large"]
+        min_size       = 5
+        max_size       = 5
+        desired_size   = 5
+        subnet_names   = ["nap-subnet-1"]
+    }]
+    eks_addons = []
+  }]
+}
+
+mock_provider "aws" {
+  source = "./tests"
+}
+
+override_data {
+  target = module.eks["eks_name"].data.aws_subnets.subnets
+  values = { ids = ["nap-subnet-1-id", "nap-subnet-2-id"] }
+}
+
+override_data {
+  target = module.eks["eks_name"].data.aws_subnet.subnet_details["nap-subnet-1"]
+  values = { id = "nap-subnet-1-id" }
+}
+
+
+run "valid_eks_nodes_subnet" {
+
+  command = apply
+
+  # Expected all cluster's subnet_ids
+  assert {
+    condition     = tolist(module.eks["eks_name"].eks_node_groups["default"].subnet_ids) == tolist(["nap-subnet-1-id", "nap-subnet-2-id"])
+    error_message = "Expected: ['nap-subnet-1', 'nap-subnet-2'] \n Actual:  ${jsonencode(module.eks["eks_name"].eks_node_groups["default"].subnet_ids)}"
+  }
+
+  # Expected only the subnet_id of the subnet defined in the node group
+  assert {
+    condition     = tolist(module.eks["eks_name"].eks_node_groups["userpool"].subnet_ids) == tolist(["nap-subnet-1-id"])
+    error_message = "Expected: ['nap-subnet-1'] \n Actual:  ${jsonencode(module.eks["eks_name"].eks_node_groups["userpool"].subnet_ids)}"
+  }
+
+  expect_failures = [check.deletion_due_time]
+}
diff --git a/modules/terraform/aws/variables.tf b/modules/terraform/aws/variables.tf
@@ -111,6 +111,7 @@ variable "eks_config_list" {
       desired_size   = number
       capacity_type  = optional(string, "ON_DEMAND")
       labels         = optional(map(string), {})
+      subnet_names   = optional(list(string), null)
       taints = optional(list(object({
         key    = string
         value  = string