Merge pull request #5 from Justin-DynamicD/node-pools

Node pool update

README.md

@@ -8,15 +8,14 @@ Unlike the the complete topology example that includes the required hub-and-spok
 |-----------------------------------------|-------|----------|
 | Virtual Network hub-and-spoke           |  ✅   |    ❌    |
 | Egress restriction using Azure Firewall |  ✅   |    ❌    |
-| Ingress Controller                      |  ✅   |    ✅    |
 | Azure Networking CNI                    |  ✅   |    ✅    |
 | Azure Active Directory Pod Identity     |  ✅   |    ✅    |
 | Default Recomended Node config          |  ✅   |    ✅    |
 | App Gateway/WAF                         |  ✅   |    ✅    |
 | Keyvault secrets provider               |  ✅   |    ✅    |
 | Azure Policy enabled                    |  ❌   |    ✅    |
 | Managed public IP option                |  ❌   |    ✅    |
-| log retention default connector         |  ❌   |    ✅    |
+| log retention rules                     |  ❌   |    ✅    |
 
 Each recomended integration is bundled into its own custom object block so it can be enabled/disabled as needed.  For example:
 
@@ -87,8 +86,6 @@ aks = {
 }
 ```
 
-
-
 ### app_gateway
 
 ```yaml
@@ -125,4 +122,4 @@ Map of tags to apply to every resource that is created.
 
 ## Outputs
 
-Comming soon
+Comming soon

agw.tf

@@ -2,11 +2,11 @@
 resource "azurerm_public_ip" "main" {
   count               = (local.app_gateway.enabled && local.app_gateway.public_ip_id == "") ? 1 : 0
   name                = local.names.agw
-  resource_group_name = local.global_settings.resource_group_name
-  location            = local.global_settings.location
+  resource_group_name = local.resource_group_name
+  location            = local.location
   allocation_method   = "Static"
   sku                 = "Standard"
-  availability_zone   = "Zone-Redundant"
+  zones               = local.zones != [] ? local.zones : null
   tags                = var.tags
 }
 
@@ -30,8 +30,8 @@ resource "azurerm_application_gateway" "main" {
 
   count               = local.app_gateway.enabled ? 1 : 0
   name                = local.names.agw
-  resource_group_name = local.global_settings.resource_group_name
-  location            = local.global_settings.location
+  resource_group_name = local.resource_group_name
+  location            = local.location
   zones               = local.zones != [] ? local.zones : null
   sku {
     name     = local.app_gateway.sku_name

aks.tf

@@ -1,4 +1,25 @@
-# see locals block for hardcoded names.
+# this local block follows Azure Documentation for node labels + taints
+# and contains thier configuration which is applied by priority
+# details: https://docs.microsoft.com/en-us/azure/aks/spot-node-pool
+
+locals {
+  aks_node_extra ={
+    Regular = {
+      labels = {}
+      taints = []
+    }
+    Spot = {
+      labels = {
+        "kubernetes.azure.com/scalesetpriority" = "spot"
+      }
+      taints =  [
+        "kubernetes.azure.com/scalesetpriority=spot:NoSchedule"
+      ]
+    }
+  }
+}
+
+
 resource "azurerm_kubernetes_cluster" "main" {
   lifecycle {
     # due to auto-scaling we need to ignore the nodecount after launch
@@ -7,12 +28,12 @@ resource "azurerm_kubernetes_cluster" "main" {
     ]
   }
   name                              = local.names.aks
-  location                          = local.global_settings.location
+  location                          = local.location
   dns_prefix                        = replace(local.names.aks, "-", "")
   resource_group_name               = data.azurerm_resource_group.source.name
-  sku_tier                          = local.aks.sku_tier
-  automatic_channel_upgrade         = local.aks.automatic_channel_upgrade != "" ? local.aks.automatic_channel_upgrade : null
-  azure_policy_enabled              = local.aks.azure_policy
+  sku_tier                          = local.sku_tier
+  automatic_channel_upgrade         = local.automatic_channel_upgrade != "" ? local.automatic_channel_upgrade : null
+  azure_policy_enabled              = local.azure_policy
   http_application_routing_enabled  = false
   role_based_access_control_enabled = true
   dynamic "ingress_application_gateway" {
@@ -31,24 +52,52 @@ resource "azurerm_kubernetes_cluster" "main" {
     }
   }
   default_node_pool {
-    name                = "default"
-    enable_auto_scaling = true
-    node_count          = local.aks.node_count
-    min_count           = local.aks.min_count
-    max_count           = local.aks.max_count
-    vm_size             = local.aks.vm_size
-    os_disk_size_gb     = local.aks.os_disk_size_gb
-    os_disk_type        = local.aks.os_disk_type
-    vnet_subnet_id      = local.aks.subnet_id
-    availability_zones  = local.zones != [] ? local.zones : null
-    tags                = local.tags
+    enable_auto_scaling          = local.node_default_pool.enable_auto_scaling
+    max_count                    = local.node_default_pool.max_count
+    min_count                    = local.node_default_pool.min_count
+    name                         = local.node_default_pool.name
+    node_count                   = local.node_default_pool.node_count
+    only_critical_addons_enabled = local.node_default_pool.only_critical_addons_enabled
+    os_disk_size_gb              = local.node_default_pool.os_disk_size_gb
+    os_disk_type                 = local.node_default_pool.os_disk_type
+    tags                         = local.tags
+    vm_size                      = local.node_default_pool.vm_size
+    vnet_subnet_id               = local.subnet_id
+    zones                        = local.zones != [] ? local.zones : null
   }
   identity {
-    type                      = "UserAssigned"
-    user_assigned_identity_id = azurerm_user_assigned_identity.main.id
+    type         = "UserAssigned"
+    identity_ids = [azurerm_user_assigned_identity.main.id]
   }
   network_profile {
     network_plugin     = "azure"
   }
   tags = local.tags
+}
+
+resource "azurerm_kubernetes_cluster_node_pool" "user" {
+  # due to auto-scaling we need to ignore the nodecount after launch
+  lifecycle {
+    ignore_changes = [
+      node_count
+    ]
+  }
+  count                 = local.node_user_pool.enabled ? 1 : 0
+  enable_auto_scaling   = local.node_user_pool.enable_auto_scaling
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id
+  max_count             = local.node_user_pool.max_count
+  min_count             = local.node_user_pool.min_count
+  mode                  = local.node_user_pool.mode
+  name                  = local.node_user_pool.name
+  node_count            = local.node_user_pool.node_count
+  node_labels           = local.aks_node_extra[local.node_user_pool.priority].labels
+  node_taints           = local.aks_node_extra[local.node_user_pool.priority].taints
+  os_disk_size_gb       = local.node_user_pool.os_disk_size_gb
+  os_disk_type          = local.node_user_pool.os_disk_type
+  priority              = local.node_user_pool.priority
+  eviction_policy       = local.node_user_pool.priority == "Spot" ? local.node_user_pool.eviction_policy : null
+  spot_max_price        = local.node_user_pool.priority == "Spot" ? local.node_user_pool.spot_max_price : null
+  tags                  = local.tags
+  vm_size               = local.node_user_pool.vm_size
+  zones                 = local.zones != [] ? local.zones : null
 }

data.tf

@@ -8,7 +8,7 @@
 # data "azurerm_client_config" "current" {}
 
 data "azurerm_resource_group" "source" {
-  name = local.global_settings.resource_group_name
+  name = local.resource_group_name
 }
 
 data "azurerm_container_registry" "list" {

identity.tf

@@ -3,8 +3,8 @@
 #  and DNS updating
 
 resource "azurerm_user_assigned_identity" "main" {
-  resource_group_name = local.global_settings.resource_group_name
-  location            = local.global_settings.location
+  resource_group_name = local.resource_group_name
+  location            = local.location
   name                = local.names.aks
   tags                = local.tags
 }
@@ -19,7 +19,7 @@ resource "azurerm_role_assignment" "attach_acr" {
 
 # grants rights to the built role as well as the subnet (only needed for kubenet, but added for completeness)
 resource "azurerm_role_assignment" "subnet" {
-    scope                = local.aks.subnet_id
+    scope                = local.subnet_id
     role_definition_name = "Network Contributor"
     principal_id         = azurerm_kubernetes_cluster.main.kubelet_identity[0].object_id
 }
@@ -36,12 +36,12 @@ resource "azurerm_role_assignment" "agwaks" {
   count                = local.app_gateway.enabled ? 1 : 0
   scope                = azurerm_application_gateway.main[0].id
   role_definition_name = "Contributor"
-  principal_id         = azurerm_kubernetes_cluster.main.addon_profile[0].ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id
+  principal_id         = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id
 }
 
 resource "azurerm_role_assignment" "agwaksrg" {
   count                = local.app_gateway.enabled ? 1 : 0
   scope                = data.azurerm_resource_group.source.id
   role_definition_name = "Reader"
-  principal_id         = azurerm_kubernetes_cluster.main.addon_profile[0].ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id
+  principal_id         = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id
 }

main.tf

@@ -4,19 +4,6 @@
 ######
 
 locals {
-  aks = defaults(var.aks, {
-    automatic_channel_upgrade = ""
-    azure_policy              = true
-    docker_bridge_cidr        = "172.17.0.1/16"
-    max_count                 = 3
-    min_count                 = 1
-    name                      = ""
-    node_count                = 2
-    os_disk_size_gb           = 128
-    os_disk_type              = "Ephemeral"
-    sku_tier                  = "Free"
-    vm_size                   = "Standard_DS3_v2"
-  })
   app_gateway = defaults(var.app_gateway, {
     enabled      = false
     name         = ""
@@ -26,8 +13,31 @@ locals {
     sku_tier     = "WAF_v2"
     subnet_id    = ""
   })
-  global_settings = defaults(var.global_settings, {
-    name_prefix = "aks-baseline"
+  node_default_pool = defaults(var.node_default_pool, {
+    enable_auto_scaling          = true
+    max_count                    = 4
+    min_count                    = 3
+    name                         = "system"
+    node_count                   = 3
+    only_critical_addons_enabled = true
+    os_disk_size_gb              = 70
+    os_disk_type                 = "Ephemeral"
+    vm_size                      = "Standard_D2ds_v5"
+  })
+  node_user_pool = defaults(var.node_user_pool, {
+    enabled             = true
+    enable_auto_scaling = true
+    max_count           = 5
+    min_count           = 2
+    mode                = "User"
+    name                = "user"
+    node_count          = 2
+    os_disk_size_gb     = 120
+    os_disk_type        = "Ephemeral"
+    priority            = "Regular"
+    eviction_policy     = "Delete"
+    spot_max_price      = -1
+    vm_size             = "Standard_D4ds_v5"
   })
   oms = defaults(var.oms, {
     enabled            = false
@@ -37,12 +47,19 @@ locals {
 
   # generate the resource names for everything based on the values offered
   names = {
-    aks = coalesce(local.aks.name, "${local.global_settings.name_prefix}-aks")
-    agw = coalesce(local.app_gateway.name, "${local.global_settings.name_prefix}-agw")
+    aks = coalesce(var.name, "${var.name_prefix}-aks")
+    agw = coalesce(local.app_gateway.name, "${var.name_prefix}-agw")
   }
 
   # these are unmodified, just dropped into locals for cconsistency
-  acr_list        = var.acr_list
-  tags            = var.tags
-  zones           = var.zones
+  acr_list                  = var.acr_list
+  automatic_channel_upgrade = var.automatic_channel_upgrade
+  azure_policy              = var.azure_policy
+  docker_bridge_cidr        = var.docker_bridge_cidr
+  location                  = var.location
+  resource_group_name       = var.resource_group_name
+  sku_tier                  = var.sku_tier
+  subnet_id                 = var.subnet_id
+  tags                      = var.tags
+  zones                     = var.zones
 }

test/main.tf

@@ -41,20 +41,22 @@ module "aks" {
   depends_on = [
     azurerm_resource_group.test
   ]
-  global_settings = {
-    location            = azurerm_resource_group.test.location
-    name_prefix         = "testaks"
-    resource_group_name = azurerm_resource_group.test.name
-  }
-  aks = {
-    os_disk_size_gb = 70
-    subnet_id       = module.myvnet.vnet_subnets["aks_nodes"].id
-    vm_size         = "Standard_D2ds_v5"
-  }
+  location            = azurerm_resource_group.test.location
+  name_prefix         = "testaks"
+  resource_group_name = azurerm_resource_group.test.name
+  subnet_id           = module.myvnet.vnet_subnets["aks_nodes"].id
   app_gateway = {
     enabled   = true
     subnet_id = module.myvnet.vnet_subnets["agw"].id
   }
+  node_default_pool = {
+    min_count  = 1
+    node_count = 1
+  }
+  node_user_pool = {
+    min_count  = 1
+    node_count = 1
+  }
   tags = {
     Project   = "AKS Baseline"
     CAF_Level = "3"