several fixes

main.tf

@@ -6,6 +6,10 @@ terraform {
       source  = "hashicorp/azurerm"
       version = "~>3.114"
     }
+    time = {
+      source  = "hashicorp/time"
+      version = "~>0.12"
+    }
     azuread = {
       source  = "hashicorp/azuread"
       version = "~>2.53"
@@ -198,10 +202,10 @@ resource "azurerm_key_vault" "main" {
   dynamic "network_acls" {
     for_each = var.key_vault_network_acls == null ? {} : { "" : "" }
     content {
-      bypass                     = var.network_acls.bypass
-      default_action             = var.network_acls.default_action
-      ip_rules                   = var.network_acls.ip_rules
-      virtual_network_subnet_ids = var.network_acls.virtual_network_subnet_ids
+      bypass                     = var.key_vault_network_acls.bypass
+      default_action             = var.key_vault_network_acls.default_action
+      ip_rules                   = var.key_vault_network_acls.ip_rules
+      virtual_network_subnet_ids = var.key_vault_network_acls.virtual_network_subnet_ids
     }
   }
 
@@ -234,7 +238,7 @@ resource "azurerm_private_endpoint" "key_vault" {
 }
 resource "azurerm_role_assignment" "key_vault_secret_user__aks" {
   scope                = azurerm_key_vault.main.id
-  principal_id         = "" # TODO
+  principal_id         = module.aks.cluster_identity.principal_id
   role_definition_name = "Key Vault Secrets User"
 }
 resource "azurerm_role_assignment" "key_vault_secret_officer__current" {
@@ -266,11 +270,19 @@ resource "azurerm_key_vault_secret" "azuread_application_client_id" {
   key_vault_id = azurerm_key_vault.main.id
   name         = format("%s-azure-client-id", var.resource_prefix)
   value        = azuread_application.main.client_id
+
+  depends_on = [
+    azurerm_role_assignment.key_vault_secret_officer__current
+  ]
 }
 resource "azurerm_key_vault_secret" "azuread_application_client_secret" {
   key_vault_id = azurerm_key_vault.main.id
   name         = format("%s-azure-client-secret", var.resource_prefix)
   value        = azuread_application.main.client_id
+
+  depends_on = [
+    azurerm_role_assignment.key_vault_secret_officer__current
+  ]
 }
 
 
@@ -495,6 +507,10 @@ resource "azurerm_key_vault_secret" "azure_openai_api_key" {
   name         = "${var.resource_prefix}-openai-api-key"
   value        = azurerm_cognitive_account.main.primary_access_key
   key_vault_id = azurerm_key_vault.main.id
+
+  depends_on = [
+    azurerm_role_assignment.key_vault_secret_officer__current
+  ]
 }
 
 
@@ -512,7 +528,7 @@ resource "azurerm_storage_account" "main" {
   account_replication_type = "LRS"
   access_tier              = "Hot"
 
-  public_network_access_enabled = false
+  public_network_access_enabled = true # TODO
   is_hns_enabled                = false
 
   tags = var.tags
@@ -524,7 +540,7 @@ resource "azurerm_storage_container" "models" {
 resource "azurerm_role_assignment" "storage_container_models__data_contributor" {
   role_definition_name = "Storage Blob Data Contributor"
   principal_id         = azuread_service_principal.main.object_id
-  scope                = azurerm_storage_container.models.id
+  scope                = azurerm_storage_container.models.resource_manager_id
 }
 resource "azurerm_private_endpoint" "blob" {
   name                = "${azurerm_storage_account.main.name}-blob"
@@ -661,6 +677,13 @@ module "aks" {
 
   tags = var.tags
 }
+resource "time_sleep" "wait_aks_creation" {
+  create_duration = "30s"
+
+  depends_on = [
+    module.aks
+  ]
+}
 # The AKS cluster identity has the Contributor role on the AKS second resource group (MC_myResourceGroup_myAKSCluster_eastus)
 # However when using a custom VNET, the AKS cluster identity needs the Network Contributor role on the VNET subnets
 # used by the system node pool and by any additional node pools.
@@ -703,6 +726,10 @@ resource "azurerm_kubernetes_cluster_node_pool" "linux_pools" {
       eviction_policy,
     ]
   }
+
+  depends_on = [
+    time_sleep.wait_aks_creation,
+  ]
 }
 
 
@@ -715,6 +742,10 @@ resource "azurerm_key_vault_secret" "jwt_signing_key" {
   key_vault_id = azurerm_key_vault.main.id
   name         = format("%s-jwt-signing-key", var.resource_prefix)
   value        = tls_private_key.jwt_signing_key.private_key_pem
+
+  depends_on = [
+    azurerm_role_assignment.key_vault_secret_officer__current
+  ]
 }
 
 
@@ -733,7 +764,7 @@ locals {
   k8s_secret_key_azure_client_secret = "azure-client-secret"
 
   helm_values = templatefile(
-    "templates/helm-values.tpl.yaml",
+    "${path.module}/templates/helm-values.tpl.yaml",
     {
       platform_domain = var.platform_domain
 
@@ -754,14 +785,14 @@ locals {
     },
   )
   secret_provider_class = templatefile(
-    "templates/secret-provider-class.tpl.yaml",
+    "${path.module}/templates/secret-provider-class.tpl.yaml",
     {
       secret_provider_class_name        = local.secret_provider_class_name
       secret_provider_class_secret_name = local.secret_provider_class_secret_name
 
       key_vault_name          = azurerm_key_vault.main.name
       tenant_id               = data.azurerm_client_config.current.tenant_id
-      aks_managed_identity_id = module.aks.key_vault_secrets_provider.secret_identity[0]
+      aks_managed_identity_id = try(module.aks.key_vault_secrets_provider.secret_identity[0].object_id, "TODO")
 
       secret_name_jwt_signing_key     = azurerm_key_vault_secret.jwt_signing_key.name
       secret_name_db_username         = azurerm_key_vault_secret.postgres_user.name

tests/dev-provisioning/apply.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ "$1" == 'init' ]
+  then
+    terraform init --backend-config backend.tfvars
+    shift 
+fi
+
+terraform apply --var-file backend.tfvars "$@"

tests/dev-provisioning/backend.tfvars.example

@@ -0,0 +1,5 @@
+tenant_id       = ""
+subscription_id = ""
+client_id       = ""
+client_secret   = ""
+

tests/dev-provisioning/main.tf

@@ -0,0 +1,89 @@
+terraform {
+  required_version = ">=1.9"
+
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~>3.114"
+    }
+    azuread = {
+      source  = "hashicorp/azuread"
+      version = "~>2.53"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~>3.6"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+
+  client_id       = var.client_id
+  client_secret   = var.client_secret
+  tenant_id       = var.tenant_id
+  subscription_id = var.subscription_id
+}
+
+# ------ Variables ------ #
+variable "resource_prefix" {
+  type = string
+}
+variable "client_id" {
+  type = string
+}
+variable "subscription_id" {
+  type = string
+}
+variable "tenant_id" {
+  type = string
+}
+variable "client_secret" {
+  type = string
+}
+variable "resource_group_name" {
+  type = string
+}
+variable "tags" {
+  type = map(any)
+}
+variable "location" {
+  type = string
+}
+
+# ------ Data Sources ------ #
+data "azuread_group" "engineering" {
+  display_name = "nebuly-engineering"
+}
+data "http" "my_ip" {
+  url = "https://ipv4.icanhazip.com"
+}
+locals {
+  my_ip = chomp(data.http.my_ip.response_body)
+}
+
+module "platform" {
+  source = "../../"
+
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  platform_domain     = "platform.azure.testing"
+
+  postgres_server_sku = {
+    tier = "GP"
+    name = "Standard_D2ads_v5"
+  }
+
+  key_vault_public_network_access_enabled = true
+  key_vault_network_acls = {
+    ip_rules                   = [local.my_ip]
+    virtual_network_subnet_ids = []
+  }
+
+  aks_cluster_admin_object_ids = [data.azuread_group.engineering.id]
+  resource_prefix              = var.resource_prefix
+
+  tags = var.tags
+}
+

tests/dev-provisioning/remote_state.tf

@@ -0,0 +1,8 @@
+ terraform {
+      backend "azurerm" {
+        resource_group_name  = "rg-shared"
+        storage_account_name = "nbllabtfstatessa"
+        container_name       = "rg-platform-inttest-tfstate"
+        key                  = "tfstate"
+      }
+    }

tests/dev-provisioning/secrets.auto.tfvars.example

@@ -0,0 +1,4 @@
+tenant_id       = ""
+subscription_id = ""
+client_id       = ""
+client_secret   = ""

tests/dev-provisioning/terraform.auto.tfvars

@@ -0,0 +1,9 @@
+### General ###
+location            = "EastUS"
+resource_prefix     = "nbltst"
+resource_group_name = "rg-platform-inttest"
+tags = {
+  "env" : "integration-test"
+  "project" : "self-hosted"
+}
+

tests/smoke_test__default_values.tftest.hcl

@@ -10,15 +10,10 @@ run "smoke_test_plan__default_values" {
     location            = "EastUS"
     platform_domain     = "intest.nebuly.ai"
 
-    # ------ PostgreSQL Database  ------ #
-    postgres_server_networking = {}
-
     # ------ Key Vault ------ #
-    key_vault_public_network_access_enabled = true
+    key_vault_public_network_access_enabled = false
 
     # ------ AKS ------ #
-    aks_net_profile_service_cidr   = "10.32.0.0/24"
-    aks_net_profile_dns_service_ip = "10.32.0.10"
     aks_cluster_admin_object_ids   = []
   }
 }

tests/smoke_test__existing_networks.tftest.hcl

@@ -24,15 +24,10 @@ run "smoke_test_plan__existing_networks" {
     virtual_network_name  = run.setup.azurerm_virtual_network.name
     subnet_name_aks_nodes = run.setup.azurerm_subnet.name
 
-    # ------ PostgreSQL Database  ------ #
-    postgres_server_networking = {}
-
     # ------ Key Vault ------ #
-    key_vault_public_network_access_enabled = true
+    key_vault_public_network_access_enabled = false
 
     # ------ AKS ------ #
-    aks_net_profile_service_cidr   = "10.32.0.0/24"
-    aks_net_profile_dns_service_ip = "10.32.0.10"
     aks_cluster_admin_object_ids   = []
   }
 }

tests/validation__networking.tftest.hcl → tests/test_values_validation.tftest.hcl

@@ -12,7 +12,7 @@ variables {
   postgres_server_networking = {}
 
   # ------ Key Vault ------ #
-  key_vault_public_network_access_enabled = true
+  key_vault_public_network_access_enabled = false
 
   # ------ AKS ------ #
   aks_net_profile_service_cidr   = "10.32.0.0/24"
@@ -47,3 +47,16 @@ run "values_validation__subnet_private_endpoints" {
     var.subnet_name_private_endpoints
   ]
 }
+
+run "values_validation__key_vault_acls" {
+  command = plan
+
+  variables {
+    # KeyVault ACLs must be provided when public access is enabled.
+    key_vault_public_network_access_enabled = true
+  }
+
+  expect_failures = [
+    var.key_vault_public_network_access_enabled
+  ]
+}

variables.tf

@@ -95,6 +95,7 @@ variable "postgres_server_networking" {
     private_dns_zone_id : optional(string, null)
     public_network_access_enabled : optional(bool, false)
   })
+  default = {}
 }
 variable "postgres_server_point_in_time_backup" {
   type = object({
@@ -182,6 +183,11 @@ variable "key_vault_sku_name" {
 variable "key_vault_public_network_access_enabled" {
   type        = bool
   description = "Can the Key Vault be accessed from the Internet?"
+
+  validation {
+    condition     = !var.key_vault_public_network_access_enabled || var.key_vault_network_acls != null
+    error_message = "You must provide network ACLs when Key Vault public network access is enabled."
+  }
 }
 variable "key_vault_soft_delete_retention_days" {
   type        = number
@@ -195,8 +201,8 @@ variable "key_vault_purge_protection_enabled" {
 }
 variable "key_vault_network_acls" {
   type = object({
-    bypass : string
-    default_action : string
+    bypass : optional(string, "AzureServices")
+    default_action : optional(string, "Deny")
     ip_rules : list(string)
     virtual_network_subnet_ids : list(string)
   })
@@ -278,7 +284,7 @@ variable "subnet_address_space_private_endpoints" {
   If `subnet_name_private_endpoints` is provided, the existing subnet is used and this variable is ignored.
   EOT
   type        = list(string)
-  default     = ["10.0.0.192/26"]
+  default     = ["10.0.8.0/26"]
 }
 variable "private_dns_zones" {
   description = <<EOT
@@ -350,10 +356,12 @@ variable "aks_api_server_allowed_ip_addresses" {
 variable "aks_net_profile_service_cidr" {
   type        = string
   description = "The Network Range used by the Kubernetes service. Must not overlap with the AKS Nodes address space. Example: 10.32.0.0/24"
+  default     = "10.32.0.0/24"
 }
 variable "aks_net_profile_dns_service_ip" {
   type        = string
   description = " IP address within the Kubernetes service address range that is used by cluster service discovery (kube-dns). Must be inluced in net_profile_cidr. Example: 10.32.0.10"
+  default     = "10.32.0.10"
 }
 variable "aks_log_analytics_workspace" {
   description = " Existing azurerm_log_analytics_workspace to attach azurerm_log_analytics_solution. Providing the config disables creation of azurerm_log_analytics_workspace."
@@ -458,7 +466,7 @@ variable "aks_worker_pools" {
       # Auto-scaling setttings
       enable_auto_scaling = true
       nodes_count : null
-      nodes_min_count = 1
+      nodes_min_count = 0
       nodes_max_count = 1
       # Tags
       tags : {}