diff --git a/README.md b/README.md index 2dba843..4600a8c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Available on [Terraform Registry](https://registry.terraform.io/modules/nebuly-a | [azuread](#provider\_azuread) | ~>2.53 | | [azurerm](#provider\_azurerm) | ~>3.114 | | [random](#provider\_random) | ~>3.6 | +| [time](#provider\_time) | ~>0.12 | | [tls](#provider\_tls) | ~>4.0 | @@ -41,14 +42,14 @@ Available on [Terraform Registry](https://registry.terraform.io/modules/nebuly-a | [aks\_cluster\_admin\_object\_ids](#input\_aks\_cluster\_admin\_object\_ids) | Object IDs that are granted the Cluster Admin role over the AKS cluster | `set(string)` | n/a | yes | | [aks\_kubernetes\_version](#input\_aks\_kubernetes\_version) | The Kubernetes version to use. | `string` | `"1.29.5"` | no | | [aks\_log\_analytics\_workspace](#input\_aks\_log\_analytics\_workspace) | Existing azurerm\_log\_analytics\_workspace to attach azurerm\_log\_analytics\_solution. Providing the config disables creation of azurerm\_log\_analytics\_workspace. | 
object({
    id                  = string
    name                = string
    location            = optional(string)
    resource_group_name = optional(string)
  })
| `null` | no | -| [aks\_net\_profile\_dns\_service\_ip](#input\_aks\_net\_profile\_dns\_service\_ip) | IP address within the Kubernetes service address range that is used by cluster service discovery (kube-dns). Must be inluced in net\_profile\_cidr. Example: 10.32.0.10 | `string` | n/a | yes | -| [aks\_net\_profile\_service\_cidr](#input\_aks\_net\_profile\_service\_cidr) | The Network Range used by the Kubernetes service. Must not overlap with the AKS Nodes address space. Example: 10.32.0.0/24 | `string` | n/a | yes | +| [aks\_net\_profile\_dns\_service\_ip](#input\_aks\_net\_profile\_dns\_service\_ip) | IP address within the Kubernetes service address range that is used by cluster service discovery (kube-dns). Must be inluced in net\_profile\_cidr. Example: 10.32.0.10 | `string` | `"10.32.0.10"` | no | +| [aks\_net\_profile\_service\_cidr](#input\_aks\_net\_profile\_service\_cidr) | The Network Range used by the Kubernetes service. Must not overlap with the AKS Nodes address space. Example: 10.32.0.0/24 | `string` | `"10.32.0.0/24"` | no | | [aks\_sku\_tier](#input\_aks\_sku\_tier) | The AKS tier. Possible values are: Free, Standard, Premium. It is recommended to use Standard or Premium for production workloads. | `string` | `"Standard"` | no | | [aks\_sys\_pool](#input\_aks\_sys\_pool) | The configuration of the AKS System Nodes Pool. | 
object({
    vm_size : string
    nodes_max_pods : number
    name : string
    availability_zones : list(string)
    disk_size_gb : number
    disk_type : string
    nodes_labels : optional(map(string), {})
    nodes_tags : optional(map(string), {})
    only_critical_addons_enabled : optional(bool, false)
    # Auto-scaling settings
    nodes_count : optional(number, null)
    enable_auto_scaling : optional(bool, false)
    agents_min_count : optional(number, null)
    agents_max_count : optional(number, null)
  })
| 
{
  "agents_max_count": 3,
  "agents_min_count": 1,
  "availability_zones": [
    "1",
    "2",
    "3"
  ],
  "disk_size_gb": 128,
  "disk_type": "Ephemeral",
  "enable_auto_scaling": true,
  "name": "system",
  "nodes_max_pods": 60,
  "only_critical_addons_enabled": false,
  "vm_size": "Standard_E4ads_v5"
}
| no | -| [aks\_worker\_pools](#input\_aks\_worker\_pools) | The worker pools of the AKS cluster, each with the respective configuration.
The default configuration uses a single worker node, with no HA. | 
map(object({
    enabled : optional(bool, true)
    vm_size : string
    priority : optional(string, "Regular")
    tags : map(string)
    max_pods : number
    disk_size_gb : optional(number, 128)
    disk_type : string
    availability_zones : list(string)
    node_taints : optional(list(string), [])
    node_labels : optional(map(string), {})
    # Auto-scaling settings
    nodes_count : optional(number, null)
    enable_auto_scaling : optional(bool, false)
    nodes_min_count : optional(number, null)
    nodes_max_count : optional(number, null)
  }))
| 
{
  "a100w01": {
    "availability_zones": [
      "1"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-ampere-a100"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 1,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC24ads_A100_v4"
  },
  "a100w02": {
    "availability_zones": [
      "2"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-ampere-a100"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 0,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC24ads_A100_v4"
  },
  "a100w03": {
    "availability_zones": [
      "3"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-ampere-a100"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 0,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC24ads_A100_v4"
  },
  "t4workers": {
    "availability_zones": [
      "1",
      "2",
      "3"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-tesla-t4"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 0,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC4as_T4_v3"
  },
  "workers01": {
    "availability_zones": [
      "1",
      "2",
      "3"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "nodes_count": 1,
    "nodes_max_count": 3,
    "nodes_min_count": 1,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_E4ads_v5"
  }
}
| no | +| [aks\_worker\_pools](#input\_aks\_worker\_pools) | The worker pools of the AKS cluster, each with the respective configuration.
The default configuration uses a single worker node, with no HA. | 
map(object({
    enabled : optional(bool, true)
    vm_size : string
    priority : optional(string, "Regular")
    tags : map(string)
    max_pods : number
    disk_size_gb : optional(number, 128)
    disk_type : string
    availability_zones : list(string)
    node_taints : optional(list(string), [])
    node_labels : optional(map(string), {})
    # Auto-scaling settings
    nodes_count : optional(number, null)
    enable_auto_scaling : optional(bool, false)
    nodes_min_count : optional(number, null)
    nodes_max_count : optional(number, null)
  }))
| 
{
  "a100w01": {
    "availability_zones": [
      "1"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-ampere-a100"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 0,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC24ads_A100_v4"
  },
  "a100w02": {
    "availability_zones": [
      "2"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-ampere-a100"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 0,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC24ads_A100_v4"
  },
  "a100w03": {
    "availability_zones": [
      "3"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-ampere-a100"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 0,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC24ads_A100_v4"
  },
  "t4workers": {
    "availability_zones": [
      "1",
      "2",
      "3"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "node_labels": {
      "nebuly.com/accelerator": "nvidia-tesla-t4"
    },
    "node_taints": [
      "nvidia.com/gpu=:NoSchedule"
    ],
    "nodes_count": null,
    "nodes_max_count": 1,
    "nodes_min_count": 0,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_NC4as_T4_v3"
  },
  "workers01": {
    "availability_zones": [
      "1",
      "2",
      "3"
    ],
    "disk_size_gb": 128,
    "disk_type": "Ephemeral",
    "enable_auto_scaling": true,
    "max_pods": 30,
    "nodes_count": 1,
    "nodes_max_count": 3,
    "nodes_min_count": 1,
    "priority": "Regular",
    "tags": {},
    "vm_size": "Standard_E4ads_v5"
  }
}
| no | | [azure\_openai\_location](#input\_azure\_openai\_location) | The Azure region where to deploy the Azure OpenAI models.
Note that the models required by Nebuly are supported only in few specific regions. For more information, you can refer to Azure documentation:
https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#standard-deployment-model-availability | `string` | `"EastUS"` | no | | [azure\_openai\_rate\_limits](#input\_azure\_openai\_rate\_limits) | The rate limits (K-tokens/minute) of the deployed Azure OpenAI models. | 
object({
    gpt_4 : number
    gpt_4o_mini : number
  })
| 
{
  "gpt_4": 100,
  "gpt_4o_mini": 100
}
| no | -| [key\_vault\_network\_acls](#input\_key\_vault\_network\_acls) | Optional configuration of network ACLs. | 
object({
    bypass : string
    default_action : string
    ip_rules : list(string)
    virtual_network_subnet_ids : list(string)
  })
| `null` | no | +| [key\_vault\_network\_acls](#input\_key\_vault\_network\_acls) | Optional configuration of network ACLs. | 
object({
    bypass : optional(string, "AzureServices")
    default_action : optional(string, "Deny")
    ip_rules : list(string)
    virtual_network_subnet_ids : list(string)
  })
| `null` | no | | [key\_vault\_private\_dns\_zone](#input\_key\_vault\_private\_dns\_zone) | Optional Private DNS Zone to link with the Key Vault when private endpoint integration is enabled. | 
object({
    id : string
    name : string
  })
| `null` | no | | [key\_vault\_private\_endpoints](#input\_key\_vault\_private\_endpoints) | Optional Private Endpoints to link with the Key Vault. | 
map(object({
    subnet_id = string
    vnet_id   = string
  }))
| `{}` | no | | [key\_vault\_public\_network\_access\_enabled](#input\_key\_vault\_public\_network\_access\_enabled) | Can the Key Vault be accessed from the Internet? | `bool` | n/a | yes | @@ -63,7 +64,7 @@ Available on [Terraform Registry](https://registry.terraform.io/modules/nebuly-a | [postgres\_server\_lock](#input\_postgres\_server\_lock) | Optionally lock the PostgreSQL server to prevent deletion. | 
object({
    enabled = optional(bool, false)
    notes   = optional(string, "Cannot be deleted.")
    name    = optional(string, "terraform-lock")
  })
| 
{
  "enabled": true
}
| no | | [postgres\_server\_maintenance\_window](#input\_postgres\_server\_maintenance\_window) | The window for performing automatic maintenance of the PostgreSQL Server. Default is Sunday at 00:00 of the timezone of the server location. | 
object({
    day_of_week : number
    start_hour : number
    start_minute : number
  })
| 
{
  "day_of_week": 0,
  "start_hour": 0,
  "start_minute": 0
}
| no | | [postgres\_server\_max\_storage\_mb](#input\_postgres\_server\_max\_storage\_mb) | The max storage allowed for the PostgreSQL Flexible Server. Possible values are 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4193280, 4194304, 8388608, 16777216 and 33553408. | `number` | `262144` | no | -| [postgres\_server\_networking](#input\_postgres\_server\_networking) | Server networking configuration.

If allowed\_ip\_ranges is not empty, then the server is accessible from
the Internet through the configured firewall rules.

If delegated\_subnet\_id or private\_dns\_zone\_id are provided, then the Server
is accessible only from the specified virutal network. | 
object({
    allowed_ip_ranges : optional(list(object({
      name : string
      start_ip_address : string
      end_ip_address : string
    })), [])
    delegated_subnet_id : optional(string, null)
    private_dns_zone_id : optional(string, null)
    public_network_access_enabled : optional(bool, false)
  })
| n/a | yes | +| [postgres\_server\_networking](#input\_postgres\_server\_networking) | Server networking configuration.

If allowed\_ip\_ranges is not empty, then the server is accessible from
the Internet through the configured firewall rules.

If delegated\_subnet\_id or private\_dns\_zone\_id are provided, then the Server
is accessible only from the specified virutal network. | 
object({
    allowed_ip_ranges : optional(list(object({
      name : string
      start_ip_address : string
      end_ip_address : string
    })), [])
    delegated_subnet_id : optional(string, null)
    private_dns_zone_id : optional(string, null)
    public_network_access_enabled : optional(bool, false)
  })
| `{}` | no | | [postgres\_server\_optional\_configurations](#input\_postgres\_server\_optional\_configurations) | Optional Flexible PostgreSQL configurations. Defaults to recommended configurations. | `map(string)` | 
{
  "intelligent_tuning": "on",
  "intelligent_tuning.metric_targets": "ALL",
  "metrics.autovacuum_diagnostics": "on",
  "metrics.collector_database_activity": "on",
  "pg_qs.query_capture_mode": "ALL",
  "pg_qs.retention_period_in_days": "7",
  "pg_qs.store_query_plans": "on",
  "pgaudit.log": "WRITE",
  "pgms_wait_sampling.query_capture_mode": "ALL",
  "track_io_timing": "on"
}
| no | | [postgres\_server\_point\_in\_time\_backup](#input\_postgres\_server\_point\_in\_time\_backup) | The backup settings of the PostgreSQL Server. | 
object({
    geo_redundant : optional(bool, true)
    retention_days : optional(number, 30)
  })
| 
{
  "geo_redundant": true,
  "retention_days": 30
}
| no | | [postgres\_server\_sku](#input\_postgres\_server\_sku) | The SKU of the PostgreSQL Server, including the Tier and the Name. Examples: B\_Standard\_B1ms, GP\_Standard\_D2s\_v3, MO\_Standard\_E4s\_v3 | 
object({
    tier : string
    name : string
  })
| 
{
  "name": "Standard_D4ds_v5",
  "tier": "GP"
}
| no | @@ -72,7 +73,7 @@ Available on [Terraform Registry](https://registry.terraform.io/modules/nebuly-a | [resource\_group\_name](#input\_resource\_group\_name) | The name of the resource group where to provision the resources. | `string` | n/a | yes | | [resource\_prefix](#input\_resource\_prefix) | The prefix that is used for generating resource names. | `string` | n/a | yes | | [subnet\_address\_space\_aks\_nodes](#input\_subnet\_address\_space\_aks\_nodes) | Address space of the new subnet in which to create the nodes of the AKS cluster.
If `subnet_name_aks_nodes` is provided, the existing subnet is used and this variable is ignored. | `list(string)` | 
[
  "10.0.0.0/22"
]
| no | -| [subnet\_address\_space\_private\_endpoints](#input\_subnet\_address\_space\_private\_endpoints) | Address space of the new subnet in which to create private endpoints.
If `subnet_name_private_endpoints` is provided, the existing subnet is used and this variable is ignored. | `list(string)` | 
[
  "10.0.0.192/26"
]
| no | +| [subnet\_address\_space\_private\_endpoints](#input\_subnet\_address\_space\_private\_endpoints) | Address space of the new subnet in which to create private endpoints.
If `subnet_name_private_endpoints` is provided, the existing subnet is used and this variable is ignored. | `list(string)` | 
[
  "10.0.8.0/26"
]
| no | | [subnet\_name\_aks\_nodes](#input\_subnet\_name\_aks\_nodes) | Optional name of the subnet to be used for provisioning AKS nodes.
If not provided, a new subnet is created. | `string` | `null` | no | | [subnet\_name\_private\_endpoints](#input\_subnet\_name\_private\_endpoints) | Optional name of the subnet to which attach the Private Endpoints.
If not provided, a new subnet is created. | `string` | `null` | no | | [tags](#input\_tags) | Common tags that are applied to all resources. | `map(string)` | `{}` | no | @@ -82,52 +83,53 @@ Available on [Terraform Registry](https://registry.terraform.io/modules/nebuly-a ## Resources -- resource.azuread_application.main (/terraform-docs/main.tf#250) -- resource.azuread_service_principal.main (/terraform-docs/main.tf#256) -- resource.azuread_service_principal_password.main (/terraform-docs/main.tf#261) -- resource.azurerm_cognitive_account.main (/terraform-docs/main.tf#445) -- resource.azurerm_cognitive_deployment.gpt_4_turbo (/terraform-docs/main.tf#464) -- resource.azurerm_cognitive_deployment.gpt_4o_mini (/terraform-docs/main.tf#479) -- resource.azurerm_key_vault.main (/terraform-docs/main.tf#184) -- resource.azurerm_key_vault_secret.azure_openai_api_key (/terraform-docs/main.tf#494) -- resource.azurerm_key_vault_secret.azuread_application_client_id (/terraform-docs/main.tf#265) -- resource.azurerm_key_vault_secret.azuread_application_client_secret (/terraform-docs/main.tf#270) -- resource.azurerm_key_vault_secret.jwt_signing_key (/terraform-docs/main.tf#714) -- resource.azurerm_key_vault_secret.postgres_password (/terraform-docs/main.tf#428) -- resource.azurerm_key_vault_secret.postgres_user (/terraform-docs/main.tf#419) -- resource.azurerm_kubernetes_cluster_node_pool.linux_pools (/terraform-docs/main.tf#675) -- resource.azurerm_management_lock.postgres_server (/terraform-docs/main.tf#362) -- resource.azurerm_monitor_metric_alert.postgres_server_alerts (/terraform-docs/main.tf#370) -- resource.azurerm_postgresql_flexible_server.main (/terraform-docs/main.tf#284) -- resource.azurerm_postgresql_flexible_server_configuration.mandatory_configurations (/terraform-docs/main.tf#335) -- resource.azurerm_postgresql_flexible_server_configuration.optional_configurations (/terraform-docs/main.tf#328) -- resource.azurerm_postgresql_flexible_server_database.analytics (/terraform-docs/main.tf#356) -- resource.azurerm_postgresql_flexible_server_database.auth (/terraform-docs/main.tf#350) -- resource.azurerm_postgresql_flexible_server_firewall_rule.main (/terraform-docs/main.tf#342) -- resource.azurerm_private_dns_zone.blob (/terraform-docs/main.tf#145) -- resource.azurerm_private_dns_zone.dfs (/terraform-docs/main.tf#163) -- resource.azurerm_private_dns_zone.file (/terraform-docs/main.tf#127) -- resource.azurerm_private_dns_zone_virtual_network_link.blob (/terraform-docs/main.tf#151) -- resource.azurerm_private_dns_zone_virtual_network_link.dfs (/terraform-docs/main.tf#169) -- resource.azurerm_private_dns_zone_virtual_network_link.file (/terraform-docs/main.tf#133) -- resource.azurerm_private_endpoint.blob (/terraform-docs/main.tf#529) -- resource.azurerm_private_endpoint.dfs (/terraform-docs/main.tf#569) -- resource.azurerm_private_endpoint.file (/terraform-docs/main.tf#549) -- resource.azurerm_private_endpoint.key_vault (/terraform-docs/main.tf#210) -- resource.azurerm_role_assignment.aks_network_contributor (/terraform-docs/main.tf#670) -- resource.azurerm_role_assignment.key_vault_secret_officer__current (/terraform-docs/main.tf#240) -- resource.azurerm_role_assignment.key_vault_secret_user__aks (/terraform-docs/main.tf#235) -- resource.azurerm_role_assignment.storage_container_models__data_contributor (/terraform-docs/main.tf#524) -- resource.azurerm_storage_account.main (/terraform-docs/main.tf#506) -- resource.azurerm_storage_container.models (/terraform-docs/main.tf#520) -- resource.azurerm_subnet.aks_nodes (/terraform-docs/main.tf#107) -- resource.azurerm_subnet.private_endpints (/terraform-docs/main.tf#115) -- resource.azurerm_virtual_network.main (/terraform-docs/main.tf#99) -- resource.random_password.postgres_server_admin_password (/terraform-docs/main.tf#279) -- resource.tls_private_key.aks (/terraform-docs/main.tf#593) -- resource.tls_private_key.jwt_signing_key (/terraform-docs/main.tf#710) -- data source.azurerm_client_config.current (/terraform-docs/main.tf#67) -- data source.azurerm_resource_group.main (/terraform-docs/main.tf#64) -- data source.azurerm_subnet.aks_nodes (/terraform-docs/main.tf#75) -- data source.azurerm_subnet.private_endpoints (/terraform-docs/main.tf#89) -- data source.azurerm_virtual_network.main (/terraform-docs/main.tf#69) +- resource.azuread_application.main (/terraform-docs/main.tf#254) +- resource.azuread_service_principal.main (/terraform-docs/main.tf#260) +- resource.azuread_service_principal_password.main (/terraform-docs/main.tf#265) +- resource.azurerm_cognitive_account.main (/terraform-docs/main.tf#457) +- resource.azurerm_cognitive_deployment.gpt_4_turbo (/terraform-docs/main.tf#476) +- resource.azurerm_cognitive_deployment.gpt_4o_mini (/terraform-docs/main.tf#491) +- resource.azurerm_key_vault.main (/terraform-docs/main.tf#188) +- resource.azurerm_key_vault_secret.azure_openai_api_key (/terraform-docs/main.tf#506) +- resource.azurerm_key_vault_secret.azuread_application_client_id (/terraform-docs/main.tf#269) +- resource.azurerm_key_vault_secret.azuread_application_client_secret (/terraform-docs/main.tf#278) +- resource.azurerm_key_vault_secret.jwt_signing_key (/terraform-docs/main.tf#741) +- resource.azurerm_key_vault_secret.postgres_password (/terraform-docs/main.tf#440) +- resource.azurerm_key_vault_secret.postgres_user (/terraform-docs/main.tf#431) +- resource.azurerm_kubernetes_cluster_node_pool.linux_pools (/terraform-docs/main.tf#698) +- resource.azurerm_management_lock.postgres_server (/terraform-docs/main.tf#374) +- resource.azurerm_monitor_metric_alert.postgres_server_alerts (/terraform-docs/main.tf#382) +- resource.azurerm_postgresql_flexible_server.main (/terraform-docs/main.tf#296) +- resource.azurerm_postgresql_flexible_server_configuration.mandatory_configurations (/terraform-docs/main.tf#347) +- resource.azurerm_postgresql_flexible_server_configuration.optional_configurations (/terraform-docs/main.tf#340) +- resource.azurerm_postgresql_flexible_server_database.analytics (/terraform-docs/main.tf#368) +- resource.azurerm_postgresql_flexible_server_database.auth (/terraform-docs/main.tf#362) +- resource.azurerm_postgresql_flexible_server_firewall_rule.main (/terraform-docs/main.tf#354) +- resource.azurerm_private_dns_zone.blob (/terraform-docs/main.tf#149) +- resource.azurerm_private_dns_zone.dfs (/terraform-docs/main.tf#167) +- resource.azurerm_private_dns_zone.file (/terraform-docs/main.tf#131) +- resource.azurerm_private_dns_zone_virtual_network_link.blob (/terraform-docs/main.tf#155) +- resource.azurerm_private_dns_zone_virtual_network_link.dfs (/terraform-docs/main.tf#173) +- resource.azurerm_private_dns_zone_virtual_network_link.file (/terraform-docs/main.tf#137) +- resource.azurerm_private_endpoint.blob (/terraform-docs/main.tf#545) +- resource.azurerm_private_endpoint.dfs (/terraform-docs/main.tf#585) +- resource.azurerm_private_endpoint.file (/terraform-docs/main.tf#565) +- resource.azurerm_private_endpoint.key_vault (/terraform-docs/main.tf#214) +- resource.azurerm_role_assignment.aks_network_contributor (/terraform-docs/main.tf#693) +- resource.azurerm_role_assignment.key_vault_secret_officer__current (/terraform-docs/main.tf#244) +- resource.azurerm_role_assignment.key_vault_secret_user__aks (/terraform-docs/main.tf#239) +- resource.azurerm_role_assignment.storage_container_models__data_contributor (/terraform-docs/main.tf#540) +- resource.azurerm_storage_account.main (/terraform-docs/main.tf#522) +- resource.azurerm_storage_container.models (/terraform-docs/main.tf#536) +- resource.azurerm_subnet.aks_nodes (/terraform-docs/main.tf#111) +- resource.azurerm_subnet.private_endpints (/terraform-docs/main.tf#119) +- resource.azurerm_virtual_network.main (/terraform-docs/main.tf#103) +- resource.random_password.postgres_server_admin_password (/terraform-docs/main.tf#291) +- resource.time_sleep.wait_aks_creation (/terraform-docs/main.tf#680) +- resource.tls_private_key.aks (/terraform-docs/main.tf#609) +- resource.tls_private_key.jwt_signing_key (/terraform-docs/main.tf#737) +- data source.azurerm_client_config.current (/terraform-docs/main.tf#71) +- data source.azurerm_resource_group.main (/terraform-docs/main.tf#68) +- data source.azurerm_subnet.aks_nodes (/terraform-docs/main.tf#79) +- data source.azurerm_subnet.private_endpoints (/terraform-docs/main.tf#93) +- data source.azurerm_virtual_network.main (/terraform-docs/main.tf#73) diff --git a/main.tf b/main.tf index 1f05f4d..bb21cea 100644 --- a/main.tf +++ b/main.tf @@ -6,6 +6,10 @@ terraform { source = "hashicorp/azurerm" version = "~>3.114" } + time = { + source = "hashicorp/time" + version = "~>0.12" + } azuread = { source = "hashicorp/azuread" version = "~>2.53" @@ -198,10 +202,10 @@ resource "azurerm_key_vault" "main" { dynamic "network_acls" { for_each = var.key_vault_network_acls == null ? {} : { "" : "" } content { - bypass = var.network_acls.bypass - default_action = var.network_acls.default_action - ip_rules = var.network_acls.ip_rules - virtual_network_subnet_ids = var.network_acls.virtual_network_subnet_ids + bypass = var.key_vault_network_acls.bypass + default_action = var.key_vault_network_acls.default_action + ip_rules = var.key_vault_network_acls.ip_rules + virtual_network_subnet_ids = var.key_vault_network_acls.virtual_network_subnet_ids } } @@ -234,7 +238,7 @@ resource "azurerm_private_endpoint" "key_vault" { } resource "azurerm_role_assignment" "key_vault_secret_user__aks" { scope = azurerm_key_vault.main.id - principal_id = "" # TODO + principal_id = module.aks.cluster_identity.principal_id role_definition_name = "Key Vault Secrets User" } resource "azurerm_role_assignment" "key_vault_secret_officer__current" { @@ -266,11 +270,19 @@ resource "azurerm_key_vault_secret" "azuread_application_client_id" { key_vault_id = azurerm_key_vault.main.id name = format("%s-azure-client-id", var.resource_prefix) value = azuread_application.main.client_id + + depends_on = [ + azurerm_role_assignment.key_vault_secret_officer__current + ] } resource "azurerm_key_vault_secret" "azuread_application_client_secret" { key_vault_id = azurerm_key_vault.main.id name = format("%s-azure-client-secret", var.resource_prefix) value = azuread_application.main.client_id + + depends_on = [ + azurerm_role_assignment.key_vault_secret_officer__current + ] } @@ -495,6 +507,10 @@ resource "azurerm_key_vault_secret" "azure_openai_api_key" { name = "${var.resource_prefix}-openai-api-key" value = azurerm_cognitive_account.main.primary_access_key key_vault_id = azurerm_key_vault.main.id + + depends_on = [ + azurerm_role_assignment.key_vault_secret_officer__current + ] } @@ -512,7 +528,7 @@ resource "azurerm_storage_account" "main" { account_replication_type = "LRS" access_tier = "Hot" - public_network_access_enabled = false + public_network_access_enabled = true # TODO is_hns_enabled = false tags = var.tags @@ -524,7 +540,7 @@ resource "azurerm_storage_container" "models" { resource "azurerm_role_assignment" "storage_container_models__data_contributor" { role_definition_name = "Storage Blob Data Contributor" principal_id = azuread_service_principal.main.object_id - scope = azurerm_storage_container.models.id + scope = azurerm_storage_container.models.resource_manager_id } resource "azurerm_private_endpoint" "blob" { name = "${azurerm_storage_account.main.name}-blob" @@ -661,6 +677,13 @@ module "aks" { tags = var.tags } +resource "time_sleep" "wait_aks_creation" { + create_duration = "30s" + + depends_on = [ + module.aks + ] +} # The AKS cluster identity has the Contributor role on the AKS second resource group (MC_myResourceGroup_myAKSCluster_eastus) # However when using a custom VNET, the AKS cluster identity needs the Network Contributor role on the VNET subnets # used by the system node pool and by any additional node pools. @@ -703,6 +726,10 @@ resource "azurerm_kubernetes_cluster_node_pool" "linux_pools" { eviction_policy, ] } + + depends_on = [ + time_sleep.wait_aks_creation, + ] } @@ -715,6 +742,10 @@ resource "azurerm_key_vault_secret" "jwt_signing_key" { key_vault_id = azurerm_key_vault.main.id name = format("%s-jwt-signing-key", var.resource_prefix) value = tls_private_key.jwt_signing_key.private_key_pem + + depends_on = [ + azurerm_role_assignment.key_vault_secret_officer__current + ] } @@ -733,7 +764,7 @@ locals { k8s_secret_key_azure_client_secret = "azure-client-secret" helm_values = templatefile( - "templates/helm-values.tpl.yaml", + "${path.module}/templates/helm-values.tpl.yaml", { platform_domain = var.platform_domain @@ -754,14 +785,14 @@ locals { }, ) secret_provider_class = templatefile( - "templates/secret-provider-class.tpl.yaml", + "${path.module}/templates/secret-provider-class.tpl.yaml", { secret_provider_class_name = local.secret_provider_class_name secret_provider_class_secret_name = local.secret_provider_class_secret_name key_vault_name = azurerm_key_vault.main.name tenant_id = data.azurerm_client_config.current.tenant_id - aks_managed_identity_id = module.aks.key_vault_secrets_provider.secret_identity[0] + aks_managed_identity_id = try(module.aks.key_vault_secrets_provider.secret_identity[0].object_id, "TODO") secret_name_jwt_signing_key = azurerm_key_vault_secret.jwt_signing_key.name secret_name_db_username = azurerm_key_vault_secret.postgres_user.name diff --git a/tests/dev-provisioning/apply.sh b/tests/dev-provisioning/apply.sh new file mode 100755 index 0000000..8cdabab --- /dev/null +++ b/tests/dev-provisioning/apply.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ "$1" == 'init' ] + then + terraform init --backend-config backend.tfvars + shift +fi + +terraform apply --var-file backend.tfvars "$@" diff --git a/tests/dev-provisioning/backend.tfvars.example b/tests/dev-provisioning/backend.tfvars.example new file mode 100644 index 0000000..5d98958 --- /dev/null +++ b/tests/dev-provisioning/backend.tfvars.example @@ -0,0 +1,5 @@ +tenant_id = "" +subscription_id = "" +client_id = "" +client_secret = "" + diff --git a/tests/dev-provisioning/main.tf b/tests/dev-provisioning/main.tf new file mode 100644 index 0000000..fe0206b --- /dev/null +++ b/tests/dev-provisioning/main.tf @@ -0,0 +1,89 @@ +terraform { + required_version = ">=1.9" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.114" + } + azuread = { + source = "hashicorp/azuread" + version = "~>2.53" + } + random = { + source = "hashicorp/random" + version = "~>3.6" + } + } +} + +provider "azurerm" { + features {} + + client_id = var.client_id + client_secret = var.client_secret + tenant_id = var.tenant_id + subscription_id = var.subscription_id +} + +# ------ Variables ------ # +variable "resource_prefix" { + type = string +} +variable "client_id" { + type = string +} +variable "subscription_id" { + type = string +} +variable "tenant_id" { + type = string +} +variable "client_secret" { + type = string +} +variable "resource_group_name" { + type = string +} +variable "tags" { + type = map(any) +} +variable "location" { + type = string +} + +# ------ Data Sources ------ # +data "azuread_group" "engineering" { + display_name = "nebuly-engineering" +} +data "http" "my_ip" { + url = "https://ipv4.icanhazip.com" +} +locals { + my_ip = chomp(data.http.my_ip.response_body) +} + +module "platform" { + source = "../../" + + location = var.location + resource_group_name = var.resource_group_name + platform_domain = "platform.azure.testing" + + postgres_server_sku = { + tier = "GP" + name = "Standard_D2ads_v5" + } + + key_vault_public_network_access_enabled = true + key_vault_network_acls = { + ip_rules = [local.my_ip] + virtual_network_subnet_ids = [] + } + + aks_cluster_admin_object_ids = [data.azuread_group.engineering.id] + resource_prefix = var.resource_prefix + + tags = var.tags +} + diff --git a/tests/dev-provisioning/remote_state.tf b/tests/dev-provisioning/remote_state.tf new file mode 100644 index 0000000..dca6f18 --- /dev/null +++ b/tests/dev-provisioning/remote_state.tf @@ -0,0 +1,8 @@ + terraform { + backend "azurerm" { + resource_group_name = "rg-shared" + storage_account_name = "nbllabtfstatessa" + container_name = "rg-platform-inttest-tfstate" + key = "tfstate" + } + } diff --git a/tests/dev-provisioning/secrets.auto.tfvars.example b/tests/dev-provisioning/secrets.auto.tfvars.example new file mode 100644 index 0000000..ad2ffcf --- /dev/null +++ b/tests/dev-provisioning/secrets.auto.tfvars.example @@ -0,0 +1,4 @@ +tenant_id = "" +subscription_id = "" +client_id = "" +client_secret = "" diff --git a/tests/dev-provisioning/terraform.auto.tfvars b/tests/dev-provisioning/terraform.auto.tfvars new file mode 100644 index 0000000..a1907e3 --- /dev/null +++ b/tests/dev-provisioning/terraform.auto.tfvars @@ -0,0 +1,9 @@ +### General ### +location = "EastUS" +resource_prefix = "nbltst" +resource_group_name = "rg-platform-inttest" +tags = { + "env" : "integration-test" + "project" : "self-hosted" +} + diff --git a/tests/smoke_test__default_values.tftest.hcl b/tests/smoke_test__default_values.tftest.hcl index 17b5030..ed637f3 100644 --- a/tests/smoke_test__default_values.tftest.hcl +++ b/tests/smoke_test__default_values.tftest.hcl @@ -10,15 +10,10 @@ run "smoke_test_plan__default_values" { location = "EastUS" platform_domain = "intest.nebuly.ai" - # ------ PostgreSQL Database ------ # - postgres_server_networking = {} - # ------ Key Vault ------ # - key_vault_public_network_access_enabled = true + key_vault_public_network_access_enabled = false # ------ AKS ------ # - aks_net_profile_service_cidr = "10.32.0.0/24" - aks_net_profile_dns_service_ip = "10.32.0.10" aks_cluster_admin_object_ids = [] } } diff --git a/tests/smoke_test__existing_networks.tftest.hcl b/tests/smoke_test__existing_networks.tftest.hcl index 90d97fb..5283f4f 100644 --- a/tests/smoke_test__existing_networks.tftest.hcl +++ b/tests/smoke_test__existing_networks.tftest.hcl @@ -24,15 +24,10 @@ run "smoke_test_plan__existing_networks" { virtual_network_name = run.setup.azurerm_virtual_network.name subnet_name_aks_nodes = run.setup.azurerm_subnet.name - # ------ PostgreSQL Database ------ # - postgres_server_networking = {} - # ------ Key Vault ------ # - key_vault_public_network_access_enabled = true + key_vault_public_network_access_enabled = false # ------ AKS ------ # - aks_net_profile_service_cidr = "10.32.0.0/24" - aks_net_profile_dns_service_ip = "10.32.0.10" aks_cluster_admin_object_ids = [] } } diff --git a/tests/validation__networking.tftest.hcl b/tests/test_values_validation.tftest.hcl similarity index 77% rename from tests/validation__networking.tftest.hcl rename to tests/test_values_validation.tftest.hcl index 3abe7b3..8681065 100644 --- a/tests/validation__networking.tftest.hcl +++ b/tests/test_values_validation.tftest.hcl @@ -12,7 +12,7 @@ variables { postgres_server_networking = {} # ------ Key Vault ------ # - key_vault_public_network_access_enabled = true + key_vault_public_network_access_enabled = false # ------ AKS ------ # aks_net_profile_service_cidr = "10.32.0.0/24" @@ -47,3 +47,16 @@ run "values_validation__subnet_private_endpoints" { var.subnet_name_private_endpoints ] } + +run "values_validation__key_vault_acls" { + command = plan + + variables { + # KeyVault ACLs must be provided when public access is enabled. + key_vault_public_network_access_enabled = true + } + + expect_failures = [ + var.key_vault_public_network_access_enabled + ] +} diff --git a/variables.tf b/variables.tf index 4aefe38..75db104 100644 --- a/variables.tf +++ b/variables.tf @@ -95,6 +95,7 @@ variable "postgres_server_networking" { private_dns_zone_id : optional(string, null) public_network_access_enabled : optional(bool, false) }) + default = {} } variable "postgres_server_point_in_time_backup" { type = object({ @@ -182,6 +183,11 @@ variable "key_vault_sku_name" { variable "key_vault_public_network_access_enabled" { type = bool description = "Can the Key Vault be accessed from the Internet?" + + validation { + condition = !var.key_vault_public_network_access_enabled || var.key_vault_network_acls != null + error_message = "You must provide network ACLs when Key Vault public network access is enabled." + } } variable "key_vault_soft_delete_retention_days" { type = number @@ -195,8 +201,8 @@ variable "key_vault_purge_protection_enabled" { } variable "key_vault_network_acls" { type = object({ - bypass : string - default_action : string + bypass : optional(string, "AzureServices") + default_action : optional(string, "Deny") ip_rules : list(string) virtual_network_subnet_ids : list(string) }) @@ -278,7 +284,7 @@ variable "subnet_address_space_private_endpoints" { If `subnet_name_private_endpoints` is provided, the existing subnet is used and this variable is ignored. EOT type = list(string) - default = ["10.0.0.192/26"] + default = ["10.0.8.0/26"] } variable "private_dns_zones" { description = <