From 6c9cb2e3f0824a72ea1051e3e08d8eadf705c8b9 Mon Sep 17 00:00:00 2001 From: Michele Zanotti Date: Mon, 5 Aug 2024 13:22:35 +0200 Subject: [PATCH] feat: generate helm values --- README.md | 44 +++--- main.tf | 12 ++ outputs.tf | 5 + templates/helm-values.tpl.yaml | 238 +++++++++++++++++++++++++++++++++ tests/smoke_test.tftest.hcl | 21 ++- tests/terraform.auto.tfvars | 7 - variables.tf | 9 ++ 7 files changed, 303 insertions(+), 33 deletions(-) create mode 100644 outputs.tf create mode 100644 templates/helm-values.tpl.yaml diff --git a/README.md b/README.md index e5b50aa..c313fb3 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,9 @@ Available on [Terraform Registry](https://registry.terraform.io/modules/nebuly-a ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [helm\_values](#output\_helm\_values) | The values.yaml file for installing Nebuly on the provisioned resources. | ## Inputs @@ -53,6 +55,7 @@ No outputs. | [key\_vault\_soft\_delete\_retention\_days](#input\_key\_vault\_soft\_delete\_retention\_days) | The number of days that items should be retained for once soft-deleted. This value can be between 7 and 90 (the default) days. | `number` | `7` | no | | [location](#input\_location) | The region where to provision the resources. | `string` | n/a | yes | | [openai\_api\_key](#input\_openai\_api\_key) | The API Key used for authenticating with OpenAI. | `string` | n/a | yes | +| [platform\_domain](#input\_platform\_domain) | The domain on which the deployed Nebuly platform will be available. | `string` | n/a | yes | | [postgres\_server\_admin\_username](#input\_postgres\_server\_admin\_username) | The username of the admin user of the PostgreSQL Server. | `string` | `"nebulyadmin"` | no | | [postgres\_server\_alert\_rules](#input\_postgres\_server\_alert\_rules) | The Azure Monitor alert rules to set on the provisioned PostgreSQL server. | 
map(object({
    description     = string
    frequency       = string
    window_size     = string
    action_group_id = string
    severity        = number

    criteria = optional(
      object({
        aggregation = string
        metric_name = string
        operator    = string
        threshold   = number
      })
    , null)
    dynamic_criteria = optional(
      object({
        aggregation       = string
        metric_name       = string
        operator          = string
        alert_sensitivity = string
      })
    , null)
  }))
| `{}` | no | | [postgres\_server\_high\_availability](#input\_postgres\_server\_high\_availability) | High-availability configuration of the DB server. Possible values for mode are: SameZone or ZoneRedundant. | 
object({
    enabled : bool
    mode : string
    standby_availability_zone : optional(string, null)
  })
| 
{
  "enabled": true,
  "mode": "SameZone"
}
| no | @@ -71,22 +74,23 @@ No outputs. ## Resources -- resource.azurerm_key_vault.main (/terraform-docs/main.tf#59) -- resource.azurerm_key_vault_secret.postgres_passwords (/terraform-docs/main.tf#272) -- resource.azurerm_key_vault_secret.postgres_users (/terraform-docs/main.tf#261) -- resource.azurerm_kubernetes_cluster_node_pool.linux_pools (/terraform-docs/main.tf#361) -- resource.azurerm_management_lock.postgres_server (/terraform-docs/main.tf#204) -- resource.azurerm_monitor_metric_alert.postgres_server_alerts (/terraform-docs/main.tf#212) -- resource.azurerm_postgresql_flexible_server.main (/terraform-docs/main.tf#130) -- resource.azurerm_postgresql_flexible_server_configuration.mandatory_configurations (/terraform-docs/main.tf#181) -- resource.azurerm_postgresql_flexible_server_configuration.optional_configurations (/terraform-docs/main.tf#174) -- resource.azurerm_postgresql_flexible_server_database.main (/terraform-docs/main.tf#196) -- resource.azurerm_postgresql_flexible_server_firewall_rule.main (/terraform-docs/main.tf#188) -- resource.azurerm_private_endpoint.key_vault (/terraform-docs/main.tf#85) -- resource.azurerm_role_assignment.key_vault_secret_officer__current (/terraform-docs/main.tf#115) -- resource.azurerm_role_assignment.key_vault_secret_user__aks (/terraform-docs/main.tf#110) -- resource.random_password.postgres_server_admin_password (/terraform-docs/main.tf#125) -- resource.tls_private_key.aks (/terraform-docs/main.tf#288) -- data source.azurerm_client_config.current (/terraform-docs/main.tf#47) -- data source.azurerm_resource_group.main (/terraform-docs/main.tf#44) -- data source.azurerm_subnet.aks_nodes (/terraform-docs/main.tf#49) +- resource.azurerm_key_vault.main (/terraform-docs/main.tf#63) +- resource.azurerm_key_vault_secret.openai_api_key (/terraform-docs/main.tf#127) +- resource.azurerm_key_vault_secret.postgres_passwords (/terraform-docs/main.tf#283) +- resource.azurerm_key_vault_secret.postgres_users (/terraform-docs/main.tf#272) +- resource.azurerm_kubernetes_cluster_node_pool.linux_pools (/terraform-docs/main.tf#372) +- resource.azurerm_management_lock.postgres_server (/terraform-docs/main.tf#215) +- resource.azurerm_monitor_metric_alert.postgres_server_alerts (/terraform-docs/main.tf#223) +- resource.azurerm_postgresql_flexible_server.main (/terraform-docs/main.tf#141) +- resource.azurerm_postgresql_flexible_server_configuration.mandatory_configurations (/terraform-docs/main.tf#192) +- resource.azurerm_postgresql_flexible_server_configuration.optional_configurations (/terraform-docs/main.tf#185) +- resource.azurerm_postgresql_flexible_server_database.main (/terraform-docs/main.tf#207) +- resource.azurerm_postgresql_flexible_server_firewall_rule.main (/terraform-docs/main.tf#199) +- resource.azurerm_private_endpoint.key_vault (/terraform-docs/main.tf#89) +- resource.azurerm_role_assignment.key_vault_secret_officer__current (/terraform-docs/main.tf#119) +- resource.azurerm_role_assignment.key_vault_secret_user__aks (/terraform-docs/main.tf#114) +- resource.random_password.postgres_server_admin_password (/terraform-docs/main.tf#136) +- resource.tls_private_key.aks (/terraform-docs/main.tf#299) +- data source.azurerm_client_config.current (/terraform-docs/main.tf#51) +- data source.azurerm_resource_group.main (/terraform-docs/main.tf#48) +- data source.azurerm_subnet.aks_nodes (/terraform-docs/main.tf#53) diff --git a/main.tf b/main.tf index 7fff633..8f8122a 100644 --- a/main.tf +++ b/main.tf @@ -401,3 +401,15 @@ resource "azurerm_kubernetes_cluster_node_pool" "linux_pools" { ] } } + + +# ------ Post provisioning ------ # +locals { + helm_values = templatefile( + "templates/helm-values.tpl.yaml", + { + platform_domain = var.platform_domain + }, + ) +} + diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..651cf05 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,5 @@ +output "helm_values" { + value = local.helm_values + sensitive = true + description = "The values.yaml file for installing Nebuly on the provisioned resources." +} diff --git a/templates/helm-values.tpl.yaml b/templates/helm-values.tpl.yaml new file mode 100644 index 0000000..b1d9910 --- /dev/null +++ b/templates/helm-values.tpl.yaml @@ -0,0 +1,238 @@ +strimzi: + enabled: true + +imagePullSecrets: + - name: nebuly-docker-pull + +backend: + image: + repository: "ghcr.io/nebuly-ai/nebuly-backend" + ingress: + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" + enabled: true + className: "nginx" + hosts: + - host: ${platform_domain} + paths: + - path: /backend(/|$)(.*) + pathType: Prefix + + volumeMounts: + - name: secrets-store + mountPath: /mnt/secrets-store + readOnly: true + volumes: + - name: secrets-store + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: nebuly-platform + +eventIngestion: + image: + repository: "ghcr.io/nebuly-ai/nebuly-event-ingestion" + + rootPath: "/event-ingestion" + + ingress: + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" + enabled: true + className: "nginx" + hosts: + - host: ${platform_domain} + paths: + - path: /event-ingestion(/|$)(.*) + pathType: Prefix + +azureml: + enabled: false + +aiModels: + registry: aws_s3 + aws: + bucketName: "" # TODO + +lionLinguist: + image: + repository: "ghcr.io/nebuly-ai/nebuly-lion-linguist" + + modelsCache: + enabled: true + storageClassName: gp2-csi + accessModes: + - ReadWriteOnce + + resources: + requests: + cpu: "1" + limits: + memory: 8Gi + nvidia.com/gpu: 1 + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nebuly.com/accelerator + operator: In + values: + - nvidia-tesla-t4 + + volumeMounts: + - name: secrets-store + mountPath: /mnt/secrets-store + readOnly: true + volumes: + - name: secrets-store + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: nebuly-platform + +ingestionWorker: + image: + repository: "ghcr.io/nebuly-ai/nebuly-ingestion-worker" + + volumeMounts: + - name: secrets-store + mountPath: /mnt/secrets-store + readOnly: true + volumes: + - name: secrets-store + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: nebuly-platform + +kafka: + external: false + zookeeper: + replicas: 1 + storage: + class: gp2-csi + replicas: 1 + config: + offsets.topic.replication.factor: 1 + replica.selector.class: org.apache.kafka.common.replica.RackAwareReplicaSelector + storage: + class: gp2-csi + size: 32Gi + resources: + limits: + memory: 6Gi + requests: + memory: 6Gi + +analyticDatabase: + server: "nbllabplatformanalytics.c72cu4g06kwz.us-east-1.rds.amazonaws.com" + name: "analytics" + existingSecret: + name: nebuly-platform-credentials + userKey: analytics-db-username + passwordKey: analytics-db-password + +auth: + image: + repository: "ghcr.io/nebuly-ai/nebuly-tenant-registry" + + postgresServer: "nbllabplatformauth.c72cu4g06kwz.us-east-1.rds.amazonaws.com" + postgresDatabase: "auth" + existingSecret: + name: nebuly-platform-credentials + postgresUserKey: auth-db-username + postgresPasswordKey: auth-db-password + jwtSigningKey: jwt-key + + loginModes: "password" + microsoft: + enabled: false + redirectUri: https://${platform_domain}/backend/auth/oauth/microsoft/callback + tenantId: "" + existingSecret: + name: microsoft-oauth-credentials + clientIdKey: microsoft-oauth-client-id + clientSecretKey: microsoft-oauth-client-secret + + volumeMounts: + - name: secrets-store + mountPath: /mnt/secrets-store + readOnly: true + volumes: + - name: secrets-store + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: nebuly-platform + ingress: + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/auth/$2" + nginx.ingress.kubernetes.io/use-regex: "true" + enabled: true + className: "nginx" + hosts: + - host: ${platform_domain} + paths: + - path: "/backend/auth(/|$)(.*)" + pathType: Prefix + +frontend: + image: + repository: "ghcr.io/nebuly-ai/nebuly-frontend" + + rootUrl: https://${platform_domain} + backendApiUrl: https://${platform_domain}/backend + authApiUrl: https://${platform_domain}/backend/auth + ingress: + enabled: true + className: "nginx" + hosts: + - host: ${platform_domain} + paths: + - path: / + pathType: Prefix + +openAi: + enabled: true + insightsGeneratorDeployment: gpt-4-1106-preview-nebuly + textEmbeddingsDeployment: ada + frustrationDetectionDeployment: gpt-4-1106-preview-nebuly + chatCompletionDeployment: gpt-4-1106-preview-nebuly + endpoint: https://openaixpanseaisandbox-useast2.openai.azure.com/ + + existingSecret: + name: nebuly-platform-credentials + apiKey: openai-api-key + +bootstrap-aws: + enabled: true + ingress-nginx: + controller: + allowSnippetAnnotations: true + config: + http-snippet: | + server { + if ($http_x_forwarded_proto = 'http') { + return 301 https://$host$request_uri; + } + } + service: + targetPorts: + http: http + https: http + annotations: + service.beta.kubernetes.io/aws-load-balancer-security-groups: "sg-0a2684cf0282660e5" + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-east-1:533267425677:certificate/6775c5fc-4e7a-4d61-9a1f-35c8455655fa" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" + cluster-autoscaler: + rbac: + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::533267425677:role/nbllabeks" diff --git a/tests/smoke_test.tftest.hcl b/tests/smoke_test.tftest.hcl index 4e223c4..e6be72f 100644 --- a/tests/smoke_test.tftest.hcl +++ b/tests/smoke_test.tftest.hcl @@ -3,7 +3,7 @@ provider "azurerm" { } run "setup" { - module { + module { source = "./tests/setup" } @@ -17,20 +17,29 @@ run "smoke_test_plan__default_values" { variables { resource_group_name = "rg-platform-inttest" - location = "EastUS" + location = "EastUS" + platform_domain = "intest.nebuly.ai" # ------ PostgreSQL Database ------ # - postgres_server_networking = { } + postgres_server_networking = {} # ------ Key Vault ------ # key_vault_public_network_access_enabled = true + # ------ External Secrets ------ # + openai_api_key = "my-key" + # ------ AKS ------ # aks_nodes_virtual_network_name = run.setup.azurerm_virtual_network.name - aks_nodes_subnet_name = run.setup.azurerm_subnet.name + aks_nodes_subnet_name = run.setup.azurerm_subnet.name - aks_net_profile_service_cidr = "10.32.0.0/24" + aks_net_profile_service_cidr = "10.32.0.0/24" aks_net_profile_dns_service_ip = "10.32.0.10" - aks_cluster_admin_object_ids = [] + aks_cluster_admin_object_ids = [] + } + + assert { + condition = strcontains(output.helm_values, var.platform_domain) + error_message = "Helm values incorrect: platform domain is not included." } } diff --git a/tests/terraform.auto.tfvars b/tests/terraform.auto.tfvars index b5edbc8..df8d1fc 100644 --- a/tests/terraform.auto.tfvars +++ b/tests/terraform.auto.tfvars @@ -6,10 +6,3 @@ tags = { "project" : "self-deploy" } - -### EKS ### -eks_kubernetes_version = "1.28" -eks_cluster_endpoint_public_access = true - -### External secrets ### -openai_api_key = "my-key" diff --git a/variables.tf b/variables.tf index cef5b5d..56160d1 100644 --- a/variables.tf +++ b/variables.tf @@ -16,6 +16,15 @@ variable "resource_group_name" { type = string description = "The name of the resource group where to provision the resources." } +variable "platform_domain" { + type = string + description = "The domain on which the deployed Nebuly platform will be available." + validation { + condition = can(regex("(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]", var.platform_domain)) + error_message = "The domain name must be a valid domain (e.g., example.com)." + } +} + # ------ PostgreSQL Databases ------ # variable "postgres_server_sku" {