Add global keyvault (Azure#654)

* Move private endpoint to a seperate module * Add shared keyvault
ahitacat · Sep 26, 2024 · 2826c88 · 2826c88
1 parent 5446256
commit 2826c88
Show file tree

Hide file tree

Showing 10 changed files with 106 additions and 79 deletions.
diff --git a/cluster-service/Makefile b/cluster-service/Makefile
@@ -5,7 +5,7 @@ CONFIG_PROFILE ?= dev
 include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk
 
 CONSUMER_NAME ?= $(shell az aks list --query "[?tags.clusterType == 'mgmt-cluster' && starts_with(resourceGroup, '$(REGIONAL_RESOURCEGROUP)')].resourceGroup" -o tsv)
-KEYVAULT_NAME ?= $(shell az keyvault list --query "[?starts_with(name, 'service-kv')].name" -g ${RESOURCEGROUP} --output tsv)
+KEYVAULT_NAME ?= $(shell az keyvault list --query "[?starts_with(name, 'service-kv')].name" -g ${SVC_KV_RESOURCEGROUP} --output tsv)
 FPA_CERT_NAME ?= firstPartyMock
 AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= "ccf5339c-61d1-402f-9c9b-d463670191f9"
 

diff --git a/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam
@@ -24,7 +24,8 @@ param deployCsInfra = false
 param csPostgresServerName = 'cs-pg-cs-integ'
 param clusterServicePostgresPrivate = false
 
-param serviceKeyVaultName = 'service-kv-cs-integ'
+param serviceKeyVaultName = 'aro-hcp-dev-svc-kv'
+param serviceKeyVaultResourceGroup = 'global'
 param serviceKeyVaultSoftDelete = true
 param serviceKeyVaultPrivate = false
 

diff --git a/dev-infrastructure/configurations/cs-integ.mk b/dev-infrastructure/configurations/cs-integ.mk
@@ -1,6 +1,6 @@
 REGION ?= westus3
 RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION)-$(AKSCONFIG)
 REGIONAL_RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION)
+SVC_KV_RESOURCEGROUP ?= global
 ARO_HCP_IMAGE_ACR ?= arohcpdev
 REGIONAL_ACR_NAME ?= arohcpdev$(shell echo $(CURRENTUSER) | sha256sum  | head -c 24)
-
diff --git a/dev-infrastructure/configurations/dev.mk b/dev-infrastructure/configurations/dev.mk
@@ -1,6 +1,7 @@
 REGION ?= westus3
 RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION)-$(AKSCONFIG)
 REGIONAL_RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION)
+SVC_KV_RESOURCEGROUP ?= global
 GLOBAL_RESOURCEGROUP ?= global
 ARO_HCP_IMAGE_ACR ?= arohcpdev
 REGIONAL_ACR_NAME ?= arohcpdev$(shell echo $(CURRENTUSER) | sha256sum  | head -c 24)

diff --git a/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam b/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam
@@ -24,7 +24,8 @@ param deployCsInfra = false
 param csPostgresServerName = 'cs-pg-aro-hcp-dev'
 param clusterServicePostgresPrivate = false
 
-param serviceKeyVaultName = 'service-kv-aro-hcp-dev'
+param serviceKeyVaultName = 'aro-hcp-dev-svc-kv'
+param serviceKeyVaultResourceGroup = 'global'
 param serviceKeyVaultSoftDelete = true
 param serviceKeyVaultPrivate = false
 

diff --git a/dev-infrastructure/configurations/svc-cluster.bicepparam b/dev-infrastructure/configurations/svc-cluster.bicepparam
@@ -25,7 +25,8 @@ param deployCsInfra = false
 param csPostgresServerName = take('cs-pg-${uniqueString(currentUserId)}', 60)
 param clusterServicePostgresPrivate = false
 
-param serviceKeyVaultName = take('service-kv-${uniqueString(currentUserId)}', 24)
+param serviceKeyVaultName = 'aro-hcp-dev-svc-kv'
+param serviceKeyVaultResourceGroup = 'global'
 param serviceKeyVaultSoftDelete = false
 param serviceKeyVaultPrivate = false
 

diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep
@@ -70,8 +70,6 @@ module aks_keyvault_builder '../modules/keyvault/keyvault.bicep' = {
     // todo: change for higher environments
     private: false
     enableSoftDelete: aksEtcdKVEnableSoftDelete
-    // AKS managed private endpoints on its own when the etcd KV is private
-    managedPrivateEndpoint: false
   }
 }
 

diff --git a/dev-infrastructure/modules/keyvault/keyvault-private-endpoint.bicep b/dev-infrastructure/modules/keyvault/keyvault-private-endpoint.bicep
@@ -0,0 +1,77 @@
+@description('Location of the endpoint.')
+param location string
+
+@description('Name of the key vault to create this endpoint for.')
+param keyVaultName string
+
+@description('ID of the subnet to create the private endpoint in.')
+param subnetId string
+
+@description('ID of the vnet, needs to correlated with subnetId.')
+param vnetId string
+
+@description('ID of the key vault.')
+param keyVaultId string
+
+//
+//   P R I V A T E   E N D P O I N T
+//
+
+var privateDnsZoneName = 'privatelink.vaultcore.azure.net'
+
+resource keyVaultPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-01-01' = {
+  name: '${keyVaultName}-pe'
+  location: location
+  properties: {
+    privateLinkServiceConnections: [
+      {
+        name: '${keyVaultName}-pe'
+        properties: {
+          groupIds: [
+            'vault'
+          ]
+          privateLinkServiceId: keyVaultId
+        }
+      }
+    ]
+    subnet: {
+      id: subnetId
+    }
+  }
+}
+
+resource keyVaultPrivateEndpointDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = {
+  name: privateDnsZoneName
+  location: 'global'
+  properties: {}
+}
+
+resource keyVaultPrivateDnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = {
+  parent: keyVaultPrivateEndpointDnsZone
+  name: uniqueString(keyVaultId)
+  location: 'global'
+  properties: {
+    registrationEnabled: false
+    virtualNetwork: {
+      id: vnetId
+    }
+  }
+}
+
+resource privateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-09-01' = {
+  parent: keyVaultPrivateEndpoint
+  name: '${keyVaultName}-dns-group'
+  properties: {
+    privateDnsZoneConfigs: [
+      {
+        name: 'config1'
+        properties: {
+          privateDnsZoneId: keyVaultPrivateEndpointDnsZone.id
+        }
+      }
+    ]
+  }
+  dependsOn: [
+    keyVaultPrivateDnsZoneVnetLink
+  ]
+}
diff --git a/dev-infrastructure/modules/keyvault/keyvault.bicep b/dev-infrastructure/modules/keyvault/keyvault.bicep
@@ -1,19 +1,15 @@
+@description('Location of the keyvault.')
 param location string
 
+@description('Name of the key vault.')
 param keyVaultName string
 
-param subnetId string = ''
-
-param vnetId string = ''
-
+@description('Toggle to enable soft delete.')
 param enableSoftDelete bool
 
+@description('Toggle to make the keyvault private.')
 param private bool
 
-// Event for some private KVs it makes sense to disable the creation of a private endpoint,
-// e.g. AKS KMS on a private KV will manage their own private endpoint setup in the nodepool RG
-param managedPrivateEndpoint bool = true
-
 resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = {
   location: location
   name: keyVaultName
@@ -35,67 +31,6 @@ resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = {
   }
 }
 
-//
-//   P R I V A T E   E N D P O I N T
-//
-
-var privateDnsZoneName = 'privatelink.vaultcore.azure.net'
-
-resource keyVaultPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-01-01' = if (managedPrivateEndpoint) {
-  name: '${keyVaultName}-pe'
-  location: location
-  properties: {
-    privateLinkServiceConnections: [
-      {
-        name: '${keyVaultName}-pe'
-        properties: {
-          groupIds: [
-            'vault'
-          ]
-          privateLinkServiceId: keyVault.id
-        }
-      }
-    ]
-    subnet: {
-      id: subnetId
-    }
-  }
-}
-
-resource keyVaultPrivateEndpointDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (managedPrivateEndpoint) {
-  name: privateDnsZoneName
-  location: 'global'
-  properties: {}
-}
-
-resource keyVaultPrivateDnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (managedPrivateEndpoint) {
-  parent: keyVaultPrivateEndpointDnsZone
-  name: uniqueString(keyVault.id)
-  location: 'global'
-  properties: {
-    registrationEnabled: false
-    virtualNetwork: {
-      id: vnetId
-    }
-  }
-}
-
-resource privateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-09-01' = if (managedPrivateEndpoint) {
-  parent: keyVaultPrivateEndpoint
-  name: '${keyVaultName}-dns-group'
-  properties: {
-    privateDnsZoneConfigs: [
-      {
-        name: 'config1'
-        properties: {
-          privateDnsZoneId: keyVaultPrivateEndpointDnsZone.id
-        }
-      }
-    ]
-  }
-  dependsOn: [
-    keyVaultPrivateDnsZoneVnetLink
-  ]
-}
+output kvId string = keyVault.id
 
 output kvName string = keyVault.name
diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep
@@ -90,6 +90,9 @@ param maestroPostgresServerStorageSizeGB int
 @description('The name of the service keyvault')
 param serviceKeyVaultName string
 
+@description('The name of the resourcegroup for the service keyvault')
+param serviceKeyVaultResourceGroup string = resourceGroup().name
+
 @description('Soft delete setting for service keyvault')
 param serviceKeyVaultSoftDelete bool = true
 
@@ -213,18 +216,26 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = {
 
 module serviceKeyVault '../modules/keyvault/keyvault.bicep' = {
   name: 'service-keyvault'
+  scope: resourceGroup(serviceKeyVaultResourceGroup)
   params: {
     location: location
     keyVaultName: serviceKeyVaultName
     private: serviceKeyVaultPrivate
     enableSoftDelete: serviceKeyVaultSoftDelete
+  }
+}
+
+module serviceKeyVaultPrivateEndpoint '../modules/keyvault/keyvault-private-endpoint.bicep' = {
+  name: 'service-keyvault-pe'
+  params: {
+    location: location
+    keyVaultName: serviceKeyVaultName
     subnetId: svcCluster.outputs.aksNodeSubnetId
     vnetId: svcCluster.outputs.aksVnetId
+    keyVaultId: serviceKeyVault.outputs.kvId
   }
 }
 
-output svcKeyVaultName string = serviceKeyVault.outputs.kvName
-
 //
 //   C L U S T E R   S E R V I C E
 //
@@ -255,6 +266,7 @@ module cs '../modules/cluster-service.bicep' = if (deployCsInfra) {
 
 module csServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = {
   name: guid(serviceKeyVaultName, 'cs', 'read')
+  scope: resourceGroup(serviceKeyVaultResourceGroup)
   params: {
     keyVaultName: serviceKeyVaultName
     roleName: 'Key Vault Secrets User'
@@ -277,6 +289,7 @@ var imageSyncManagedIdentityPrincipalId = filter(
 
 module imageServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = {
   name: guid(serviceKeyVaultName, 'imagesync', 'read')
+  scope: resourceGroup(serviceKeyVaultResourceGroup)
   params: {
     keyVaultName: serviceKeyVaultName
     roleName: 'Key Vault Secrets User'