diff --git a/cluster-service/Makefile b/cluster-service/Makefile index b4f9b4606..473b2bb23 100644 --- a/cluster-service/Makefile +++ b/cluster-service/Makefile @@ -5,7 +5,7 @@ CONFIG_PROFILE ?= dev include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk CONSUMER_NAME ?= $(shell az aks list --query "[?tags.clusterType == 'mgmt-cluster' && starts_with(resourceGroup, '$(REGIONAL_RESOURCEGROUP)')].resourceGroup" -o tsv) -KEYVAULT_NAME ?= $(shell az keyvault list --query "[?starts_with(name, 'service-kv')].name" -g ${RESOURCEGROUP} --output tsv) +KEYVAULT_NAME ?= $(shell az keyvault list --query "[?starts_with(name, 'service-kv')].name" -g ${SVC_KV_RESOURCEGROUP} --output tsv) FPA_CERT_NAME ?= firstPartyMock AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= "ccf5339c-61d1-402f-9c9b-d463670191f9" diff --git a/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam index ea2da30b2..584c8c0b3 100644 --- a/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam +++ b/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam @@ -24,7 +24,8 @@ param deployCsInfra = false param csPostgresServerName = 'cs-pg-cs-integ' param clusterServicePostgresPrivate = false -param serviceKeyVaultName = 'service-kv-cs-integ' +param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' +param serviceKeyVaultResourceGroup = 'global' param serviceKeyVaultSoftDelete = true param serviceKeyVaultPrivate = false diff --git a/dev-infrastructure/configurations/cs-integ.mk b/dev-infrastructure/configurations/cs-integ.mk index c37ab4fcd..ca1de3066 100644 --- a/dev-infrastructure/configurations/cs-integ.mk +++ b/dev-infrastructure/configurations/cs-integ.mk @@ -1,6 +1,6 @@ REGION ?= westus3 RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION)-$(AKSCONFIG) REGIONAL_RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION) +SVC_KV_RESOURCEGROUP ?= global ARO_HCP_IMAGE_ACR ?= arohcpdev REGIONAL_ACR_NAME ?= arohcpdev$(shell echo $(CURRENTUSER) | sha256sum | head -c 24) - diff --git a/dev-infrastructure/configurations/dev.mk b/dev-infrastructure/configurations/dev.mk index fd29cbd3f..15b6247de 100644 --- a/dev-infrastructure/configurations/dev.mk +++ b/dev-infrastructure/configurations/dev.mk @@ -1,6 +1,7 @@ REGION ?= westus3 RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION)-$(AKSCONFIG) REGIONAL_RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION) +SVC_KV_RESOURCEGROUP ?= global GLOBAL_RESOURCEGROUP ?= global ARO_HCP_IMAGE_ACR ?= arohcpdev REGIONAL_ACR_NAME ?= arohcpdev$(shell echo $(CURRENTUSER) | sha256sum | head -c 24) diff --git a/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam b/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam index facce2787..4f65ef57b 100644 --- a/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam +++ b/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam @@ -24,7 +24,8 @@ param deployCsInfra = false param csPostgresServerName = 'cs-pg-aro-hcp-dev' param clusterServicePostgresPrivate = false -param serviceKeyVaultName = 'service-kv-aro-hcp-dev' +param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' +param serviceKeyVaultResourceGroup = 'global' param serviceKeyVaultSoftDelete = true param serviceKeyVaultPrivate = false diff --git a/dev-infrastructure/configurations/svc-cluster.bicepparam b/dev-infrastructure/configurations/svc-cluster.bicepparam index d830f9bd9..074759299 100644 --- a/dev-infrastructure/configurations/svc-cluster.bicepparam +++ b/dev-infrastructure/configurations/svc-cluster.bicepparam @@ -25,7 +25,8 @@ param deployCsInfra = false param csPostgresServerName = take('cs-pg-${uniqueString(currentUserId)}', 60) param clusterServicePostgresPrivate = false -param serviceKeyVaultName = take('service-kv-${uniqueString(currentUserId)}', 24) +param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' +param serviceKeyVaultResourceGroup = 'global' param serviceKeyVaultSoftDelete = false param serviceKeyVaultPrivate = false diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep index e5525dc58..6a4298bdd 100644 --- a/dev-infrastructure/modules/aks-cluster-base.bicep +++ b/dev-infrastructure/modules/aks-cluster-base.bicep @@ -70,8 +70,6 @@ module aks_keyvault_builder '../modules/keyvault/keyvault.bicep' = { // todo: change for higher environments private: false enableSoftDelete: aksEtcdKVEnableSoftDelete - // AKS managed private endpoints on its own when the etcd KV is private - managedPrivateEndpoint: false } } diff --git a/dev-infrastructure/modules/keyvault/keyvault-private-endpoint.bicep b/dev-infrastructure/modules/keyvault/keyvault-private-endpoint.bicep new file mode 100644 index 000000000..1bda0990a --- /dev/null +++ b/dev-infrastructure/modules/keyvault/keyvault-private-endpoint.bicep @@ -0,0 +1,77 @@ +@description('Location of the endpoint.') +param location string + +@description('Name of the key vault to create this endpoint for.') +param keyVaultName string + +@description('ID of the subnet to create the private endpoint in.') +param subnetId string + +@description('ID of the vnet, needs to correlated with subnetId.') +param vnetId string + +@description('ID of the key vault.') +param keyVaultId string + +// +// P R I V A T E E N D P O I N T +// + +var privateDnsZoneName = 'privatelink.vaultcore.azure.net' + +resource keyVaultPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-01-01' = { + name: '${keyVaultName}-pe' + location: location + properties: { + privateLinkServiceConnections: [ + { + name: '${keyVaultName}-pe' + properties: { + groupIds: [ + 'vault' + ] + privateLinkServiceId: keyVaultId + } + } + ] + subnet: { + id: subnetId + } + } +} + +resource keyVaultPrivateEndpointDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: privateDnsZoneName + location: 'global' + properties: {} +} + +resource keyVaultPrivateDnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + parent: keyVaultPrivateEndpointDnsZone + name: uniqueString(keyVaultId) + location: 'global' + properties: { + registrationEnabled: false + virtualNetwork: { + id: vnetId + } + } +} + +resource privateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-09-01' = { + parent: keyVaultPrivateEndpoint + name: '${keyVaultName}-dns-group' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: keyVaultPrivateEndpointDnsZone.id + } + } + ] + } + dependsOn: [ + keyVaultPrivateDnsZoneVnetLink + ] +} diff --git a/dev-infrastructure/modules/keyvault/keyvault.bicep b/dev-infrastructure/modules/keyvault/keyvault.bicep index 750924a3d..5c2eab1af 100644 --- a/dev-infrastructure/modules/keyvault/keyvault.bicep +++ b/dev-infrastructure/modules/keyvault/keyvault.bicep @@ -1,19 +1,15 @@ +@description('Location of the keyvault.') param location string +@description('Name of the key vault.') param keyVaultName string -param subnetId string = '' - -param vnetId string = '' - +@description('Toggle to enable soft delete.') param enableSoftDelete bool +@description('Toggle to make the keyvault private.') param private bool -// Event for some private KVs it makes sense to disable the creation of a private endpoint, -// e.g. AKS KMS on a private KV will manage their own private endpoint setup in the nodepool RG -param managedPrivateEndpoint bool = true - resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = { location: location name: keyVaultName @@ -35,67 +31,6 @@ resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = { } } -// -// P R I V A T E E N D P O I N T -// - -var privateDnsZoneName = 'privatelink.vaultcore.azure.net' - -resource keyVaultPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-01-01' = if (managedPrivateEndpoint) { - name: '${keyVaultName}-pe' - location: location - properties: { - privateLinkServiceConnections: [ - { - name: '${keyVaultName}-pe' - properties: { - groupIds: [ - 'vault' - ] - privateLinkServiceId: keyVault.id - } - } - ] - subnet: { - id: subnetId - } - } -} - -resource keyVaultPrivateEndpointDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (managedPrivateEndpoint) { - name: privateDnsZoneName - location: 'global' - properties: {} -} - -resource keyVaultPrivateDnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (managedPrivateEndpoint) { - parent: keyVaultPrivateEndpointDnsZone - name: uniqueString(keyVault.id) - location: 'global' - properties: { - registrationEnabled: false - virtualNetwork: { - id: vnetId - } - } -} - -resource privateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-09-01' = if (managedPrivateEndpoint) { - parent: keyVaultPrivateEndpoint - name: '${keyVaultName}-dns-group' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: keyVaultPrivateEndpointDnsZone.id - } - } - ] - } - dependsOn: [ - keyVaultPrivateDnsZoneVnetLink - ] -} +output kvId string = keyVault.id output kvName string = keyVault.name diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index 8ff03c854..98866cfaa 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -90,6 +90,9 @@ param maestroPostgresServerStorageSizeGB int @description('The name of the service keyvault') param serviceKeyVaultName string +@description('The name of the resourcegroup for the service keyvault') +param serviceKeyVaultResourceGroup string = resourceGroup().name + @description('Soft delete setting for service keyvault') param serviceKeyVaultSoftDelete bool = true @@ -213,18 +216,26 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = { module serviceKeyVault '../modules/keyvault/keyvault.bicep' = { name: 'service-keyvault' + scope: resourceGroup(serviceKeyVaultResourceGroup) params: { location: location keyVaultName: serviceKeyVaultName private: serviceKeyVaultPrivate enableSoftDelete: serviceKeyVaultSoftDelete + } +} + +module serviceKeyVaultPrivateEndpoint '../modules/keyvault/keyvault-private-endpoint.bicep' = { + name: 'service-keyvault-pe' + params: { + location: location + keyVaultName: serviceKeyVaultName subnetId: svcCluster.outputs.aksNodeSubnetId vnetId: svcCluster.outputs.aksVnetId + keyVaultId: serviceKeyVault.outputs.kvId } } -output svcKeyVaultName string = serviceKeyVault.outputs.kvName - // // C L U S T E R S E R V I C E // @@ -255,6 +266,7 @@ module cs '../modules/cluster-service.bicep' = if (deployCsInfra) { module csServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = { name: guid(serviceKeyVaultName, 'cs', 'read') + scope: resourceGroup(serviceKeyVaultResourceGroup) params: { keyVaultName: serviceKeyVaultName roleName: 'Key Vault Secrets User' @@ -277,6 +289,7 @@ var imageSyncManagedIdentityPrincipalId = filter( module imageServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = { name: guid(serviceKeyVaultName, 'imagesync', 'read') + scope: resourceGroup(serviceKeyVaultResourceGroup) params: { keyVaultName: serviceKeyVaultName roleName: 'Key Vault Secrets User'