refactor metrics infrastructure

- remove metrics from regional bicep template
- split up metrics and aks cluster templates
- use acr published bicep modules

.github/workflows/aro-hcp-dev-env-cd.yml

@@ -110,6 +110,14 @@
                 --parameters currentUserId="${GITHUB_ACTOR}" \
                 --parameters regionalDNSSubdomain="${REGION}"
 
+              # metrics infrastructure
+              az deployment group create \
+                --name "metrics-infra-${GITHUB_RUN_ID}" \
+                --resource-group "${REGIONAL_RESOURCEGROUP}" \
+                --template-file modules/metrics/metrics.bicep \
+                --parameters configurations/mvp-metrics.bicepparam \
+                --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}"
+
     deploy_service_cluster_rg:
       if: github.event.pull_request.merged == true
       needs:
@@ -146,7 +154,6 @@
                 --template-file templates/svc-cluster.bicep \
                 --parameters configurations/mvp-svc-cluster.bicepparam \
                 --parameters currentUserId="${GITHUB_ACTOR}" \
-                --parameters azureMonitorWorkspaceResourceId=$(az monitor account show --resource-group ${REGIONAL_RESOURCEGROUP} --name aro-hcp-monitor --query id -o tsv) \
                 --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"
 
               SVC_CLUSTER_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value)
@@ -164,6 +171,14 @@
                 --parameters kvNames="['${SVC_KV_NAME}']" \
                 --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }}
 
+              # enable aks metrics
+              AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID} --output tsv --query properties.outputs.monitorId.value)
+              GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID} --output tsv --query properties.outputs.grafanaId.value)
+
+              az aks update --name "${SVC_CLUSTER_NAME}" --resource-group "${SC_RESOURCEGROUP}" \
+                --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \
+                --grafana-resource-id "${GRAFANA_RESOURCE_ID}"
+
     deploy_management_cluster_rg:
       if: github.event.pull_request.merged == true
       needs:
@@ -200,7 +215,6 @@
                 --template-file templates/mgmt-cluster.bicep \
                 --parameters configurations/mvp-mgmt-cluster.bicepparam \
                 --parameters currentUserId="${GITHUB_ACTOR}" \
-                --parameters azureMonitorWorkspaceResourceId=$(az monitor account show --resource-group ${REGIONAL_RESOURCEGROUP} --name aro-hcp-monitor --query id -o tsv) \
                 --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"
 
               MGMT_CLUSTER_NAME=$(az deployment group show --resource-group "${MC_RESOURCEGROUP}" --name "mgmt-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value)
@@ -214,6 +228,14 @@
                 --parameters grantCosmosAccess=false \
                 --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }}
 
+              # enable aks metrics
+              AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID} --output tsv --query properties.outputs.monitorId.value)
+              GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID} --output tsv --query properties.outputs.grafanaId.value)
+
+              az aks update --name "${MGMT_CLUSTER_NAME}" --resource-group "${MC_RESOURCEGROUP}" \
+                --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \
+                --grafana-resource-id "${GRAFANA_RESOURCE_ID}"
+
     build_push_frontend:
       permissions:
         id-token: 'write'

.github/workflows/bicep-what-if.yml

@@ -70,7 +70,6 @@ jobs:
               --template-file templates/svc-cluster.bicep \
               --parameters configurations/mvp-svc-cluster.bicepparam \
               --parameters currentUserId="${GITHUB_ACTOR}" \
-              --parameters azureMonitorWorkspaceResourceId=$(az monitor account show --resource-group ${REGIONAL_RESOURCEGROUP} --name aro-hcp-monitor --query id -o tsv) \
               --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"
 
             # management cluster
@@ -80,5 +79,12 @@ jobs:
               --template-file templates/mgmt-cluster.bicep \
               --parameters configurations/mvp-mgmt-cluster.bicepparam \
               --parameters currentUserId="${GITHUB_ACTOR}" \
-              --parameters azureMonitorWorkspaceResourceId=$(az monitor account show --resource-group ${REGIONAL_RESOURCEGROUP} --name aro-hcp-monitor --query id -o tsv) \
               --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"
+
+            # metrics infrastructure
+            az deployment group what-if \
+              --name "metrics-infra-${GITHUB_RUN_ID}" \
+              --resource-group "${REGIONAL_RESOURCEGROUP}" \
+              --template-file modules/metrics/metrics.bicep \
+              --parameters configurations/mvp-metrics.bicepparam \
+              --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}"

dev-infrastructure/Makefile

@@ -103,7 +103,6 @@ endif
 			configurations/$(AKSCONFIG).bicepparam \
 		--parameters \
 			currentUserId=$(CURRENTUSER) \
-			azureMonitorWorkspaceResourceId=$(shell az monitor account show --resource-group $(REGIONAL_RESOURCEGROUP) --name aro-hcp-monitor --query id -o tsv) \
 			regionalResourceGroup=$(REGIONAL_RESOURCEGROUP)
 .PHONY: cluster
 
@@ -156,6 +155,28 @@ endif
 	@scripts/cs-miwi-pg-connect.sh $(RESOURCEGROUP) "maestro-pg-" "maestro-server" "maestro" "maestro"
 .PHONY: maestro-miwi-pg-connect
 
+metrics-infra: regionalRg
+	 CURRENTUSER=$(CURRENTUSER) az deployment group create \
+		--name "metrics-infra" \
+		--resource-group $(REGIONAL_RESOURCEGROUP) \
+		--template-file modules/metrics/metrics.bicep \
+		$(PROMPT_TO_CONFIRM) \
+		--parameters \
+		  globalResourceGroup=$(REGIONAL_RESOURCEGROUP) \
+		--parameters configurations/dev-metrics.bicepparam
+.PHONY: metrics-infra 
+
+enable-aks-metrics: metrics-infra
+ifndef AKSCONFIG
+	$(error "Must set AKSCONFIG")
+endif
+	az aks update --enable-azure-monitor-metrics \
+	--resource-group $(RESOURCEGROUP) \
+	--name aro-hcp-$(AKSCONFIG) \
+	--azure-monitor-workspace-resource-id $(shell az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.monitorId.value) \
+	--grafana-resource-id $(shell az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.grafanaId.value)
+.PHONY: enable-aks-metrics
+
 aks.kubeconfig:
 ifndef AKSCONFIG
 	$(error "Must set AKSCONFIG")

dev-infrastructure/configurations/dev-metrics.bicepparam

@@ -0,0 +1,7 @@
+using '../modules/metrics/metrics.bicep'
+
+param grafanaName = take('aro-hcp-grafana-${uniqueString(readEnvironmentVariable('CURRENTUSER', ''))}', 23)
+param msiName = 'aro-hcp-metrics-msi-${take(uniqueString(readEnvironmentVariable('CURRENTUSER', '')), 5)}'
+
+// overriden in makefile
+param globalResourceGroup = ''

dev-infrastructure/configurations/mgmt-cluster.bicepparam

@@ -27,4 +27,3 @@ param acrPullResourceGroups = ['global']
 // These parameters are always overriden in the Makefile
 param currentUserId = ''
 param regionalResourceGroup = ''
-param azureMonitorWorkspaceResourceId = ''

dev-infrastructure/configurations/mvp-metrics.bicepparam

@@ -0,0 +1,7 @@
+using '../modules/metrics/metrics.bicep'
+
+param grafanaName = 'aro-hcp-grafana'
+param msiName = 'aro-hcp-metrics-msi'
+
+// overriden in makefile
+param globalResourceGroup = ''

dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam

@@ -27,4 +27,3 @@ param acrPullResourceGroups = [regionalResourceGroup, 'global']
 // These parameters are always overridden in the Makefile
 param currentUserId = ''
 param regionalResourceGroup = ''
-param azureMonitorWorkspaceResourceId = ''

dev-infrastructure/configurations/mvp-region.bicepparam

@@ -11,8 +11,5 @@ param maestroKeyVaultName = 'maestro-kv-aro-hcp-dev'
 param maestroEventGridNamespacesName = 'maestro-eventgrid-aro-hcp-dev'
 param maestroEventGridMaxClientSessionsPerAuthName = 4
 
-// metrics
-param grafanaName = 'aro-hcp-grafana'
-
 // This parameter is always overriden in the Makefile
 param currentUserId = ''

dev-infrastructure/configurations/mvp-svc-cluster.bicepparam

@@ -35,4 +35,3 @@ param imageSyncAcrResourceGroupNames = ['global']
 // These parameters are always overridden in the Makefile
 param currentUserId = ''
 param regionalResourceGroup = ''
-param azureMonitorWorkspaceResourceId = ''

dev-infrastructure/configurations/region.bicepparam

@@ -9,8 +9,5 @@ param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24
 param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24)
 param maestroEventGridMaxClientSessionsPerAuthName = 4
 
-// metrics
-param grafanaName = take('aro-hcp-grafana-${uniqueString(currentUserId)}', 23)
-
 // These parameters are always overriden in the Makefile
 param currentUserId = ''

dev-infrastructure/configurations/svc-cluster.bicepparam

@@ -36,4 +36,3 @@ param clustersServiceAcrResourceGroupNames = ['global']
 // These parameters are always overriden in the Makefile
 param currentUserId = ''
 param regionalResourceGroup = ''
-param azureMonitorWorkspaceResourceId = ''

dev-infrastructure/modules/aks-cluster-base.bicep

@@ -47,11 +47,6 @@ param userOsDiskSizeGB int = 32
 
 param acrPullResourceGroups array = []
 
-// Metric Params
-param azureMonitorWorkspaceResourceId string
-param metricLabelsAllowlist string = ''
-param metricAnnotationsAllowList string = ''
-
 @description('Perform cryptographic operations using keys. Only works for key vaults that use the Azure role-based access control permission model.')
 var keyVaultCryptoUserId = subscriptionResourceId(
   'Microsoft.Authorization/roleDefinitions',
@@ -337,15 +332,6 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ
       nodeOSUpgradeChannel: 'NodeImage'
       upgradeChannel: 'patch'
     }
-    azureMonitorProfile: {
-      metrics: {
-        enabled: true
-        kubeStateMetrics: {
-          metricLabelsAllowlist: metricLabelsAllowlist
-          metricAnnotationsAllowList: metricAnnotationsAllowList
-        }
-      }
-    }
     disableLocalAccounts: true
     dnsPrefix: dnsPrefix
     enableRBAC: true
@@ -471,16 +457,6 @@ resource uami_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federate
   }
 ]
 
-module aksMetrics '../modules/metrics/aks-azure-monitor-metrics.bicep' = {
-  name: 'aks-metrics-${aksClusterName}'
-  params: {
-    azureMonitorWorkspaceResourceId: azureMonitorWorkspaceResourceId
-    azureMonitorWorkspaceLocation: location
-    clusterResourceId: aksCluster.id
-    clusterLocation: aksCluster.location
-  }
-}
-
 // Outputs
 output userAssignedIdentities array = [
   for i in range(0, length(workloadIdentities)): {