Merge pull request #2 from helderpinto/helderpinto/dev/aoe-diskrecsbug

Fixed underutilized Premium SSD disk recommendation not reporting results
helderpinto · Jul 23, 2024 · 607160c · 607160c
2 parents 7c24859 + c2e1efb
commit 607160c
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 18 deletions.
diff --git a/.github/workflows/aoe-cd-dev.yml b/.github/workflows/aoe-cd-dev.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - dev
     paths:
+      - 'docs/deploy/optimization-engine/**'
       - 'src/optimization-engine/**'       
 permissions:
       id-token: write

diff --git a/.github/workflows/aoe-cd-prod.yml b/.github/workflows/aoe-cd-prod.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - main
     paths:
+      - 'docs/deploy/optimization-engine/**'
       - 'src/optimization-engine/**'       
 permissions:
       id-token: write

diff --git a/.github/workflows/aoe-cd-test.yml b/.github/workflows/aoe-cd-test.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - features/aoe
     paths:
+      - 'docs/deploy/optimization-engine/**'
       - 'src/optimization-engine/**'       
 permissions:
       id-token: write

diff --git a/src/optimization-engine/azuredeploy-nested.bicep b/src/optimization-engine/azuredeploy-nested.bicep
@@ -734,7 +734,7 @@ var csvParameterizedExports = [
     exportJobId: monitorDiskIOPSAvgExportJobId
     parameters: {
       ResourceType: 'microsoft.compute/disks'
-      ARGFilter: 'sku.name =~ \'Premium_LRS\' and properties.diskState != \'Unattached\''
+      ARGFilter: 'sku.name startswith \'Premium_\' and properties.diskState =~ \'Attached\''
       TimeSpan: '01:00:00'
       aggregationType: 'Average'
       AggregationOfType: 'Maximum'
@@ -748,7 +748,7 @@ var csvParameterizedExports = [
     exportJobId: monitorDiskMBPsAvgExportJobId
     parameters: {
       ResourceType: 'microsoft.compute/disks'
-      ARGFilter: 'sku.name =~ \'Premium_LRS\' and properties.diskState != \'Unattached\''
+      ARGFilter: 'sku.name startswith \'Premium_\' and properties.diskState =~ \'Attached\''
       TimeSpan: '01:00:00'
       aggregationType: 'Average'
       AggregationOfType: 'Maximum'
@@ -1179,7 +1179,7 @@ var runbooks = [
   }
   {
     name: diskOptimizationsRecommendationsRunbookName
-    version: '1.1.1.0'
+    version: '1.1.2.0'
     description: 'Generates Disk optimizations recommendations'
     type: 'PowerShell'
     scriptUri: uri(templateLocation, 'runbooks/recommendations/${diskOptimizationsRecommendationsRunbookName}.ps1')

diff --git a/...optimization-engine/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 b/...optimization-engine/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1
@@ -6,7 +6,7 @@ function Find-DiskMonthlyPrice {
         [string] $DiskSizeTier
     )
 
-    $diskSkus = $SKUPriceSheet | Where-Object { $_.MeterName_s.Replace(" Disks","") -eq $DiskSizeTier }
+    $diskSkus = $SKUPriceSheet | Where-Object { $_.MeterName_s.Replace(" Disks","").Replace(" Disk","") -eq $DiskSizeTier }
     $targetMonthlyPrice = [double]::MaxValue
     if ($diskSkus)
     {
@@ -182,7 +182,7 @@ try
     $baseQuery = @"
     $pricesheetTableName
     | where TimeGenerated > ago(14d)
-    | where MeterCategory_s == 'Storage' and MeterSubCategory_s endswith "Managed Disks" and MeterName_s endswith "Disks" and MeterRegion_s == '$pricesheetRegion' and PriceType_s == 'Consumption'
+    | where MeterCategory_s == 'Storage' and MeterSubCategory_s contains "Managed Disk" and (MeterName_s endswith "Disk" or MeterName_s endswith "Disks") and MeterName_s !has 'Special' and MeterRegion_s == '$pricesheetRegion' and PriceType_s == 'Consumption'
     | distinct MeterName_s, MeterSubCategory_s, MeterCategory_s, MeterRegion_s, UnitPrice_s, UnitOfMeasure_s
 "@    
 
@@ -222,7 +222,8 @@ $baseQuery = @"
     | summarize MaxIOPSMetric = max(todouble(MetricValue_s)) by ResourceId
     | join kind=inner ( 
         $disksTableName
-        | where TimeGenerated > ago(1d) and DiskState_s != 'Unattached' and SKU_s startswith 'Premium'
+        | where TimeGenerated > ago(1d) and DiskState_s =~ 'Attached' and SKU_s startswith 'Premium'
+        | extend DiskTier_s = strcat(DiskTier_s, ' ', tostring(split(SKU_s, '_')[1]))
         | project ResourceId=InstanceId_s, DiskName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, Tags_s, MaxIOPSDisk=toint(DiskIOPS_s), DiskSizeGB_s, SKU_s, DiskTier_s, DiskType_s
     ) on ResourceId
     | project-away ResourceId1
@@ -234,7 +235,8 @@ $baseQuery = @"
         | summarize MaxMBsMetric = max(todouble(MetricValue_s)/1024/1024) by ResourceId
         | join kind=inner ( 
             $disksTableName
-            | where TimeGenerated > ago(1d) and DiskState_s != 'Unattached' and SKU_s startswith 'Premium'
+            | where TimeGenerated > ago(1d) and DiskState_s =~ 'Attached' and SKU_s startswith 'Premium'
+            | extend DiskTier_s = strcat(DiskTier_s, ' ', tostring(split(SKU_s, '_')[1]))
             | project ResourceId=InstanceId_s, DiskName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, Tags_s, MaxMBsDisk=toint(DiskThroughput_s), DiskSizeGB_s, SKU_s, DiskTier_s, DiskType_s
         ) on ResourceId
         | project-away ResourceId1
@@ -280,7 +282,7 @@ foreach ($result in $results)
     $targetSku = $null
     $currentDiskTier = $null
 
-    if ([string]::IsNullOrEmpty($result.DiskTier_s)) # older disks do not have Tier info in their properties
+    if ([string]::IsNullOrEmpty($result.DiskTier_s) -or $result.DiskTier_s.Trim().Length -le 3) # older disks do not have Tier info in their properties
     {
         $currentSkuCandidates = @()
         foreach ($sku in $skus)
@@ -294,13 +296,14 @@ foreach ($result in $results)
             if ($sku.Name -eq $result.SKU_s -and $skuMinSizeGB -lt [int]$result.DiskSizeGB_s -and $skuMaxSizeGB -ge [int]$result.DiskSizeGB_s `
             -and [int]$skuMaxIOps -eq [int]$result.MaxIOPSDisk -and [int]$skuMaxBandwidthMBps -eq [int]$result.MaxMBsDisk)
             {
-                if ($null -eq $skuPricesFound[$sku.Size])
+                $skuSize = $sku.Size + " " + $result.SKU_s.Split("_")[1]
+                if ($null -eq $skuPricesFound[$skuSize])
                 {
-                    $skuPricesFound[$sku.Size] = Find-DiskMonthlyPrice -DiskSizeTier $sku.Size -SKUPriceSheet $pricesheetEntries
+                    $skuPricesFound[$sku.Size] = Find-DiskMonthlyPrice -DiskSizeTier $skuSize -SKUPriceSheet $pricesheetEntries
                 }
 
                 $currentSkuCandidate = New-Object PSObject -Property @{
-                    Name = $sku.Size
+                    Name = $skuSize
                     MaxSizeGB = $skuMaxSizeGB
                 }    
 
@@ -334,16 +337,17 @@ foreach ($result in $results)
         if ($sku.Name -eq $targetSkuPerfTier -and $skuMinSizeGB -lt [int]$result.DiskSizeGB_s -and $skuMaxSizeGB -ge [int]$result.DiskSizeGB_s `
                 -and [double]$skuMaxIOps -ge [double]$result.MaxIOPSMetric -and [double]$skuMaxBandwidthMBps -ge [double]$result.MaxMBsMetric)
         {
+            $skuSize = $sku.Size + " " + $targetSkuPerfTier.Split("_")[1]
             if ($null -eq $skuPricesFound[$sku.Size])
             {
-                $skuPricesFound[$sku.Size] = Find-DiskMonthlyPrice -DiskSizeTier $sku.Size -SKUPriceSheet $pricesheetEntries
+                $skuPricesFound[$skuSize] = Find-DiskMonthlyPrice -DiskSizeTier $skuSize -SKUPriceSheet $pricesheetEntries
             }
 
-            if ($skuPricesFound[$sku.Size] -lt [double]::MaxValue -and $skuPricesFound[$sku.Size] -lt $skuPricesFound[$currentDiskTier])
+            if ($skuPricesFound[$skuSize] -lt [double]::MaxValue -and $skuPricesFound[$skuSize] -lt $skuPricesFound[$currentDiskTier])
             {
                 $targetSkuCandidate = New-Object PSObject -Property @{
-                    Name = $sku.Size
-                    MonthlyPrice = $skuPricesFound[$sku.Size]
+                    Name = $skuSize
+                    MonthlyPrice = $skuPricesFound[$skuSize]
                     MaxSizeGB = $skuMaxSizeGB
                     MaxIOPS = $skuMaxIOps
                     MaxMBps = $skuMaxBandwidthMBps

diff --git a/src/optimization-engine/upgrade-manifest.json b/src/optimization-engine/upgrade-manifest.json
@@ -663,7 +663,7 @@
                     "schedule": "AzureOptimization_ExportMonitorDiskIOPSHourly",
                     "parameters": {
                         "ResourceType": "microsoft.compute/disks",
-                        "ARGFilter": "sku.name =~ 'Premium_LRS' and properties.diskState != 'Unattached'",
+                        "ARGFilter": "sku.name startswith 'Premium_' and properties.diskState =~ 'Attached'",
                         "TimeSpan": "01:00:00",
                         "aggregationType": "Average",
                         "AggregationOfType": "Maximum",
@@ -675,7 +675,7 @@
                     "schedule": "AzureOptimization_ExportMonitorDiskMBPsHourly",
                     "parameters": {
                         "ResourceType": "microsoft.compute/disks",
-                        "ARGFilter": "sku.name =~ 'Premium_LRS' and properties.diskState != 'Unattached'",
+                        "ARGFilter": "sku.name startswith 'Premium_' and properties.diskState =~ 'Attached'",
                         "TimeSpan": "01:00:00",
                         "aggregationType": "Average",
                         "AggregationOfType": "Maximum",
@@ -969,7 +969,7 @@
         {
             "runbook": {
                 "name": "runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1",
-                "version": "1.1.1.0"
+                "version": "1.1.2.0"
             },
             "requiredVariables": [
                 {