diff --git a/.github/workflows/aoe-cd-dev.yml b/.github/workflows/aoe-cd-dev.yml index f12fe0efb..8d6f785a1 100644 --- a/.github/workflows/aoe-cd-dev.yml +++ b/.github/workflows/aoe-cd-dev.yml @@ -20,7 +20,7 @@ jobs: AOE_WORKSPACENAME: ${{ secrets.AOE_WORKSPACENAME }} AOE_WORKSPACERG: ${{ secrets.AOE_WORKSPACERG }} steps: - - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub for the ${{ github.ref }} branch of the ${{ github.repository }} repository!" + - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub for the ${{ github.ref.name }} branch of the ${{ github.repository }} repository!" - name: Installing modules shell: pwsh run: | @@ -57,5 +57,5 @@ jobs: shell: pwsh run: | Set-Location ./src/optimization-engine - ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/microsoft/finops-toolkit/dev/src/optimization-engine/azuredeploy.bicep" + ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref.name }}/src/optimization-engine/azuredeploy.bicep" - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/.github/workflows/aoe-cd-prod.yml b/.github/workflows/aoe-cd-prod.yml index 62a775576..2dc7e9f47 100644 --- a/.github/workflows/aoe-cd-prod.yml +++ b/.github/workflows/aoe-cd-prod.yml @@ -18,7 +18,7 @@ jobs: AOE_LOCATION: ${{ secrets.AOE_LOCATION }} AOE_NAMEPREFIX: ${{ secrets.AOE_NAMEPREFIX }} steps: - - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub for the ${{ github.ref }} branch of the ${{ github.repository }} repository!" + - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub for the ${{ github.ref.name }} branch of the ${{ github.repository }} repository!" - name: Installing modules shell: pwsh run: | @@ -55,5 +55,5 @@ jobs: shell: pwsh run: | Set-Location ./src/optimization-engine - ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json + ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref.name }}/src/optimization-engine/azuredeploy.bicep" - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/.github/workflows/aoe-cd-test.yml b/.github/workflows/aoe-cd-test.yml index 6f6804b71..f8c0583d0 100644 --- a/.github/workflows/aoe-cd-test.yml +++ b/.github/workflows/aoe-cd-test.yml @@ -18,7 +18,7 @@ jobs: AOE_LOCATION: ${{ secrets.AOE_LOCATION }} AOE_NAMEPREFIX: ${{ secrets.AOE_NAMEPREFIX }} steps: - - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub for the ${{ github.ref }} branch of the ${{ github.repository }} repository!" + - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub for the ${{ github.ref.name }} branch of the ${{ github.repository }} repository!" - name: Installing modules shell: pwsh run: | @@ -55,5 +55,5 @@ jobs: shell: pwsh run: | Set-Location ./src/optimization-engine - ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/microsoft/finops-toolkit/features/aoe/src/optimization-engine/azuredeploy.bicep" + ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref.name }}/src/optimization-engine/azuredeploy.bicep" - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/docs/deploy/finops-hub-0.2.1-rc.2.json b/docs/deploy/finops-hub-0.2.1-rc.2.json index 92bb92359..929aef644 100644 --- a/docs/deploy/finops-hub-0.2.1-rc.2.json +++ b/docs/deploy/finops-hub-0.2.1-rc.2.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "13980213354947249584" + "version": "0.24.24.22086", + "templateHash": "6371350577264419703" } }, "parameters": { @@ -91,8 +91,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "3688631684062478281" + "version": "0.24.24.22086", + "templateHash": "2610829918662778812" } }, "parameters": { @@ -162,7 +162,7 @@ "dataFactorySuffix": "[format('-{0}', variables('uniqueSuffix'))]", "dataFactoryName": "[replace(format('{0}{1}', take(variables('dataFactoryPrefix'), sub(63, length(variables('dataFactorySuffix')))), variables('dataFactorySuffix')), '--', '-')]", "telemetryId": "00f120b5-2007-6120-0000-40b000000000", - "finOpsToolkitVersion": "0.2.1-rc.2" + "finOpsToolkitVersion": "placeholder" }, "resources": [ { @@ -234,8 +234,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "3183312799881178788" + "version": "0.24.24.22086", + "templateHash": "9609675916139055971" } }, "parameters": { @@ -291,7 +291,7 @@ } }, "variables": { - "$fxv#0": "0.2.1-rc.2", + "$fxv#0": "placeholder", "$fxv#1": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nWrite-Output \"Updating settings.json file...\"\r\nWrite-Output \" Storage account: $env:storageAccountName\"\r\nWrite-Output \" Container: $env:containerName\"\r\n\r\n$validateScopes = { $_.Length -gt 45 }\r\n\r\n# Initialize variables\r\n$fileName = 'settings.json'\r\n$filePath = Join-Path -Path . -ChildPath $fileName\r\n$newScopes = $env:exportScopes.Split('|') | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n\r\n# Get storage context\r\n$storageContext = @{\r\n Context = New-AzStorageContext -StorageAccountName $env:storageAccountName -UseConnectedAccount\r\n Container = $env:containerName\r\n}\r\n\r\n# Download existing settings, if they exist\r\n$blob = Get-AzStorageBlobContent @storageContext -Blob $fileName -Destination $filePath -Force\r\nif ($blob) {\r\n Write-Output \"Existing settings.json file found. Updating...\"\r\n $text = Get-Content $filePath -Raw\r\n Write-Output \"---------\"\r\n Write-Output $text\r\n Write-Output \"---------\"\r\n $json = $text | ConvertFrom-Json\r\n\r\n # Rename exportScopes to scopes + convert to object array\r\n if ($json.exportScopes) {\r\n Write-Output \" Updating exportScopes...\"\r\n if ($json.exportScopes[0] -is [string]) {\r\n Write-Output \" Converting string array to object array...\"\r\n $json.exportScopes = $json.exportScopes | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n if (-not ($json.exportScopes -is [array])) {\r\n Write-Output \" Converting single object to object array...\"\r\n $json.exportScopes = @($json.exportScopes)\r\n }\r\n }\r\n\r\n Write-Output \" Renaming to 'scopes'...\"\r\n $json | Add-Member -MemberType NoteProperty -Name scopes -Value $json.exportScopes\r\n $json.PSObject.Properties.Remove('exportScopes')\r\n }\r\n}\r\n\r\n# Set default if not found\r\nif (!$json) {\r\n Write-Output \"No existing settings.json file found. Creating new file...\"\r\n $json = [ordered]@{\r\n '$schema' = 'https://aka.ms/finops/hubs/settings-schema'\r\n type = 'HubInstance'\r\n version = ''\r\n learnMore = 'https://aka.ms/finops/hubs'\r\n scopes = @()\r\n }\r\n}\r\n\r\n# Updating settings\r\nWrite-Output \"Updating version to $env:ftkVersion...\"\r\n$json.version = $env:ftkVersion\r\nif ($newScopes) {\r\n Write-Output \"Merging $($newScopes.Count) scopes...\"\r\n $json.scopes = Compare-Object -ReferenceObject $json.scopes -DifferenceObject $newScopes -Property scope -PassThru -IncludeEqual\r\n\r\n # Remove the SideIndicator property from the Compare-Object output\r\n $json.scopes | ForEach-Object { $_.PSObject.Properties.Remove('SideIndicator') } | ConvertTo-Json\r\n\r\n if (-not ($json.scopes -is [array])) {\r\n $json.scopes = @($json.scopes)\r\n }\r\n Write-Output \"$($json.scopes.Count) scopes found.\"\r\n}\r\n$text = $json | ConvertTo-Json\r\nWrite-Output \"---------\"\r\nWrite-Output $text\r\nWrite-Output \"---------\"\r\n$text | Out-File $filePath\r\n\r\n# Upload new/updated settings\r\nWrite-Output \"Uploading settings.json file...\"\r\nSet-AzStorageBlobContent @storageContext -File $filePath -Force\r\n", "safeHubName": "[replace(replace(toLower(parameters('hubName')), '-', ''), '_', '')]", "storageAccountSuffix": "[parameters('uniqueSuffix')]", @@ -515,8 +515,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "5687422994519942055" + "version": "0.24.24.22086", + "templateHash": "5738194981634133446" } }, "parameters": { @@ -1829,8 +1829,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "8584269155107147116" + "version": "0.24.24.22086", + "templateHash": "10770478197596540923" } }, "parameters": { diff --git a/docs/deploy/finops-hub-latest.json b/docs/deploy/finops-hub-latest.json index 92bb92359..929aef644 100644 --- a/docs/deploy/finops-hub-latest.json +++ b/docs/deploy/finops-hub-latest.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "13980213354947249584" + "version": "0.24.24.22086", + "templateHash": "6371350577264419703" } }, "parameters": { @@ -91,8 +91,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "3688631684062478281" + "version": "0.24.24.22086", + "templateHash": "2610829918662778812" } }, "parameters": { @@ -162,7 +162,7 @@ "dataFactorySuffix": "[format('-{0}', variables('uniqueSuffix'))]", "dataFactoryName": "[replace(format('{0}{1}', take(variables('dataFactoryPrefix'), sub(63, length(variables('dataFactorySuffix')))), variables('dataFactorySuffix')), '--', '-')]", "telemetryId": "00f120b5-2007-6120-0000-40b000000000", - "finOpsToolkitVersion": "0.2.1-rc.2" + "finOpsToolkitVersion": "placeholder" }, "resources": [ { @@ -234,8 +234,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "3183312799881178788" + "version": "0.24.24.22086", + "templateHash": "9609675916139055971" } }, "parameters": { @@ -291,7 +291,7 @@ } }, "variables": { - "$fxv#0": "0.2.1-rc.2", + "$fxv#0": "placeholder", "$fxv#1": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nWrite-Output \"Updating settings.json file...\"\r\nWrite-Output \" Storage account: $env:storageAccountName\"\r\nWrite-Output \" Container: $env:containerName\"\r\n\r\n$validateScopes = { $_.Length -gt 45 }\r\n\r\n# Initialize variables\r\n$fileName = 'settings.json'\r\n$filePath = Join-Path -Path . -ChildPath $fileName\r\n$newScopes = $env:exportScopes.Split('|') | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n\r\n# Get storage context\r\n$storageContext = @{\r\n Context = New-AzStorageContext -StorageAccountName $env:storageAccountName -UseConnectedAccount\r\n Container = $env:containerName\r\n}\r\n\r\n# Download existing settings, if they exist\r\n$blob = Get-AzStorageBlobContent @storageContext -Blob $fileName -Destination $filePath -Force\r\nif ($blob) {\r\n Write-Output \"Existing settings.json file found. Updating...\"\r\n $text = Get-Content $filePath -Raw\r\n Write-Output \"---------\"\r\n Write-Output $text\r\n Write-Output \"---------\"\r\n $json = $text | ConvertFrom-Json\r\n\r\n # Rename exportScopes to scopes + convert to object array\r\n if ($json.exportScopes) {\r\n Write-Output \" Updating exportScopes...\"\r\n if ($json.exportScopes[0] -is [string]) {\r\n Write-Output \" Converting string array to object array...\"\r\n $json.exportScopes = $json.exportScopes | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n if (-not ($json.exportScopes -is [array])) {\r\n Write-Output \" Converting single object to object array...\"\r\n $json.exportScopes = @($json.exportScopes)\r\n }\r\n }\r\n\r\n Write-Output \" Renaming to 'scopes'...\"\r\n $json | Add-Member -MemberType NoteProperty -Name scopes -Value $json.exportScopes\r\n $json.PSObject.Properties.Remove('exportScopes')\r\n }\r\n}\r\n\r\n# Set default if not found\r\nif (!$json) {\r\n Write-Output \"No existing settings.json file found. Creating new file...\"\r\n $json = [ordered]@{\r\n '$schema' = 'https://aka.ms/finops/hubs/settings-schema'\r\n type = 'HubInstance'\r\n version = ''\r\n learnMore = 'https://aka.ms/finops/hubs'\r\n scopes = @()\r\n }\r\n}\r\n\r\n# Updating settings\r\nWrite-Output \"Updating version to $env:ftkVersion...\"\r\n$json.version = $env:ftkVersion\r\nif ($newScopes) {\r\n Write-Output \"Merging $($newScopes.Count) scopes...\"\r\n $json.scopes = Compare-Object -ReferenceObject $json.scopes -DifferenceObject $newScopes -Property scope -PassThru -IncludeEqual\r\n\r\n # Remove the SideIndicator property from the Compare-Object output\r\n $json.scopes | ForEach-Object { $_.PSObject.Properties.Remove('SideIndicator') } | ConvertTo-Json\r\n\r\n if (-not ($json.scopes -is [array])) {\r\n $json.scopes = @($json.scopes)\r\n }\r\n Write-Output \"$($json.scopes.Count) scopes found.\"\r\n}\r\n$text = $json | ConvertTo-Json\r\nWrite-Output \"---------\"\r\nWrite-Output $text\r\nWrite-Output \"---------\"\r\n$text | Out-File $filePath\r\n\r\n# Upload new/updated settings\r\nWrite-Output \"Uploading settings.json file...\"\r\nSet-AzStorageBlobContent @storageContext -File $filePath -Force\r\n", "safeHubName": "[replace(replace(toLower(parameters('hubName')), '-', ''), '_', '')]", "storageAccountSuffix": "[parameters('uniqueSuffix')]", @@ -515,8 +515,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "5687422994519942055" + "version": "0.24.24.22086", + "templateHash": "5738194981634133446" } }, "parameters": { @@ -1829,8 +1829,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "8584269155107147116" + "version": "0.24.24.22086", + "templateHash": "10770478197596540923" } }, "parameters": { diff --git a/docs/deploy/governance-workbook-0.2.1-rc.2.json b/docs/deploy/governance-workbook-0.2.1-rc.2.json index a6563907c..1316308bf 100644 --- a/docs/deploy/governance-workbook-0.2.1-rc.2.json +++ b/docs/deploy/governance-workbook-0.2.1-rc.2.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "11237752380577820438" + "version": "0.24.24.22086", + "templateHash": "12278846045479855901" } }, "parameters": { @@ -8035,7 +8035,7 @@ "workbookJson": "[string(variables('$fxv#0'))]", "workbookId": "907", "telemetryId": "[format('00f120b5-2007-6120-0000-{0}30126b006', variables('workbookId'))]", - "finOpsToolkitVersion": "0.2.1-rc.2" + "finOpsToolkitVersion": "placeholder" }, "resources": [ { diff --git a/docs/deploy/governance-workbook-latest.json b/docs/deploy/governance-workbook-latest.json index a6563907c..1316308bf 100644 --- a/docs/deploy/governance-workbook-latest.json +++ b/docs/deploy/governance-workbook-latest.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "11237752380577820438" + "version": "0.24.24.22086", + "templateHash": "12278846045479855901" } }, "parameters": { @@ -8035,7 +8035,7 @@ "workbookJson": "[string(variables('$fxv#0'))]", "workbookId": "907", "telemetryId": "[format('00f120b5-2007-6120-0000-{0}30126b006', variables('workbookId'))]", - "finOpsToolkitVersion": "0.2.1-rc.2" + "finOpsToolkitVersion": "placeholder" }, "resources": [ { diff --git a/docs/deploy/optimization-engine/azuredeploy-nested.bicep b/docs/deploy/optimization-engine/0.2.1-rc.2/azuredeploy-nested.bicep similarity index 99% rename from docs/deploy/optimization-engine/azuredeploy-nested.bicep rename to docs/deploy/optimization-engine/0.2.1-rc.2/azuredeploy-nested.bicep index 81b35eccd..cfed41bd4 100644 --- a/docs/deploy/optimization-engine/azuredeploy-nested.bicep +++ b/docs/deploy/optimization-engine/0.2.1-rc.2/azuredeploy-nested.bicep @@ -536,6 +536,7 @@ var csvExports = [ ingestTimeOffset: 'PT2H' ingestFrequency: 'Hour' ingestJobId: monitorIngestJobId + exportSchedule: null exportJobId: 'dummy' } { diff --git a/docs/deploy/optimization-engine/azuredeploy.bicep b/docs/deploy/optimization-engine/0.2.1-rc.2/azuredeploy.bicep similarity index 100% rename from docs/deploy/optimization-engine/azuredeploy.bicep rename to docs/deploy/optimization-engine/0.2.1-rc.2/azuredeploy.bicep diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-AADObjectsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-AADObjectsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-AADObjectsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-AADObjectsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGAppGatewayPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGAppGatewayPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGAppGatewayPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGAppGatewayPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGAppServicePlanPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGAppServicePlanPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGAppServicePlanPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGAppServicePlanPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGAvailabilitySetPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGAvailabilitySetPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGAvailabilitySetPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGAvailabilitySetPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGLoadBalancerPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGLoadBalancerPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGLoadBalancerPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGLoadBalancerPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGManagedDisksPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGManagedDisksPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGManagedDisksPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGManagedDisksPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGNICPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGNICPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGNICPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGNICPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGNSGPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGNSGPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGNSGPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGNSGPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGPublicIpPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGPublicIpPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGPublicIpPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGPublicIpPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGResourceContainersPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGResourceContainersPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGResourceContainersPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGResourceContainersPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGSqlDatabasePropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGSqlDatabasePropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGSqlDatabasePropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGSqlDatabasePropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGUnmanagedDisksPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGUnmanagedDisksPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGUnmanagedDisksPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGUnmanagedDisksPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGVMSSPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGVMSSPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGVMSSPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGVMSSPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGVNetPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGVNetPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGVNetPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGVNetPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGVirtualMachinesPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGVirtualMachinesPropertiesToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ARGVirtualMachinesPropertiesToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ARGVirtualMachinesPropertiesToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-AdvisorRecommendationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-AdvisorRecommendationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-AdvisorRecommendationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-AdvisorRecommendationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-AzMonitorMetricsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-AzMonitorMetricsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-AzMonitorMetricsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-AzMonitorMetricsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-PolicyComplianceToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-PolicyComplianceToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-PolicyComplianceToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-PolicyComplianceToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-PriceSheetToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-PriceSheetToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-PriceSheetToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-PriceSheetToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-RBACAssignmentsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-RBACAssignmentsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-RBACAssignmentsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-RBACAssignmentsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ReservationsPriceToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ReservationsPriceToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ReservationsPriceToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ReservationsPriceToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-ReservationsUsageToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ReservationsUsageToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-ReservationsUsageToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-ReservationsUsageToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Export-SavingsPlansUsageToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-SavingsPlansUsageToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Export-SavingsPlansUsageToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Export-SavingsPlansUsageToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/maintenance/CleanUp-OlderRecommendationsFromSqlServer.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/maintenance/CleanUp-OlderRecommendationsFromSqlServer.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/maintenance/CleanUp-OlderRecommendationsFromSqlServer.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/maintenance/CleanUp-OlderRecommendationsFromSqlServer.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AADExpiringCredentialsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AADExpiringCredentialsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AADExpiringCredentialsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AADExpiringCredentialsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-ARMOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-ARMOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-ARMOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-ARMOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AdvisorAsIsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AdvisorAsIsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AdvisorAsIsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AdvisorAsIsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AdvisorCostAugmentedToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AdvisorCostAugmentedToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AdvisorCostAugmentedToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AdvisorCostAugmentedToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AppServiceOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AppServiceOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-AppServiceOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-AppServiceOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-StorageAccountOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-StorageAccountOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-StorageAccountOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-StorageAccountOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-UnattachedDisksToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-UnattachedDisksToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-UnattachedDisksToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-UnattachedDisksToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-UnusedAppGWsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-UnusedAppGWsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-UnusedAppGWsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-UnusedAppGWsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-UnusedLoadBalancersToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-UnusedLoadBalancersToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-UnusedLoadBalancersToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-UnusedLoadBalancersToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VMOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VMOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VMOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VMOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VMSSOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VMSSOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VMSSOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VMSSOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VMsHighAvailabilityToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VMsHighAvailabilityToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VMsHighAvailabilityToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VMsHighAvailabilityToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VNetOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VNetOptimizationsToBlobStorage.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/recommendations/Recommend-VNetOptimizationsToBlobStorage.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/recommendations/Recommend-VNetOptimizationsToBlobStorage.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/remediations/Remediate-AdvisorRightSizeFiltered.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/remediations/Remediate-AdvisorRightSizeFiltered.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/remediations/Remediate-AdvisorRightSizeFiltered.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/remediations/Remediate-AdvisorRightSizeFiltered.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/remediations/Remediate-LongDeallocatedVMsFiltered.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/remediations/Remediate-LongDeallocatedVMsFiltered.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/remediations/Remediate-LongDeallocatedVMsFiltered.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/remediations/Remediate-LongDeallocatedVMsFiltered.ps1 diff --git a/docs/deploy/optimization-engine/runbooks/remediations/Remediate-UnattachedDisksFiltered.ps1 b/docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/remediations/Remediate-UnattachedDisksFiltered.ps1 similarity index 100% rename from docs/deploy/optimization-engine/runbooks/remediations/Remediate-UnattachedDisksFiltered.ps1 rename to docs/deploy/optimization-engine/0.2.1-rc.2/runbooks/remediations/Remediate-UnattachedDisksFiltered.ps1 diff --git a/docs/deploy/optimization-engine/azuredeploy-nested-0.2.1-rc.2.bicep b/docs/deploy/optimization-engine/latest/azuredeploy-nested.bicep similarity index 99% rename from docs/deploy/optimization-engine/azuredeploy-nested-0.2.1-rc.2.bicep rename to docs/deploy/optimization-engine/latest/azuredeploy-nested.bicep index 81b35eccd..cfed41bd4 100644 --- a/docs/deploy/optimization-engine/azuredeploy-nested-0.2.1-rc.2.bicep +++ b/docs/deploy/optimization-engine/latest/azuredeploy-nested.bicep @@ -536,6 +536,7 @@ var csvExports = [ ingestTimeOffset: 'PT2H' ingestFrequency: 'Hour' ingestJobId: monitorIngestJobId + exportSchedule: null exportJobId: 'dummy' } { diff --git a/docs/deploy/optimization-engine/azuredeploy-0.2.1-rc.2.bicep b/docs/deploy/optimization-engine/latest/azuredeploy.bicep similarity index 100% rename from docs/deploy/optimization-engine/azuredeploy-0.2.1-rc.2.bicep rename to docs/deploy/optimization-engine/latest/azuredeploy.bicep diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AADObjectsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AADObjectsToBlobStorage.ps1 new file mode 100644 index 000000000..8bde11357 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AADObjectsToBlobStorage.ps1 @@ -0,0 +1,509 @@ +param( + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName, + + [Parameter(Mandatory = $false)] + [string] $groupFilter, + + [Parameter(Mandatory = $false)] + [string] $userFilter +) + +$ErrorActionPreference = "Stop" + +function Build-CredObjectWithDates { + param ( + [object] $appObject + ) + + $credObjects = @() + + foreach ($obj in $appObject.KeyCredentials) + { + $credObject = New-Object PSObject -Property @{ + DisplayName = $obj.DisplayName + KeyId = $obj.KeyId + KeyType = $obj.Type + StartDate = (Get-Date($obj.StartDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + EndDate = (Get-Date($obj.EndDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $credObjects += $credObject + } + + foreach ($obj in $appObject.PasswordCredentials) + { + $credObject = New-Object PSObject -Property @{ + DisplayName = $obj.DisplayName + KeyId = $obj.KeyId + KeyType = "Password" + StartDate = (Get-Date($obj.StartDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + EndDate = (Get-Date($obj.EndDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $credObjects += $credObject + } + + return $credObjects +} + +function Build-PrincipalNames { + param ( + [object] $appObject + ) + + $principalNames = @() + + if ($appObject.Web.HomePageUrl) + { + $principalNames += $appObject.Web.HomePageUrl + } + + foreach ($obj in $appObject.IdentifierUris) + { + $principalNames += $obj + } + + foreach ($obj in $appObject.ServicePrincipalNames) + { + $principalNames += $obj + } + + foreach ($obj in $appObject.AlternativeNames) + { + $principalNames += $obj + } + + return $principalNames +} + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_AADObjectsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "aadobjectsexports" +} + +# Application,ServicePrincipal,User,Group +$aadObjectsFilter = Get-AutomationVariable -Name "AzureOptimization_AADObjectsFilter" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($aadObjectsFilter)) +{ + $aadObjectsFilter = "Application,ServicePrincipal" +} + +$groupFilterVariable = Get-AutomationVariable -Name "AzureOptimization_AADObjectsGroupFilter" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($groupFilter) -and -not([string]::IsNullOrEmpty($groupFilterVariable))) +{ + $groupFilter = $groupFilterVariable +} + +$userFilterVariable = Get-AutomationVariable -Name "AzureOptimization_AADObjectsUserFilter" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($userFilter) -and -not([string]::IsNullOrEmpty($userFilterVariable))) +{ + $userFilter = $userFilterVariable +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +#workaround for https://github.com/microsoftgraph/msgraph-sdk-powershell/issues/888 +$localPath = [System.Environment]::GetFolderPath([System.Environment+SpecialFolder]::UserProfile) +if (-not(get-item "$localPath\.graph\" -ErrorAction SilentlyContinue)) +{ + New-Item -Type Directory "$localPath\.graph" +} + +Import-Module Microsoft.Graph.Authentication +Import-Module Microsoft.Graph.Users +Import-Module Microsoft.Graph.Applications +Import-Module Microsoft.Graph.Groups + +switch ($cloudEnvironment) { + "AzureUSGovernment" { + $graphEnvironment = "USGov" + break + } + "AzureChinaCloud" { + $graphEnvironment = "China" + break + } + "AzureGermanCloud" { + $graphEnvironment = "Germany" + break + } + Default { + $graphEnvironment = "Global" + } +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Microsoft Graph with $externalCredentialName external credential..." + Connect-MgGraph -TenantId $externalTenantId -ClientSecretCredential $externalCredential -Environment $graphEnvironment -NoWelcome +} +else +{ + "Logging in to Microsoft Graph..." + Connect-MgGraph -Identity -Environment $graphEnvironment -NoWelcome +} + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +$aadObjectsTypes = $aadObjectsFilter.Split(",") + +$fileDate = $datetime.ToString("yyyyMMdd") + +if ("Application" -in $aadObjectsTypes) +{ + $aadObjects = @() + + "Getting AAD applications..." + $apps = Get-MgApplication -All -ExpandProperty Owners -Property Id,AppId,CreatedDateTime,DeletedDateTime,DisplayName,KeyCredentials,PasswordCredentials,Owners,PublisherDomain,Web,IdentifierUris + "Found $($apps.Count) AAD applications" + + foreach ($app in $apps) + { + $owners = $null + if ($app.Owners.Count -gt 0) + { + $owners = ($app.Owners | Where-Object { [string]::IsNullOrEmpty($_.DeletedDateTime) }).Id | ConvertTo-Json -Compress + } + $createdDate = $null + if ($app.CreatedDateTime) + { + $createdDate = (Get-Date($app.CreatedDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $deletedDate = $null + if ($app.DeletedDateTime) + { + $deletedDate = (Get-Date($app.DeletedDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $aadObject = New-Object PSObject -Property @{ + Timestamp = $timestamp + TenantGuid = $tenantId + Cloud = $cloudEnvironment + ObjectId = $app.Id + ObjectType = "Application" + ObjectSubType = "N/A" + DisplayName = $app.DisplayName + SecurityEnabled = "N/A" + ApplicationId = $app.AppId + Keys = (Build-CredObjectWithDates -appObject $app) | ConvertTo-Json -Compress + PrincipalNames = (Build-PrincipalNames -appObject $app) | ConvertTo-Json -Compress + Owners = $owners + CreatedDate = $createdDate + DeletedDate = $deletedDate + } + $aadObjects += $aadObject + } + + $jsonExportPath = "$fileDate-$tenantId-aadobjects-apps.json" + $csvExportPath = "$fileDate-$tenantId-aadobjects-apps.csv" + + $aadObjects | ConvertTo-Json -Depth 3 -Compress | Out-File $jsonExportPath + "Exported to JSON: $($aadObjects.Count) lines" + $aadObjectsJson = Get-Content -Path $jsonExportPath | ConvertFrom-Json + "JSON Import: $($aadObjectsJson.Count) lines" + $aadObjectsJson | Export-Csv -NoTypeInformation -Path $csvExportPath + "Export to $csvExportPath" + + $csvBlobName = $csvExportPath + $csvProperties = @{"ContentType" = "text/csv"}; + + Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Uploaded $csvBlobName to Blob Storage..." + + Remove-Item -Path $csvExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $csvExportPath from local disk..." + + Remove-Item -Path $jsonExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $jsonExportPath from local disk..." +} + +if ("ServicePrincipal" -in $aadObjectsTypes) +{ + $aadObjects = @() + + "Getting AAD service principals..." + $spns = Get-MgServicePrincipal -All -ExpandProperty Owners -Property Id,AppId,DeletedDateTime,DisplayName,KeyCredentials,PasswordCredentials,Owners,ServicePrincipalNames,ServicePrincipalType,AccountEnabled,AlternativeNames + "Found $($spns.Count) AAD service principals" + + foreach ($spn in $spns) + { + $owners = $null + if ($spn.Owners.Count -gt 0) + { + $owners = ($spn.Owners | Where-Object { [string]::IsNullOrEmpty($_.DeletedDateTime) }).Id | ConvertTo-Json -Compress + } + $deletedDate = $null + if ($spn.DeletedDateTime) + { + $deletedDate = (Get-Date($spn.DeletedDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $aadObject = New-Object PSObject -Property @{ + Timestamp = $timestamp + TenantGuid = $tenantId + Cloud = $cloudEnvironment + ObjectId = $spn.Id + ObjectType = "ServicePrincipal" + ObjectSubType = $spn.ServicePrincipalType + DisplayName = $spn.DisplayName + SecurityEnabled = $spn.AccountEnabled + ApplicationId = $spn.AppId + Keys = (Build-CredObjectWithDates -appObject $spn) | ConvertTo-Json -Compress + PrincipalNames = (Build-PrincipalNames -appObject $spn) | ConvertTo-Json -Compress + Owners = $owners + DeletedDate = $deletedDate + } + $aadObjects += $aadObject + } + + $jsonExportPath = "$fileDate-$tenantId-aadobjects-spns.json" + $csvExportPath = "$fileDate-$tenantId-aadobjects-spns.csv" + + $aadObjects | ConvertTo-Json -Depth 3 -Compress | Out-File $jsonExportPath + "Exported to JSON: $($aadObjects.Count) lines" + $aadObjectsJson = Get-Content -Path $jsonExportPath | ConvertFrom-Json + "JSON Import: $($aadObjectsJson.Count) lines" + $aadObjectsJson | Export-Csv -NoTypeInformation -Path $csvExportPath + "Export to $csvExportPath" + + $csvBlobName = $csvExportPath + $csvProperties = @{"ContentType" = "text/csv"}; + + Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Uploaded $csvBlobName to Blob Storage..." + + Remove-Item -Path $csvExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $csvExportPath from local disk..." + + Remove-Item -Path $jsonExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $jsonExportPath from local disk..." +} + +if ("User" -in $aadObjectsTypes) +{ + $aadObjects = @() + + if ([string]::IsNullOrEmpty($userFilter)) + { + "Getting AAD users..." + $users = Get-MgUser -All -Property Id,AccountEnabled,DisplayName,UserPrincipalName,UserType,CreatedDateTime,DeletedDateTime + } + else + { + "Getting AAD users with filter $userFilter..." + $users = Get-MgUser -Filter $userFilter -All -Property Id,AccountEnabled,DisplayName,UserPrincipalName,UserType,CreatedDateTime,DeletedDateTime + } + "Found $($users.Count) AAD users" + + foreach ($user in $users) + { + $createdDate = $null + if ($user.CreatedDateTime) + { + $createdDate = (Get-Date($user.CreatedDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $deletedDate = $null + if ($user.DeletedDateTime) + { + $deletedDate = (Get-Date($user.DeletedDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $aadObject = New-Object PSObject -Property @{ + Timestamp = $timestamp + TenantGuid = $tenantId + Cloud = $cloudEnvironment + ObjectId = $user.Id + ObjectType = "User" + ObjectSubType = $user.UserType + DisplayName = $user.DisplayName + SecurityEnabled = $user.AccountEnabled + PrincipalNames = $user.UserPrincipalName + CreatedDate = $createdDate + DeletedDate = $deletedDate + } + $aadObjects += $aadObject + } + + $jsonExportPath = "$fileDate-$tenantId-aadobjects-users.json" + $csvExportPath = "$fileDate-$tenantId-aadobjects-users.csv" + + $aadObjects | Export-Csv -NoTypeInformation -Path $csvExportPath + "Export to $csvExportPath" + + $csvBlobName = $csvExportPath + $csvProperties = @{"ContentType" = "text/csv"}; + + Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Uploaded $csvBlobName to Blob Storage..." + + Remove-Item -Path $csvExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $csvExportPath from local disk..." +} + +if ("Group" -in $aadObjectsTypes) +{ + $aadObjects = @() + + if ([string]::IsNullOrEmpty($groupFilter)) + { + "Getting AAD groups..." + $groups = Get-MgGroup -All -ExpandProperty Members -Property Id,SecurityEnabled,DisplayName,Members,CreatedDateTime,DeletedDateTime,GroupTypes + } + else + { + "Getting AAD groups with filter $groupFilter..." + $groups = Get-MgGroup -Filter $groupFilter -All -ExpandProperty Members -Property Id,SecurityEnabled,DisplayName,Members,CreatedDateTime,DeletedDateTime,GroupTypes + } + "Found $($groups.Count) AAD groups" + + foreach ($group in $groups) + { + $groupMembers = $null + if ($group.Members.Count -gt 0) + { + $groupMembers = $group.Members.Id | ConvertTo-Json -Compress + } + $createdDate = $null + if ($group.CreatedDateTime) + { + $createdDate = (Get-Date($group.CreatedDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $deletedDate = $null + if ($group.DeletedDateTime) + { + $deletedDate = (Get-Date($group.DeletedDateTime)).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + $aadObject = New-Object PSObject -Property @{ + Timestamp = $timestamp + TenantGuid = $tenantId + Cloud = $cloudEnvironment + ObjectId = $group.Id + ObjectType = "Group" + ObjectSubType = $group.GroupTypes | ConvertTo-Json -Compress + DisplayName = $group.DisplayName + SecurityEnabled = $group.SecurityEnabled + PrincipalNames = $groupMembers + CreatedDate = $createdDate + DeletedDate = $deletedDate + } + $aadObjects += $aadObject + } + + $jsonExportPath = "$fileDate-$tenantId-aadobjects-groups.json" + $csvExportPath = "$fileDate-$tenantId-aadobjects-groups.csv" + + $aadObjects | ConvertTo-Json -Depth 3 -Compress | Out-File $jsonExportPath + "Exported to JSON: $($aadObjects.Count) lines" + $aadObjectsJson = Get-Content -Path $jsonExportPath | ConvertFrom-Json + "JSON Import: $($aadObjectsJson.Count) lines" + $aadObjectsJson | Export-Csv -NoTypeInformation -Path $csvExportPath + "Export to $csvExportPath" + + $csvBlobName = $csvExportPath + $csvProperties = @{"ContentType" = "text/csv"}; + + Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Uploaded $csvBlobName to Blob Storage..." + + Remove-Item -Path $csvExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $csvExportPath from local disk..." + + Remove-Item -Path $jsonExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $jsonExportPath from local disk..." +} + +"DONE!" \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAppGatewayPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAppGatewayPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..226208aa8 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAppGatewayPropertiesToBlobStorage.ps1 @@ -0,0 +1,231 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGAppGatewayContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argappgwexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allAppGWs = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$appGWsTotal = @() +$resultsSoFar = 0 + +Write-Output "Querying for Application Gateways properties" + +$argQuery = @" +resources +| where type =~ 'Microsoft.Network/applicationGateways' +| extend gatewayIPsCount = array_length(properties.gatewayIPConfigurations) +| extend frontendIPsCount = array_length(properties.frontendIPConfigurations) +| extend frontendPortsCount = array_length(properties.frontendPorts) +| extend backendPoolsCount = array_length(properties.backendAddressPools) +| extend httpSettingsCount = array_length(properties.backendHttpSettingsCollection) +| extend httpListenersCount = array_length(properties.httpListeners) +| extend urlPathMapsCount = array_length(properties.urlPathMaps) +| extend requestRoutingRulesCount = array_length(properties.requestRoutingRules) +| extend probesCount = array_length(properties.probes) +| extend rewriteRulesCount = array_length(properties.rewriteRuleSets) +| extend redirectConfsCount = array_length(properties.redirectConfigurations) +| project id, name, resourceGroup, subscriptionId, tenantId, location, zones, skuName = properties.sku.name, skuTier = properties.sku.tier, skuCapacity = properties.sku.capacity, enableHttp2 = properties.enableHttp2, gatewayIPsCount, frontendIPsCount, frontendPortsCount, httpSettingsCount, httpListenersCount, backendPoolsCount, urlPathMapsCount, requestRoutingRulesCount, probesCount, rewriteRulesCount, redirectConfsCount, tags +| join kind=leftouter ( + resources + | where type =~ 'Microsoft.Network/applicationGateways' + | mvexpand backendPools = properties.backendAddressPools + | extend backendIPCount = array_length(backendPools.properties.backendIPConfigurations) + | extend backendAddressesCount = array_length(backendPools.properties.backendAddresses) + | summarize backendIPCount = sum(backendIPCount), backendAddressesCount = sum(backendAddressesCount) by id +) on id +| project-away id1 +| order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $appGWs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $appGWs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($appGWs -and $appGWs.GetType().Name -eq "PSResourceGraphResponse") + { + $appGWs = $appGWs.Data + } + $resultsCount = $appGWs.Count + $resultsSoFar += $resultsCount + $appGWsTotal += $appGWs + +} while ($resultsCount -eq $ARGPageSize) + +Write-Output "Found $($appGWsTotal.Count) Application Gateway entries" + +<# + Building CSV entries +#> + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +foreach ($appGW in $appGWsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $appGW.tenantId + SubscriptionGuid = $appGW.subscriptionId + ResourceGroupName = $appGW.resourceGroup.ToLower() + InstanceName = $appGW.name.ToLower() + InstanceId = $appGW.id.ToLower() + SkuName = $appGW.skuName + SkuTier = $appGW.skuTier + SkuCapacity = $appGW.skuCapacity + Location = $appGW.location + Zones = $appGW.zones + EnableHttp2 = $appGW.enableHttp2 + GatewayIPsCount = $appGW.gatewayIPsCount + FrontendIPsCount = $appGW.frontendIPsCount + FrontendPortsCount = $appGW.frontendPortsCount + BackendIPCount = $appGW.backendIPCount + BackendAddressesCount = $appGW.backendAddressesCount + HttpSettingsCount = $appGW.httpSettingsCount + HttpListenersCount = $appGW.httpListenersCount + BackendPoolsCount = $appGW.backendPoolsCount + ProbesCount = $appGW.probesCount + UrlPathMapsCount = $appGW.urlPathMapsCount + RequestRoutingRulesCount = $appGW.requestRoutingRulesCount + RewriteRulesCount = $appGW.rewriteRulesCount + RedirectConfsCount = $appGW.redirectConfsCount + StatusDate = $statusDate + Tags = $appGW.tags + } + + $allAppGWs += $logentry +} + +<# + Actually exporting CSV to Azure Storage +#> + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-appgws-$subscriptionSuffix.csv" + +$allAppGWs | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAppServicePlanPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAppServicePlanPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..b81e05bbd --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAppServicePlanPropertiesToBlobStorage.ps1 @@ -0,0 +1,209 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGAppServicePlanContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argappserviceplanexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allasp = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$aspTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for App Service Plan properties" + +$argQuery = @" + resources + | where type =~ 'microsoft.web/serverfarms' + | extend skuName = sku.name, skuTier = sku.tier, skuCapacity = sku.capacity, skuFamily = sku.family, skuSize = sku.size + | extend computeMode = properties.computeMode, zoneRedundant = properties.zoneRedundant + | extend numberOfWorkers = properties.numberOfWorkers, currentNumberOfWorkers = properties.currentNumberOfWorkers, maximumNumberOfWorkers = properties.maximumNumberOfWorkers + | extend numberOfSites = properties.numberOfSites, planName = properties.planName + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $asp = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $asp = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($asp -and $asp.GetType().Name -eq "PSResourceGraphResponse") + { + $asp = $asp.Data + } + $resultsCount = $asp.Count + $resultsSoFar += $resultsCount + $aspTotal += $asp + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($aspTotal.Count) App Service Plan entries" + +foreach ($asplan in $aspTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $asplan.tenantId + SubscriptionGuid = $asplan.subscriptionId + ResourceGroupName = $asplan.resourceGroup.ToLower() + ZoneRedundant = $asplan.zoneRedundant + Location = $asplan.location + AppServicePlanName = $asplan.name.ToLower() + InstanceId = $asplan.id.ToLower() + Kind = $asplan.kind + SkuName = $asplan.skuName + SkuTier = $asplan.skuTier + SkuCapacity = $asplan.skuCapacity + SkuFamily = $asplan.skuFamily + SkuSize = $asplan.skuSize + ComputeMode = $asplan.computeMode + NumberOfWorkers = $asplan.numberOfWorkers + CurrentNumberOfWorkers = $asplan.currentNumberOfWorkers + MaximumNumberOfWorkers = $asplan.maximumNumberOfWorkers + NumberOfSites = $asplan.numberOfSites + PlanName = $asplan.planName + Tags = $asplan.tags + StatusDate = $statusDate + } + + $allasp += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-asp-$subscriptionSuffix.csv" + +$allasp | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAvailabilitySetPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAvailabilitySetPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..d13e82b75 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGAvailabilitySetPropertiesToBlobStorage.ps1 @@ -0,0 +1,198 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGAvailabilitySetContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argavailsetexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allAvSets = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$avSetsTotal = @() +$resultsSoFar = 0 + +Write-Output "Querying for Availability Set properties" + +$argQuery = @" +resources +| where type =~ 'Microsoft.Compute/availabilitySets' +| project id, name, location, resourceGroup, subscriptionId, tenantId, skuName = tostring(sku.name), faultDomains = tostring(properties.platformFaultDomainCount), updateDomains = tostring(properties.platformUpdateDomainCount), vmCount = array_length(properties.virtualMachines), tags, zones +| order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $avSets = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $avSets = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($avSets -and $avSets.GetType().Name -eq "PSResourceGraphResponse") + { + $avSets = $avSets.Data + } + $resultsCount = $avSets.Count + $resultsSoFar += $resultsCount + $avSetsTotal += $avSets + +} while ($resultsCount -eq $ARGPageSize) + +Write-Output "Found $($avSetsTotal.Count) Availability Set entries" + +<# + Building CSV entries +#> + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +foreach ($avSet in $avSetsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $avSet.tenantId + SubscriptionGuid = $avSet.subscriptionId + ResourceGroupName = $avSet.resourceGroup.ToLower() + InstanceName = $avSet.name.ToLower() + InstanceId = $avSet.id.ToLower() + SkuName = $avSet.skuName + Location = $avSet.location + FaultDomains = $avSet.faultDomains + UpdateDomains = $avSet.updateDomains + VmCount = $avSet.vmCount + StatusDate = $statusDate + Tags = $avSet.tags + Zones = $avSet.zones + } + + $allAvSets += $logentry +} + +<# + Actually exporting CSV to Azure Storage +#> + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-availsets-$subscriptionSuffix.csv" + +$allAvSets | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGLoadBalancerPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGLoadBalancerPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..103ba7cf6 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGLoadBalancerPropertiesToBlobStorage.ps1 @@ -0,0 +1,222 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGLoadBalancerContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "arglbexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allLBs = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$LBsTotal = @() +$resultsSoFar = 0 + +Write-Output "Querying for Load Balancer properties" + +$argQuery = @" +resources +| where type =~ 'Microsoft.Network/loadBalancers' +| extend lbType = iif(properties.frontendIPConfigurations contains 'publicIPAddress', 'Public', iif(properties.frontendIPConfigurations contains 'privateIPAddress', 'Internal', 'Unknown')) +| extend lbRulesCount = array_length(properties.loadBalancingRules) +| extend frontendIPsCount = array_length(properties.frontendIPConfigurations) +| extend inboundNatRulesCount = array_length(properties.inboundNatRules) +| extend outboundRulesCount = array_length(properties.outboundRules) +| extend inboundNatPoolsCount = array_length(properties.inboundNatPools) +| extend backendPoolsCount = array_length(properties.backendAddressPools) +| extend probesCount = array_length(properties.probes) +| project id, name, resourceGroup, subscriptionId, tenantId, location, skuName = sku.name, skuTier = sku.tier, lbType, lbRulesCount, frontendIPsCount, inboundNatRulesCount, outboundRulesCount, inboundNatPoolsCount, backendPoolsCount, probesCount, tags +| join kind=leftouter ( + resources + | where type =~ 'Microsoft.Network/loadBalancers' + | mvexpand backendPools = properties.backendAddressPools + | extend backendIPCount = array_length(backendPools.properties.backendIPConfigurations) + | extend backendAddressesCount = array_length(backendPools.properties.loadBalancerBackendAddresses) + | summarize backendIPCount = sum(backendIPCount), backendAddressesCount = sum(backendAddressesCount) by id +) on id +| project-away id1 +| order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $LBs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $LBs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($LBs -and $LBs.GetType().Name -eq "PSResourceGraphResponse") + { + $LBs = $LBs.Data + } + $resultsCount = $LBs.Count + $resultsSoFar += $resultsCount + $LBsTotal += $LBs + +} while ($resultsCount -eq $ARGPageSize) + +Write-Output "Found $($LBsTotal.Count) Load Balancer entries" + +<# + Building CSV entries +#> + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +foreach ($lb in $LBsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $lb.tenantId + SubscriptionGuid = $lb.subscriptionId + ResourceGroupName = $lb.resourceGroup.ToLower() + InstanceName = $lb.name.ToLower() + InstanceId = $lb.id.ToLower() + SkuName = $lb.skuName + SkuTier = $lb.skuTier + Location = $lb.location + LbType = $lb.lbType + LbRulesCount = $lb.lbRulesCount + InboundNatRulesCount = $lb.inboundNatRulesCount + OutboundRulesCount = $lb.outboundRulesCount + FrontendIPsCount = $lb.frontendIPsCount + BackendIPCount = $lb.backendIPCount + BackendAddressesCount = $lb.backendAddressesCount + InboundNatPoolsCount = $lb.inboundNatPoolsCount + BackendPoolsCount = $lb.backendPoolsCount + ProbesCount = $lb.probesCount + StatusDate = $statusDate + Tags = $lb.tags + } + + $allLBs += $logentry +} + +<# + Actually exporting CSV to Azure Storage +#> + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-lbs-$subscriptionSuffix.csv" + +$allLBs | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGManagedDisksPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGManagedDisksPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..3ba66de18 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGManagedDisksPropertiesToBlobStorage.ps1 @@ -0,0 +1,232 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGDiskContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argdiskexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$alldisks = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$mdisksTotal = @() +$resultsSoFar = 0 + +<# + Getting all Managed Disks properties with Azure Resource Graph query +#> + +Write-Output "Querying for ARM Managed Disks properties" + +$argQuery = @" + resources + | where type =~ 'Microsoft.Compute/disks' + | extend DiskId = tolower(id), OwnerVmId = tolower(managedBy) + | join kind=leftouter ( + resources + | where type =~ 'Microsoft.Compute/virtualMachines' and array_length(properties.storageProfile.dataDisks) > 0 + | extend OwnerVmId = tolower(id) + | mv-expand DataDisks = properties.storageProfile.dataDisks + | extend DiskId = tolower(DataDisks.managedDisk.id), diskCaching = tostring(DataDisks.caching), diskType = 'Data' + | project DiskId, OwnerVmId, diskCaching, diskType + | union ( + resources + | where type =~ 'Microsoft.Compute/virtualMachines' + | extend OwnerVmId = tolower(id) + | extend DiskId = tolower(properties.storageProfile.osDisk.managedDisk.id), diskCaching = tostring(properties.storageProfile.osDisk.caching), diskType = 'OS' + | project DiskId, OwnerVmId, diskCaching, diskType + ) + ) on OwnerVmId, DiskId + | project-away OwnerVmId, DiskId, OwnerVmId1, DiskId1 + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $mdisks = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $mdisks = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($mdisks -and $mdisks.GetType().Name -eq "PSResourceGraphResponse") + { + $mdisks = $mdisks.Data + } + $resultsCount = $mdisks.Count + $resultsSoFar += $resultsCount + $mdisksTotal += $mdisks + +} while ($resultsCount -eq $ARGPageSize) + +Write-Output "Found $($mdisksTotal.Count) Managed Disk entries" + +<# + Building CSV entries +#> + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +foreach ($disk in $mdisksTotal) +{ + $ownerVmId = $null + if ($null -ne $disk.managedBy) + { + $ownerVmId = $disk.managedBy.ToLower() + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $disk.tenantId + SubscriptionGuid = $disk.subscriptionId + ResourceGroupName = $disk.resourceGroup.ToLower() + DiskName = $disk.name.ToLower() + InstanceId = $disk.id.ToLower() + Location = $disk.location + OwnerVMId = $ownerVmId + DeploymentModel = "Managed" + DiskType = $disk.diskType + TimeCreated = $disk.properties.timeCreated + DiskIOPS = $disk.properties.diskIOPSReadWrite + DiskThroughput = $disk.properties.diskMBpsReadWrite + DiskTier = $disk.properties.tier + DiskState = $disk.properties.diskState + EncryptionType = $disk.properties.encryption.type + Zones = $disk.zones + Caching = $disk.diskCaching + DiskSizeGB = $disk.properties.diskSizeGB + SKU = $disk.sku.name + StatusDate = $statusDate + Tags = $disk.tags + } + + $alldisks += $logentry +} + +<# + Actually exporting CSV to Azure Storage +#> + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-disks-$subscriptionSuffix.csv" + +$alldisks | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGNICPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGNICPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..6ae3d1946 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGNICPropertiesToBlobStorage.ps1 @@ -0,0 +1,235 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGNICContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argnicexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allnics = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$nicsTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for NIC properties" + +$argQuery = @" + resources + | where type =~ 'microsoft.network/networkinterfaces' + | extend isPrimary = properties.primary + | extend enableAcceleratedNetworking = properties.enableAcceleratedNetworking + | extend enableIPForwarding = properties.enableIPForwarding + | extend tapConfigurationsCount = array_length(properties.tapConfigurations) + | extend hostedWorkloadsCount = array_length(properties.hostedWorkloads) + | extend internalDomainNameSuffix = properties.dnsSettings.internalDomainNameSuffix + | extend appliedDnsServers = properties.dnsSettings.appliedDnsServers + | extend dnsServers = properties.dnsSettings.dnsServers + | extend ownerVMId = tolower(properties.virtualMachine.id) + | extend ownerPEId = tolower(properties.privateEndpoint.id) + | extend macAddress = properties.macAddress + | extend nicType = properties.nicType + | extend nicNsgId = tolower(properties.networkSecurityGroup.id) + | mv-expand ipconfigs = properties.ipConfigurations + | project-away properties + | extend privateIPAddressVersion = tostring(ipconfigs.properties.privateIPAddressVersion) + | extend privateIPAllocationMethod = tostring(ipconfigs.properties.privateIPAllocationMethod) + | extend isIPConfigPrimary = tostring(ipconfigs.properties.primary) + | extend privateIPAddress = tostring(ipconfigs.properties.privateIPAddress) + | extend publicIPId = tolower(ipconfigs.properties.publicIPAddress.id) + | extend IPConfigName = tostring(ipconfigs.name) + | extend subnetId = tolower(ipconfigs.properties.subnet.id) + | project-away ipconfigs + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $nics = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $nics = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($nics -and $nics.GetType().Name -eq "PSResourceGraphResponse") + { + $nics = $nics.Data + } + $resultsCount = $nics.Count + $resultsSoFar += $resultsCount + $nicsTotal += $nics + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($nicsTotal.Count) ARM VNet nic entries" + +foreach ($nic in $nicsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $nic.tenantId + SubscriptionGuid = $nic.subscriptionId + ResourceGroupName = $nic.resourceGroup.ToLower() + Location = $nic.location + Name = $nic.name.ToLower() + InstanceId = $nic.id.ToLower() + IsPrimary = $nic.isPrimary + EnableAcceleratedNetworking = $nic.enableAcceleratedNetworking + EnableIPForwarding = $nic.enableIPForwarding + TapConfigurationsCount = $nic.tapConfigurationsCount + HostedWorkloadsCount = $nic.hostedWorkloadsCount + InternalDomainNameSuffix = $nic.internalDomainNameSuffix + AppliedDnsServers = $nic.appliedDnsServers + DnsServers = $nic.dnsServers + OwnerVMId = $nic.ownerVMId + OwnerPEId = $nic.ownerPEId + MacAddress = $nic.macAddress + NicType = $nic.nicType + NicNSGId = $nic.nicNsgId + PrivateIPAddressVersion = $nic.privateIPAddressVersion + PrivateIPAllocationMethod = $nic.privateIPAllocationMethod + IsIPConfigPrimary = $nic.isIPConfigPrimary + PrivateIPAddress = $nic.privateIPAddress + PublicIPId = $nic.publicIPId + IPConfigName = $nic.IPConfigName + SubnetId = $nic.subnetId + Tags = $nic.tags + StatusDate = $statusDate + } + + $allnics += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-nics-$subscriptionSuffix.csv" + +$allnics | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGNSGPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGNSGPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..763dbe32d --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGNSGPropertiesToBlobStorage.ps1 @@ -0,0 +1,217 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGNSGContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argnsgexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allnsgRules = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$nsgRulesTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for NSG properties" + +$argQuery = @" +resources +| where type =~ 'Microsoft.Network/networkSecurityGroups' +| extend nicCount = iif(isnotempty(properties.networkInterfaces),array_length(properties.networkInterfaces),0) +| extend subnetCount = iif(isnotempty(properties.subnets),array_length(properties.subnets),0) +| mvexpand securityRules = properties.securityRules +| extend ruleName = tolower(securityRules.name) +| extend ruleProtocol = tolower(securityRules.properties.protocol) +| extend ruleDirection = tolower(securityRules.properties.direction) +| extend rulePriority = toint(securityRules.properties.priority) +| extend ruleAccess = tolower(securityRules.properties.access) +| extend ruleDestinationAddresses = tolower(iif(array_length(securityRules.properties.destinationAddressPrefixes) > 0,strcat_array(securityRules.properties.destinationAddressPrefixes, ','),securityRules.properties.destinationAddressPrefix)) +| extend ruleSourceAddresses = tolower(iif(array_length(securityRules.properties.sourceAddressPrefixes) > 0,strcat_array(securityRules.properties.sourceAddressPrefixes, ','),securityRules.properties.sourceAddressPrefix)) +| extend ruleDestinationPorts = iif(array_length(securityRules.properties.destinationPortRanges) > 0,strcat_array(securityRules.properties.destinationPortRanges, ','),securityRules.properties.destinationPortRange) +| extend ruleSourcePorts = iif(array_length(securityRules.properties.sourcePortRanges) > 0,strcat_array(securityRules.properties.sourcePortRanges, ','),securityRules.properties.sourcePortRange) +| extend ruleId = tolower(securityRules.id) +| project-away securityRules, properties +| order by ruleId asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $nsgRules = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $nsgRules = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($nsgRules -and $nsgRules.GetType().Name -eq "PSResourceGraphResponse") + { + $nsgRules = $nsgRules.Data + } + $resultsCount = $nsgRules.Count + $resultsSoFar += $resultsCount + $nsgRulesTotal += $nsgRules + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($nsgRulesTotal.Count) ARM NSG entries" + +foreach ($nsgRule in $nsgRulesTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $nsgRule.tenantId + SubscriptionGuid = $nsgRule.subscriptionId + ResourceGroupName = $nsgRule.resourceGroup.ToLower() + Location = $nsgRule.location + NSGName = $nsgRule.name.ToLower() + InstanceId = $nsgRule.id.ToLower() + NicCount = $nsgRule.nicCount + SubnetCount = $nsgRule.subnetCount + RuleName = $nsgRule.ruleName + RuleProtocol = $nsgRule.ruleProtocol + RuleDirection = $nsgRule.ruleDirection + RulePriority = $nsgRule.rulePriority + RuleAccess = $nsgRule.ruleAccess + RuleDestinationAddresses = $nsgRule.ruleDestinationAddresses + RuleSourceAddresses = $nsgRule.ruleSourceAddresses + RuleDestinationPorts = $nsgRule.ruleDestinationPorts + RuleSourcePorts = $nsgRule.ruleSourcePorts + Tags = $nsgRule.tags + StatusDate = $statusDate + } + + $allnsgRules += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-nsgrules-$subscriptionSuffix.csv" + +$allnsgRules | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGPublicIpPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGPublicIpPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..d33a8330f --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGPublicIpPropertiesToBlobStorage.ps1 @@ -0,0 +1,275 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGPublicIpContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argpublicipexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allpips = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$pipsTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for ARM Public IP properties" + +$argQuery = @" +resources +| where type =~ 'microsoft.network/publicipaddresses' +| extend skuName = tolower(sku.name) +| extend skuTier = tolower(sku.tier) +| extend allocationMethod = tolower(properties.publicIPAllocationMethod) +| extend addressVersion = tolower(properties.publicIPAddressVersion) +| extend associatedResourceId = iif(isnotempty(properties.ipConfiguration.id),tolower(properties.ipConfiguration.id),tolower(properties.natGateway.id)) +| extend ipAddress = tostring(properties.ipAddress) +| extend fqdn = tolower(properties.dnsSettings.fqdn) +| extend publicIpPrefixId = tostring(properties.publicIPPrefix.id) +| order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $pips = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $pips = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($pips -and $pips.GetType().Name -eq "PSResourceGraphResponse") + { + $pips = $pips.Data + } + $resultsCount = $pips.Count + $resultsSoFar += $resultsCount + $pipsTotal += $pips + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($pipsTotal.Count) ARM Public IP entries" + +foreach ($pip in $pipsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $pip.tenantId + SubscriptionGuid = $pip.subscriptionId + ResourceGroupName = $pip.resourceGroup.ToLower() + Location = $pip.location + Name = $pip.name.ToLower() + InstanceId = $pip.id.ToLower() + Model = "ARM" + SkuName = $pip.skuName + SkuTier = $pip.skuTier + AllocationMethod = $pip.allocationMethod + AddressVersion = $pip.addressVersion + AssociatedResourceId = $pip.associatedResourceId + PublicIpPrefixId = $pip.publicIpPrefixId + IPAddress = $pip.ipAddress + FQDN = $pip.fqdn + Zones = $pip.zones + Tags = $pip.tags + StatusDate = $statusDate + } + + $allpips += $logentry +} + +$pipsTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for Classic Reserved IP properties" + +$argQuery = @" +resources +| where type =~ 'microsoft.classicnetwork/reservedips' +| extend ipAddress = tostring(properties.ipAddress) +| extend allocationMethod = 'static' +| extend addressVersion = 'ipv4' +| extend associatedResourceId = tolower(properties.attachedTo.id) +| extend ipAddress = tostring(properties.ipAddress) +| order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $pips = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $pips = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($pips -and $pips.GetType().Name -eq "PSResourceGraphResponse") + { + $pips = $pips.Data + } + $resultsCount = $pips.Count + $resultsSoFar += $resultsCount + $pipsTotal += $pips + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($pipsTotal.Count) Classic Reserved IP entries" + +foreach ($pip in $pipsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $pip.tenantId + SubscriptionGuid = $pip.subscriptionId + ResourceGroupName = $pip.resourceGroup.ToLower() + Location = $pip.location + Name = $pip.name.ToLower() + InstanceId = $pip.id.ToLower() + Model = "Classic" + AllocationMethod = $pip.allocationMethod + AddressVersion = $pip.addressVersion + AssociatedResourceId = $pip.associatedResourceId + IPAddress = $pip.ipAddress + StatusDate = $statusDate + } + + $allpips += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-publicips-$subscriptionSuffix.csv" + +$allpips | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGResourceContainersPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGResourceContainersPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..b7609cda6 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGResourceContainersPropertiesToBlobStorage.ps1 @@ -0,0 +1,272 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGResourceContainersContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argrescontainersexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allResourceContainers = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$rgsTotal = @() +$subsTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for resource groups..." + +$argQuery = @" + resourcecontainers + | where type == "microsoft.resources/subscriptions/resourcegroups" + | join kind=leftouter ( + resources + | summarize ResourceCount= count() by subscriptionId, resourceGroup + ) on subscriptionId, resourceGroup + | extend ResourceCount = iif(isempty(ResourceCount), 0, ResourceCount) + | project id, name, type, tenantId, location, subscriptionId, managedBy, tags, properties, ResourceCount + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $rgs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $rgs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($rgs -and $rgs.GetType().Name -eq "PSResourceGraphResponse") + { + $rgs = $rgs.Data + } + $resultsCount = $rgs.Count + $resultsSoFar += $resultsCount + $rgsTotal += $rgs + +} while ($resultsCount -eq $ARGPageSize) + +$resultsSoFar = 0 + +Write-Output "Querying for subscriptions" + +$argQuery = @" + resourcecontainers + | where type == "microsoft.resources/subscriptions" + | join kind=leftouter ( + resources + | summarize ResourceCount= count() by subscriptionId + ) on subscriptionId + | extend ResourceCount = iif(isempty(ResourceCount), 0, ResourceCount) + | project id, name, type, tenantId, subscriptionId, managedBy, tags, properties, ResourceCount + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $subs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $subs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($subs -and $subs.GetType().Name -eq "PSResourceGraphResponse") + { + $subs = $subs.Data + } + $resultsCount = $subs.Count + $resultsSoFar += $resultsCount + $subsTotal += $subs + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($rgsTotal.Count) RG entries" + +foreach ($rg in $rgsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $rg.tenantId + SubscriptionGuid = $rg.subscriptionId + Location = $rg.location + ContainerType = $rg.type + ContainerName = $rg.name.ToLower() + InstanceId = $rg.id.ToLower() + ResourceCount = $rg.ResourceCount + ManagedBy = $rg.managedBy + ContainerProperties = $rg.properties | ConvertTo-Json -Compress + Tags = $rg.tags + StatusDate = $statusDate + } + + $allResourceContainers += $logentry +} + +Write-Output "Building $($subsTotal.Count) subscription entries" + +foreach ($sub in $subsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $sub.tenantId + SubscriptionGuid = $sub.subscriptionId + Location = $sub.location + ContainerType = $sub.type + ContainerName = $sub.name.ToLower() + InstanceId = $sub.id.ToLower() + ResourceCount = $sub.ResourceCount + ManagedBy = $sub.managedBy + ContainerProperties = $sub.properties | ConvertTo-Json -Compress + Tags = $sub.tags + StatusDate = $statusDate + } + + $allResourceContainers += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$jsonExportPath = "$today-rescontainers-$subscriptionSuffix.json" +$csvExportPath = "$today-rescontainers-$subscriptionSuffix.csv" + +$allResourceContainers | ConvertTo-Json -Depth 3 -Compress | Out-File $jsonExportPath +Write-Output "Exported to JSON: $($allResourceContainers.Count) lines" +$allResourceContainersJson = Get-Content -Path $jsonExportPath | ConvertFrom-Json +Write-Output "JSON Import: $($allResourceContainersJson.Count) lines" +$allResourceContainersJson | Export-Csv -NoTypeInformation -Path $csvExportPath +Write-Output "Export to $csvExportPath" + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGSqlDatabasePropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGSqlDatabasePropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..adf0979eb --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGSqlDatabasePropertiesToBlobStorage.ps1 @@ -0,0 +1,204 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGSqlDatabaseContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argsqldbexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$alldbs = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$dbsTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for SQL Databases properties" + +$argQuery = @" + resources + | where type =~ 'microsoft.sql/servers/databases' and name != 'master' + | extend skuName = sku.name, skuTier = sku.tier, skuCapacity = sku.capacity + | extend storageAccountType = properties.storageAccountType, licenseType = properties.licenseType, serviceObjectiveName = properties.currentServiceObjectiveName + | extend zoneRedundant = properties.zoneRedundant, maxSizeBytes = properties.maxSizeBytes, maxLogSizeBytes = properties.maxLogSizeBytes + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $dbs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $dbs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($dbs -and $dbs.GetType().Name -eq "PSResourceGraphResponse") + { + $dbs = $dbs.Data + } + $resultsCount = $dbs.Count + $resultsSoFar += $resultsCount + $dbsTotal += $dbs + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($dbsTotal.Count) SQL Database entries" + +foreach ($db in $dbsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $db.tenantId + SubscriptionGuid = $db.subscriptionId + ResourceGroupName = $db.resourceGroup.ToLower() + ZoneRedundant = $db.zoneRedundant + Location = $db.location + DBName = $db.name.ToLower() + InstanceId = $db.id.ToLower() + SkuName = $db.skuName + SkuTier = $db.skuTier + SkuCapacity = $db.skuCapacity + ServiceObjectiveName = $db.serviceObjectiveName + StorageAccountType = $db.storageAccountType + LicenseType = $db.licenseType + MaxSizeBytes = $db.maxSizeBytes + MaxLogSizeBytes = $db.maxLogSizeBytes + Tags = $db.tags + StatusDate = $statusDate + } + + $alldbs += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-sqldbs-$subscriptionSuffix.csv" + +$alldbs | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGUnmanagedDisksPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGUnmanagedDisksPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..e48c45406 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGUnmanagedDisksPropertiesToBlobStorage.ps1 @@ -0,0 +1,236 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGVhdContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argvhdexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$alldisks = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$mdisksTotal = @() +$resultsSoFar = 0 + +Write-Output "Querying for ARM Unmanaged OS Disks properties" + +$argQuery = @" +resources +| where type =~ 'Microsoft.Compute/virtualMachines' and isnull(properties.storageProfile.osDisk.managedDisk) +| extend diskType = 'OS', diskCaching = tostring(properties.storageProfile.osDisk.caching), diskSize = tostring(properties.storageProfile.osDisk.diskSizeGB) +| extend vhdUriParts = split(tostring(properties.storageProfile.osDisk.vhd.uri),'/') +| extend diskStorageAccountName = tostring(split(vhdUriParts[2],'.')[0]), diskContainerName = tostring(vhdUriParts[3]), diskVhdName = tostring(vhdUriParts[4]) +| order by id, diskStorageAccountName, diskContainerName, diskVhdName +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $mdisks = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $mdisks = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($mdisks -and $mdisks.GetType().Name -eq "PSResourceGraphResponse") + { + $mdisks = $mdisks.Data + } + $resultsCount = $mdisks.Count + $resultsSoFar += $resultsCount + $mdisksTotal += $mdisks + +} while ($resultsCount -eq $ARGPageSize) + +$resultsSoFar = 0 + +Write-Output "Found $($mdisksTotal.Count) Unmanaged OS Disk entries" + +Write-Output "Querying for ARM Unmanaged Data Disks properties" + +$argQuery = @" +resources +| where type =~ 'Microsoft.Compute/virtualMachines' and isnull(properties.storageProfile.osDisk.managedDisk) +| mvexpand dataDisks = properties.storageProfile.dataDisks +| extend diskType = 'Data', diskCaching = tostring(dataDisks.caching), diskSize = tostring(dataDisks.diskSizeGB) +| extend vhdUriParts = split(tostring(dataDisks.vhd.uri),'/') +| extend diskStorageAccountName = tostring(split(vhdUriParts[2],'.')[0]), diskContainerName = tostring(vhdUriParts[3]), diskVhdName = tostring(vhdUriParts[4]) +| order by id, diskStorageAccountName, diskContainerName, diskVhdName +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $mdisks = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $mdisks = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($mdisks -and $mdisks.GetType().Name -eq "PSResourceGraphResponse") + { + $mdisks = $mdisks.Data + } + $resultsCount = $mdisks.Count + $resultsSoFar += $resultsCount + $mdisksTotal += $mdisks + +} while ($resultsCount -eq $ARGPageSize) + +Write-Output "Found overall $($mdisksTotal.Count) Unmanaged Disk entries" + +<# + Building CSV entries +#> + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +foreach ($disk in $mdisksTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $disk.tenantId + SubscriptionGuid = $disk.subscriptionId + ResourceGroupName = $disk.resourceGroup.ToLower() + DiskName = $disk.diskVhdName.ToLower() + InstanceId = ($disk.diskStorageAccountName + "/" + $disk.diskContainerName + "/" + $disk.diskVhdName).ToLower() + OwnerVMId = $disk.id.ToLower() + Location = $disk.location + DeploymentModel = "Unmanaged" + DiskType = $disk.diskType + Caching = $disk.diskCaching + DiskSizeGB = $disk.diskSize + StatusDate = $statusDate + Tags = $disk.tags + } + + $alldisks += $logentry +} + +<# + Actually exporting CSV to Azure Storage +#> + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-vhds-$subscriptionSuffix.csv" + +$alldisks | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVMSSPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVMSSPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..81f39771a --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVMSSPropertiesToBlobStorage.ps1 @@ -0,0 +1,239 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGVMSSContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argvmssexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +Write-Output "Getting VM sizes details for $referenceRegion" +$sizes = Get-AzVMSize -Location $referenceRegion + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allvmss = @() + +if ($TargetSubscription) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = "-" + $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$armVmssTotal = @() + +$resultsSoFar = 0 + +$argQuery = @" +resources +| where type =~ 'microsoft.compute/virtualmachinescalesets' +| project id, tenantId, name, location, resourceGroup, subscriptionId, skUName = tostring(sku.name), + computerNamePrefix = tostring(properties.virtualMachineProfile.osProfile.computerNamePrefix), + usesManagedDisks = iif(isnull(properties.virtualMachineProfile.storageProfile.osDisk.managedDisk), 'false', 'true'), + capacity = tostring(sku.capacity), priority = tostring(properties.virtualMachineProfile.priority), tags, zones, + osType = iif(isnotnull(properties.virtualMachineProfile.osProfile.linuxConfiguration), "Linux", "Windows"), + osDiskSize = tostring(properties.virtualMachineProfile.storageProfile.osDisk.diskSizeGB), + osDiskCaching = tostring(properties.virtualMachineProfile.storageProfile.osDisk.caching), + osDiskSKU = tostring(properties.virtualMachineProfile.storageProfile.osDisk.managedDisk.storageAccountType), + dataDiskCount = iif(isnotnull(properties.virtualMachineProfile.storageProfile.dataDisks), array_length(properties.virtualMachineProfile.storageProfile.dataDisks), 0), + nicCount = array_length(properties.virtualMachineProfile.networkProfile.networkInterfaceConfigurations), + imagePublisher = iif(isnotempty(properties.virtualMachineProfile.storageProfile.imageReference.publisher),tostring(properties.virtualMachineProfile.storageProfile.imageReference.publisher),'Custom'), + imageOffer = iif(isnotempty(properties.virtualMachineProfile.storageProfile.imageReference.offer),tostring(properties.virtualMachineProfile.storageProfile.imageReference.offer),tostring(properties.virtualMachineProfile.storageProfile.imageReference.id)), + imageSku = tostring(properties.virtualMachineProfile.storageProfile.imageReference.sku), + imageVersion = tostring(properties.virtualMachineProfile.storageProfile.imageReference.version), + imageExactVersion = tostring(properties.virtualMachineProfile.storageProfile.imageReference.exactVersion), + singlePlacementGroup = tostring(properties.singlePlacementGroup), + upgradePolicy = tostring(properties.upgradePolicy.mode), + overProvision = tostring(properties.overprovision), + platformFaultDomainCount = tostring(properties.platformFaultDomainCount), + zoneBalance = tostring(properties.zoneBalance) +| order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $armVmss = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $armVmss = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + + if ($armVmss -and $armVmss.GetType().Name -eq "PSResourceGraphResponse") + { + $armVmss = $armVmss.Data + } + $resultsCount = $armVmss.Count + $resultsSoFar += $resultsCount + $armVmssTotal += $armVmss + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($armVmssTotal.Count) VMSS entries" + +foreach ($vmss in $armVmssTotal) +{ + $vmSize = $sizes | Where-Object {$_.name -eq $vmss.skUName} + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $vmss.tenantId + SubscriptionGuid = $vmss.subscriptionId + ResourceGroupName = $vmss.resourceGroup.ToLower() + Zones = $vmss.zones + Location = $vmss.location + VMSSName = $vmss.name.ToLower() + ComputerNamePrefix = $vmss.computerNamePrefix.ToLower() + InstanceId = $vmss.id.ToLower() + VMSSSize = $vmSize.name.ToLower() + CoresCount = $vmSize.NumberOfCores + MemoryMB = $vmSize.MemoryInMB + OSType = $vmss.osType + DataDiskCount = $vmss.dataDiskCount + NicCount = $vmss.nicCount + StatusDate = $statusDate + Tags = $vmss.tags + Capacity = $vmss.capacity + Priority = $vmss.priority + OSDiskSize = $vmss.osDiskSize + OSDiskCaching = $vmss.osDiskCaching + OSDiskSKU = $vmss.osDiskSKU + SinglePlacementGroup = $vmss.singlePlacementGroup + UpgradePolicy = $vmss.upgradePolicy + OverProvision = $vmss.overProvision + PlatformFaultDomainCount = $vmss.platformFaultDomainCount + ZoneBalance = $vmss.zoneBalance + UsesManagedDisks = $vmss.usesManagedDisks + ImagePublisher = $vmss.imagePublisher + ImageOffer = $vmss.imageOffer + ImageSku = $vmss.imageSku + ImageVersion = $vmss.imageVersion + ImageExactVersion = $vmss.imageExactVersion + } + + $allvmss += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-vmss-$subscriptionSuffix.csv" + +$allvmss | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVNetPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVNetPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..94301bfad --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVNetPropertiesToBlobStorage.ps1 @@ -0,0 +1,308 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGVNetContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argvnetexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allsubnets = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$subnetsTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for ARM VNet properties" + +$argQuery = @" + resources + | where type =~ 'microsoft.network/virtualnetworks' + | mv-expand subnets = properties.subnets limit 400 + | extend peeringsCount = array_length(properties.virtualNetworkPeerings) + | extend vnetPrefixes = properties.addressSpace.addressPrefixes + | extend dnsServers = properties.dhcpOptions.dnsServers + | extend enableDdosProtection = properties.enableDdosProtection + | project-away properties + | extend subnetPrefix = tostring(subnets.properties.addressPrefix) + | extend subnetDelegationsCount = array_length(subnets.properties.delegations) + | extend subnetUsedIPs = iif(isnotempty(subnets.properties.ipConfigurations), array_length(subnets.properties.ipConfigurations), 0) + | extend subnetTotalPrefixIPs = pow(2, 32 - toint(split(subnetPrefix,'/')[1])) - 5 + | extend subnetNsgId = tolower(subnets.properties.networkSecurityGroup.id) + | project id, vnetName = name, resourceGroup, subscriptionId, tenantId, location, vnetPrefixes, dnsServers, subnetName = tolower(tostring(subnets.name)), subnetPrefix, subnetDelegationsCount, subnetTotalPrefixIPs, subnetUsedIPs, subnetNsgId, peeringsCount, enableDdosProtection, tags + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $subnets = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $subnets = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($subnets -and $subnets.GetType().Name -eq "PSResourceGraphResponse") + { + $subnets = $subnets.Data + } + $resultsCount = $subnets.Count + $resultsSoFar += $resultsCount + $subnetsTotal += $subnets + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($subnetsTotal.Count) ARM VNet subnet entries" + +foreach ($subnet in $subnetsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $subnet.tenantId + SubscriptionGuid = $subnet.subscriptionId + ResourceGroupName = $subnet.resourceGroup.ToLower() + Location = $subnet.location + VNetName = $subnet.vnetName.ToLower() + InstanceId = $subnet.id.ToLower() + Model = "ARM" + VNetPrefixes = $subnet.vnetPrefixes + DNSServers = $subnet.dnsServers + PeeringsCount = $subnet.peeringsCount + EnableDdosProtection = $subnet.enableDdosProtection + SubnetName = $subnet.subnetName + SubnetPrefix = $subnet.subnetPrefix + SubnetDelegationsCount = $subnet.subnetDelegationsCount + SubnetTotalPrefixIPs = $subnet.subnetTotalPrefixIPs + SubnetUsedIPs = $subnet.subnetUsedIPs + SubnetNSGId = $subnet.subnetNsgId + Tags = $subnet.tags + StatusDate = $statusDate + } + + $allsubnets += $logentry +} + +$subnetsTotal = @() + +$resultsSoFar = 0 + +Write-Output "Querying for Classic VNet properties" + +$argQuery = @" + resources + | where type =~ 'microsoft.classicnetwork/virtualnetworks' + | extend vNetId = tolower(id) + | mv-expand subnets = properties.subnets limit 400 + | extend subnetName = tolower(tostring(subnets.name)) + | join kind=leftouter ( + resources + | where type =~ 'microsoft.network/virtualnetworks' + | mvexpand peerings = properties.virtualNetworkPeerings limit 400 + | extend vNetId = tolower(tostring(peerings.properties.remoteVirtualNetwork.id)) + | where vNetId has "microsoft.classicnetwork" + | summarize vNetPeerings=count() by vNetId + ) on vNetId + | extend peeringsCount = iif(isnotempty(vNetPeerings), vNetPeerings, 0) + | extend vnetPrefixes = properties.addressSpace.addressPrefixes + | extend dnsServers = properties.dhcpOptions.dnsServers + | project-away properties + | extend subnetPrefix = tostring(subnets.addressPrefix) + | join kind=leftouter ( + resources + | where type =~ 'microsoft.classiccompute/virtualmachines' + | extend networkProfile = properties.networkProfile + | mvexpand subnets = networkProfile.virtualNetwork.subnetNames limit 400 + | extend subnetName = tolower(tostring(subnets)) + | project id, vNetId = tolower(tostring(networkProfile.virtualNetwork.id)), subnetName + | summarize subnetUsedIPs = count() by vNetId, subnetName + ) on vNetId and subnetName + | extend subnetUsedIPs = iif(isnotempty(subnetUsedIPs), subnetUsedIPs, 0) + | extend subnetTotalPrefixIPs = pow(2, 32 - toint(split(subnetPrefix,'/')[1])) - 5 + | extend enableDdosProtection = 'false' + | project vNetId, vnetName = name, resourceGroup, subscriptionId, tenantId, location, vnetPrefixes, dnsServers, subnetName, subnetPrefix, subnetTotalPrefixIPs, subnetUsedIPs, peeringsCount, enableDdosProtection + | order by vNetId asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $subnets = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $subnets = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($subnets -and $subnets.GetType().Name -eq "PSResourceGraphResponse") + { + $subnets = $subnets.Data + } + $resultsCount = $subnets.Count + $resultsSoFar += $resultsCount + $subnetsTotal += $subnets + +} while ($resultsCount -eq $ARGPageSize) + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($subnetsTotal.Count) Classic VNet subnet entries" + +foreach ($subnet in $subnetsTotal) +{ + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $subnet.tenantId + SubscriptionGuid = $subnet.subscriptionId + ResourceGroupName = $subnet.resourceGroup.ToLower() + Location = $subnet.location + VNetName = $subnet.vnetName.ToLower() + InstanceId = $subnet.vNetId.ToLower() + Model = "Classic" + VNetPrefixes = $subnet.vnetPrefixes + DNSServers = $subnet.dnsServers + PeeringsCount = $subnet.peeringsCount + EnableDdosProtection = $subnet.enableDdosProtection + SubnetName = $subnet.subnetName + SubnetPrefix = $subnet.subnetPrefix + SubnetTotalPrefixIPs = $subnet.subnetTotalPrefixIPs + SubnetUsedIPs = $subnet.subnetUsedIPs + StatusDate = $statusDate + } + + $allsubnets += $logentry +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-vnetsubnets-$subscriptionSuffix.csv" + +$allsubnets | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVirtualMachinesPropertiesToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVirtualMachinesPropertiesToBlobStorage.ps1 new file mode 100644 index 000000000..a30511aad --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ARGVirtualMachinesPropertiesToBlobStorage.ps1 @@ -0,0 +1,340 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ARGVMContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "argvmexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +# get list of all VM sizes +Write-Output "Getting VM sizes details for $referenceRegion" +$sizes = Get-AzVMSize -Location $referenceRegion + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allvms = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +$armVmsTotal = @() +$classicVmsTotal = @() + +$resultsSoFar = 0 + +<# + Getting all ARM VMs properties with Azure Resource Graph query +#> + +Write-Output "Querying for ARM VM properties" + +$argQuery = @" + resources + | where type =~ 'Microsoft.Compute/virtualMachines' + | extend dataDiskCount = array_length(properties.storageProfile.dataDisks), nicCount = array_length(properties.networkProfile.networkInterfaces) + | extend usesManagedDisks = iif(isnull(properties.storageProfile.osDisk.managedDisk), 'false', 'true') + | extend availabilitySetId = tostring(properties.availabilitySet.id) + | extend bootDiagnosticsEnabled = tostring(properties.diagnosticsProfile.bootDiagnostics.enabled) + | extend bootDiagnosticsStorageAccount = split(split(properties.diagnosticsProfile.bootDiagnostics.storageUri, '/')[2],'.')[0] + | extend powerState = tostring(properties.extended.instanceView.powerState.code) + | extend imagePublisher = iif(isnotempty(properties.storageProfile.imageReference.publisher),tostring(properties.storageProfile.imageReference.publisher),'Custom') + | extend imageOffer = iif(isnotempty(properties.storageProfile.imageReference.offer),tostring(properties.storageProfile.imageReference.offer),tostring(properties.storageProfile.imageReference.id)) + | extend imageSku = tostring(properties.storageProfile.imageReference.sku) + | extend imageVersion = tostring(properties.storageProfile.imageReference.version) + | extend imageExactVersion = tostring(properties.storageProfile.imageReference.exactVersion) + | extend osName = tostring(properties.extended.instanceView.osName) + | extend osVersion = tostring(properties.extended.instanceView.osVersion) + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $armVms = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $armVms = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($armVms -and $armVms.GetType().Name -eq "PSResourceGraphResponse") + { + $armVms = $armVms.Data + } + $resultsCount = $armVms.Count + $resultsSoFar += $resultsCount + $armVmsTotal += $armVms + +} while ($resultsCount -eq $ARGPageSize) + +$resultsSoFar = 0 + +<# + Getting all Classic VMs properties with Azure Resource Graph query +#> + +Write-Output "Querying for Classic VM properties" + +$argQuery = @" + resources + | where type =~ 'Microsoft.ClassicCompute/virtualMachines' + | extend dataDiskCount = iif(isnotnull(properties.storageProfile.dataDisks), array_length(properties.storageProfile.dataDisks), 0), nicCount = iif(isnotnull(properties.networkProfile.virtualNetwork.networkInterfaces), array_length(properties.networkProfile.virtualNetwork.networkInterfaces) + 1, 1) + | extend usesManagedDisks = 'false' + | extend availabilitySetId = tostring(properties.hardwareProfile.availabilitySet) + | extend bootDiagnosticsEnabled = tostring(properties.debugProfile.bootDiagnosticsEnabled) + | extend bootDiagnosticsStorageAccount = split(split(properties.debugProfile.serialOutputBlobUri, '/')[2],'.')[0] + | extend powerState = tostring(properties.instanceView.status) + | extend imageOffer = tostring(properties.storageProfile.operatingSystemDisk.sourceImageName) + | order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $classicVms = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $classicVms = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($classicVms -and $classicVms.GetType().Name -eq "PSResourceGraphResponse") + { + $classicVms = $classicVms.Data + } + $resultsCount = $classicVms.Count + $resultsSoFar += $resultsCount + $classicVmsTotal += $classicVms + +} while ($resultsCount -eq $ARGPageSize) + +<# + Merging ARM + Classic VMs, enriching VM size details and building CSV entries +#> + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +Write-Output "Building $($armVmsTotal.Count) ARM VM entries" + +foreach ($vm in $armVmsTotal) +{ + $vmSize = $sizes | Where-Object {$_.name -eq $vm.properties.hardwareProfile.vmSize} + + $avSetId = $null + if ($vm.availabilitySetId) + { + $avSetId = $vm.availabilitySetId.ToLower() + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $vm.tenantId + SubscriptionGuid = $vm.subscriptionId + ResourceGroupName = $vm.resourceGroup.ToLower() + Zones = $vm.zones + Location = $vm.location + VMName = $vm.name.ToLower() + DeploymentModel = 'ARM' + InstanceId = $vm.id.ToLower() + VMSize = $vm.properties.hardwareProfile.vmSize + CoresCount = $vmSize.NumberOfCores + MemoryMB = $vmSize.MemoryInMB + OSType = $vm.properties.storageProfile.osDisk.osType + LicenseType = $vm.properties.licenseType + DataDiskCount = $vm.dataDiskCount + NicCount = $vm.nicCount + UsesManagedDisks = $vm.usesManagedDisks + AvailabilitySetId = $avSetId + BootDiagnosticsEnabled = $vm.bootDiagnosticsEnabled + BootDiagnosticsStorageAccount = $vm.bootDiagnosticsStorageAccount + StatusDate = $statusDate + PowerState = $vm.powerState + ImagePublisher = $vm.imagePublisher + ImageOffer = $vm.imageOffer + ImageSku = $vm.imageSku + ImageVersion = $vm.imageVersion + ImageExactVersion = $vm.imageExactVersion + OSName = $vm.osName + OSVersion = $vm.osVersion + Tags = $vm.tags + } + + $allvms += $logentry +} + +Write-Output "Building $($classicVmsTotal.Count) Classic VM entries" + +foreach ($vm in $classicVmsTotal) +{ + $vmSize = $sizes | Where-Object {$_.name -eq $vm.properties.hardwareProfile.size} + + $avSetId = $null + if ($vm.availabilitySetId) + { + $avSetId = $vm.availabilitySetId.ToLower() + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $vm.tenantId + SubscriptionGuid = $vm.subscriptionId + ResourceGroupName = $vm.resourceGroup.ToLower() + VMName = $vm.name.ToLower() + DeploymentModel = 'Classic' + Location = $vm.location + InstanceId = $vm.id.ToLower() + VMSize = $vm.properties.hardwareProfile.size + CoresCount = $vmSize.NumberOfCores + MemoryMB = $vmSize.MemoryInMB + OSType = $vm.properties.storageProfile.operatingSystemDisk.operatingSystem + LicenseType = "N/A" + DataDiskCount = $vm.dataDiskCount + NicCount = $vm.nicCount + UsesManagedDisks = $vm.usesManagedDisks + AvailabilitySetId = $avSetId + BootDiagnosticsEnabled = $vm.bootDiagnosticsEnabled + BootDiagnosticsStorageAccount = $vm.bootDiagnosticsStorageAccount + PowerState = $vm.powerState + StatusDate = $statusDate + ImagePublisher = $vm.imagePublisher + ImageOffer = $vm.imageOffer + ImageSku = $vm.imageSku + ImageVersion = $vm.imageVersion + ImageExactVersion = $vm.imageExactVersion + OSName = $vm.osName + OSVersion = $vm.osVersion + Tags = $null + } + + $allvms += $logentry +} + +<# + Actually exporting CSV to Azure Storage +#> + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-vms-$subscriptionSuffix.csv" + +$allvms | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AdvisorRecommendationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AdvisorRecommendationsToBlobStorage.ps1 new file mode 100644 index 000000000..cc82c3e90 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AdvisorRecommendationsToBlobStorage.ps1 @@ -0,0 +1,247 @@ +param( + [Parameter(Mandatory = $false)] + [string] $targetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_AdvisorContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "advisorexports" +} + +$CategoryFilter = Get-AutomationVariable -Name "AzureOptimization_AdvisorFilter" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($CategoryFilter)) +{ + $CategoryFilter = "HighAvailability,Security,Performance,OperationalExcellence" # comma-separated list of categories +} +$CategoryFilter += ",Cost" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +Write-Output "Getting subscriptions target $TargetSubscription" + +$tenantId = (Get-AzContext).Tenant.Id + +$ARGPageSize = 1000 + +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $scope = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" -and $_.SubscriptionPolicies.QuotaId -notlike "AAD*" } | ForEach-Object { "$($_.Id)"} + $scope = $tenantId +} + + +<# + Getting Advisor recommendations for each subscription and building CSV entries +#> + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +$recommendationsARG = @() + +$resultsSoFar = 0 + +$FinalCategoryFilter = "" + +if (-not([string]::IsNullOrEmpty($CategoryFilter))) +{ + $categories = $CategoryFilter.Split(',') + for ($i = 0; $i -lt $categories.Count; $i++) + { + $categories[$i] = "'" + $categories[$i] + "'" + } + $FinalCategoryFilter = " and properties.category in (" + ($categories -join ",") + ")" +} + +$argQuery = @" +advisorresources +| where type == 'microsoft.advisor/recommendations' +| where isnull(properties.suppressionIds)$FinalCategoryFilter +| extend resourceId = tostring(split(tolower(id),'/providers/microsoft.advisor')[0]) +| join kind=leftouter (resources | project resourceId=tolower(id), resourceTags=tags) on resourceId +| project id, category = properties.category, impact = properties.impact, impactedArea = properties.impactedField, + description = properties.shortDescription.problem, recommendationText = properties.shortDescription.solution, + recommendationTypeId = properties.recommendationTypeId, instanceName = properties.impactedValue, + additionalInfo = properties.extendedProperties, tags=resourceTags +| order by id asc +"@ + +do +{ + if ($resultsSoFar -eq 0) + { + $recs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $recs = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($recs -and $recs.GetType().Name -eq "PSResourceGraphResponse") + { + $recs = $recs.Data + } + $resultsCount = $recs.Count + $resultsSoFar += $resultsCount + $recommendationsARG += $recs + +} while ($resultsCount -eq $ARGPageSize) + +Write-Output "Building $($recommendationsARG.Count) recommendations entries" + +$recommendations = @() + +foreach ($advisorRecommendation in $recommendationsARG) +{ + $resourceIdParts = $advisorRecommendation.id.Split('/') + if ($resourceIdParts.Count -ge 9) + { + # if the Resource ID is made of 9 parts, then the recommendation is relative to a specific Azure resource + $realResourceIdParts = $resourceIdParts[0..8] + $instanceId = ($realResourceIdParts -join "/").ToLower() + $resourceGroup = $realResourceIdParts[4].ToLower() + $subscriptionId = $realResourceIdParts[2] + } + else + { + # otherwise it is not a resource-specific recommendation (e.g., reservations) + $resourceGroup = "notavailable" + $instanceId = $advisorRecommendation.id.ToLower() + $subscriptionId = $resourceIdParts[2] + } + + if (-not([string]::IsNullOrEmpty($advisorRecommendation.additionalInfo))) + { + $additionalInfo = $advisorRecommendation.additionalInfo | ConvertTo-Json -Compress + } + else + { + $additionalInfo = $null + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + Category = $advisorRecommendation.category + Impact = $advisorRecommendation.impact + ImpactedArea = $advisorRecommendation.impactedArea + Description = $advisorRecommendation.description + RecommendationText = $advisorRecommendation.recommendationText + RecommendationTypeId = $advisorRecommendation.recommendationTypeId + InstanceId = $instanceId + InstanceName = $advisorRecommendation.instanceName + Tags = $advisorRecommendation.tags + AdditionalInfo = $additionalInfo + ResourceGroup = $resourceGroup + SubscriptionGuid = $subscriptionId + TenantGuid = $tenantId + } + + $recommendations += $recommendation +} + +Write-Output "Found $($recommendations.Count) ($CategoryFilter) recommendations..." + +$fileDate = $datetime.ToString("yyyyMMdd") +$advisorFilter = $CategoryFilter.Replace(',','').ToLower() +$csvExportPath = "$fileDate-$advisorFilter-$scope.csv" + +$recommendations | Export-Csv -NoTypeInformation -Path $csvExportPath +Write-Output "Export to $csvExportPath" + +$csvBlobName = $csvExportPath +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." + +Write-Output "DONE!" \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AzMonitorMetricsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AzMonitorMetricsToBlobStorage.ps1 new file mode 100644 index 000000000..e31dd034f --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-AzMonitorMetricsToBlobStorage.ps1 @@ -0,0 +1,296 @@ +Param ( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $true)] + [string] $ResourceType, # ARM resource type + + [Parameter(Mandatory = $false)] + [string] $ARGFilter, # e.g., name != 'master' and sku.tier in ('Basic','Standard','Premium') + + [Parameter(Mandatory = $true)] + [string] $MetricNames, # comma-separated metrics names (use Get-AzMetricDefinition for a list of supported metric names for a given resource) + + [Parameter(Mandatory = $true)] + [ValidateSet("Maximum", "Minimum", "Average", "Total")] + [string] $AggregationType, + + [Parameter(Mandatory = $false)] + [ValidateSet("Default", "Maximum", "Minimum", "Average", "Total")] + [string] $AggregationOfType = "Default", + + [Parameter(Mandatory = $true)] + [string] $TimeSpan, # [d.]hh:mm:ss + + [Parameter(Mandatory = $true)] + [string] $TimeGrain, # [d.]hh:mm:ss (00:01:00, 00:05:00, 00:15:00, 00:30:00, 01:00:00, 06:00:00, 12:00:00, 1.00:00:00, 7.00:00:00, 30.00:00:00) + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_AzMonitorContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "azmonitorexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +if (-not([string]::IsNullOrEmpty($TargetSubscription))) { + $subscriptions = $TargetSubscription + $subscriptionSuffix = "-" + $TargetSubscription +} +else { + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +[TimeSpan]::Parse($TimeGrain) | Out-Null +$TimeSpanObj = [TimeSpan]::Parse("-$TimeSpan") + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Querying for $ResourceType with page size $ARGPageSize and target subscription $TargetSubscription..." + +$allResources = @() + +$resultsSoFar = 0 + +$argWhere = "" +if (-not([string]::IsNullOrEmpty($ARGFilter))) +{ + $argWhere = " and $ARGFilter" +} + +$argQuery = @" +resources +| where type =~ '$ResourceType'$argWhere +| project id, name, subscriptionId, resourceGroup, tenantId +| order by id asc +"@ + +do { + if ($resultsSoFar -eq 0) { + $resources = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else { + $resources = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($resources -and $resources.GetType().Name -eq "PSResourceGraphResponse") + { + $resources = $resources.Data + } + $resultsCount = $resources.Count + $resultsSoFar += $resultsCount + $allResources += $resources + +} while ($resultsCount -eq $ARGPageSize) + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Found $($allResources.Count) resources." + +$metrics = $MetricNames.Split(',') + +$queryDate = Get-Date +$utcNow = $queryDate.ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +$utcAgo = $queryDate.Add($TimeSpanObj).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + +$customMetrics = @() + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Analyzing resources for $MetricNames metrics ($AggregationType with $TimeGrain time grain) since $utcAgo..." + +foreach ($resource in $allResources) { + $valuesAggregation = @() + $foundResource = $true + foreach ($metric in $metrics) { + $metricValues = Get-AzMetric -ResourceId $resource.id -MetricName $metric -TimeGrain $TimeGrain -AggregationType $AggregationType ` + -StartTime $utcAgo -EndTime $utcNow -WarningAction SilentlyContinue -ErrorAction Continue + if ($metricValues.Data) { + if ($valuesAggregation.Count -eq 0) { + $valuesAggregation = $metricValues.Data."$AggregationType" + } + else { + for ($i = 0; $i -lt $valuesAggregation.Count; $i++) { + if ($metricValues.Data.Count -gt 1) + { + $valuesAggregation[$i] += $metricValues.Data[$i]."$AggregationType" + } + else + { + $valuesAggregation += $metricValues.Data."$AggregationType" + } + } + } + } + + if (-not($metricValues.Id)) + { + $foundResource = $false + } + } + + if ($foundResource) + { + $aggregatedValue = $null + $finalAggregationType = $AggregationType + if ($AggregationOfType -ne "Default") + { + $finalAggregationType = $AggregationOfType + } + if ($valuesAggregation.Count -gt 0) { + switch ($finalAggregationType) { + "Maximum" { + $aggregatedValue = ($valuesAggregation | Measure-Object -Maximum).Maximum + } + "Minimum" { + $aggregatedValue = ($valuesAggregation | Measure-Object -Minimum).Minimum + } + "Average" { + $aggregatedValue = ($valuesAggregation | Measure-Object -Average).Average + } + "Total" { + $aggregatedValue = ($valuesAggregation | Measure-Object -Sum).Sum + } + } + } + + $customMetric = New-Object PSObject -Property @{ + Timestamp = $utcNow + Cloud = $cloudEnvironment + TenantGuid = $resource.tenantId + SubscriptionGuid = $resource.subscriptionId + ResourceGroupName = $resource.resourceGroup.ToLower() + ResourceName = $resource.name.ToLower() + ResourceId = $resource.id.ToLower() + MetricNames = $MetricNames + AggregationType = $AggregationType + AggregationOfType = $AggregationOfType + MetricValue = $aggregatedValue + TimeGrain = $TimeGrain + TimeSpan = $TimeSpan + } + + $customMetrics += $customMetric + } +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Found $($customMetrics.Count) resources to collect metrics from..." + +$metricMoment = $queryDate.Add($TimeSpanObj).ToUniversalTime().ToString("yyyyMMddHHmmss") +$ResourceTypeName = $ResourceType.Split('/')[1].ToLower() +$MetricName = $MetricNames.Replace(',','').Replace(' ','').Replace('/','').ToLower() +$AggregationOfTypeName = "" +if ($AggregationOfType -ne "Default") +{ + $AggregationOfTypeName = ("-$AggregationOfType").ToLower() +} +$AggregationTypeName = "$($AggregationType.ToLower())$AggregationOfTypeName" +$csvExportPath = "$metricMoment-metrics-$ResourceTypeName-$MetricName-$AggregationTypeName-$subscriptionSuffix.csv" + +$ci = [CultureInfo]::new([System.Threading.Thread]::CurrentThread.CurrentCulture.Name) +if ($ci.NumberFormat.NumberDecimalSeparator -ne '.') +{ + Write-Output "Current culture ($($ci.Name)) does not use . as decimal separator" + $ci.NumberFormat.NumberDecimalSeparator = '.' + [System.Threading.Thread]::CurrentThread.CurrentCulture = $ci +} + +$customMetrics | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." + diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 new file mode 100644 index 000000000..eac393878 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 @@ -0,0 +1,790 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName, + + [Parameter(Mandatory = $false)] + [string] $targetStartDate, # YYYY-MM-DD format + + [Parameter(Mandatory = $false)] + [string] $targetEndDate # YYYY-MM-DD format +) + +$ErrorActionPreference = "Stop" +$global:hadErrors = $false +$global:scopesWithErrors = @() + +function Authenticate-AzureWithOption { + param ( + [string] $authOption = "ManagedIdentity", + [string] $cloudEnv = "AzureCloud", + [string] $clientID + ) + + switch ($authOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnv -AccountId $clientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnv + break + } + } +} + +function Generate-CostDetails { + param ( + [string] $ScopeId, + [string] $ScopeName + ) + + $MaxTries = 20 # The typical Retry-After is set to 20 seconds. We'll give ~6 minutes overall to download the cost details report + $hadErrors = $false + + $CostDetailsApiPath = "$ScopeId/providers/Microsoft.CostManagement/generateCostDetailsReport?api-version=2022-05-01" + $body = "{ `"metric`": `"$consumptionMetric`", `"timePeriod`": { `"start`": `"$targetStartDate`", `"end`": `"$targetEndDate`" } }" + $result = Invoke-AzRestMethod -Path $CostDetailsApiPath -Method POST -Payload $body + $requestResultPath = $result.Headers.Location.PathAndQuery + if ($result.StatusCode -in (200,202)) + { + $tries = 0 + $requestSuccess = $false + + Write-Output "Obtained cost detail results endpoint: $requestResultPath..." + + Write-Output "Was told to wait $($result.Headers.RetryAfter.Delta.TotalSeconds) seconds." + + $sleepSeconds = 60 + if ($result.Headers.RetryAfter.Delta.TotalSeconds -gt 0) + { + $sleepSeconds = $result.Headers.RetryAfter.Delta.TotalSeconds + } + + do + { + $tries++ + Write-Output "Checking whether export is ready (try $tries)..." + + Start-Sleep -Seconds $sleepSeconds + $downloadResult = Invoke-AzRestMethod -Method GET -Path $requestResultPath + + if ($downloadResult.StatusCode -eq 200) + { + + Write-Output "Export is ready. Proceeding with CSV download..." + + $downloadBlobJson = $downloadResult.Content | ConvertFrom-Json + + $blobCounter = 0 + foreach ($blob in $downloadBlobJson.manifest.blobs) + { + $blobCounter++ + + Write-Output "Downloading blob $blobCounter..." + + $csvExportPath = "$env:TEMP\$targetStartDate-$ScopeName-$consumptionMetric-$blobCounter.csv" + $finalCsvExportPath = "$env:TEMP\$targetStartDate-$ScopeName-$consumptionMetric-$blobCounter-final.csv" + + Invoke-WebRequest -Uri $blob.blobLink -OutFile $csvExportPath + + Write-Output "Blob downloaded to $csvExportPath successfully." + + $r = [IO.File]::OpenText($csvExportPath) + $w = [System.IO.StreamWriter]::new($finalCsvExportPath) + + # header normalization between MCA and EA + $headerConversion = @{ + additionalInfo = "AdditionalInfo"; + billingAccountId = "BillingAccountId"; + billingAccountName = "BillingAccountName"; + billingCurrency = "BillingCurrencyCode"; + billingPeriodEndDate = "BillingPeriodEndDate"; + billingPeriodStartDate = "BillingPeriodStartDate"; + billingProfileId = "BillingProfileId"; + billingProfileName = "BillingProfileName"; + chargeType = "ChargeType"; + consumedService = "ConsumedService"; + costAllocationRuleName = "CostAllocationRuleName"; + costCenter = "CostCenter"; + costInBillingCurrency = "CostInBillingCurrency"; + date = "Date"; + effectivePrice = "EffectivePrice"; + frequency = "Frequency"; + invoiceSectionId = "InvoiceSectionId"; + invoiceSectionName = "InvoiceSectionName"; + isAzureCreditEligible = "IsAzureCreditEligible"; + meterCategory = "MeterCategory"; + meterId = "MeterId"; + meterName = "MeterName"; + meterRegion = "MeterRegion"; + meterSubCategory = "MeterSubCategory"; + offerId = "OfferId"; + pricingModel = "PricingModel"; + productOrderId = "ProductOrderId"; + productOrderName = "ProductOrderName"; + publisherName = "PublisherName"; + publisherType = "PublisherType"; + quantity = "Quantity"; + reservationId = "ReservationId"; + reservationName = "ReservationName"; + resourceGroupName = "ResourceGroup"; + resourceLocation = "ResourceLocation"; + serviceFamily = "ServiceFamily"; + serviceInfo1 = "ServiceInfo1"; + serviceInfo2 = "ServiceInfo2"; + subscriptionName = "SubscriptionName"; + tags = "Tags"; + term = "Term"; + unitOfMeasure = "UnitOfMeasure"; + unitPrice = "UnitPrice" + } + + $lineCounter = 0 + while ($r.Peek() -ge 0) { + $line = $r.ReadLine() + $lineCounter++ + if ($lineCounter -eq 1) + { + $headers = $line.Split(",") + + for ($i = 0; $i -lt $headers.Length; $i++) + { + $header = $headers[$i] + if ($headerConversion.ContainsKey($header)) + { + $headers[$i] = $headerConversion[$header] + } + } + + $line = $headers -join "," + + $w.WriteLine($line) + } + else + { + $w.WriteLine($line) + } + } + $r.Dispose() + $w.Close() + + $csvBlobName = [System.IO.Path]::GetFileName($finalCsvExportPath) + $csvProperties = @{"ContentType" = "text/csv"}; + Set-AzStorageBlobContent -File $finalCsvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + + Remove-Item -Path $csvExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "[$now] Removed $csvExportPath from local disk..." + + Remove-Item -Path $finalCsvExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "[$now] Removed $finalCsvExportPath from local disk..." + } + + $requestSuccess = $true + } + elseif ($downloadResult.StatusCode -eq 202) + { + Write-Output "Was told to wait a bit more... $($downloadResult.Headers.RetryAfter.Delta.TotalSeconds) seconds." + + $sleepSeconds = 60 + if ($downloadResult.Headers.RetryAfter.Delta.TotalSeconds -gt 0) + { + $sleepSeconds = $downloadResult.Headers.RetryAfter.Delta.TotalSeconds + } + } + elseif ($downloadResult.StatusCode -eq 401) + { + Write-Output "Had an authentication issue. Will login again and sleep just a couple of seconds." + + if ($authenticationOption -eq "UserAssignedManagedIdentity") + { + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment -clientID $uamiClientID + } + else + { + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment + } + + $sleepSeconds = 2 + } + else + { + $global:hadErrors = $true + $global:scopesWithErrors += $ScopeName + Write-Warning "Got an unexpected response code: $($downloadResult.StatusCode)" + } + } + while (-not($requestSuccess) -and $tries -lt $MaxTries) + + if (-not($requestSuccess)) + { + $global:hadErrors = $true + $global:scopesWithErrors += $ScopeName + if ($tries -eq $MaxTries) + { + Write-Warning "Reached maximum number of tries. Aborting..." + } + else + { + Write-Warning "Error returned by the Download Cost Details API. Status Code: $($downloadResult.StatusCode). Message: $($downloadResult.Content)" + } + } + else + { + Write-Output "Export download processing complete." + } + } + else + { + if ($result.StatusCode -ne 204) + { + $global:hadErrors = $true + $global:scopesWithErrors += $ScopeName + Write-Warning "Error returned by the Generate Cost Details API. Status Code: $($result.StatusCode). Message: $($result.Content)" + } + else + { + Write-Output "Request returned 204 No Content" + } + } +} + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ConsumptionContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "consumptionexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") + +$consumptionMetric = Get-AutomationVariable -Name "AzureOptimization_ConsumptionMetric" -ErrorAction SilentlyContinue # AmortizedCost|ActualCost +if ([string]::IsNullOrEmpty($consumptionMetric)) +{ + $consumptionMetric = "AmortizedCost" +} + +$consumptionAPIOption = Get-AutomationVariable -Name "AzureOptimization_ConsumptionAPIOption" -ErrorAction SilentlyContinue # CostDetails|UsageDetails +if ([string]::IsNullOrEmpty($consumptionAPIOption)) +{ + $consumptionAPIOption = "CostDetails" +} + +$consumptionScope = Get-AutomationVariable -Name "AzureOptimization_ConsumptionScope" -ErrorAction SilentlyContinue # Subscription|BillingAccount +if ([string]::IsNullOrEmpty($consumptionScope)) +{ + "Consumption Scope not specified, defaulting to Subscription" + $consumptionScope = "Subscription" +} +else +{ + "Consumption Scope is $consumptionScope" + if ($consumptionScope -eq "BillingAccount") + { + $BillingAccountID = Get-AutomationVariable -Name "AzureOptimization_BillingAccountID" + } + else + { + if ($consumptionScope -ne "Subscription") + { + throw "Invalid value for AzureOptimization_ConsumptionScope. Valid values are 'Subscription' or 'BillingAccount'." + } + } +} + +if ($cloudEnvironment -eq "AzureChinaCloud") +{ + $chinaEAEnrollment = Get-AutomationVariable -Name "AzureOptimization_AzureChinaEAEnrollment" -ErrorAction SilentlyContinue + $chinaEAKey = Get-AutomationVariable -Name "AzureOptimization_AzureChinaEAKey" -ErrorAction SilentlyContinue +} + +"Logging in to Azure with $authenticationOption..." + +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment -clientID $uamiClientID +} +else +{ + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +# compute start+end dates + +if ([string]::IsNullOrEmpty($targetStartDate) -or [string]::IsNullOrEmpty($targetEndDate)) +{ + $targetStartDate = (Get-Date).Date.AddDays($consumptionOffsetDays * -1).ToString("yyyy-MM-dd") + $targetEndDate = $targetStartDate +} + +if ($consumptionScope -eq "Subscription") +{ + if (-not([string]::IsNullOrEmpty($TargetSubscription))) + { + $subscriptions = Get-AzSubscription -SubscriptionId $TargetSubscription + } + else + { + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } + } + "Exporting consumption data from $targetStartDate to $targetEndDate for $($subscriptions.Count) subscriptions..." +} +else +{ + "Exporting consumption data from $targetStartDate to $targetEndDate for Billing Account ID $BillingAccountID..." +} + + +# for each subscription, get billing data + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($chinaEAEnrollment)) -and -not([string]::IsNullOrEmpty($chinaEAKey))) +{ + $targetMonth = $targetStartDate.Substring(0,7) + $consumption = $null + $billingEntries = @() + + $BillingApiUri = "https://ea.azure.cn/rest/$chinaEAEnrollment/usage-report?month=$targetMonth&type=detail&fmt=Csv" + $PricesheetApiUri = "https://ea.azure.cn/rest/$chinaEAEnrollment/usage-report?month=$targetMonth&type=pricesheet&fmt=Csv" + + $Headers = @{} + $Headers.Add("Authorization","Bearer $chinaEAKey") + + Write-Output "Getting pricesheet for month $targetMonth (EA enrollment $chinaEAEnrollment)..." + + Invoke-RestMethod -Method Get -Uri $PricesheetApiUri -Headers $Headers -OutFile "pricesheet-$targetMonth.csv" + + Write-Output "Pricesheet data exported to disk as CSV." + + $csvFile = Get-Content -Path "pricesheet-$targetMonth.csv" + + Write-Output "Pricesheet data imported from disk as string." + + Remove-Item -Path "pricesheet-$targetMonth.csv" -Force + + Write-Output "Removed pricesheet-$targetMonth.csv from local disk..." + + $csvFile2 = $csvFile[2..($csvFile.Count-1)] + $headerLine = $csvFile2[0] + $columnHeaders = $headerLine.Split(",") + for ($i = 0; $i -lt $columnHeaders.Count; $i++) + { + if($columnHeaders[$i] -match '.+\((?.+)\)') + { + $columnHeaders[$i] = $Matches.ColumnName + } + } + $csvFile2[0] = $columnHeaders -join "," + + Write-Output "Removed first 2 lines and replaced header." + + $pricesheet = $csvFile2 | ConvertFrom-Csv + + Write-Output "Starting Azure China billing export process from $targetStartDate to $targetEndDate (month $targetMonth) for EA enrollment $chinaEAEnrollment..." + + $tries = 0 + $requestSuccess = $false + do + { + try { + $tries++ + Invoke-RestMethod -Method Get -Uri $BillingApiUri -Headers $Headers -OutFile "usagedetails-$targetStartDate.csv" + + Write-Output "Consumption data exported to disk as CSV." + + $csvFile = Get-Content -Path "usagedetails-$targetStartDate.csv" + + Write-Output "Consumption data imported from disk as string." + + Remove-Item -Path "usagedetails-$targetStartDate.csv" -Force + + Write-Output "Removed usagedetails-$targetStartDate.csv from local disk..." + + $csvFile2 = $csvFile[2..($csvFile.Count-1)] + $headerLine = $csvFile2[0] + $columnHeaders = $headerLine.Split(",") + for ($i = 0; $i -lt $columnHeaders.Count; $i++) + { + if($columnHeaders[$i] -match '.+\((?.+)\)') + { + $columnHeaders[$i] = $Matches.ColumnName + } + } + $csvFile2[0] = $columnHeaders -join "," + + Write-Output "Removed first 2 lines and replaced header." + + $consumption = $csvFile2 | ConvertFrom-Csv + $requestSuccess = $true + } + catch { + $ErrorMessage = $_.Exception.Message + Write-Warning "Error getting consumption data: $ErrorMessage. $tries of 3 tries. Waiting 60 seconds..." + Start-Sleep -s 60 + } + + } while ( -not($requestSuccess) -and $tries -lt 3 ) + + if (-not($requestSuccess)) + { + throw "Failed consumption export" + } + + Write-Output "Consumption data in memory as CSV. Processing lines..." + + foreach ($consumptionLine in $consumption) + { + $usageDate = [Datetime]::ParseExact($consumptionLine.Date, 'MM/dd/yyyy', $null).ToString("yyyy-MM-dd") + + if ($usageDate -ge $targetStartDate -and $usageDate -le $targetEndDate -and ($subscriptions.Count -gt 1 -or $subscriptions.Id -eq $consumptionLine.SubscriptionGuid)) + { + $instanceId = $null + $instanceName = $null + if ($null -ne $consumptionLine.'Instance ID') + { + $instanceId = $consumptionLine.'Instance ID'.ToLower() + $idParts = $consumptionLine.'Instance ID'.Split("/") + $instanceName = $idParts[$idParts.Count-1].ToLower() + } + + $rgName = $null + if ($null -ne $consumptionLine.'Resource Group') + { + $rgName = $consumptionLine.'Resource Group'.ToLower() + } + + $convertedCost = 0.0 + if ([double]$consumptionLine.ExtendedCost -ne 0) + { + $convertedCost = [double]$consumptionLine.ExtendedCost + } + $convertedPrice = 0.0 + if ([double]$consumptionLine.ResourceRate -ne 0) + { + $convertedPrice = [double]$consumptionLine.ResourceRate + } + + $unitPrice = 0.0 + $partNumber = "N/A" + foreach ($priceItem in $pricesheet) + { + if ($priceItem.Service -eq $consumptionLine.Product) + { + $partNumber = $priceItem.'Part Number' + if ($consumptionLine.'Meter Category' -eq "Virtual Machines") + { + $tempUnitPrice = [double] $priceItem.'Unit Price' + $uom = $priceItem.'Unit of Measure' + $currentUnitHours = [int] (Select-String -InputObject $uom -Pattern "^\d+").Matches[0].Value + if ($currentUnitHours -gt 0) + { + $unitPrice = [double] ($tempUnitPrice / $currentUnitHours) + } + } + else + { + $unitPrice = $convertedPrice + } + break + } + } + + $billingEntry = New-Object PSObject -Property @{ + Timestamp = $timestamp + SubscriptionId = $consumptionLine.SubscriptionGuid + ResourceGroup = $rgName + ResourceName = $instanceName + ResourceId = $instanceId + Date = $consumptionLine.Date + Tags = $consumptionLine.Tags + AdditionalInfo = $consumptionLine.AdditionalInfo + BillingCurrencyCode = "CNY" + ChargeType = "Usage" + ConsumedService = $consumptionLine.'Consumed Service' + CostInBillingCurrency = $convertedCost + EffectivePrice = $convertedPrice + Frequency = "UsageBased" + MeterCategory = $consumptionLine.'Meter Category' + MeterId = $consumptionLine.'Meter ID' + MeterName = $consumptionLine.'Meter Name' + MeterSubCategory = $consumptionLine.'Meter Sub-Category' + PartNumber = $partNumber + ProductName = $consumptionLine.Product + Quantity = $consumptionLine.'Consumed Quantity' + UnitOfMeasure = $consumptionLine.'Unit of Measure' + UnitPrice = $unitPrice + ResourceLocation = $consumptionLine.'Resource Location' + AccountOwnerId = $consumptionLine.AccountOwnerId + } + + $billingEntries += $billingEntry + } + } + + if ($targetStartDate -ne $targetEndDate) + { + $targetStartDate = "$targetStartDate-$targetEndDate" + } + + $csvExportPath = "$targetStartDate-eachina.csv" + + $billingEntries | Export-Csv -Path $csvExportPath -NoTypeInformation + + Write-Output "Exported $($billingEntries.Count) entries as CSV to $csvExportPath" + + $csvBlobName = $csvExportPath + $csvProperties = @{"ContentType" = "text/csv"}; + Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + Write-Output "Uploaded to blob storage!" + + Remove-Item -Path $csvExportPath -Force + + Write-Output "Removed $csvExportPath from local disk..." +} +else +{ + if ($consumptionScope -eq "Subscription") + { + $CostDetailsSupportedQuotaIDs = @('EnterpriseAgreement_2014-09-01','Internal_2014-09-01','CSP_2015-05-01') + $ConsumptionSupportedQuotaIDs = @('PayAsYouGo_2014-09-01','MSDN_2014-09-01') + + foreach ($subscription in $subscriptions) + { + $subscriptionQuotaID = $subscription.SubscriptionPolicies.QuotaId + + if ($subscriptionQuotaID -in $ConsumptionSupportedQuotaIDs -or $consumptionAPIOption -eq "UsageDetails") + { + $consumption = $null + $billingEntries = @() + + $ConsumptionApiPath = "/subscriptions/$($subscription.Id)/providers/Microsoft.Consumption/usageDetails?api-version=2021-10-01&metric=$($consumptionMetric.ToLower())&%24expand=properties%2FmeterDetails%2Cproperties%2FadditionalInfo&%24filter=properties%2FusageStart%20ge%20%27$targetStartDate%27%20and%20properties%2FusageEnd%20le%20%27$targetEndDate%27" + + "Starting consumption export process from $targetStartDate to $targetEndDate for subscription $($subscription.Name)..." + + do + { + if (-not([string]::IsNullOrEmpty($consumption.nextLink))) + { + $ConsumptionApiPath = $consumption.nextLink.Substring($consumption.nextLink.IndexOf("/subscriptions/")) + } + $tries = 0 + $requestSuccess = $false + do + { + try { + $tries++ + $consumption = (Invoke-AzRestMethod -Path $ConsumptionApiPath -Method GET).Content | ConvertFrom-Json + $requestSuccess = $true + } + catch { + $ErrorMessage = $_.Exception.Message + Write-Warning "Error getting consumption data: $ErrorMessage. $tries of 3 tries. Waiting 60 seconds..." + Start-Sleep -s 60 + } + } while ( -not($requestSuccess) -and $tries -lt 3 ) + + foreach ($consumptionLine in $consumption.value) + { + if ((Get-Date $consumptionLine.properties.date).ToString("yyyy-MM-dd") -ge $targetStartDate -and (Get-Date $consumptionLine.properties.date).ToString("yyyy-MM-dd") -le $targetEndDate) + { + if ($consumptionLine.tags) + { + $tags = $consumptionLine.tags | ConvertTo-Json -Compress + } + else + { + $tags = $null + } + + $billingEntry = New-Object PSObject -Property @{ + Timestamp = $timestamp + AccountName = $consumptionLine.properties.accountName + AccountOwnerId = $consumptionLine.properties.accountOwnerId + AdditionalInfo = $consumptionLine.properties.additionalInfo + benefitId = $consumptionLine.properties.benefitId + benefitName = $consumptionLine.properties.benefitName + BillingAccountId = $consumptionLine.properties.billingAccountId + BillingAccountName = $consumptionLine.properties.billingAccountName + BillingCurrencyCode = $consumptionLine.properties.billingCurrency + BillingPeriodEndDate= $consumptionLine.properties.billingPeriodEndDate + BillingPeriodStartDate= $consumptionLine.properties.billingPeriodStartDate + BillingProfileId = $consumptionLine.properties.billingProfileId + BillingProfileName= $consumptionLine.properties.billingProfileName + ChargeType = $consumptionLine.properties.chargeType + ConsumedService = $consumptionLine.properties.consumedService + CostAllocationRuleName = $consumptionLine.properties.costAllocationRuleName + CostCenter = $consumptionLine.properties.costCenter + CostInBillingCurrency = $consumptionLine.properties.cost + Date = (Get-Date $consumptionLine.properties.date).ToString("MM/dd/yyyy") + EffectivePrice = $consumptionLine.properties.effectivePrice + Frequency = $consumptionLine.properties.frequency + InvoiceSectionName = $consumptionLine.properties.invoiceSection + IsAzureCreditEligible = $consumptionLine.properties.isAzureCreditEligible + MeterCategory = $consumptionLine.properties.meterDetails.meterCategory + MeterId = $consumptionLine.properties.meterId + MeterName = $consumptionLine.properties.meterDetails.meterName + MeterRegion = $consumptionLine.properties.meterDetails.meterRegion + MeterSubCategory = $consumptionLine.properties.meterDetails.meterSubCategory + OfferId = $consumptionLine.properties.offerId + PartNumber = $consumptionLine.properties.partNumber + PayGPrice = $consumptionLine.properties.PayGPrice + PlanName = $consumptionLine.properties.planName + PricingModel = $consumptionLine.properties.pricingModel + ProductName = $consumptionLine.properties.product + PublisherName = $consumptionLine.properties.publisherName + PublisherType = $consumptionLine.properties.publisherType + Quantity = $consumptionLine.properties.quantity + ReservationId = $consumptionLine.properties.reservationId + ReservationName = $consumptionLine.properties.reservationName + ResourceGroup = $consumptionLine.properties.resourceGroup + ResourceId = $consumptionLine.properties.resourceId + ResourceLocation = $consumptionLine.properties.resourceLocation + ResourceName = $consumptionLine.properties.resourceName + ServiceFamily = $consumptionLine.properties.meterDetails.serviceFamily + SubscriptionId = $consumptionLine.properties.subscriptionId + SubscriptionName = $consumptionLine.properties.subscriptionName + Tags = $tags + Term = $consumptionLine.properties.term + UnitOfMeasure = $consumptionLine.properties.meterDetails.unitOfMeasure + UnitPrice = $consumptionLine.properties.unitPrice + } + $billingEntries += $billingEntry + } + } + } + while ($requestSuccess -and -not([string]::IsNullOrEmpty($consumption.nextLink))) + + if ($requestSuccess) + { + "Generated $($billingEntries.Count) entries..." + + "Uploading CSV to Storage" + + $ci = [CultureInfo]::new([System.Threading.Thread]::CurrentThread.CurrentCulture.Name) + if ($ci.NumberFormat.NumberDecimalSeparator -ne '.') + { + "Current culture ($($ci.Name)) does not use . as decimal separator" + $ci.NumberFormat.NumberDecimalSeparator = '.' + [System.Threading.Thread]::CurrentThread.CurrentCulture = $ci + } + + $csvExportPath = "$targetStartDate-$($subscription.Id)-$consumptionMetric.csv" + + $billingEntries | Export-Csv -Path $csvExportPath -NoTypeInformation + + $csvBlobName = $csvExportPath + $csvProperties = @{"ContentType" = "text/csv"}; + Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Uploaded $csvBlobName to Blob Storage..." + + Remove-Item -Path $csvExportPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + "[$now] Removed $csvExportPath from local disk..." + } + else + { + $global:hadErrors = $true + $global:scopesWithErrors += $ScopeName + Write-Warning "Failed to get consumption data for subscription $($subscription.Name)..." + } + } + elseif ($subscriptionQuotaID -in $CostDetailsSupportedQuotaIDs -or $consumptionAPIOption -eq "CostDetails") + { + "Starting cost details export process from $targetStartDate to $targetEndDate for subscription $($subscription.Name)..." + Generate-CostDetails -ScopeId "/subscriptions/$($subscription.Id)" -ScopeName $subscription.Id + } + else + { + $global:hadErrors = $true + $global:scopesWithErrors += $ScopeName + Write-Warning "Subscription quota $subscriptionQuotaID not supported" + } + } + } + else + { + "Starting cost details export process from $targetStartDate to $targetEndDate for Billing Account ID $BillingAccountID..." + Generate-CostDetails -ScopeId "/providers/Microsoft.Billing/billingAccounts/$BillingAccountID" -ScopeName $BillingAccountID + } +} + +if ($global:hadErrors) +{ + $scopesWithErrorsString = $global:scopesWithErrors -join "," + throw "There were errors during the export process with the following scopes: $scopesWithErrorsString. Please check the output for details." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-PolicyComplianceToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-PolicyComplianceToBlobStorage.ps1 new file mode 100644 index 000000000..d287e7b6b --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-PolicyComplianceToBlobStorage.ps1 @@ -0,0 +1,644 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetSubscription, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName, + + [Parameter(Mandatory = $false)] + [ValidateSet("ARG", "ARM")] + [string] $PolicyStatesEndpoint = "ARG" +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" -ErrorAction SilentlyContinue # e.g., westeurope +if ([string]::IsNullOrEmpty($referenceRegion)) +{ + $referenceRegion = "westeurope" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_PolicyStatesContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "policystateexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$ARGPageSize = 1000 + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +$cloudSuffix = "" + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudSuffix = $externalCloudEnvironment.ToLower() + "-" + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +$allpolicyStates = @() + +Write-Output "Getting subscriptions target $TargetSubscription" +if (-not([string]::IsNullOrEmpty($TargetSubscription))) +{ + $subscriptions = $TargetSubscription + $subscriptionSuffix = $TargetSubscription +} +else +{ + $subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" -and $_.SubscriptionPolicies.QuotaId -notlike "AAD*" } | ForEach-Object { "$($_.Id)"} + $subscriptionSuffix = $cloudSuffix + "all-" + $tenantId +} + +Write-Output "Building Policy display names..." + +$policyAssignments = @{} +$policyInitiatives = @{} +$policyDefinitions = @{} +$excludedAssignmentScopes = @() +$allInitiatives = @() + +if ($PolicyStatesEndpoint -eq "ARG") +{ + $resultsSoFar = 0 + + $argQuery = @" + policyresources + | where type =~ 'microsoft.authorization/policyassignments' + | extend displayName = iif(isnotempty(properties.displayName), tostring(properties.displayName), 'N/A') + | distinct id, displayName + | order by id asc +"@ + + $argAssignmentsTotal = @() + + do + { + if ($resultsSoFar -eq 0) + { + $argAssignments = Search-AzGraph -Query $argQuery -First $ARGPageSize -UseTenantScope + } + else + { + $argAssignments = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -UseTenantScope + } + if ($argAssignments -and $argAssignments.GetType().Name -eq "PSResourceGraphResponse") + { + $argAssignments = $argAssignments.Data + } + $resultsCount = $argAssignments.Count + $resultsSoFar += $resultsCount + $argAssignmentsTotal += $argAssignments + + } while ($resultsCount -eq $ARGPageSize) + + Write-Output "Building $($argAssignmentsTotal.Count) assignment entries" + + foreach ($assignment in $argAssignmentsTotal) + { + $policyAssignments.Add($assignment.id, $assignment.displayName) + } + + $resultsSoFar = 0 + + $argQuery = @" + policyresources + | where type =~ 'microsoft.authorization/policysetdefinitions' + | extend displayName = iif(isnotempty(properties.displayName), tostring(properties.displayName), 'N/A') + | distinct id, displayName + | order by id asc +"@ + + $argInitiativesTotal = @() + + do + { + if ($resultsSoFar -eq 0) + { + $argInitiatives = Search-AzGraph -Query $argQuery -First $ARGPageSize -UseTenantScope + } + else + { + $argInitiatives = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -UseTenantScope + } + if ($argInitiatives -and $argInitiatives.GetType().Name -eq "PSResourceGraphResponse") + { + $argInitiatives = $argInitiatives.Data + } + $resultsCount = $argInitiatives.Count + $resultsSoFar += $resultsCount + $argInitiativesTotal += $argInitiatives + + } while ($resultsCount -eq $ARGPageSize) + + Write-Output "Building $($argInitiativesTotal.Count) initiative entries" + + foreach ($initiative in $argInitiativesTotal) + { + $policyInitiatives.Add($initiative.id, $initiative.displayName) + } + + $resultsSoFar = 0 + + $argQuery = @" + policyresources + | where type =~ 'microsoft.authorization/policydefinitions' + | extend displayName = iif(isnotempty(properties.displayName), tostring(properties.displayName), 'N/A') + | distinct id, displayName + | order by id asc +"@ + + $argDefinitionsTotal = @() + + do + { + if ($resultsSoFar -eq 0) + { + $argDefinitions = Search-AzGraph -Query $argQuery -First $ARGPageSize -UseTenantScope + } + else + { + $argDefinitions = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -UseTenantScope + } + if ($argDefinitions -and $argDefinitions.GetType().Name -eq "PSResourceGraphResponse") + { + $argDefinitions = $argDefinitions.Data + } + $resultsCount = $argDefinitions.Count + $resultsSoFar += $resultsCount + $argDefinitionsTotal += $argDefinitions + + } while ($resultsCount -eq $ARGPageSize) + + Write-Output "Building $($argDefinitionsTotal.Count) definition entries" + + foreach ($definition in $argDefinitionsTotal) + { + $policyDefinitions.Add($definition.id, $definition.displayName) + } +} +else +{ + foreach ($sub in $subscriptions) + { + Select-AzSubscription -SubscriptionId $sub | Out-Null + $assignments = Get-AzPolicyAssignment -IncludeDescendent + foreach ($assignment in $assignments) + { + if (-not($policyAssignments[$assignment.PolicyAssignmentId])) + { + $assignmentName = $assignment.Properties.DisplayName + if([string]::IsNullOrWhiteSpace($assignmentName)) { + $policyAssignments.Add($assignment.PolicyAssignmentId, 'N/A') + } + else { + $policyAssignments.Add($assignment.PolicyAssignmentId, $assignmentName) + } + } + if ($assignment.Properties.NotScopes -and -not($excludedAssignmentScopes | Where-Object { $_.PolicyAssignmentId -eq $assignment.PolicyAssignmentId })) + { + $excludedAssignmentScopes += $assignment + } + } + + $initiatives = Get-AzPolicySetDefinition + foreach ($initiative in $initiatives) + { + if (-not($policyInitiatives[$initiative.PolicySetDefinitionId])) + { + $setDefinitionName = $initiative.Properties.DisplayName + if([string]::IsNullOrWhiteSpace($setDefinitionName)) { + $policyInitiatives.Add($initiative.PolicySetDefinitionId, 'N/A') + } + else { + $policyInitiatives.Add($initiative.PolicySetDefinitionId, $setDefinitionName) + } + } + if (-not($allInitiatives | Where-Object { $_.PolicySetDefinitionId -eq $initiative.PolicySetDefinitionId })) + { + $allInitiatives += $initiative + } + } + + $definitions = Get-AzPolicyDefinition + foreach ($definition in $definitions) + { + if (-not($policyDefinitions[$definition.PolicyDefinitionId])) + { + $definitionName = $initiative.Properties.DisplayName + if([string]::IsNullOrWhiteSpace($definitionName)) { + $policyDefinitions.Add($definition.PolicyDefinitionId, 'N/A') + } + else { + $policyDefinitions.Add($definition.PolicyDefinitionId, $definitionName) + } + } + } + } +} + +$policyStatesTotal = @() + +Write-Output "Querying for Policy states using $PolicyStatesEndpoint endpoint..." + +if ($PolicyStatesEndpoint -eq "ARG") +{ + $resultsSoFar = 0 + + $argQuery = @" + policyresources + | where type =~ 'microsoft.policyinsights/policystates' + | extend complianceState = tostring(properties.complianceState) + | extend complianceReason = tostring(properties.complianceReasonCode) + | where complianceState != 'Compliant' and complianceReason !contains 'ResourceNotFound' + | extend effect = tostring(properties.policyDefinitionAction) + | extend assignmentId = tolower(properties.policyAssignmentId) + | extend definitionId = tolower(properties.policyDefinitionId) + | extend definitionReferenceId = tolower(properties.policyDefinitionReferenceId) + | extend initiativeId = tolower(properties.policySetDefinitionId) + | extend resourceId = tolower(properties.resourceId) + | extend resourceType = tostring(properties.resourceType) + | extend evaluatedOn = todatetime(properties.timestamp) + | summarize StatesCount = count() by id, tenantId, subscriptionId, resourceGroup, resourceId, resourceType, complianceState, complianceReason, effect, assignmentId, definitionReferenceId, definitionId, initiativeId, evaluatedOn + | union ( policyresources + | where type =~ 'microsoft.policyinsights/policystates' + | extend complianceState = tostring(properties.complianceState) + | where complianceState == 'Compliant' + | extend effect = tostring(properties.policyDefinitionAction) + | extend assignmentId = tolower(properties.policyAssignmentId) + | extend definitionId = tolower(properties.policyDefinitionId) + | extend definitionReferenceId = tolower(properties.policyDefinitionReferenceId) + | extend initiativeId = tolower(properties.policySetDefinitionId) + | summarize StatesCount = count() by tenantId, subscriptionId, complianceState, effect, assignmentId, definitionReferenceId, definitionId, initiativeId + ) + | join kind=leftouter ( + resources + | project resourceId=tolower(id), tags + ) on resourceId + | project-away resourceId1 + | order by id asc +"@ + + do + { + if ($resultsSoFar -eq 0) + { + $policyStates = Search-AzGraph -Query $argQuery -First $ARGPageSize -Subscription $subscriptions + } + else + { + $policyStates = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -Subscription $subscriptions + } + if ($policyStates -and $policyStates.GetType().Name -eq "PSResourceGraphResponse") + { + $policyStates = $policyStates.Data + } + $resultsCount = $policyStates.Count + $resultsSoFar += $resultsCount + $policyStatesTotal += $policyStates + + } while ($resultsCount -eq $ARGPageSize) + + Write-Output "Building $($policyStatesTotal.Count) policyState entries" +} +else +{ + foreach ($sub in $subscriptions) + { + Select-AzSubscription -SubscriptionId $sub | Out-Null + $policyStates = Get-AzPolicyState -All + + $nonCompliantStates = $policyStates | Where-Object { $_.ComplianceState -ne "Compliant" } + + foreach ($policyState in $nonCompliantStates) + { + $policyStateObject = New-Object PSObject -Property @{ + tenantId = $tenantId + subscriptionId = $sub + resourceGroup = $policyState.ResourceGroup + resourceId = $policyState.ResourceId + resourceType = $policyState.ResourceType + complianceState = $policyState.ComplianceState + complianceReason = $policyState.AdditionalProperties.complianceReasonCode + effect = $policyState.PolicyDefinitionAction + assignmentId = $policyState.PolicyAssignmentId + initiativeId = $policyState.PolicySetDefinitionId + definitionId = $policyState.PolicyDefinitionId + definitionReferenceId = $policyState.PolicyDefinitionReferenceId + evaluatedOn = $policyState.Timestamp + StatesCount = 1 + } + $policyStatesTotal += $policyStateObject + } + + $compliantStates = $policyStates | Where-Object { $_.ComplianceState -eq "Compliant" } ` + | Group-Object PolicyDefinitionAction, PolicyAssignmentId, PolicyDefinitionId, PolicyDefinitionReferenceId, PolicySetDefinitionId + + foreach ($policyState in $compliantStates) + { + $compliantStateProps = $policyState.Name.Split(',') + $definitionReferenceId = $null + if ($compliantStateProps[3]) + { + $definitionReferenceId = $compliantStateProps[3].Trim().ToLower() + } + $initiativeId = $null + if ($compliantStateProps[4]) + { + $initiativeId = $compliantStateProps[4].Trim().ToLower() + } + + $policyStateObject = New-Object PSObject -Property @{ + tenantId = $tenantId + subscriptionId = $sub + complianceState = "Compliant" + effect = $compliantStateProps[0] + assignmentId = $compliantStateProps[1].Trim().ToLower() + definitionId = $compliantStateProps[2].Trim().ToLower() + definitionReferenceId = $definitionReferenceId + initiativeId = $initiativeId + StatesCount = $policyState.Count + } + $policyStatesTotal += $policyStateObject + } + } + + Write-Output "Building $($policyStatesTotal.Count) policyState entries" +} + +$datetime = (Get-Date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") +$statusDate = $datetime.ToString("yyyy-MM-dd") + +foreach ($policyState in $policyStatesTotal) +{ + $resourceGroup = $null + if ($policyState.resourceGroup) + { + $resourceGroup = $policyState.resourceGroup.ToLower() + } + + if (-not([string]::IsNullOrEmpty($policyState.tags))) + { + $tags = $policyState.tags | ConvertTo-Json -Compress + } + else + { + $tags = $null + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $policyState.tenantId + SubscriptionGuid = $policyState.subscriptionId + ResourceGroupName = $resourceGroup + ResourceId = $policyState.resourceId + ResourceType = $policyState.resourceType + ComplianceState = $policyState.complianceState + ComplianceReason = $policyState.complianceReason + Effect = $policyState.effect + AssignmentId = $policyState.assignmentId + AssignmentName = $policyAssignments[$policyState.assignmentId] + InitiativeId = $policyState.initiativeId + InitiativeName = $policyInitiatives[$policyState.initiativeId] + DefinitionId = $policyState.definitionId + DefinitionName = $policyDefinitions[$policyState.definitionId] + DefinitionReferenceId = $policyState.definitionReferenceId + EvaluatedOn = $policyState.evaluatedOn + StatesCount = $policyState.StatesCount + Tags = $tags + StatusDate = $statusDate + } + + $allpolicyStates += $logentry +} + +if ($PolicyStatesEndpoint -eq "ARG") +{ + $resultsSoFar = 0 + + $argQuery = @" + policyresources + | where type =~ 'microsoft.authorization/policyassignments' + | where array_length(properties.notScopes) > 0 + | mv-expand notScope = properties.notScopes + | extend policyAssignmentId = tolower(id) + | extend assignmentPolicyDefinitionId = tolower(properties.policyDefinitionId) + | join kind=leftouter ( + policyresources + | where type =~ 'microsoft.authorization/policysetdefinitions' + | mv-expand policyDefinition = properties.policyDefinitions + | project policySetDefinitionId = tolower(id), policyDefinitionId = tolower(policyDefinition.policyDefinitionId), policyDefinitionReferenceId = tolower(policyDefinition.policyDefinitionReferenceId) + ) on `$left.assignmentPolicyDefinitionId == `$right.policySetDefinitionId + | project policyAssignmentId, notScope, assignmentPolicyDefinitionId, policySetDefinitionId, policyDefinitionId, policyDefinitionReferenceId + | order by policyDefinitionReferenceId, tostring(notScope) +"@ + + do + { + if ($resultsSoFar -eq 0) + { + $argExcludedAssignments = Search-AzGraph -Query $argQuery -First $ARGPageSize -UseTenantScope + } + else + { + $argExcludedAssignments = Search-AzGraph -Query $argQuery -First $ARGPageSize -Skip $resultsSoFar -UseTenantScope + } + if ($argExcludedAssignments -and $argExcludedAssignments.GetType().Name -eq "PSResourceGraphResponse") + { + $argExcludedAssignments = $argExcludedAssignments.Data + } + $resultsCount = $argExcludedAssignments.Count + $resultsSoFar += $resultsCount + $excludedAssignmentScopes += $argExcludedAssignments + + } while ($resultsCount -eq $ARGPageSize) + + Write-Output "Adding excluded scopes from $($excludedAssignmentScopes.Count) assignments" + + foreach ($excludedAssignmentScope in $excludedAssignmentScopes) + { + if (-not([String]::IsNullOrEmpty($excludedAssignmentScope.policySetDefinitionId))) + { + $initiativeId = $excludedAssignmentScope.policySetDefinitionId + $initiativeName = $policyInitiatives[$initiativeId] + $definitionReferenceId = $excludedAssignmentScope.policyDefinitionReferenceId + $definitionId = $excludedAssignmentScope.policyDefinitionId + } + else + { + $initiativeId = $null + $initiativeName = $null + $definitionReferenceId = $null + $definitionId = $excludedAssignmentScope.assignmentPolicyDefinitionId + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $tenantId + ResourceId = $excludedAssignmentScope.notScope + ComplianceState = 'Excluded' + AssignmentId = $excludedAssignmentScope.policyAssignmentId + AssignmentName = $policyAssignments[$excludedAssignmentScope.policyAssignmentId] + InitiativeId = $initiativeId + InitiativeName = $initiativeName + DefinitionId = $definitionId + DefinitionName = $policyDefinitions[$definitionId] + DefinitionReferenceId = $definitionReferenceId + StatusDate = $statusDate + } + + $allpolicyStates += $logentry + } +} +else +{ + Write-Output "Adding excluded scopes from $($excludedAssignmentScopes.Count) assignments" + + foreach ($excludedAssignment in $excludedAssignmentScopes) + { + $excludedIDs = @() + $excludedInitiative = $allInitiatives | Where-Object { $_.PolicySetDefinitionId -eq $excludedAssignment.Properties.PolicyDefinitionId } + if ($excludedInitiative) + { + $excludedDefinitions = $excludedInitiative.Properties.PolicyDefinitions + foreach ($excludedDefinition in $excludedDefinitions) + { + $excludedIDs += "$($excludedDefinition.policyDefinitionId)|$($excludedDefinition.policyDefinitionReferenceId)" + } + } + else + { + $excludedIDs += $excludedAssignment.Properties.PolicyDefinitionId + } + + foreach ($excludedID in $excludedIDs) + { + $excludedIDParts = $excludedID.Split('|') + $definitionId = $excludedIDParts[0].ToLower() + $definitionReferenceId = $null + if (-not([string]::IsNullOrEmpty($excludedIDParts[1]))) + { + $definitionReferenceId = $excludedIDParts[1].ToLower() + } + + $initiativeId = $null + $initiativeName = $null + if ($excludedInitiative) + { + $initiativeId = $excludedInitiative.PolicySetDefinitionId.ToLower() + $initiativeName = $policyInitiatives[$initiativeId] + } + + foreach ($notScope in $excludedAssignment.Properties.NotScopes) + { + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + TenantGuid = $tenantId + ResourceId = $notScope.ToLower() + ComplianceState = 'Excluded' + AssignmentId = $excludedAssignment.PolicyAssignmentId.ToLower() + AssignmentName = $policyAssignments[$excludedAssignment.PolicyAssignmentId] + InitiativeId = $initiativeId + InitiativeName = $initiativeName + DefinitionId = $definitionId + DefinitionName = $policyDefinitions[$definitionId] + DefinitionReferenceId = $definitionReferenceId + StatusDate = $statusDate + } + + $allpolicyStates += $logentry + } + } + } +} + +Write-Output "Uploading CSV to Storage" + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-policyStates-$subscriptionSuffix.csv" + +$allpolicyStates | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +Write-Output "Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +Write-Output "Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-PriceSheetToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-PriceSheetToBlobStorage.ps1 new file mode 100644 index 000000000..e315cd995 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-PriceSheetToBlobStorage.ps1 @@ -0,0 +1,452 @@ +param( + [Parameter(Mandatory = $false)] + [string] $BillingAccountID, + + [Parameter(Mandatory = $false)] + [string] $BillingProfileID, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName, + + [Parameter(Mandatory = $false)] + [string] $billingPeriod, # YYYYMM format + + [Parameter(Mandatory = $false)] + [string] $meterCategories, # comma-separated meter categories (e.g., "Virtual Machines,Storage") + + [Parameter(Mandatory = $false)] + [string] $meterRegions # comma-separated billing meter regions (e.g., "EU North,EU West") +) + +$ErrorActionPreference = "Stop" + +function Authenticate-AzureWithOption { + param ( + [string] $authOption = "ManagedIdentity", + [string] $cloudEnv = "AzureCloud", + [string] $clientID + ) + + switch ($authOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnv -AccountId $clientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnv + break + } + } +} + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_PriceSheetContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "pricesheetexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$meterCategoriesVar = Get-AutomationVariable -Name "AzureOptimization_PriceSheetMeterCategories" -ErrorAction SilentlyContinue +$meterRegionsVar = Get-AutomationVariable -Name "AzureOptimization_PriceSheetMeterRegions" -ErrorAction SilentlyContinue +$BillingAccountIDVar = Get-AutomationVariable -Name "AzureOptimization_BillingAccountID" -ErrorAction SilentlyContinue +$BillingProfileIDVar = Get-AutomationVariable -Name "AzureOptimization_BillingProfileID" -ErrorAction SilentlyContinue + +"Logging in to Azure with $authenticationOption..." + +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment -clientID $uamiClientID +} +else +{ + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +# compute billing period + +if ([string]::IsNullOrEmpty($billingPeriod)) +{ + $billingPeriod = (Get-Date).Date.AddDays($consumptionOffsetDays * -1).ToString("yyyyMM") +} + +$exportDate = (Get-Date).ToUniversalTime().ToString("yyyyMMdd") + +if ([string]::IsNullOrEmpty($BillingAccountID) -and -not([string]::IsNullOrEmpty($BillingAccountIDVar))) +{ + $BillingAccountID = $BillingAccountIDVar +} + +if ([string]::IsNullOrEmpty($BillingProfileID) -and -not([string]::IsNullOrEmpty($BillingProfileIDVar))) +{ + $BillingProfileID = $BillingProfileIDVar +} + +$mcaBillingAccountIdRegex = "([A-Za-z0-9]+(-[A-Za-z0-9]+)+):([A-Za-z0-9]+(-[A-Za-z0-9]+)+)_[0-9]{4}-[0-9]{2}-[0-9]{2}" +$mcaBillingProfileIdRegex = "([A-Za-z0-9]+(-[A-Za-z0-9]+)+)" + +if ([string]::IsNullOrEmpty($BillingAccountID)) +{ + throw "Billing Account ID undefined. Use either the AzureOptimization_BillingAccountID variable or the BillingAccountID parameter" +} +else { + if ($BillingAccountID -match $mcaBillingAccountIdRegex) + { + if ([string]::IsNullOrEmpty($BillingProfileID)) + { + throw "Billing Profile ID undefined for MCA. Use either the AzureOptimization_BillingProfileID variable or the BillingProfileID parameter" + } + if (-not($BillingProfileID -match $mcaBillingProfileIdRegex)) + { + throw "Billing Profile ID does not follow pattern for MCA: ([A-Za-z0-9]+(-[A-Za-z0-9]+)+)" + } + } +} + +if (-not([string]::IsNullOrEmpty($meterCategoriesVar))) +{ + $meterCategories = $meterCategoriesVar +} + +if (-not([string]::IsNullOrEmpty($meterRegionsVar))) +{ + $meterRegions = $meterRegionsVar +} + +$meterCategoryFilters = $null +$meterRegionFilters = $null + +if (-not([string]::IsNullOrEmpty($meterCategories))) +{ + $meterCategoryFilters = $meterCategories.Split(',') +} + +if (-not([string]::IsNullOrEmpty($meterRegions))) +{ + $meterRegionFilters = $meterRegions.Split(',') +} + +function Generate-Pricesheet { + param ( + [string] $InputCSVPath, + [string] $OutputCSVPath, + [string] $HeaderLine + ) + + # header normalization between MCA and EA + $headerConversion = @{ + 'Meter ID' = "MeterID"; + meterId = "MeterID"; + 'Meter name' = "MeterName"; + meterName = "MeterName"; + 'Meter category' = "MeterCategory"; + meterCategory = "MeterCategory"; + 'Meter sub-category' = "MeterSubCategory"; + meterSubCategory = "MeterSubCategory"; + 'Meter region' = "MeterRegion"; + meterRegion = "MeterRegion"; + 'Unit of measure' = "UnitOfMeasure"; + unitOfMeasure = "UnitOfMeasure"; + 'Part number' = "PartNumber"; + 'Unit price' = "UnitPrice"; + unitPrice = "UnitPrice"; + 'Currency code' = "CurrencyCode"; + currency = "CurrencyCode"; + 'Included quantity' = "IncludedQuantity"; + includedQuantity = "IncludedQuantity"; + 'Offer Id' = "OfferId"; + Term = "Term"; + 'Price type' = "PriceType"; + priceType = "PriceType" + } + + $r = [IO.File]::OpenText($InputCSVPath) + $w = [System.IO.StreamWriter]::new($OutputCSVPath) + $lineCounter = 0 + while ($r.Peek() -ge 0) { + $line = $r.ReadLine() + $lineCounter++ + if ($lineCounter -eq $HeaderLine) + { + $headers = $line.Split(",") + + for ($i = 0; $i -lt $headers.Length; $i++) + { + $header = $headers[$i] + if ($headerConversion.ContainsKey($header)) + { + $headers[$i] = $headerConversion[$header] + } + } + + $line = $headers -join "," + + if (-not($line -match "SubCategory")) + { + throw "Pricesheet format has changed at line $HeaderLine - $line" + } + + Write-Output "New headers: $line" + + $w.WriteLine($line) + } + else + { + if ($lineCounter -gt $HeaderLine) + { + $categoryWriteLine = $categoryWriteLineDefault + $regionWriteLine = $regionWriteLineDefault + + foreach ($meterCategory in $meterCategoryFilters) + { + if ($line -match ",$meterCategory,") + { + $categoryWriteLine = $true + break + } + } + + foreach ($meterRegion in $meterRegionFilters) + { + if ($line -match ",$meterRegion,") + { + $regionWriteLine = $true + break + } + } + + if ($categoryWriteLine -eq $true -and $regionWriteLine -eq $true) + { + $w.WriteLine($line) + } + } + } + } + $r.Dispose() + $w.Close() + + $csvBlobName = [System.IO.Path]::GetFileName($OutputCSVPath) + $csvProperties = @{"ContentType" = "text/csv"}; + Set-AzStorageBlobContent -File $OutputCSVPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + + Remove-Item -Path $InputCSVPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "[$now] Removed $InputCSVPath from local disk..." + + Remove-Item -Path $OutputCSVPath -Force + + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "[$now] Removed $OutputCSVPath from local disk..." +} + +Write-Output "Starting pricesheet export process for $billingPeriod billing period for Billing Account $BillingAccountID..." + +$MaxTries = 30 # The typical Retry-After is set to 20 seconds. We'll give 10 minutes overall to download the pricesheet report + +if ($BillingAccountID -match $mcaBillingAccountIdRegex) +{ + $PriceSheetApiPath = "/providers/Microsoft.Billing/billingAccounts/$BillingAccountID/billingProfiles/$BillingProfileID/providers/Microsoft.CostManagement/pricesheets/default/download?api-version=2023-03-01&format=csv" + $result = Invoke-AzRestMethod -Path $PriceSheetApiPath -Method POST +} +else +{ + $PriceSheetApiPath = "/providers/Microsoft.Billing/billingAccounts/$BillingAccountID/billingPeriods/$billingPeriod/providers/Microsoft.Consumption/pricesheets/download?api-version=2022-06-01&ln=en" + $result = Invoke-AzRestMethod -Path $PriceSheetApiPath -Method GET +} + +$requestResultPath = $result.Headers.Location.PathAndQuery +if ($result.StatusCode -in (200,202)) +{ + $tries = 0 + $requestSuccess = $false + + Write-Output "Obtained pricesheet results endpoint: $requestResultPath..." + + Write-Output "Was told to wait $($result.Headers.RetryAfter.Delta.TotalSeconds) seconds." + + $sleepSeconds = 60 + if ($result.Headers.RetryAfter.Delta.TotalSeconds -gt 0) + { + $sleepSeconds = $result.Headers.RetryAfter.Delta.TotalSeconds + } + + do + { + $tries++ + Write-Output "Checking whether export is ready (try $tries)..." + + Start-Sleep -Seconds $sleepSeconds + $downloadResult = Invoke-AzRestMethod -Method GET -Path $requestResultPath + + if ($downloadResult.StatusCode -eq 200) + { + Write-Output "Filtering data with meter categories $meterCategories and meter regions $meterRegions to $finalCsvExportPath..." + + $categoryWriteLineDefault = $true + if ($meterCategoryFilters.Count -gt 0) + { + $categoryWriteLineDefault = $false + } + $regionWriteLineDefault = $true + if ($meterRegionFilters.Count -gt 0) + { + $regionWriteLineDefault = $false + } + + Write-Output "Defaulting to meter categories writes $($categoryWriteLineDefault) and meter regions writes $($regionWriteLineDefault)..." + + if ($BillingAccountID -match $mcaBillingAccountIdRegex) + { + Write-Output "Export is ready. Proceeding with ZIP download..." + $downloadUrl = ($downloadResult.Content | ConvertFrom-Json).publishedEntity.properties.downloadUrl + $zipExportPath = "$env:TEMP\pricesheet-$BillingProfileID-$exportDate.zip" + $zipExpandPath = "$env:TEMP\pricesheet" + Invoke-WebRequest -Uri $downloadUrl -OutFile $zipExportPath + Write-Output "Blob downloaded to $zipExportPath successfully." + Expand-Archive -LiteralPath $zipExportPath -DestinationPath $zipExpandPath -Force + Write-Output "Zip expanded to $zipExpandPath successfully." + $csvFiles = Get-ChildItem -Path $zipExpandPath -Filter *.csv -Recurse + foreach ($csvFile in $csvFiles) + { + $csvExportPath = $csvFile.FullName + $finalCsvExportPath = "$env:TEMP\$($csvFile.Name)-final.csv" + Generate-Pricesheet -InputCSVPath $csvExportPath -OutputCSVPath $finalCsvExportPath -HeaderLine 1 + } + Remove-Item -Path $zipExportPath -Force + $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "[$now] Removed $zipExportPath from local disk..." + } + else + { + Write-Output "Export is ready. Proceeding with CSV download..." + $downloadUrl = ($downloadResult.Content | ConvertFrom-Json).properties.downloadUrl + $csvExportPath = "$env:TEMP\pricesheet-$billingPeriod-$BillingAccountID.csv" + $finalCsvExportPath = "$env:TEMP\pricesheet-$billingPeriod-$BillingAccountID$($meterCategories.Replace(',',''))$($meterRegions.Replace(',',''))-$exportDate-final.csv" + Invoke-WebRequest -Uri $downloadUrl -OutFile $csvExportPath + Write-Output "Blob downloaded to $csvExportPath successfully." + Generate-Pricesheet -InputCSVPath $csvExportPath -OutputCSVPath $finalCsvExportPath -HeaderLine 3 + } + + $requestSuccess = $true + } + elseif ($downloadResult.StatusCode -eq 202) + { + Write-Output "Was told to wait a bit more... $($downloadResult.Headers.RetryAfter.Delta.TotalSeconds) seconds." + + $sleepSeconds = 60 + if ($downloadResult.Headers.RetryAfter.Delta.TotalSeconds -gt 0) + { + $sleepSeconds = $downloadResult.Headers.RetryAfter.Delta.TotalSeconds + } + } + elseif ($downloadResult.StatusCode -eq 401) + { + Write-Output "Had an authentication issue. Will login again and sleep just a couple of seconds." + + if ($authenticationOption -eq "UserAssignedManagedIdentity") + { + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment -clientID $uamiClientID + } + else + { + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment + } + + $sleepSeconds = 2 + } + else + { + Write-Output "Got an unexpected response code: $($downloadResult.StatusCode)" + } + } + while (-not($requestSuccess) -and $tries -lt $MaxTries) + + if ($tries -ge $MaxTries) + { + throw "Couldn't complete request before the alloted number of $MaxTries retries" + } + + if (-not($requestSuccess)) + { + throw "Error returned by the Download PriceSheet API. Status Code: $($downloadResult.StatusCode). Message: $($downloadResult.Content)" + } + else + { + Write-Output "Export download processing complete." + } +} +else +{ + if ($result.StatusCode -ne 204) + { + throw "Error returned by the Download PriceSheet API. Status Code: $($result.StatusCode). Message: $($result.Content)" + } + else + { + Write-Output "Request returned 204 No Content" + } +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-RBACAssignmentsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-RBACAssignmentsToBlobStorage.ps1 new file mode 100644 index 000000000..f5000dec2 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-RBACAssignmentsToBlobStorage.ps1 @@ -0,0 +1,258 @@ +param( + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RBACAssignmentsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "rbacexports" +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +$subscriptions = Get-AzSubscription | Where-Object { $_.State -eq "Enabled" } + +$roleAssignments = @() + +"Iterating through all reachable subscriptions..." + +foreach ($subscription in $subscriptions) { + + Select-AzSubscription -SubscriptionId $subscription.Id -TenantId $tenantId | Out-Null + + $assignments = Get-AzRoleAssignment -IncludeClassicAdministrators -ErrorAction Continue + "Found $($assignments.Count) assignments for $($subscription.Name) subscription..." + + foreach ($assignment in $assignments) { + if ($null -eq $assignment.ObjectId -and $assignment.Scope.Contains($subscription.Id)) + { + $assignmentEntry = New-Object PSObject -Property @{ + Timestamp = $timestamp + TenantGuid = $tenantId + Cloud = $cloudEnvironment + Model = "AzureClassic" + PrincipalId = $assignment.SignInName + Scope = $assignment.Scope + RoleDefinition = $assignment.RoleDefinitionName + } + $roleAssignments += $assignmentEntry + } + else + { + $duplicateRoleAssignment = $roleAssignments | Where-Object { $_.PrincipalId -eq $assignment.ObjectId -and $_.Scope -eq $assignment.Scope -and $_.RoleDefinition -eq $assignment.RoleDefinitionName} + if (-not($duplicateRoleAssignment)) + { + $assignmentEntry = New-Object PSObject -Property @{ + Timestamp = $timestamp + TenantGuid = $tenantId + Cloud = $cloudEnvironment + Model = "AzureRM" + PrincipalId = $assignment.ObjectId + Scope = $assignment.Scope + RoleDefinition = $assignment.RoleDefinitionName + } + $roleAssignments += $assignmentEntry + } + } + } +} + +$fileDate = $datetime.ToString("yyyyMMdd") +$jsonExportPath = "$fileDate-$tenantId-rbacassignments.json" +$csvExportPath = "$fileDate-$tenantId-rbacassignments.csv" + +$roleAssignments | ConvertTo-Json -Depth 3 -Compress | Out-File $jsonExportPath +"Exported to JSON: $($roleAssignments.Count) lines" +$rbacObjectsJson = Get-Content -Path $jsonExportPath | ConvertFrom-Json +"JSON Import: $($rbacObjectsJson.Count) lines" +$rbacObjectsJson | Export-Csv -NoTypeInformation -Path $csvExportPath +"Export to $csvExportPath" + +$csvBlobName = $csvExportPath +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +"[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +"[$now] Removed $csvExportPath from local disk..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +"[$now] Removed $jsonExportPath from local disk..." + +$roleAssignments = @() + +"Getting Microsoft Entra ID roles..." + +#workaround for https://github.com/microsoftgraph/msgraph-sdk-powershell/issues/888 +$localPath = [System.Environment]::GetFolderPath([System.Environment+SpecialFolder]::UserProfile) +if (-not(get-item "$localPath\.graph\" -ErrorAction SilentlyContinue)) +{ + New-Item -Type Directory "$localPath\.graph" +} + +Import-Module Microsoft.Graph.Identity.DirectoryManagement + +switch ($cloudEnvironment) { + "AzureUSGovernment" { + $graphEnvironment = "USGov" + break + } + "AzureChinaCloud" { + $graphEnvironment = "China" + break + } + "AzureGermanCloud" { + $graphEnvironment = "Germany" + break + } + Default { + $graphEnvironment = "Global" + } +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Microsoft Graph with $externalCredentialName external credential..." + Connect-MgGraph -TenantId $externalTenantId -ClientSecretCredential $externalCredential -Environment $graphEnvironment -NoWelcome +} +else +{ + "Logging in to Microsoft Graph..." + Connect-MgGraph -Identity -Environment $graphEnvironment -NoWelcome +} + +$domainName = (Get-MgDomain | Where-Object { $_.IsVerified -and $_.IsDefault } | Select-Object -First 1).Id + +$roles = Get-MgDirectoryRole -ExpandProperty Members -Property DisplayName,Members +foreach ($role in $roles) +{ + $roleMembers = $role.Members | Where-Object { -not($_.DeletedDateTime) } + foreach ($roleMember in $roleMembers) + { + $assignmentEntry = New-Object PSObject -Property @{ + Timestamp = $timestamp + TenantGuid = $tenantId + Cloud = $cloudEnvironment + Model = "AzureAD" + PrincipalId = $roleMember.Id + Scope = $domainName + RoleDefinition = $role.DisplayName + } + $roleAssignments += $assignmentEntry + } +} + +$fileDate = $datetime.ToString("yyyyMMdd") +$jsonExportPath = "$fileDate-$tenantId-aadrbacassignments.json" +$csvExportPath = "$fileDate-$tenantId-aadrbacassignments.csv" + +$roleAssignments | ConvertTo-Json -Depth 3 -Compress | Out-File $jsonExportPath +"Exported to JSON: $($roleAssignments.Count) lines" +$rbacObjectsJson = Get-Content -Path $jsonExportPath | ConvertFrom-Json +"JSON Import: $($rbacObjectsJson.Count) lines" +$rbacObjectsJson | Export-Csv -NoTypeInformation -Path $csvExportPath +"Export to $csvExportPath" + +$csvBlobName = $csvExportPath +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +"[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +"[$now] Removed $csvExportPath from local disk..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +"[$now] Removed $jsonExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ReservationsPriceToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ReservationsPriceToBlobStorage.ps1 new file mode 100644 index 000000000..9e4fa5554 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ReservationsPriceToBlobStorage.ps1 @@ -0,0 +1,146 @@ +param( + [Parameter(Mandatory = $false)] + [string] $Filter = "serviceName eq 'Virtual Machines' and priceType eq 'Reservation'" # e.g., serviceName eq 'Virtual Machines' and priceType eq 'Reservation' and armRegionName eq 'northeurope' +) + +$ErrorActionPreference = "Stop" + +function Authenticate-AzureWithOption { + param ( + [string] $authOption = "ManagedIdentity", + [string] $cloudEnv = "AzureCloud", + [string] $clientID + ) + + switch ($authOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnv -AccountId $clientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnv + break + } + } +} + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ReservationsPriceContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "reservationspriceexports" +} + +$filterVar = Get-AutomationVariable -Name "AzureOptimization_RetailPricesFilter" -ErrorAction SilentlyContinue +$currencyCode = Get-AutomationVariable -Name "AzureOptimization_RetailPricesCurrencyCode" + +"Logging in to Azure with $authenticationOption..." + +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment -clientID $uamiClientID +} +else +{ + Authenticate-AzureWithOption -authOption $authenticationOption -cloudEnv $cloudEnvironment +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +if (-not([string]::IsNullOrEmpty($filterVar))) +{ + $Filter = $filterVar +} + +Write-Output "Starting retails prices export process with $currencyCode currency code and filter: $Filter ..." + +$RetailPricesApiPath = "https://prices.azure.com/api/retail/prices?currencyCode='$currencyCode'&`$filter=$Filter" + +$prices = @() + +do +{ + $Response = Invoke-RestMethod -Method Get -Uri $RetailPricesApiPath + if ($Response.Items.Count -gt 0) + { + $prices += $Response.Items + } + $RetailPricesApiPath = $Response.NextPageLink +} while ($Response.NextPageLink) + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyyMMdd") + +$fileFriendlyFilter = $Filter.Replace(" ","").Replace("'","") +$csvExportPath = "reservationsprice-$timestamp-$fileFriendlyFilter.csv" + +$ci = [CultureInfo]::new([System.Threading.Thread]::CurrentThread.CurrentCulture.Name) +if ($ci.NumberFormat.NumberDecimalSeparator -ne '.') +{ + Write-Output "Current culture ($($ci.Name)) does not use . as decimal separator" + $ci.NumberFormat.NumberDecimalSeparator = '.' + [System.Threading.Thread]::CurrentThread.CurrentCulture = $ci +} + +$prices | Export-Csv -NoTypeInformation -Path $csvExportPath + +Write-Output "Reservations price CSV exported to $csvExportPath successfully." + +$csvBlobName = $csvExportPath +$csvProperties = @{"ContentType" = "text/csv"}; +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ReservationsUsageToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ReservationsUsageToBlobStorage.ps1 new file mode 100644 index 000000000..80cdb3985 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-ReservationsUsageToBlobStorage.ps1 @@ -0,0 +1,304 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetScope, + + [Parameter(Mandatory = $false)] + [string] $BillingAccountID, + + [Parameter(Mandatory = $false)] + [string] $BillingProfileID, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName, + + [Parameter(Mandatory = $false)] + [string] $targetStartDate, # YYYY-MM-DD format + + [Parameter(Mandatory = $false)] + [string] $targetEndDate # YYYY-MM-DD format +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_ReservationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "reservationsexports" +} + +$BillingAccountIDVar = Get-AutomationVariable -Name "AzureOptimization_BillingAccountID" -ErrorAction SilentlyContinue +$BillingProfileIDVar = Get-AutomationVariable -Name "AzureOptimization_BillingProfileID" -ErrorAction SilentlyContinue + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") + +if ([string]::IsNullOrEmpty($BillingAccountID) -and -not([string]::IsNullOrEmpty($BillingAccountIDVar))) +{ + $BillingAccountID = $BillingAccountIDVar +} + +if ([string]::IsNullOrEmpty($BillingProfileID) -and -not([string]::IsNullOrEmpty($BillingProfileIDVar))) +{ + $BillingProfileID = $BillingProfileIDVar +} + +$mcaBillingAccountIdRegex = "([A-Za-z0-9]+(-[A-Za-z0-9]+)+):([A-Za-z0-9]+(-[A-Za-z0-9]+)+)_[0-9]{4}-[0-9]{2}-[0-9]{2}" +$mcaBillingProfileIdRegex = "([A-Za-z0-9]+(-[A-Za-z0-9]+)+)" + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +# compute start+end dates + +if ([string]::IsNullOrEmpty($targetStartDate) -or [string]::IsNullOrEmpty($targetEndDate)) +{ + $targetStartDate = (Get-Date).Date.AddDays($consumptionOffsetDays * -1).ToString("yyyy-MM-dd") + $targetEndDate = $targetStartDate +} + +if (-not([string]::IsNullOrEmpty($TargetScope))) +{ + $scope = $TargetScope +} +else +{ + if ([string]::IsNullOrEmpty($BillingAccountID)) + { + throw "Billing Account ID undefined. Use either the AzureOptimization_BillingAccountID variable or the BillingAccountID parameter" + } + if ($BillingAccountID -match $mcaBillingAccountIdRegex) + { + if ([string]::IsNullOrEmpty($BillingProfileID)) + { + throw "Billing Profile ID undefined for MCA. Use either the AzureOptimization_BillingProfileID variable or the BillingProfileID parameter" + } + if (-not($BillingProfileID -match $mcaBillingProfileIdRegex)) + { + throw "Billing Profile ID does not follow pattern for MCA: ([A-Za-z0-9]+(-[A-Za-z0-9]+)+)" + } + $scope = "/providers/Microsoft.Billing/billingaccounts/$BillingAccountID/billingProfiles/$BillingProfileID" + } + else + { + $scope = "/providers/Microsoft.Billing/billingaccounts/$BillingAccountID" + } +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Starting reservations export process from $targetStartDate to $targetEndDate for scope $scope..." + +# get reservations details + +$reservationsDetailsResponse = $null +$reservationsDetails = @() +$reservationsDetailsPath = "$scope/reservations?api-version=2020-05-01&&refreshSummary=true" + +do +{ + if (-not([string]::IsNullOrEmpty($reservationsDetailsResponse.nextLink))) + { + $reservationsDetailsPath = $reservationsDetailsResponse.nextLink.Substring($reservationsDetailsResponse.nextLink.IndexOf("/providers/")) + } + + $result = Invoke-AzRestMethod -Path $reservationsDetailsPath -Method GET + + if (-not($result.StatusCode -in (200, 201, 202))) + { + throw "Error while getting reservations details: $($result.Content)" + } + + $reservationsDetailsResponse = $result.Content | ConvertFrom-Json + if ($reservationsDetailsResponse.value) + { + $reservationsDetails += $reservationsDetailsResponse.value + } +} +while (-not([string]::IsNullOrEmpty($reservationsDetailsResponse.nextLink))) + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Found $($reservationsDetails.Count) reservation details." + +# get reservations usage + +$reservationsUsage = @() +if ($BillingAccountID -match $mcaBillingAccountIdRegex) +{ + $reservationsUsagePath = "$scope/providers/Microsoft.Consumption/reservationSummaries?api-version=2023-05-01&startDate=$targetStartDate&endDate=$targetEndDate&grain=daily" +} +else +{ + $reservationsUsagePath = "$scope/providers/Microsoft.Consumption/reservationSummaries?api-version=2023-05-01&`$filter=properties/UsageDate ge $targetStartDate and properties/UsageDate le $targetEndDate&grain=daily" +} + +$result = Invoke-AzRestMethod -Path $reservationsUsagePath -Method GET + +if (-not($result.StatusCode -in (200, 201, 202))) +{ + throw "Error while getting reservations usage: $($result.Content)" +} + +$reservationsUsageResponse = $result.Content | ConvertFrom-Json +if ($reservationsUsageResponse.value) +{ + $reservationsUsage += $reservationsUsageResponse.value +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Found $($reservationsUsage.Count) reservation usages." + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +$reservations = @() + +foreach ($usage in $reservationsUsage) +{ + $reservationResourceId = "/providers/microsoft.capacity/reservationorders/$($usage.properties.reservationOrderId)/reservations/$($usage.properties.reservationId)" + $reservationDetail = $reservationsDetails | Where-Object { $_.id -eq $reservationResourceId } + $reservationEntry = New-Object PSObject -Property @{ + ReservationResourceId = $reservationResourceId + ReservationOrderId = $usage.properties.reservationOrderId + ReservationId = $usage.properties.reservationId + DisplayName = $reservationDetail.properties.displayName + SKUName = $usage.properties.skuName + Location = $reservationDetail.location + ResourceType = $reservationDetail.properties.reservedResourceType + AppliedScopeType = $reservationDetail.properties.userFriendlyAppliedScopeType + Term = $reservationDetail.properties.term + ProvisioningState = $reservationDetail.properties.displayProvisioningState + RenewState = $reservationDetail.properties.userFriendlyRenewState + PurchaseDate = $reservationDetail.properties.purchaseDate + ExpiryDate = $reservationDetail.properties.expiryDate + Archived = $reservationDetail.properties.archived + ReservedHours = $usage.properties.reservedHours + UsedHours = $usage.properties.usedHours + UsageDate = $usage.properties.usageDate + MinUtilPercentage = $usage.properties.minUtilizationPercentage + AvgUtilPercentage = $usage.properties.avgUtilizationPercentage + MaxUtilPercentage = $usage.properties.maxUtilizationPercentage + PurchasedQuantity = $usage.properties.purchasedQuantity + RemainingQuantity = $usage.properties.remainingQuantity + TotalReservedQuantity = $usage.properties.totalReservedQuantity + UsedQuantity = $usage.properties.usedQuantity + UtilizedPercentage = $usage.properties.utilizedPercentage + UtilTrend = $reservationDetail.properties.utilization.trend + Util1Days = ($reservationDetail.properties.utilization.aggregates | Where-Object { $_.grain -eq 1 }).value + Util7Days = ($reservationDetail.properties.utilization.aggregates | Where-Object { $_.grain -eq 7 }).value + Util30Days = ($reservationDetail.properties.utilization.aggregates | Where-Object { $_.grain -eq 30 }).value + Scope = $scope + TenantGuid = $tenantId + Cloud = $cloudEnvironment + CollectedDate = $timestamp + Timestamp = $timestamp + } + $reservations += $reservationEntry +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Generated $($reservations.Count) entries..." + +if ($BillingAccountID -match $mcaBillingAccountIdRegex) +{ + $csvExportPath = "$targetStartDate-$BillingProfileID.csv" +} +else +{ + $csvExportPath = "$targetStartDate-$BillingAccountID-$($scope.Split('/')[-1]).csv" +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploading CSV to Storage" + +$ci = [CultureInfo]::new([System.Threading.Thread]::CurrentThread.CurrentCulture.Name) +if ($ci.NumberFormat.NumberDecimalSeparator -ne '.') +{ + Write-Output "Current culture ($($ci.Name)) does not use . as decimal separator" + $ci.NumberFormat.NumberDecimalSeparator = '.' + [System.Threading.Thread]::CurrentThread.CurrentCulture = $ci +} + +$reservations | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath +$csvProperties = @{"ContentType" = "text/csv"}; +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-SavingsPlansUsageToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-SavingsPlansUsageToBlobStorage.ps1 new file mode 100644 index 000000000..a5eb70ab2 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Export-SavingsPlansUsageToBlobStorage.ps1 @@ -0,0 +1,252 @@ +param( + [Parameter(Mandatory = $false)] + [string] $TargetScope, + + [Parameter(Mandatory = $false)] + [string] $BillingAccountID, + + [Parameter(Mandatory = $false)] + [string] $BillingProfileID, + + [Parameter(Mandatory = $false)] + [string] $externalCloudEnvironment, + + [Parameter(Mandatory = $false)] + [string] $externalTenantId, + + [Parameter(Mandatory = $false)] + [string] $externalCredentialName +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkEnv = Get-AutomationVariable -Name "AzureOptimization_StorageSinkEnvironment" -ErrorAction SilentlyContinue +if (-not($storageAccountSinkEnv)) +{ + $storageAccountSinkEnv = $cloudEnvironment +} +$storageAccountSinkKeyCred = Get-AutomationPSCredential -Name "AzureOptimization_StorageSinkKey" -ErrorAction SilentlyContinue +$storageAccountSinkKey = $null +if ($storageAccountSinkKeyCred) +{ + $storageAccountSink = $storageAccountSinkKeyCred.UserName + $storageAccountSinkKey = $storageAccountSinkKeyCred.GetNetworkCredential().Password +} + +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_SavingsPlansContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ + $storageAccountSinkContainer = "savingsplansexports" +} + +$BillingAccountIDVar = Get-AutomationVariable -Name "AzureOptimization_BillingAccountID" -ErrorAction SilentlyContinue +$BillingProfileIDVar = Get-AutomationVariable -Name "AzureOptimization_BillingProfileID" -ErrorAction SilentlyContinue + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + $externalCredential = Get-AutomationPSCredential -Name $externalCredentialName +} + +if ([string]::IsNullOrEmpty($BillingAccountID) -and -not([string]::IsNullOrEmpty($BillingAccountIDVar))) +{ + $BillingAccountID = $BillingAccountIDVar +} + +if ([string]::IsNullOrEmpty($BillingProfileID) -and -not([string]::IsNullOrEmpty($BillingProfileIDVar))) +{ + $BillingProfileID = $BillingProfileIDVar +} + +$mcaBillingAccountIdRegex = "([A-Za-z0-9]+(-[A-Za-z0-9]+)+):([A-Za-z0-9]+(-[A-Za-z0-9]+)+)_[0-9]{4}-[0-9]{2}-[0-9]{2}" +$mcaBillingProfileIdRegex = "([A-Za-z0-9]+(-[A-Za-z0-9]+)+)" + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +if (-not($storageAccountSinkKey)) +{ + Write-Output "Getting Storage Account context with login" + Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId + $saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context +} +else +{ + Write-Output "Getting Storage Account context with key" + $saCtx = New-AzStorageContext -StorageAccountName $storageAccountSink -StorageAccountKey $storageAccountSinkKey -Environment $storageAccountSinkEnv +} + +if (-not([string]::IsNullOrEmpty($externalCredentialName))) +{ + "Logging in to Azure with $externalCredentialName external credential..." + Connect-AzAccount -ServicePrincipal -EnvironmentName $externalCloudEnvironment -Tenant $externalTenantId -Credential $externalCredential + $cloudEnvironment = $externalCloudEnvironment +} + +$tenantId = (Get-AzContext).Tenant.Id + +if (-not([string]::IsNullOrEmpty($TargetScope))) +{ + $scope = $TargetScope +} +else +{ + if ([string]::IsNullOrEmpty($BillingAccountID)) + { + throw "Billing Account ID undefined. Use either the AzureOptimization_BillingAccountID variable or the BillingAccountID parameter" + } + if ($BillingAccountID -match $mcaBillingAccountIdRegex) + { + if ([string]::IsNullOrEmpty($BillingProfileID)) + { + throw "Billing Profile ID undefined for MCA. Use either the AzureOptimization_BillingProfileID variable or the BillingProfileID parameter" + } + if (-not($BillingProfileID -match $mcaBillingProfileIdRegex)) + { + throw "Billing Profile ID does not follow pattern for MCA: ([A-Za-z0-9]+(-[A-Za-z0-9]+)+)" + } + #$scope = "/providers/Microsoft.BillingBenefits" + $scope = "/providers/Microsoft.Billing/billingaccounts/$BillingAccountID" + } + else + { + $scope = "/providers/Microsoft.Billing/billingaccounts/$BillingAccountID" + } +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Starting savings plans export process for scope $scope..." + +$savingsPlansUsage = @() +if ($BillingAccountID -match $mcaBillingAccountIdRegex) +{ + #$savingsPlansUsagePath = "$scope/savingsPlans?api-version=2022-11-01&refreshsummary=true&take=100" + $savingsPlansUsagePath = "$scope/savingsPlans?api-version=2022-10-01-privatepreview&refreshsummary=true&take=100&`$filter=(properties/billingProfileId eq '/providers/Microsoft.Billing/billingAccounts/$BillingAccountID/billingProfiles/$BillingProfileID')" +} +else +{ + $savingsPlansUsagePath = "$scope/savingsPlans?api-version=2020-12-15-privatepreview&refreshsummary=true&take=100" +} + +$result = Invoke-AzRestMethod -Path $savingsPlansUsagePath -Method GET + +if (-not($result.StatusCode -in (200, 201, 202))) +{ + throw "Error while getting savings plans usage: $($result.Content)" +} + +$savingsPlansUsageResponse = $result.Content | ConvertFrom-Json +if ($savingsPlansUsageResponse.value) +{ + $savingsPlansUsage += $savingsPlansUsageResponse.value +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Found $($savingsPlansUsage.Count) savings plans usages." + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +$savingsPlans = @() + +foreach ($usage in $savingsPlansUsage) +{ + $savingsPlanEntry = New-Object PSObject -Property @{ + SavingsPlanResourceId = $usage.id + SavingsPlanOrderId = $usage.id.Substring(0,$usage.id.IndexOf("/savingsPlans/")) + SavingsPlanId = $usage.id.Split("/")[-1] + DisplayName = $usage.properties.displayName + SKUName = $usage.sku.name + Term = $usage.properties.term + ProvisioningState = $usage.properties.displayProvisioningState + AppliedScopeType = $usage.properties.userFriendlyAppliedScopeType + RenewState = $usage.properties.renew + PurchaseDate = $usage.properties.purchaseDateTime + BenefitStart = $usage.properties.benefitStartTime + ExpiryDate = $usage.properties.expiryDateTime + EffectiveDate = $usage.properties.effectiveDateTime + BillingScopeId = $usage.properties.billingScopeId + BillingAccountId = $usage.properties.billingAccountId + BillingProfileId = $usage.properties.billingProfileId + BillingPlan = $usage.properties.billingProfileId + CommitmentGrain = $usage.properties.commitment.grain + CommitmentCurrencyCode = $usage.properties.commitment.currencyCode + CommitmentAmount = $usage.properties.commitment.amount + UtilTrend = $usage.properties.utilization.trend + Util1Days = ($usage.properties.utilization.aggregates | Where-Object { $_.grain -eq 1 }).value + Util7Days = ($usage.properties.utilization.aggregates | Where-Object { $_.grain -eq 7 }).value + Util30Days = ($usage.properties.utilization.aggregates | Where-Object { $_.grain -eq 30 }).value + Scope = $scope + TenantGuid = $tenantId + Cloud = $cloudEnvironment + CollectedDate = $timestamp + Timestamp = $timestamp + } + $savingsPlans += $savingsPlanEntry +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Generated $($savingsPlans.Count) entries..." + +$targetDate = $datetime.ToString("yyyy-MM-dd") + +if ($BillingAccountID -match $mcaBillingAccountIdRegex) +{ + $csvExportPath = "$targetDate-$BillingProfileID.csv" +} +else +{ + $csvExportPath = "$targetDate-$BillingAccountID.csv" +} + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploading CSV to Storage" + +$ci = [CultureInfo]::new([System.Threading.Thread]::CurrentThread.CurrentCulture.Name) +if ($ci.NumberFormat.NumberDecimalSeparator -ne '.') +{ + Write-Output "Current culture ($($ci.Name)) does not use . as decimal separator" + $ci.NumberFormat.NumberDecimalSeparator = '.' + [System.Threading.Thread]::CurrentThread.CurrentCulture = $ci +} + +$savingsPlans | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath +$csvProperties = @{"ContentType" = "text/csv"}; +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $csvBlobName to Blob Storage..." + +Remove-Item -Path $csvExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $csvExportPath from local disk..." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 new file mode 100644 index 000000000..e45dac978 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 @@ -0,0 +1,329 @@ +param( + [Parameter(Mandatory = $true)] + [string] $StorageSinkContainer +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$sharedKey = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceKey" +$LogAnalyticsChunkSize = [int] (Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsChunkSize" -ErrorAction SilentlyContinue) +if (-not($LogAnalyticsChunkSize -gt 0)) +{ + $LogAnalyticsChunkSize = 6000 +} +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = $StorageSinkContainer +$StorageBlobsPageSize = [int] (Get-AutomationVariable -Name "AzureOptimization_StorageBlobsPageSize" -ErrorAction SilentlyContinue) +if (-not($StorageBlobsPageSize -gt 0)) +{ + $StorageBlobsPageSize = 1000 +} + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +#region Functions + +# Function to create the authorization signature +Function Build-OMSSignature ($workspaceId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) { + $xHeaders = "x-ms-date:" + $date + $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource + $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) + $keyBytes = [Convert]::FromBase64String($sharedKey) + $sha256 = New-Object System.Security.Cryptography.HMACSHA256 + $sha256.Key = $keyBytes + $calculatedHash = $sha256.ComputeHash($bytesToHash) + $encodedHash = [Convert]::ToBase64String($calculatedHash) + $authorization = 'SharedKey {0}:{1}' -f $workspaceId, $encodedHash + return $authorization +} + +# Function to create and post the request +Function Post-OMSData($workspaceId, $sharedKey, $body, $logType, $TimeStampField, $AzureEnvironment) { + $method = "POST" + $contentType = "application/json" + $resource = "/api/logs" + $rfc1123date = [DateTime]::UtcNow.ToString("r") + $contentLength = $body.Length + $signature = Build-OMSSignature ` + -workspaceId $workspaceId ` + -sharedKey $sharedKey ` + -date $rfc1123date ` + -contentLength $contentLength ` + -method $method ` + -contentType $contentType ` + -resource $resource + + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" + if ($AzureEnvironment -eq "AzureChinaCloud") + { + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.cn" + $resource + "?api-version=2016-04-01" + } + if ($AzureEnvironment -eq "AzureUSGovernment") + { + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.us" + $resource + "?api-version=2016-04-01" + } + if ($AzureEnvironment -eq "AzureGermanCloud") + { + throw "Azure Germany isn't suported for the Log Analytics Data Collector API" + } + + $OMSheaders = @{ + "Authorization" = $signature; + "Log-Type" = $logType; + "x-ms-date" = $rfc1123date; + "time-generated-field" = $TimeStampField; + } + + Try { + + $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + } + catch { + if ($_.Exception.Response.StatusCode.Value__ -eq 401) { + "REAUTHENTICATING" + + $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + } + else + { + return $_.Exception.Response.StatusCode.Value__ + } + } + + return $response.StatusCode +} +#endregion Functions + +# get reference to storage sink +Write-Output "Getting blobs list from $storageAccountSink storage account ($storageAccountSinkContainer container)..." +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +$allblobs = @() + +$continuationToken = $null +do +{ + $blobs = Get-AzStorageBlob -Container $storageAccountSinkContainer -MaxCount $StorageBlobsPageSize -ContinuationToken $continuationToken -Context $sa.Context | Sort-Object -Property LastModified + if ($blobs.Count -le 0) { break } + $allblobs += $blobs + $continuationToken = $blobs[$blobs.Count -1].ContinuationToken; +} +While ($null -ne $continuationToken) + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE StorageContainerName = '$storageAccountSinkContainer'" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$Conn.Close() +$Conn.Dispose() + +if ($controlRows.Count -eq 0 -or -not($controlRows[0].LastProcessedDateTime)) +{ + throw "Could not find a valid ingestion control row for $storageAccountSinkContainer" +} + +$controlRow = $controlRows[0] +$lastProcessedLine = $controlRow.LastProcessedLine +$lastProcessedDateTime = $controlRow.LastProcessedDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +$LogAnalyticsSuffix = $controlRow.LogAnalyticsSuffix +$logname = $lognamePrefix + $LogAnalyticsSuffix + +Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the $($logname)_CL table..." + +$newProcessedTime = $null + +$unprocessedBlobs = @() + +foreach ($blob in $allblobs) { + $blobLastModified = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + if ($lastProcessedDateTime -lt $blobLastModified -or ` + ($lastProcessedDateTime -eq $blobLastModified -and $lastProcessedLine -gt 0)) { + Write-Output "$($blob.Name) found (modified on $blobLastModified)" + $unprocessedBlobs += $blob + } +} + +$unprocessedBlobs = $unprocessedBlobs | Sort-Object -Property LastModified + +Write-Output "Found $($unprocessedBlobs.Count) new blobs to process..." + +foreach ($blob in $unprocessedBlobs) { + $newProcessedTime = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "About to process $($blob.Name)..." + $blobFilePath = "$env:TEMP\$($blob.Name)" + Get-AzStorageBlobContent -CloudBlob $blob.ICloudBlob -Context $sa.Context -Force -Destination $blobFilePath | Out-Null + + $r = [IO.File]::OpenText($blobFilePath) + + $linesProcessed = 0 + $lineCounter = 0 + $chunkLines = @() + + while ($r.Peek() -ge 0) + { + $line = $r.ReadLine() + if ($lineCounter -eq 0) + { + $header = $line + $chunkLines += $line + } + else + { + $linesProcessed++ + } + if ($lastProcessedLine -lt $linesProcessed -and $lineCounter -gt 0) + { + $chunkLines += $line + } + if (($lineCounter -eq $LogAnalyticsChunkSize -or $r.Peek() -lt 0) -and $linesProcessed -gt 0) + { + $csvObject = $chunkLines | ConvertFrom-Csv + $jsonObject = ConvertTo-Json -InputObject $csvObject + + $res = Post-OMSData -workspaceId $workspaceId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($jsonObject)) -logType $logname -TimeStampField "Timestamp" -AzureEnvironment $cloudEnvironment + if ($res -ge 200 -and $res -lt 300) + { + Write-Output "Succesfully uploaded $lineCounter $LogAnalyticsSuffix rows to Log Analytics" + if ($r.Peek() -lt 0) { + $lastProcessedLine = -1 + } + else { + $lastProcessedLine = $linesProcessed - 1 + } + + $updatedLastProcessedLine = $lastProcessedLine + $updatedLastProcessedDateTime = $lastProcessedDateTime + if ($r.Peek() -lt 0) { + $updatedLastProcessedDateTime = $newProcessedTime + } + $lastProcessedDateTime = $updatedLastProcessedDateTime + Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" + $sqlStatement = "UPDATE [$LogAnalyticsIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandText = $sqlStatement + $Cmd.CommandTimeout=120 + $Cmd.ExecuteReader() + $Conn.Close() + $Conn.Dispose() + } + else + { + Write-Warning "Failed to upload $lineCounter $LogAnalyticsSuffix rows. Error code: $res" + $r.Dispose() + Remove-Item -Path $blobFilePath -Force + throw + } + + $chunkLines = @() + $chunkLines += $header + $lineCounter = 1 + } + else + { + $lineCounter++ + } + } + $r.Dispose() + + if ($linesProcessed -eq 0) + { + Write-Output "No rows found" + $updatedLastProcessedLine = -1 + $updatedLastProcessedDateTime = $newProcessedTime + Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" + $sqlStatement = "UPDATE [$LogAnalyticsIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandText = $sqlStatement + $Cmd.CommandTimeout=120 + $Cmd.ExecuteReader() + $Conn.Close() + $Conn.Dispose() + } + else + { + Write-Output "Processed $linesProcessed row(s) in total." + } + + Remove-Item -Path $blobFilePath -Force +} + +Write-Output "DONE" \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/maintenance/CleanUp-OlderRecommendationsFromSqlServer.ps1 b/docs/deploy/optimization-engine/latest/runbooks/maintenance/CleanUp-OlderRecommendationsFromSqlServer.ps1 new file mode 100644 index 000000000..9220678b0 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/maintenance/CleanUp-OlderRecommendationsFromSqlServer.ps1 @@ -0,0 +1,53 @@ +$ErrorActionPreference = "Stop" + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} +$RecommendationsMaxAge = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendationsMaxAgeInDays" -ErrorAction SilentlyContinue) +if (-not($RecommendationsMaxAge -gt 0)) +{ + $RecommendationsMaxAge = 365 +} + +$recommendationsTable = "Recommendations" + +$tries = 0 +$connectionSuccess = $false + +Write-Output "Cleaning up recommendations older than $RecommendationsMaxAge days..." + +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = 0 + $Cmd.CommandText = "DELETE FROM [dbo].[$recommendationsTable] WHERE GeneratedDate < GETDATE()-$RecommendationsMaxAge" + $DeletedRows = $Cmd.ExecuteNonQuery() + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } + finally { + $Conn.Close() + $Conn.Dispose() + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +Write-Output "Cleaned up $DeletedRows recommendations." \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 new file mode 100644 index 000000000..391ed24c9 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 @@ -0,0 +1,321 @@ +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$sharedKey = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceKey" +$LogAnalyticsChunkSize = [int] (Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsChunkSize" -ErrorAction SilentlyContinue) +if (-not($LogAnalyticsChunkSize -gt 0)) +{ + $LogAnalyticsChunkSize = 6000 +} +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} +$StorageBlobsPageSize = [int] (Get-AutomationVariable -Name "AzureOptimization_StorageBlobsPageSize" -ErrorAction SilentlyContinue) +if (-not($StorageBlobsPageSize -gt 0)) +{ + $StorageBlobsPageSize = 1000 +} + +#region Functions + +# Function to create the authorization signature +Function Build-OMSSignature ($workspaceId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) { + $xHeaders = "x-ms-date:" + $date + $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource + $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) + $keyBytes = [Convert]::FromBase64String($sharedKey) + $sha256 = New-Object System.Security.Cryptography.HMACSHA256 + $sha256.Key = $keyBytes + $calculatedHash = $sha256.ComputeHash($bytesToHash) + $encodedHash = [Convert]::ToBase64String($calculatedHash) + $authorization = 'SharedKey {0}:{1}' -f $workspaceId, $encodedHash + return $authorization +} + +# Function to create and post the request +Function Post-OMSData($workspaceId, $sharedKey, $body, $logType, $TimeStampField, $AzureEnvironment) { + $method = "POST" + $contentType = "application/json" + $resource = "/api/logs" + $rfc1123date = [DateTime]::UtcNow.ToString("r") + $contentLength = $body.Length + $signature = Build-OMSSignature ` + -workspaceId $workspaceId ` + -sharedKey $sharedKey ` + -date $rfc1123date ` + -contentLength $contentLength ` + -method $method ` + -contentType $contentType ` + -resource $resource + + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" + if ($AzureEnvironment -eq "AzureChinaCloud") + { + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.cn" + $resource + "?api-version=2016-04-01" + } + if ($AzureEnvironment -eq "AzureUSGovernment") + { + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.us" + $resource + "?api-version=2016-04-01" + } + if ($AzureEnvironment -eq "AzureGermanCloud") + { + throw "Azure Germany isn't suported for the Log Analytics Data Collector API" + } + + $OMSheaders = @{ + "Authorization" = $signature; + "Log-Type" = $logType; + "x-ms-date" = $rfc1123date; + "time-generated-field" = $TimeStampField; + } + + Try { + + $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + } + catch { + if ($_.Exception.Response.StatusCode.Value__ -eq 401) { + "REAUTHENTICATING" + + $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + } + else + { + return $_.Exception.Response.StatusCode.Value__ + } + } + + return $response.StatusCode +} +#endregion Functions + + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +# get reference to storage sink +Write-Output "Getting reference to $storageAccountSink storage account (recommendations exports sink)" +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context + +$allblobs = @() + +Write-Output "Getting blobs list..." +$continuationToken = $null +do +{ + $blobs = Get-AzStorageBlob -Container $storageAccountSinkContainer -MaxCount $StorageBlobsPageSize -ContinuationToken $continuationToken -Context $saCtx | Sort-Object -Property LastModified + if ($blobs.Count -le 0) { break } + $allblobs += $blobs + $continuationToken = $blobs[$blobs.Count -1].ContinuationToken; +} +While ($null -ne $continuationToken) + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE StorageContainerName = '$storageAccountSinkContainer'" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$Conn.Close() +$Conn.Dispose() + +if ($controlRows.Count -eq 0 -or -not($controlRows[0].LastProcessedDateTime)) +{ + throw "Could not find a valid ingestion control row for $storageAccountSinkContainer" +} + +$controlRow = $controlRows[0] +$lastProcessedLine = $controlRow.LastProcessedLine +$lastProcessedDateTime = $controlRow.LastProcessedDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +$LogAnalyticsSuffix = $controlRow.LogAnalyticsSuffix +$logname = $lognamePrefix + $LogAnalyticsSuffix + +Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the $($logname)_CL table..." + +$newProcessedTime = $null + +$unprocessedBlobs = @() + +foreach ($blob in $allblobs) { + $blobLastModified = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + if ($lastProcessedDateTime -lt $blobLastModified -or ` + ($lastProcessedDateTime -eq $blobLastModified -and $lastProcessedLine -gt 0)) { + Write-Output "$($blob.Name) found (modified on $blobLastModified)" + $unprocessedBlobs += $blob + } +} + +$unprocessedBlobs = $unprocessedBlobs | Sort-Object -Property LastModified + +Write-Output "Found $($unprocessedBlobs.Count) new blobs to process..." + +foreach ($blob in $unprocessedBlobs) { + $newProcessedTime = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "About to process $($blob.Name)..." + Get-AzStorageBlobContent -CloudBlob $blob.ICloudBlob -Context $saCtx -Force + $jsonObject = Get-Content -Path $blob.Name | ConvertFrom-Json + Write-Output "Blob contains $($jsonObject.Count) results..." + + if ($null -eq $jsonObject) + { + $recCount = 0 + } + elseif ($null -eq $jsonObject.Count) + { + $recCount = 1 + } + else + { + $recCount = $jsonObject.Count + } + + $linesProcessed = 0 + $jsonObjectSplitted = @() + + if ($recCount -gt 1) + { + for ($i = 0; $i -lt $recCount; $i += $LogAnalyticsChunkSize) { + $jsonObjectSplitted += , @($jsonObject[$i..($i + ($LogAnalyticsChunkSize - 1))]); + } + } + else + { + $jsonObjectArray = @() + $jsonObjectArray += $jsonObject + $jsonObjectSplitted += , $jsonObjectArray + } + + for ($j = 0; $j -lt $jsonObjectSplitted.Count; $j++) + { + if ($jsonObjectSplitted[$j]) + { + $currentObjectLines = $jsonObjectSplitted[$j].Count + if ($lastProcessedLine -lt $linesProcessed) + { + for ($i = 0; $i -lt $jsonObjectSplitted[$j].Count; $i++) + { + $jsonObjectSplitted[$j][$i].RecommendationDescription = $jsonObjectSplitted[$j][$i].RecommendationDescription.Replace("'", "") + $jsonObjectSplitted[$j][$i].RecommendationAction = $jsonObjectSplitted[$j][$i].RecommendationAction.Replace("'", "") + $jsonObjectSplitted[$j][$i].AdditionalInfo = $jsonObjectSplitted[$j][$i].AdditionalInfo | ConvertTo-Json -Compress + $jsonObjectSplitted[$j][$i].Tags = $jsonObjectSplitted[$j][$i].Tags | ConvertTo-Json -Compress + } + + $jsonObject = ConvertTo-Json -InputObject $jsonObjectSplitted[$j] + $res = Post-OMSData -workspaceId $workspaceId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($jsonObject)) -logType $logname -TimeStampField "Timestamp" -AzureEnvironment $cloudEnvironment + If ($res -ge 200 -and $res -lt 300) { + Write-Output "Succesfully uploaded $currentObjectLines $LogAnalyticsSuffix rows to Log Analytics" + $linesProcessed += $currentObjectLines + if ($j -eq ($jsonObjectSplitted.Count - 1)) { + $lastProcessedLine = -1 + } + else { + $lastProcessedLine = $linesProcessed - 1 + } + + $updatedLastProcessedLine = $lastProcessedLine + $updatedLastProcessedDateTime = $lastProcessedDateTime + if ($j -eq ($jsonObjectSplitted.Count - 1)) { + $updatedLastProcessedDateTime = $newProcessedTime + } + $lastProcessedDateTime = $updatedLastProcessedDateTime + Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" + $sqlStatement = "UPDATE [$LogAnalyticsIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandText = $sqlStatement + $Cmd.CommandTimeout=120 + $Cmd.ExecuteReader() + $Conn.Close() + $Conn.Dispose() + } + Else { + $linesProcessed += $currentObjectLines + Write-Warning "Failed to upload $currentObjectLines $LogAnalyticsSuffix rows. Error code: $res" + throw + } + } + else + { + $linesProcessed += $currentObjectLines + } + } + } + + Remove-Item -Path $blob.Name -Force +} + +Write-Output "DONE" \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 new file mode 100644 index 000000000..f7041940f --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 @@ -0,0 +1,285 @@ +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} +$ChunkSize = [int] (Get-AutomationVariable -Name "AzureOptimization_SQLServerInsertSize" -ErrorAction SilentlyContinue) +if (-not($ChunkSize -gt 0)) +{ + $ChunkSize = 900 +} +$SqlTimeout = 120 + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} +$StorageBlobsPageSize = [int] (Get-AutomationVariable -Name "AzureOptimization_StorageBlobsPageSize" -ErrorAction SilentlyContinue) +if (-not($StorageBlobsPageSize -gt 0)) +{ + $StorageBlobsPageSize = 1000 +} + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +# get reference to storage sink +Write-Output "Getting reference to $storageAccountSink storage account (recommendations exports sink)" +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context + +$allblobs = @() + +Write-Output "Getting blobs list..." +$continuationToken = $null +do +{ + $blobs = Get-AzStorageBlob -Container $storageAccountSinkContainer -MaxCount $StorageBlobsPageSize -ContinuationToken $continuationToken -Context $saCtx | Sort-Object -Property LastModified + if ($blobs.Count -le 0) { break } + $allblobs += $blobs + $continuationToken = $blobs[$blobs.Count -1].ContinuationToken; +} +While ($null -ne $continuationToken) + +$SqlServerIngestControlTable = "SqlServerIngestControl" +$recommendationsTable = "Recommendations" + +$tries = 0 +$connectionSuccess = $false + +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$SqlServerIngestControlTable] WHERE StorageContainerName = '$storageAccountSinkContainer' and SqlTableName = '$recommendationsTable'" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +if ($controlRows.Count -eq 0) +{ + throw "Could not find a control row for $storageAccountSinkContainer container and $recommendationsTable table." +} + +$controlRow = $controlRows[0] +$lastProcessedLine = $controlRow.LastProcessedLine +$lastProcessedDateTime = $controlRow.LastProcessedDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + +$Conn.Close() +$Conn.Dispose() + +Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the Recommendations SQL table..." + +$newProcessedTime = $null + +$unprocessedBlobs = @() + +foreach ($blob in $allblobs) { + $blobLastModified = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + if ($lastProcessedDateTime -lt $blobLastModified -or ` + ($lastProcessedDateTime -eq $blobLastModified -and $lastProcessedLine -gt 0)) { + Write-Output "$($blob.Name) found (modified on $blobLastModified)" + $unprocessedBlobs += $blob + } +} + +$unprocessedBlobs = $unprocessedBlobs | Sort-Object -Property LastModified + +Write-Output "Found $($unprocessedBlobs.Count) new blobs to process..." + +foreach ($blob in $unprocessedBlobs) { + $newProcessedTime = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") + Write-Output "About to process $($blob.Name)..." + Get-AzStorageBlobContent -CloudBlob $blob.ICloudBlob -Context $saCtx -Force + $jsonObject = Get-Content -Path $blob.Name | ConvertFrom-Json + Write-Output "Blob contains $($jsonObject.Count) results..." + + if ($null -eq $jsonObject) + { + $recCount = 0 + } + elseif ($null -eq $jsonObject.Count) + { + $recCount = 1 + } + else + { + $recCount = $jsonObject.Count + } + + $linesProcessed = 0 + $jsonObjectSplitted = @() + + if ($recCount -gt 1) + { + for ($i = 0; $i -lt $recCount; $i += $ChunkSize) { + $jsonObjectSplitted += , @($jsonObject[$i..($i + ($ChunkSize - 1))]); + } + } + else + { + $jsonObjectArray = @() + $jsonObjectArray += $jsonObject + $jsonObjectSplitted += , $jsonObjectArray + } + + for ($j = 0; $j -lt $jsonObjectSplitted.Count; $j++) + { + if ($jsonObjectSplitted[$j]) + { + $currentObjectLines = $jsonObjectSplitted[$j].Count + if ($lastProcessedLine -lt $linesProcessed) + { + $sqlStatement = "INSERT INTO [$recommendationsTable]" + $sqlStatement += " (RecommendationId, GeneratedDate, Cloud, Category, ImpactedArea, Impact, RecommendationType, RecommendationSubType," + $sqlStatement += " RecommendationSubTypeId, RecommendationDescription, RecommendationAction, InstanceId, InstanceName, AdditionalInfo," + $sqlStatement += " ResourceGroup, SubscriptionGuid, SubscriptionName, TenantGuid, FitScore, Tags, DetailsUrl) VALUES" + for ($i = 0; $i -lt $jsonObjectSplitted[$j].Count; $i++) + { + $jsonObjectSplitted[$j][$i].RecommendationDescription = $jsonObjectSplitted[$j][$i].RecommendationDescription.Replace("'", "") + $jsonObjectSplitted[$j][$i].RecommendationAction = $jsonObjectSplitted[$j][$i].RecommendationAction.Replace("'", "") + if ($null -ne $jsonObjectSplitted[$j][$i].InstanceName) + { + $jsonObjectSplitted[$j][$i].InstanceName = $jsonObjectSplitted[$j][$i].InstanceName.Replace("'", "") + } + $additionalInfoString = $jsonObjectSplitted[$j][$i].AdditionalInfo | ConvertTo-Json -Compress + $tagsString = $jsonObjectSplitted[$j][$i].Tags | ConvertTo-Json -Compress + $subscriptionGuid = "NULL" + if ($jsonObjectSplitted[$j][$i].SubscriptionGuid) + { + $subscriptionGuid = "'$($jsonObjectSplitted[$j][$i].SubscriptionGuid)'" + } + $subscriptionName = "NULL" + if ($jsonObjectSplitted[$j][$i].SubscriptionName) + { + $subscriptionName = $jsonObjectSplitted[$j][$i].SubscriptionName.Replace("'", "") + $subscriptionName = "'$subscriptionName'" + } + $resourceGroup = "NULL" + if ($jsonObjectSplitted[$j][$i].ResourceGroup) + { + $resourceGroup = "'$($jsonObjectSplitted[$j][$i].ResourceGroup)'" + } + $sqlStatement += " (NEWID(), CONVERT(DATETIME, '$($jsonObjectSplitted[$j][$i].Timestamp)'), '$($jsonObjectSplitted[$j][$i].Cloud)'" + $sqlStatement += ", '$($jsonObjectSplitted[$j][$i].Category)', '$($jsonObjectSplitted[$j][$i].ImpactedArea)'" + $sqlStatement += ", '$($jsonObjectSplitted[$j][$i].Impact)', '$($jsonObjectSplitted[$j][$i].RecommendationType)'" + $sqlStatement += ", '$($jsonObjectSplitted[$j][$i].RecommendationSubType)', '$($jsonObjectSplitted[$j][$i].RecommendationSubTypeId)'" + $sqlStatement += ", '$($jsonObjectSplitted[$j][$i].RecommendationDescription)', '$($jsonObjectSplitted[$j][$i].RecommendationAction)'" + $sqlStatement += ", '$($jsonObjectSplitted[$j][$i].InstanceId)', '$($jsonObjectSplitted[$j][$i].InstanceName)', '$additionalInfoString'" + $sqlStatement += ", $resourceGroup, $subscriptionGuid, $subscriptionName, '$($jsonObjectSplitted[$j][$i].TenantGuid)'" + $sqlStatement += ", $($jsonObjectSplitted[$j][$i].FitScore), '$tagsString', '$($jsonObjectSplitted[$j][$i].DetailsURL)')" + if ($i -ne ($jsonObjectSplitted[$j].Count-1)) + { + $sqlStatement += "," + } + } + + $Conn2 = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn2.Open() + + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn2 + $Cmd.CommandText = $sqlStatement + $Cmd.CommandTimeout=120 + try + { + $Cmd.ExecuteReader() + } + catch + { + Write-Output "Failed statement: $sqlStatement" + throw + } + + $Conn2.Close() + + $linesProcessed += $currentObjectLines + Write-Output "Processed $linesProcessed lines..." + if ($j -eq ($jsonObjectSplitted.Count - 1)) { + $lastProcessedLine = -1 + } + else { + $lastProcessedLine = $linesProcessed - 1 + } + + $updatedLastProcessedLine = $lastProcessedLine + $updatedLastProcessedDateTime = $lastProcessedDateTime + if ($j -eq ($jsonObjectSplitted.Count - 1)) { + $updatedLastProcessedDateTime = $newProcessedTime + } + $lastProcessedDateTime = $updatedLastProcessedDateTime + Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" + $sqlStatement = "UPDATE [$SqlServerIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandText = $sqlStatement + $Cmd.CommandTimeout=$SqlTimeout + $Cmd.ExecuteReader() + $Conn.Close() + } + else + { + $linesProcessed += $currentObjectLines + } + } + } + + Remove-Item -Path $blob.Name -Force +} + +Write-Output "DONE" \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 new file mode 100644 index 000000000..2ebbe91fb --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 @@ -0,0 +1,224 @@ +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$sharedKey = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceKey" +$LogAnalyticsChunkSize = [int] (Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsChunkSize" -ErrorAction SilentlyContinue) +if (-not($LogAnalyticsChunkSize -gt 0)) +{ + $LogAnalyticsChunkSize = 6000 +} +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$SqlTimeout = 300 +$FiltersTable = "Filters" + +#region Functions + +# Function to create the authorization signature +Function Build-OMSSignature ($workspaceId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) { + $xHeaders = "x-ms-date:" + $date + $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource + $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) + $keyBytes = [Convert]::FromBase64String($sharedKey) + $sha256 = New-Object System.Security.Cryptography.HMACSHA256 + $sha256.Key = $keyBytes + $calculatedHash = $sha256.ComputeHash($bytesToHash) + $encodedHash = [Convert]::ToBase64String($calculatedHash) + $authorization = 'SharedKey {0}:{1}' -f $workspaceId, $encodedHash + return $authorization +} + +# Function to create and post the request +Function Post-OMSData($workspaceId, $sharedKey, $body, $logType, $TimeStampField, $AzureEnvironment) { + $method = "POST" + $contentType = "application/json" + $resource = "/api/logs" + $rfc1123date = [DateTime]::UtcNow.ToString("r") + $contentLength = $body.Length + $signature = Build-OMSSignature ` + -workspaceId $workspaceId ` + -sharedKey $sharedKey ` + -date $rfc1123date ` + -contentLength $contentLength ` + -method $method ` + -contentType $contentType ` + -resource $resource + + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" + if ($AzureEnvironment -eq "AzureChinaCloud") + { + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.cn" + $resource + "?api-version=2016-04-01" + } + if ($AzureEnvironment -eq "AzureUSGovernment") + { + $uri = "https://" + $workspaceId + ".ods.opinsights.azure.us" + $resource + "?api-version=2016-04-01" + } + if ($AzureEnvironment -eq "AzureGermanCloud") + { + throw "Azure Germany isn't suported for the Log Analytics Data Collector API" + } + + $OMSheaders = @{ + "Authorization" = $signature; + "Log-Type" = $logType; + "x-ms-date" = $rfc1123date; + "time-generated-field" = $TimeStampField; + } + + Try { + + $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + } + catch { + if ($_.Exception.Response.StatusCode.Value__ -eq 401) { + "REAUTHENTICATING" + + $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + } + else + { + return $_.Exception.Response.StatusCode.Value__ + } + } + + return $response.StatusCode +} +#endregion Functions + +Write-Output "Getting excluded recommendation sub-type IDs..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$FiltersTable] WHERE IsEnabled = 1 AND (FilterEndDate IS NULL OR FilterEndDate > GETDATE())" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $filters = New-Object System.Data.DataTable + $sqlAdapter.Fill($filters) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$Conn.Close() +$Conn.Dispose() + +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +$filterObjects = @() + +$filterObject = New-Object PSObject -Property @{ + Timestamp = $timestamp + FilterId = (New-Guid).Guid + RecommendationSubTypeId = [System.Guid]::empty.Guid + FilterType = "Dummy" + InstanceId = [System.Guid]::empty.Guid + InstanceName = "Dummy" + FilterStartDate = "2019-01-01T00:00:00.000Z" + FilterEndDate = "2199-12-31T23:59:59.000Z" + Author = "AOE" + Notes = "This is a dummy suppression required to build the full suppressions schema in Log Analytics" +} +$filterObjects += $filterObject + +foreach ($filter in $filters) +{ + $filterEndDate = $null + if (-not([string]::IsNullOrEmpty($filter.FilterEndDate))) + { + Write-Output $filter.FilterEndDate + $filterEndDate = $filter.FilterEndDate.ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + else + { + $filterEndDate = "2199-12-31T23:59:59.000Z" + } + + $filterStartDate = $null + if (-not([string]::IsNullOrEmpty($filter.FilterStartDate))) + { + $filterStartDate = $filter.FilterStartDate.ToString("yyyy-MM-ddTHH:mm:00.000Z") + } + else + { + $filterStartDate = "2019-01-01T00:00:00.000Z" + } + + $instanceId = $null + $instanceName = $null + $ObjectGuid = [System.Guid]::empty + if ([System.Guid]::TryParse($filter.InstanceId, [System.Management.Automation.PSReference]$ObjectGuid)) + { + $instanceId = $filter.InstanceId + } + else + { + $instanceName = $filter.InstanceId + } + + $filterObject = New-Object PSObject -Property @{ + Timestamp = $timestamp + FilterId = $filter.FilterId + RecommendationSubTypeId = $filter.RecommendationSubTypeId + FilterType = $filter.FilterType + InstanceId = $instanceId + InstanceName = $instanceName + FilterStartDate = $filterStartDate + FilterEndDate = $filterEndDate + Author = $filter.Author + Notes = $filter.Notes + } + $filterObjects += $filterObject +} + +$filtersJson = $filterObjects | ConvertTo-Json + +$LogAnalyticsSuffix = "SuppressionsV1" +$logname = $lognamePrefix + $LogAnalyticsSuffix + +$res = Post-OMSData -workspaceId $workspaceId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($filtersJson)) -logType $logname -TimeStampField "Timestamp" -AzureEnvironment $cloudEnvironment +If ($res -ge 200 -and $res -lt 300) { + Write-Output "Succesfully uploaded $($filterObjects.Count) $LogAnalyticsSuffix rows to Log Analytics" +} +Else { + Write-Warning "Failed to upload $($filterObjects.Count) $LogAnalyticsSuffix rows. Error code: $res" + throw +} diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AADExpiringCredentialsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AADExpiringCredentialsToBlobStorage.ps1 new file mode 100644 index 000000000..00e81c592 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AADExpiringCredentialsToBlobStorage.ps1 @@ -0,0 +1,369 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$expiringCredsDays = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendationAADMinCredValidityDays") +$notExpiringCredsDays = ([int] (Get-AutomationVariable -Name "AzureOptimization_RecommendationAADMaxCredValidityYears")) * 365 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('AADObjects')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$aadObjectsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AADObjects' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $aadObjectsTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 1 + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +# Execute the expiring creds recommendation query against Log Analytics + +$baseQuery = @" + let expiryInterval = $($expiringCredsDays)d; + let AppsAndKeys = materialize ($aadObjectsTableName + | where TimeGenerated > ago(1d) + | where ObjectType_s in ('Application','ServicePrincipal') + | where ObjectSubType_s != 'ManagedIdentity' + | where Keys_s startswith '[' + | extend Keys = parse_json(Keys_s) + | project-away Keys_s + | mv-expand Keys + | evaluate bag_unpack(Keys) + | union ( + $aadObjectsTableName + | where TimeGenerated > ago(1d) + | where ObjectType_s in ('Application','ServicePrincipal') + | where ObjectSubType_s != 'ManagedIdentity' + | where isnotempty(Keys_s) and Keys_s !startswith '[' + | extend Keys = parse_json(Keys_s) + | project-away Keys_s + | evaluate bag_unpack(Keys) + ) + ); + let ExpirationInRisk = AppsAndKeys + | where EndDate < now()+expiryInterval + | project ApplicationId_g, KeyId, RiskDate = EndDate; + let NotInRisk = AppsAndKeys + | where EndDate > now()+expiryInterval + | project ApplicationId_g, KeyId, ComfortDate = EndDate; + let ApplicationsInRisk = ExpirationInRisk + | join kind=leftouter ( NotInRisk ) on ApplicationId_g + | where isempty(ComfortDate) + | summarize ExpiresOn = max(RiskDate) by ApplicationId_g; + AppsAndKeys + | join kind=inner (ApplicationsInRisk) on ApplicationId_g + | summarize ExpiresOn = max(EndDate) by ApplicationId_g, ObjectType_s, DisplayName_s, Cloud_s, KeyType, TenantGuid_g + | order by ExpiresOn desc +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.ApplicationId_g + $detailsURL = "https://portal.azure.$azureTld/#blade/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/Credentials/appId/$queryInstanceId" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["ObjectType"] = $result.ObjectType_s + $additionalInfoDictionary["KeyType"] = $result.KeyType + $additionalInfoDictionary["ExpiresOn"] = $result.ExpiresOn + + $fitScore = 5 + + $tags = @{} + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.AzureActiveDirectory/objects" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "AADExpiringCredentials" + RecommendationSubTypeId = "3292c489-2782-498b-aad0-a4cef50f6ca2" + RecommendationDescription = "Microsoft Entra application with credentials expired or about to expire" + RecommendationAction = "Update the Microsoft Entra application credential before the expiration date" + InstanceId = $result.ApplicationId_g + InstanceName = $result.DisplayName_s + AdditionalInfo = $additionalInfoDictionary + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "aadexpiringcerts-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +# Execute the not expiring in less than X years creds recommendation query against Log Analytics + +$baseQuery = @" + let expiryInterval = $($notExpiringCredsDays)d; + let AppsAndKeys = materialize ($aadObjectsTableName + | where TimeGenerated > ago(1d) + | where ObjectSubType_s != 'ManagedIdentity' + | where Keys_s startswith '[' + | extend Keys = parse_json(Keys_s) + | project-away Keys_s + | mv-expand Keys + | evaluate bag_unpack(Keys) + | union ( + $aadObjectsTableName + | where TimeGenerated > ago(1d) + | where ObjectSubType_s != 'ManagedIdentity' + | where isnotempty(Keys_s) and Keys_s !startswith '[' + | extend Keys = parse_json(Keys_s) + | project-away Keys_s + | evaluate bag_unpack(Keys) + ) + ); + AppsAndKeys + | where EndDate > now()+expiryInterval + | project ApplicationId_g, ObjectType_s, DisplayName_s, Cloud_s, KeyType, TenantGuid_g, EndDate +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.ApplicationId_g + $detailsURL = "https://portal.azure.$azureTld/#blade/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/Credentials/appId/$queryInstanceId" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["ObjectType"] = $result.ObjectType_s + $additionalInfoDictionary["KeyType"] = $result.KeyType + $additionalInfoDictionary["ExpiresOn"] = $result.EndDate + + $fitScore = 5 + + $tags = @{} + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Security" + ImpactedArea = "Microsoft.AzureActiveDirectory/objects" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "AADNotExpiringCredentials" + RecommendationSubTypeId = "ecd969c8-3f16-481a-9577-5ed32e5e1a1d" + RecommendationDescription = "Microsoft Entra application with credentials expiration not set or too far in time" + RecommendationAction = "Update the Microsoft Entra application credential with a shorter expiration date" + InstanceId = $result.ApplicationId_g + InstanceName = $result.DisplayName_s + AdditionalInfo = $additionalInfoDictionary + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "aadnotexpiringcerts-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-ARMOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-ARMOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..d959112c2 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-ARMOptimizationsToBlobStorage.ps1 @@ -0,0 +1,515 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$assignmentsPercentageThresholdVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationRBACAssignmentsPercentageThreshold" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($assignmentsPercentageThresholdVar) -or $assignmentsPercentageThresholdVar -eq 0) +{ + $assignmentsPercentageThreshold = 80 +} +else +{ + $assignmentsPercentageThreshold = [int] $assignmentsPercentageThresholdVar +} + +$assignmentsSubscriptionsLimitVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationRBACSubscriptionsAssignmentsLimit" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($assignmentsSubscriptionsLimitVar) -or $assignmentsSubscriptionsLimitVar -eq 0) +{ + $assignmentsSubscriptionsLimit = 4000 +} +else +{ + $assignmentsSubscriptionsLimit = [int] $assignmentsSubscriptionsLimitVar +} + +$assignmentsMgmtGroupsLimitVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationRBACMgmtGroupsAssignmentsLimit" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($assignmentsMgmtGroupsLimitVar) -or $assignmentsMgmtGroupsLimitVar -eq 0) +{ + $assignmentsMgmtGroupsLimit = 500 +} +else +{ + $assignmentsMgmtGroupsLimit = [int] $assignmentsMgmtGroupsLimitVar +} + +$rgPercentageThresholdVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationResourceGroupsPerSubPercentageThreshold" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($rgPercentageThresholdVar) -or $rgPercentageThresholdVar -eq 0) +{ + $rgPercentageThreshold = 80 +} +else +{ + $rgPercentageThreshold = [int] $rgPercentageThresholdVar +} + +$rgLimitVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationResourceGroupsPerSubLimit" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($rgLimitVar) -or $rgLimitVar -eq 0) +{ + $rgLimit = 980 +} +else +{ + $rgLimit = [int] $rgLimitVar +} + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('RBACAssignments','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$rbacTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'RBACAssignments' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $rbacTableName and $subscriptionsTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +$assignmentsThreshold = $assignmentsSubscriptionsLimit * ($assignmentsPercentageThreshold / 100) + +Write-Output "Looking for subscriptions with more than $assignmentsPercentageThreshold% of the $assignmentsSubscriptionsLimit RBAC assignments limit..." + +$baseQuery = @" + $rbacTableName + | where TimeGenerated > ago(1d) and Model_s == 'AzureRM' and Scope_s startswith '/subscriptions/' + | extend SubscriptionGuid_g = tostring(split(Scope_s, '/')[2]) + | summarize AssignmentsCount=count() by SubscriptionGuid_g, TenantGuid_g, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s, Tags_s, InstanceId_s + ) on SubscriptionGuid_g + | where AssignmentsCount >= $assignmentsThreshold +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/users" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["assignmentsCount"] = $result.AssignmentsCount + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Resources/subscriptions" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "HighRBACAssignmentsSubscriptions" + RecommendationSubTypeId = "c6a88d8c-3242-44b0-9793-c91897ef68bc" + RecommendationDescription = "Subscriptions close to the maximum limit of RBAC assignments" + RecommendationAction = "Remove unneeded RBAC assignments or use group-based (or nested group-based) assignments" + InstanceId = $result.InstanceId_s + InstanceName = $result.SubscriptionName + AdditionalInfo = $additionalInfoDictionary + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "subscriptionsrbaclimits-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +$assignmentsThreshold = $assignmentsMgmtGroupsLimit * ($assignmentsPercentageThreshold / 100) + +Write-Output "Looking for management groups with more than $assignmentsPercentageThreshold% of the $assignmentsMgmtGroupsLimit RBAC assignments limit..." + +$baseQuery = @" + $rbacTableName + | where TimeGenerated > ago(1d) and Model_s == 'AzureRM' and Scope_s has 'managementGroups' + | extend ManagementGroupId = tostring(split(Scope_s, '/')[4]) + | summarize AssignmentsCount=count() by ManagementGroupId, TenantGuid_g, Scope_s, Cloud_s + | where AssignmentsCount >= $assignmentsThreshold +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/blade/Microsoft_Azure_ManagementGroups/ManagementGroupBrowseBlade/MGBrowse_overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["assignmentsCount"] = $result.AssignmentsCount + + $fitScore = 5 + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Management/managementGroups" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "HighRBACAssignmentsManagementGroups" + RecommendationSubTypeId = "b36dea3e-ef21-45a9-a704-6f629fab236d" + RecommendationDescription = "Management Groups close to the maximum limit of RBAC assignments" + RecommendationAction = "Remove unneeded RBAC assignments or use group-based (or nested group-based) assignments" + InstanceId = $result.Scope_s + InstanceName = $result.ManagementGroupId + AdditionalInfo = $additionalInfoDictionary + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "mgmtgroupsrbaclimits-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +$rgThreshold = $rgLimit * ($rgPercentageThreshold / 100) + +Write-Output "Looking for subscriptions with more than $rgPercentageThreshold% of the $rgLimit Resource Groups limit..." + +$baseQuery = @" + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions/resourceGroups' + | summarize RGCount=count() by SubscriptionGuid_g, TenantGuid_g, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s, Tags_s, InstanceId_s + ) on SubscriptionGuid_g + | where RGCount >= $rgThreshold +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/resourceGroups" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["resourceGroupsCount"] = $result.RGCount + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Resources/subscriptions" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "HighResourceGroupCountSubscriptions" + RecommendationSubTypeId = "4468da8d-1e72-4998-b6d2-3bc38ddd9330" + RecommendationDescription = "Subscriptions close to the maximum limit of resource groups" + RecommendationAction = "Remove unneeded resource groups or split your resource groups across multiple subscriptions" + InstanceId = $result.InstanceId_s + InstanceName = $result.SubscriptionName + AdditionalInfo = $additionalInfoDictionary + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "subscriptionsrglimits-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AdvisorAsIsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AdvisorAsIsToBlobStorage.ps1 new file mode 100644 index 000000000..2b1de2d5d --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AdvisorAsIsToBlobStorage.ps1 @@ -0,0 +1,311 @@ +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +# must be less than or equal to the advisor exports frequency +$daysBackwards = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendAdvisorPeriodInDays" -ErrorAction SilentlyContinue) +if (-not($daysBackwards -gt 0)) { + $daysBackwards = 7 +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$CategoryFilter = Get-AutomationVariable -Name "AzureOptimization_AdvisorFilter" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($CategoryFilter)) +{ + $CategoryFilter = "HighAvailability,Security,Performance,OperationalExcellence" # comma-separated list of categories +} + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" +$FiltersTable = "Filters" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGVirtualMachine','AzureAdvisor','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$advisorTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureAdvisor' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $subscriptionsTableName and $advisorTableName" + +$Conn.Close() +$Conn.Dispose() + +Write-Output "Getting excluded recommendation sub-type IDs..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$FiltersTable] WHERE FilterType = 'Exclude' AND IsEnabled = 1 AND (FilterEndDate IS NULL OR FilterEndDate > GETDATE())" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $filters = New-Object System.Data.DataTable + $sqlAdapter.Fill($filters) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$Conn.Close() +$Conn.Dispose() + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +# Execute the recommendation query against Log Analytics + +$FinalCategoryFilter = "" + +if (-not([string]::IsNullOrEmpty($CategoryFilter))) +{ + $categories = $CategoryFilter.Split(',') + for ($i = 0; $i -lt $categories.Count; $i++) + { + $categories[$i] = "'" + $categories[$i] + "'" + } + $FinalCategoryFilter = " and Category in (" + ($categories -join ",") + ")" +} + +$baseQuery = @" +let advisorInterval = $($daysBackwards)d; +$advisorTableName +| where todatetime(TimeGenerated) > ago(advisorInterval)$FinalCategoryFilter +| extend AdvisorRecIdIndex = indexof(InstanceId_s, '/providers/microsoft.advisor/recommendations') +| extend InstanceName_s = iif(isnotempty(InstanceName_s),InstanceName_s,iif(AdvisorRecIdIndex > 0, split(substring(InstanceId_s, 0, AdvisorRecIdIndex),'/')[-1], split(InstanceId_s,'/')[-1])) +| summarize by InstanceId_s, InstanceName_s, Category, Description_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroup, Cloud_s, AdditionalInfo_s, RecommendationText_s, ImpactedArea_s, Impact_s, RecommendationTypeId_g, Tags_s +| join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s +) on SubscriptionGuid_g +"@ + +Write-Output "Getting $CategoryFilter recommendations for $($daysBackwards)d Advisor..." + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $daysBackwards) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + throw "Execution aborted" +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +Write-Output "Generating fit score..." + +foreach ($result in $results) { + + if ($filters | Where-Object { $_.RecommendationSubTypeId -eq $result.RecommendationTypeId_g}) + { + continue + } + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $additionalInfoDictionary = @{} + if (-not([string]::IsNullOrEmpty($result.AdditionalInfo_s))) + { + ($result.AdditionalInfo_s | ConvertFrom-Json).PsObject.Properties | ForEach-Object { $additionalInfoDictionary[$_.Name] = $_.Value } + } + + $fitScore = 5 + + $queryInstanceId = $result.InstanceId_s + + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $recommendationSubType = "Advisor" + $result.Category + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = $result.Category + ImpactedArea = $result.ImpactedArea_s + Impact = $result.Impact_s + RecommendationType = "BestPractices" + RecommendationSubType = $recommendationSubType + RecommendationSubTypeId = $result.RecommendationTypeId_g + RecommendationDescription = $result.Description_s.Replace("'","") + RecommendationAction = $result.RecommendationText_s.Replace("'","") + InstanceId = $result.InstanceId_s + InstanceName = $result.InstanceName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +Write-Output "Exporting final $($recommendations.Count) results as a JSON file..." + +$fileDate = $datetime.ToString("yyyyMMdd") +$jsonExportPath = "advisor-asis-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +Write-Output "Uploading $jsonExportPath to blob storage..." + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json" }; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AdvisorCostAugmentedToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AdvisorCostAugmentedToBlobStorage.ps1 new file mode 100644 index 000000000..fd3d91b12 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AdvisorCostAugmentedToBlobStorage.ps1 @@ -0,0 +1,899 @@ +$ErrorActionPreference = "Stop" + +function Find-SkuHourlyPrice { + param ( + [object[]] $SKUPriceSheet, + [string] $SKUName + ) + + $skuPriceObject = $null + + if ($SKUPriceSheet) + { + $skuNameParts = $SKUName.Split('_') + + if ($skuNameParts.Count -eq 3) # e.g., Standard_D1_v2 + { + $skuNameFilter = "*" + $skuNameParts[1] + " *" + $skuVersionFilter = "*" + $skuNameParts[2] + $skuPrices = $SKUPriceSheet | Where-Object { $_.MeterName_s -like $skuNameFilter ` + -and $_.MeterName_s -notlike '*Low Priority' -and $_.MeterName_s -notlike '*Expired' ` + -and $_.MeterName_s -like $skuVersionFilter -and $_.MeterSubCategory_s -notlike '*Windows' -and $_.UnitPrice_s -ne 0 } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + if ($skuPrices.Count -gt 2) # D1-like scenarios + { + $skuFilter = "*" + $skuNameParts[1] + " " + $skuNameParts[2] + "*" + $skuPrices = $skuPrices | Where-Object { $_.MeterName_s -like $skuFilter } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + } + } + + if ($skuNameParts.Count -eq 2) # e.g., Standard_D1 + { + $skuNameFilter = "*" + $skuNameParts[1] + "*" + + $skuPrices = $SKUPriceSheet | Where-Object { $_.MeterName_s -like $skuNameFilter ` + -and $_.MeterName_s -notlike '*Low Priority' -and $_.MeterName_s -notlike '*Expired' ` + -and $_.MeterName_s -notlike '* v*' -and $_.MeterSubCategory_s -notlike '*Windows' -and $_.UnitPrice_s -ne 0 } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + if ($skuPrices.Count -gt 2) # D1-like scenarios + { + $skuFilterLeft = "*" + $skuNameParts[1] + "/*" + $skuFilterRight = "*/" + $skuNameParts[1] + "*" + $skuPrices = $skuPrices | Where-Object { $_.MeterName_s -like $skuFilterLeft -or $_.MeterName_s -like $skuFilterRight } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + } + } + } + + $targetHourlyPrice = [double]::MaxValue + if ($null -ne $skuPriceObject) + { + $targetUnitHours = [int] (Select-String -InputObject $skuPriceObject.UnitOfMeasure_s -Pattern "^\d+").Matches[0].Value + if ($targetUnitHours -gt 0) + { + $targetHourlyPrice = [double] ($skuPriceObject.UnitPrice_s / $targetUnitHours) + } + } + + return $targetHourlyPrice +} + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" + +# must be less than or equal to the advisor exports frequency +$daysBackwards = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendAdvisorPeriodInDays" -ErrorAction SilentlyContinue) +if (-not($daysBackwards -gt 0)) { + $daysBackwards = 7 +} + +$perfDaysBackwards = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendPerfPeriodInDays" -ErrorAction SilentlyContinue) +if (-not($perfDaysBackwards -gt 0)) { + $perfDaysBackwards = 7 +} + +$perfTimeGrain = Get-AutomationVariable -Name "AzureOptimization_RecommendPerfTimeGrain" -ErrorAction SilentlyContinue +if (-not($perfTimeGrain)) { + $perfTimeGrain = "1h" +} + +# percentiles variables +$cpuPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileCpu" -ErrorAction SilentlyContinue) +if (-not($cpuPercentile -gt 0)) { + $cpuPercentile = 99 +} +$memoryPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileMemory" -ErrorAction SilentlyContinue) +if (-not($memoryPercentile -gt 0)) { + $memoryPercentile = 99 +} +$networkPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileNetwork" -ErrorAction SilentlyContinue) +if (-not($networkPercentile -gt 0)) { + $networkPercentile = 99 +} +$diskPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileDisk" -ErrorAction SilentlyContinue) +if (-not($diskPercentile -gt 0)) { + $diskPercentile = 99 +} + +# perf thresholds variables +$cpuPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuPercentageThreshold -gt 0)) { + $cpuPercentageThreshold = 30 +} +$memoryPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdMemoryPercentage" -ErrorAction SilentlyContinue) +if (-not($memoryPercentageThreshold -gt 0)) { + $memoryPercentageThreshold = 50 +} +$networkMpbsThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdNetworkMbps" -ErrorAction SilentlyContinue) +if (-not($networkMpbsThreshold -gt 0)) { + $networkMpbsThreshold = 750 +} + +# perf thresholds variables (shutdown) +$cpuPercentageShutdownThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuShutdownPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuPercentageShutdownThreshold -gt 0)) { + $cpuPercentageShutdownThreshold = 5 +} +$memoryPercentageShutdownThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdMemoryShutdownPercentage" -ErrorAction SilentlyContinue) +if (-not($memoryPercentageShutdownThreshold -gt 0)) { + $memoryPercentageShutdownThreshold = 100 +} +$networkMpbsShutdownThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdNetworkShutdownMbps" -ErrorAction SilentlyContinue ) +if (-not($networkMpbsShutdownThreshold -gt 0)) { + $networkMpbsShutdownThreshold = 10 +} + +$rightSizeRecommendationId = Get-AutomationVariable -Name "AzureOptimization_RecommendationAdvisorCostRightSizeId" -ErrorAction SilentlyContinue +if (-not($rightSizeRecommendationId)) { + $rightSizeRecommendationId = 'e10b1381-5f0a-47ff-8c7b-37bd13d7c974' +} + +$additionalPerfWorkspaces = Get-AutomationVariable -Name "AzureOptimization_RightSizeAdditionalPerfWorkspaces" -ErrorAction SilentlyContinue + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" +$FiltersTable = "Filters" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGVirtualMachine','AzureAdvisor','AzureConsumption','ARGResourceContainers','Pricesheet')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$vmsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGVirtualMachine' }).LogAnalyticsSuffix + "_CL" +$advisorTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureAdvisor' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" +$pricesheetTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'Pricesheet' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $vmsTableName, $subscriptionsTableName, $advisorTableName, $pricesheetTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +Write-Output "Getting excluded recommendation sub-type IDs..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$FiltersTable] WHERE FilterType = 'Exclude' AND IsEnabled = 1 AND (FilterEndDate IS NULL OR FilterEndDate > GETDATE())" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $filters = New-Object System.Data.DataTable + $sqlAdapter.Fill($filters) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + + +Write-Output "Getting Virtual Machine SKUs for the $referenceRegion region..." +# Get all the VM SKUs information for the reference Azure region +$skus = Get-AzComputeResourceSku -Location $referenceRegion | Where-Object { $_.ResourceType -eq "virtualMachines" } + +Write-Output "Getting the current Pricesheet..." + +if ($cloudEnvironment -eq "AzureCloud") +{ + $pricesheetRegion = "EU West" +} + +try +{ + $pricesheetEntries = @() + + $baseQuery = @" + $pricesheetTableName + | where TimeGenerated > ago(14d) + | where MeterCategory_s == 'Virtual Machines' and MeterRegion_s == '$pricesheetRegion' and PriceType_s == 'Consumption' + | distinct MeterName_s, MeterSubCategory_s, MeterCategory_s, MeterRegion_s, UnitPrice_s, UnitOfMeasure_s +"@ + + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days 14) -Wait 600 -IncludeStatistics + $pricesheetEntries = [System.Linq.Enumerable]::ToArray($queryResults.Results) + + Write-Output "Query finished with $($pricesheetEntries.Count) results." + Write-Output "Query statistics: $($queryResults.Statistics.query)" +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + Write-Output "Consumption pricesheet not available, will estimate savings based in cores count..." +} + +$linuxMemoryPerfAdditionalWorkspaces = "" +$windowsMemoryPerfAdditionalWorkspaces = "" +$processorPerfAdditionalWorkspaces = "" +$windowsNetworkPerfAdditionalWorkspaces = "" +$diskPerfAdditionalWorkspaces = "" +if ($additionalPerfWorkspaces) +{ + $additionalWorkspaces = $additionalPerfWorkspaces.Split(",") + foreach ($additionalWorkspace in $additionalWorkspaces) { + $additionalWorkspace = $additionalWorkspace.Trim() + $linuxMemoryPerfAdditionalWorkspaces += @" + | union ( workspace('$additionalWorkspace').Perf + | where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) + | where CounterName == '% Used Memory' + | extend WorkspaceId = TenantId + | summarize hint.strategy=shuffle PMemoryPercentage = percentile(CounterValue, memoryPercentileValue) by _ResourceId, WorkspaceId) +"@ + $windowsMemoryPerfAdditionalWorkspaces += @" + | union ( workspace('$additionalWorkspace').Perf + | where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) + | where CounterName == 'Available MBytes' + | extend WorkspaceId = TenantId + | project TimeGenerated, MemoryAvailableMBs = CounterValue, _ResourceId, WorkspaceId) +"@ + $processorPerfAdditionalWorkspaces += @" + | union ( workspace('$additionalWorkspace').Perf + | where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) + | where ObjectName == 'Processor' and CounterName == '% Processor Time' and InstanceName == '_Total' + | extend WorkspaceId = TenantId + | summarize hint.strategy=shuffle PCPUPercentage = percentile(CounterValue, cpuPercentileValue) by _ResourceId, WorkspaceId) +"@ + $windowsNetworkPerfAdditionalWorkspaces += @" + | union ( workspace('$additionalWorkspace').Perf + | where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) + | where CounterName == 'Bytes Total/sec' + | extend WorkspaceId = TenantId + | summarize hint.strategy=shuffle PCounter = percentile(CounterValue, networkPercentileValue) by InstanceName, _ResourceId, WorkspaceId + | summarize PNetwork = sum(PCounter) by _ResourceId, WorkspaceId) +"@ + $diskPerfAdditionalWorkspaces += @" + | union ( workspace('$additionalWorkspace').Perf + | where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) + | where CounterName in ('Disk Reads/sec', 'Disk Writes/sec', 'Disk Read Bytes/sec', 'Disk Write Bytes/sec') and InstanceName !in ('_Total', 'D:', '/mnt/resource', '/mnt') + | extend WorkspaceId = TenantId + | summarize hint.strategy=shuffle PCounter = percentile(CounterValue, diskPercentileValue) by bin(TimeGenerated, perfTimeGrain), CounterName, InstanceName, _ResourceId, WorkspaceId + | summarize SumPCounter = sum(PCounter) by CounterName, TimeGenerated, _ResourceId, WorkspaceId + | summarize MaxPReadIOPS = maxif(SumPCounter, CounterName == 'Disk Reads/sec'), + MaxPWriteIOPS = maxif(SumPCounter, CounterName == 'Disk Writes/sec'), + MaxPReadMiBps = (maxif(SumPCounter, CounterName == 'Disk Read Bytes/sec') / 1024 / 1024), + MaxPWriteMiBps = (maxif(SumPCounter, CounterName == 'Disk Write Bytes/sec') / 1024 / 1024) by _ResourceId, WorkspaceId) +"@ + } +} + +# Execute the recommendation query against Log Analytics + +$baseQuery = @" +let advisorInterval = $($daysBackwards)d; +let perfInterval = $($perfDaysBackwards)d; +let perfTimeGrain = $perfTimeGrain; +let cpuPercentileValue = $cpuPercentile; +let memoryPercentileValue = $memoryPercentile; +let networkPercentileValue = $networkPercentile; +let diskPercentileValue = $diskPercentile; +let rightSizeRecommendationId = '$rightSizeRecommendationId'; +let billingInterval = 30d; +let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(30d) | summarize max(todatetime(Date_s)))); +let stime = etime-billingInterval; +let RightSizeInstanceIds = materialize($advisorTableName +| where todatetime(TimeGenerated) > ago(advisorInterval) and Category == 'Cost' and RecommendationTypeId_g == rightSizeRecommendationId +| distinct InstanceId_s); +let LinuxMemoryPerf = Perf +| where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) +| where CounterName == '% Used Memory' +| extend WorkspaceId = TenantId +| summarize hint.strategy=shuffle PMemoryPercentage = percentile(CounterValue, memoryPercentileValue) by _ResourceId, WorkspaceId$linuxMemoryPerfAdditionalWorkspaces; +let WindowsMemoryPerf = Perf +| where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) +| where CounterName == 'Available MBytes' +| extend WorkspaceId = TenantId +| project TimeGenerated, MemoryAvailableMBs = CounterValue, _ResourceId, WorkspaceId$windowsMemoryPerfAdditionalWorkspaces; +let MemoryPerf = $vmsTableName +| where TimeGenerated > ago(1d) +| distinct InstanceId_s, MemoryMB_s +| join kind=inner hint.strategy=broadcast ( + WindowsMemoryPerf +) on `$left.InstanceId_s == `$right._ResourceId +| extend MemoryPercentage = todouble(toint(MemoryMB_s) - toint(MemoryAvailableMBs)) / todouble(MemoryMB_s) * 100 +| summarize hint.strategy=shuffle PMemoryPercentage = percentile(MemoryPercentage, memoryPercentileValue) by _ResourceId, WorkspaceId +| union LinuxMemoryPerf; +let ProcessorPerf = Perf +| where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) +| where ObjectName == 'Processor' and CounterName == '% Processor Time' and InstanceName == '_Total' +| extend WorkspaceId = TenantId +| summarize hint.strategy=shuffle PCPUPercentage = percentile(CounterValue, cpuPercentileValue) by _ResourceId, WorkspaceId$processorPerfAdditionalWorkspaces; +let WindowsNetworkPerf = Perf +| where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) +| where CounterName == 'Bytes Total/sec' +| extend WorkspaceId = TenantId +| summarize hint.strategy=shuffle PCounter = percentile(CounterValue, networkPercentileValue) by InstanceName, _ResourceId, WorkspaceId +| summarize PNetwork = sum(PCounter) by _ResourceId, WorkspaceId$windowsNetworkPerfAdditionalWorkspaces; +let DiskPerf = Perf +| where TimeGenerated > ago(perfInterval) and _ResourceId in (RightSizeInstanceIds) +| where CounterName in ('Disk Reads/sec', 'Disk Writes/sec', 'Disk Read Bytes/sec', 'Disk Write Bytes/sec') and InstanceName !in ('_Total', 'D:', '/mnt/resource', '/mnt') +| extend WorkspaceId = TenantId +| summarize hint.strategy=shuffle PCounter = percentile(CounterValue, diskPercentileValue) by bin(TimeGenerated, perfTimeGrain), CounterName, InstanceName, _ResourceId, WorkspaceId +| summarize SumPCounter = sum(PCounter) by CounterName, TimeGenerated, _ResourceId, WorkspaceId +| summarize MaxPReadIOPS = maxif(SumPCounter, CounterName == 'Disk Reads/sec'), + MaxPWriteIOPS = maxif(SumPCounter, CounterName == 'Disk Writes/sec'), + MaxPReadMiBps = (maxif(SumPCounter, CounterName == 'Disk Read Bytes/sec') / 1024 / 1024), + MaxPWriteMiBps = (maxif(SumPCounter, CounterName == 'Disk Write Bytes/sec') / 1024 / 1024) by _ResourceId, WorkspaceId$diskPerfAdditionalWorkspaces; +$advisorTableName +| where todatetime(TimeGenerated) > ago(advisorInterval) and Category == 'Cost' +| extend AdvisorRecIdIndex = indexof(InstanceId_s, '/providers/microsoft.advisor/recommendations') +| extend InstanceName_s = iif(isnotempty(InstanceName_s),InstanceName_s,iif(AdvisorRecIdIndex > 0, split(substring(InstanceId_s, 0, AdvisorRecIdIndex),'/')[-1], split(InstanceId_s,'/')[-1])) +| distinct InstanceId_s, InstanceName_s, Description_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroup, Cloud_s, AdditionalInfo_s, RecommendationText_s, ImpactedArea_s, Impact_s, RecommendationTypeId_g, Tags_s +| join kind=leftouter ( + $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | extend VMConsumedQuantity = iif(ResourceId contains 'virtualmachines' and MeterCategory_s == 'Virtual Machines', todouble(Quantity_s), 0.0) + | extend VMPrice = iif(ResourceId contains 'virtualmachines' and MeterCategory_s == 'Virtual Machines', todouble(EffectivePrice_s), 0.0) + | extend FinalCost = iif(ResourceId contains 'virtualmachines', VMPrice * VMConsumedQuantity, todouble(CostInBillingCurrency_s)) + | extend InstanceId_s = tolower(ResourceId) + | summarize Last30DaysCost = sum(FinalCost), Last30DaysQuantity = sum(VMConsumedQuantity) by InstanceId_s +) on InstanceId_s +| join kind=leftouter ( + $vmsTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, NicCount_s, DataDiskCount_s +) on InstanceId_s +| where RecommendationTypeId_g != rightSizeRecommendationId or (RecommendationTypeId_g == rightSizeRecommendationId and toint(NicCount_s) >= 0 and toint(DataDiskCount_s) >= 0) +| join kind=leftouter hint.strategy=broadcast ( MemoryPerf ) on `$left.InstanceId_s == `$right._ResourceId +| join kind=leftouter hint.strategy=broadcast ( ProcessorPerf ) on `$left.InstanceId_s == `$right._ResourceId +| join kind=leftouter hint.strategy=broadcast ( WindowsNetworkPerf ) on `$left.InstanceId_s == `$right._ResourceId +| join kind=leftouter hint.strategy=broadcast ( DiskPerf ) on `$left.InstanceId_s == `$right._ResourceId +| extend MaxPIOPS = MaxPReadIOPS + MaxPWriteIOPS, MaxPMiBps = MaxPReadMiBps + MaxPWriteMiBps +| extend PNetworkMbps = PNetwork * 8 / 1000 / 1000 +| distinct Last30DaysCost, Last30DaysQuantity, InstanceId_s, InstanceName_s, Description_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroup, Cloud_s, AdditionalInfo_s, RecommendationText_s, ImpactedArea_s, Impact_s, RecommendationTypeId_g, NicCount_s, DataDiskCount_s, PMemoryPercentage, PCPUPercentage, PNetworkMbps, MaxPIOPS, MaxPMiBps, Tags_s, WorkspaceId +| join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s +) on SubscriptionGuid_g +"@ + +Write-Output "Will run the following query (use this query against the LA workspace for troubleshooting): $baseQuery" + +Write-Output "Getting cost recommendations for $($daysBackwards)d Advisor and $($perfDaysBackwards)d Perf history and a $perfTimeGrain time grain..." + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + throw "Execution aborted" +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +$skuPricesFound = @{} + +Write-Output "Generating fit score..." + +foreach ($result in $results) { + + if ($filters | Where-Object { $_.RecommendationSubTypeId -eq $result.RecommendationTypeId_g}) + { + continue + } + + $queryInstanceId = $result.InstanceId_s + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $additionalInfoDictionary = @{} + if (-not([string]::IsNullOrEmpty($result.AdditionalInfo_s))) + { + ($result.AdditionalInfo_s | ConvertFrom-Json).PsObject.Properties | ForEach-Object { $additionalInfoDictionary[$_.Name] = $_.Value } + } + + # Fixing reservation model inconsistencies + if (-not([string]::IsNullOrEmpty($additionalInfoDictionary["location"]))) + { + $additionalInfoDictionary["region"] = $additionalInfoDictionary["location"] + } + if (-not([string]::IsNullOrEmpty($additionalInfoDictionary["targetResourceCount"]))) + { + $additionalInfoDictionary["qty"] = $additionalInfoDictionary["targetResourceCount"] + } + if (-not([string]::IsNullOrEmpty($additionalInfoDictionary["vmSize"]))) + { + $additionalInfoDictionary["displaySKU"] = $additionalInfoDictionary["vmSize"] + } + + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + + $fitScore = 5 + $hasCpuRamPerfMetrics = $false + + if ($additionalInfoDictionary.targetSku -and $result.RecommendationTypeId_g -eq $rightSizeRecommendationId) { + $additionalInfoDictionary["SupportsDataDisksCount"] = "true" + $additionalInfoDictionary["DataDiskCount"] = "$($result.DataDiskCount_s)" + $additionalInfoDictionary["SupportsNICCount"] = "true" + $additionalInfoDictionary["NicCount"] = "$($result.NicCount_s)" + $additionalInfoDictionary["SupportsIOPS"] = "true" + $additionalInfoDictionary["MetricIOPS"] = "$($result.MaxPIOPS)" + $additionalInfoDictionary["SupportsMiBps"] = "true" + $additionalInfoDictionary["MetricMiBps"] = "$($result.MaxPMiBps)" + $additionalInfoDictionary["BelowCPUThreshold"] = "true" + $additionalInfoDictionary["MetricCPUPercentage"] = "$($result.PCPUPercentage)" + $additionalInfoDictionary["BelowMemoryThreshold"] = "true" + $additionalInfoDictionary["MetricMemoryPercentage"] = "$($result.PMemoryPercentage)" + $additionalInfoDictionary["BelowNetworkThreshold"] = "true" + $additionalInfoDictionary["MetricNetworkMbps"] = "$($result.PNetworkMbps)" + + $targetSku = $null + if ($additionalInfoDictionary.targetSku -ne "Shutdown") { + $currentSku = $skus | Where-Object { $_.Name -eq $additionalInfoDictionary.currentSku } + $currentSkuvCPUs = [int]($currentSku.Capabilities | Where-Object { $_.Name -eq 'vCPUsAvailable' }).Value + $targetSku = $skus | Where-Object { $_.Name -eq $additionalInfoDictionary.targetSku } + $targetSkuvCPUs = [int]($targetSku.Capabilities | Where-Object { $_.Name -eq 'vCPUsAvailable' }).Value + $targetMaxDataDiskCount = [int]($targetSku.Capabilities | Where-Object { $_.Name -eq 'MaxDataDiskCount' }).Value + if ($targetMaxDataDiskCount -gt 0) { + if (-not([string]::isNullOrEmpty($result.DataDiskCount_s))) { + if ([int]$result.DataDiskCount_s -gt $targetMaxDataDiskCount) { + $fitScore = 1 + $additionalInfoDictionary["SupportsDataDisksCount"] = "false:needs$($result.DataDiskCount_s)-max$targetMaxDataDiskCount" + } + } + else { + $fitScore -= 1 + $additionalInfoDictionary["SupportsDataDisksCount"] = "unknown:max$targetMaxDataDiskCount" + } + } + else { + $fitScore -= 1 + $additionalInfoDictionary["SupportsDataDisksCount"] = "unknown:needs$($result.DataDiskCount_s)" + } + $targetMaxNICCount = [int]($targetSku.Capabilities | Where-Object { $_.Name -eq 'MaxNetworkInterfaces' }).Value + if ($targetMaxNICCount -gt 0) { + if (-not([string]::isNullOrEmpty($result.NicCount_s))) { + if ([int]$result.NicCount_s -gt $targetMaxNICCount) { + $fitScore = 1 + $additionalInfoDictionary["SupportsNICCount"] = "false:needs$($result.NicCount_s)-max$targetMaxNICCount" + } + } + else { + $fitScore -= 1 + $additionalInfoDictionary["SupportsNICCount"] = "unknown:max$targetMaxNICCount" + } + } + else { + $fitScore -= 1 + $additionalInfoDictionary["SupportsNICCount"] = "unknown:needs$($result.NicCount_s)" + } + $targetUncachedDiskIOPS = [int]($targetSku.Capabilities | Where-Object { $_.Name -eq 'UncachedDiskIOPS' }).Value + if ($targetUncachedDiskIOPS -gt 0) { + if (-not([string]::isNullOrEmpty($result.MaxPIOPS))) { + if ([double]$result.MaxPIOPS -ge [double]$targetUncachedDiskIOPS) { + $fitScore -= 1 + $additionalInfoDictionary["SupportsIOPS"] = "false:needs$($result.MaxPIOPS)-max$targetUncachedDiskIOPS" + } + } + else { + $fitScore -= 0.5 + $additionalInfoDictionary["SupportsIOPS"] = "unknown:max$targetUncachedDiskIOPS" + } + } + else { + $fitScore -= 1 + $additionalInfoDictionary["SupportsIOPS"] = "unknown:needs$($result.MaxPIOPS)" + } + $targetUncachedDiskMiBps = [double]([int]($targetSku.Capabilities | Where-Object { $_.Name -eq 'UncachedDiskBytesPerSecond' }).Value) / 1024 / 1024 + if ($targetUncachedDiskMiBps -gt 0) { + if (-not([string]::isNullOrEmpty($result.MaxPMiBps))) { + if ([double]$result.MaxPMiBps -ge $targetUncachedDiskMiBps) { + $fitScore -= 1 + $additionalInfoDictionary["SupportsMiBps"] = "false:needs$($result.MaxPMiBps)-max$targetUncachedDiskMiBps" + } + } + else { + $fitScore -= 0.5 + $additionalInfoDictionary["SupportsMiBps"] = "unknown:max$targetUncachedDiskMiBps" + } + } + else { + $additionalInfoDictionary["SupportsMiBps"] = "unknown:needs$($result.MaxPMiBps)" + } + + $savingCoefficient = [double] $currentSkuvCPUs / $targetSkuvCPUs + + if ($savingCoefficient -gt 1) + { + $targetSkuSavingsMonthly = [double]$result.Last30DaysCost - ([double]$result.Last30DaysCost / $savingCoefficient) + } + else + { + $targetSkuSavingsMonthly = [double]$result.Last30DaysCost / 2 + } + + if ($targetSku -and $null -eq $skuPricesFound[$targetSku.Name]) + { + $skuPricesFound[$targetSku.Name] = Find-SkuHourlyPrice -SKUName $targetSku.Name -SKUPriceSheet $pricesheetEntries + } + + $tentativeTargetSkuSavingsMonthly = -1 + + if ($targetSku -and $skuPricesFound[$targetSku.Name] -gt 0 -and $skuPricesFound[$targetSku.Name] -lt [double]::MaxValue) + { + $targetSkuPrice = $skuPricesFound[$targetSku.Name] + + if ($null -eq $skuPricesFound[$currentSku.Name]) + { + $skuPricesFound[$currentSku.Name] = Find-SkuHourlyPrice -SKUName $currentSku.Name -SKUPriceSheet $pricesheetEntries + } + + if ($skuPricesFound[$currentSku.Name] -gt 0) + { + $currentSkuPrice = $skuPricesFound[$currentSku.Name] + $tentativeTargetSkuSavingsMonthly = ($currentSkuPrice * [double] $result.Last30DaysQuantity) - ($targetSkuPrice * [double] $result.Last30DaysQuantity) + } + else + { + $tentativeTargetSkuSavingsMonthly = [double]$result.Last30DaysCost - ($targetSkuPrice * [double] $result.Last30DaysQuantity) + } + } + + if ($tentativeTargetSkuSavingsMonthly -ge 0) + { + $targetSkuSavingsMonthly = $tentativeTargetSkuSavingsMonthly + } + + if ($targetSkuSavingsMonthly -eq [double]::PositiveInfinity) + { + $targetSkuSavingsMonthly = [double] $result.Last30DaysCost / 2 + } + + $savingsMonthly = $targetSkuSavingsMonthly + + } + else + { + $savingsMonthly = [double]$result.Last30DaysCost + } + + $cpuThreshold = $cpuPercentageThreshold + $memoryThreshold = $memoryPercentageThreshold + $networkThreshold = $networkMpbsThreshold + if ($additionalInfoDictionary.targetSku -eq "Shutdown") { + $cpuThreshold = $cpuPercentageShutdownThreshold + $memoryThreshold = $memoryPercentageShutdownThreshold + $networkThreshold = $networkMpbsShutdownThreshold + } + + if (-not([string]::isNullOrEmpty($result.PCPUPercentage))) { + if ([double]$result.PCPUPercentage -ge [double]$cpuThreshold) { + $fitScore -= 0.5 + $additionalInfoDictionary["BelowCPUThreshold"] = "false:needs$($result.PCPUPercentage)-max$cpuThreshold" + } + $hasCpuRamPerfMetrics = $true + } + else { + $fitScore -= 0.5 + $additionalInfoDictionary["BelowCPUThreshold"] = "unknown:max$cpuThreshold" + } + if (-not([string]::isNullOrEmpty($result.PMemoryPercentage))) { + if ([double]$result.PMemoryPercentage -ge [double]$memoryThreshold) { + $fitScore -= 0.5 + $additionalInfoDictionary["BelowMemoryThreshold"] = "false:needs$($result.PMemoryPercentage)-max$memoryThreshold" + } + $hasCpuRamPerfMetrics = $true + } + else { + $fitScore -= 0.5 + $additionalInfoDictionary["BelowMemoryThreshold"] = "unknown:max$memoryThreshold" + } + if (-not([string]::isNullOrEmpty($result.PNetworkMbps))) { + if ([double]$result.PNetworkMbps -ge [double]$networkThreshold) { + $fitScore -= 0.1 + $additionalInfoDictionary["BelowNetworkThreshold"] = "false:needs$($result.PNetworkMbps)-max$networkThreshold" + } + } + else { + $fitScore -= 0.1 + $additionalInfoDictionary["BelowNetworkThreshold"] = "unknown:max$networkThreshold" + } + + $fitScore = [Math]::max(0.0, $fitScore) + } + else + { + if (-not([string]::IsNullOrEmpty($additionalInfoDictionary["annualSavingsAmount"]))) + { + $savingsMonthly = [double] $additionalInfoDictionary["annualSavingsAmount"] / 12 + } + else + { + if ($result.RecommendationTypeId_g -eq $rightSizeRecommendationId) + { + $savingsMonthly = [double] $result.Last30DaysCost + } + else + { + $savingsMonthly = 0.0 # unknown + } + } + } + + $additionalInfoDictionary["savingsAmount"] = [double] $savingsMonthly + + $queryInstanceId = $result.InstanceId_s + if (-not($hasCpuRamPerfMetrics)) + { + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + } + else + { + $queryWorkspace = "" + if (-not([string]::IsNullOrEmpty($result.WorkspaceId)) -and $result.WorkspaceId -ne $workspaceId) + { + $queryWorkspace = "workspace('$($result.WorkspaceId)')." + } + + $queryText = @" + let perfInterval = $($perfDaysBackwards)d; + let armId = tolower(`'$queryInstanceId`'); + let gInt = $perfTimeGrain; + let LinuxMemoryPerf = $($queryWorkspace)Perf + | where TimeGenerated > ago(perfInterval) + | where CounterName == '% Used Memory' and _ResourceId =~ armId + | project TimeGenerated, MemoryPercentage = CounterValue; + let WindowsMemoryPerf = $($queryWorkspace)Perf + | where TimeGenerated > ago(perfInterval) + | where CounterName == 'Available MBytes' and _ResourceId =~ armId + | extend MemoryAvailableMBs = CounterValue, InstanceId = tolower(_ResourceId) + | project TimeGenerated, MemoryAvailableMBs, InstanceId; + let MemoryPerf = WindowsMemoryPerf + | join kind=inner ( + $vmsTableName + | where TimeGenerated > ago(1d) + | extend InstanceId = tolower(InstanceId_s) + | distinct InstanceId, MemoryMB_s + ) on InstanceId + | extend MemoryPercentage = todouble(toint(MemoryMB_s) - toint(MemoryAvailableMBs)) / todouble(MemoryMB_s) * 100 + | project TimeGenerated, MemoryPercentage + | union LinuxMemoryPerf + | summarize P$($memoryPercentile)MemoryPercentage = percentile(MemoryPercentage, $memoryPercentile) by bin(TimeGenerated, gInt); + let ProcessorPerf = $($queryWorkspace)Perf + | where TimeGenerated > ago(perfInterval) + | where CounterName == '% Processor Time' and InstanceName == '_Total' and _ResourceId =~ armId + | summarize P$($cpuPercentile)CPUPercentage = percentile(CounterValue, $cpuPercentile) by bin(TimeGenerated, gInt); + MemoryPerf + | join kind=inner (ProcessorPerf) on TimeGenerated + | render timechart +"@ + + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = $result.ImpactedArea_s + Impact = $result.Impact_s + RecommendationType = "Saving" + RecommendationSubType = "AdvisorCost" + RecommendationSubTypeId = $result.RecommendationTypeId_g + RecommendationDescription = $result.Description_s + RecommendationAction = $result.RecommendationText_s + InstanceId = $result.InstanceId_s + InstanceName = $result.InstanceName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +Write-Output "Exporting final $($recommendations.Count) results as a JSON file..." + +$fileDate = $datetime.ToString("yyyyMMdd") +$jsonExportPath = "advisor-cost-augmented-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +Write-Output "Uploading $jsonExportPath to blob storage..." + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json" }; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AppServiceOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AppServiceOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..11f8dbf56 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-AppServiceOptimizationsToBlobStorage.ps1 @@ -0,0 +1,693 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$perfDaysBackwards = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendPerfPeriodInDays" -ErrorAction SilentlyContinue) +if (-not($perfDaysBackwards -gt 0)) { + $perfDaysBackwards = 7 +} + +$perfTimeGrain = Get-AutomationVariable -Name "AzureOptimization_RecommendPerfTimeGrain" -ErrorAction SilentlyContinue +if (-not($perfTimeGrain)) { + $perfTimeGrain = "1h" +} + +# percentiles variables +$cpuPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileCpu" -ErrorAction SilentlyContinue) +if (-not($cpuPercentile -gt 0)) { + $cpuPercentile = 99 +} +$memoryPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileMemory" -ErrorAction SilentlyContinue) +if (-not($memoryPercentile -gt 0)) { + $memoryPercentile = 99 +} + +# perf thresholds variables +$cpuPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuPercentageThreshold -gt 0)) { + $cpuPercentageThreshold = 30 +} +$memoryPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdMemoryPercentage" -ErrorAction SilentlyContinue) +if (-not($memoryPercentageThreshold -gt 0)) { + $memoryPercentageThreshold = 50 +} +$cpuDegradedMaxPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuDegradedMaxPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuDegradedMaxPercentageThreshold -gt 0)) { + $cpuDegradedMaxPercentageThreshold = 95 +} +$cpuDegradedAvgPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuDegradedAvgPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuDegradedAvgPercentageThreshold -gt 0)) { + $cpuDegradedAvgPercentageThreshold = 75 +} +$memoryDegradedPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdMemoryDegradedPercentage" -ErrorAction SilentlyContinue) +if (-not($memoryDegradedPercentageThreshold -gt 0)) { + $memoryDegradedPercentageThreshold = 90 +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('AppServicePlans','MonitorMetrics','AzureConsumption','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$appServicePlansTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AppServicePlans' }).LogAnalyticsSuffix + "_CL" +$metricsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'MonitorMetrics' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $appServicePlansTableName, $subscriptionsTableName, $metricsTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +# Execute the recommendation query against Log Analytics +Write-Output "Looking for underused App Service Plans, with less than $cpuPercentageThreshold% CPU and $memoryPercentageThreshold% RAM usage..." + +$baseQuery = @" + let billingInterval = 30d; + let perfInterval = $($perfDaysBackwards)d; + let cpuPercentileValue = $cpuPercentile; + let memoryPercentileValue = $memoryPercentile; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(30d) | summarize max(todatetime(Date_s)))); + let stime = etime-billingInterval; + + let BilledPlans = $consumptionTableName + | where todatetime(Date_s) between (stime..etime) and ResourceId has 'microsoft.web/serverfarms' + | extend ConsumedQuantity = todouble(Quantity_s) + | extend FinalCost = todouble(EffectivePrice_s) * ConsumedQuantity + | extend InstanceId_s = tolower(ResourceId) + | summarize Last30DaysCost = sum(FinalCost), Last30DaysQuantity = sum(ConsumedQuantity) by InstanceId_s; + + let ProcessorPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where ResourceId has 'microsoft.web/serverfarms' + | where MetricNames_s == "CpuPercentage" and AggregationType_s == 'Maximum' + | extend InstanceId_s = ResourceId + | summarize PCPUPercentage = percentile(todouble(MetricValue_s), cpuPercentileValue) by InstanceId_s; + + let MemoryPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where ResourceId has 'microsoft.web/serverfarms' + | where MetricNames_s == "MemoryPercentage" and AggregationType_s == 'Maximum' + | extend InstanceId_s = ResourceId + | summarize PMemoryPercentage = percentile(todouble(MetricValue_s), memoryPercentileValue) by InstanceId_s; + + $appServicePlansTableName + | where TimeGenerated > ago(1d) and ComputeMode_s == 'Dedicated' and SkuTier_s != 'Free' + | distinct InstanceId_s, AppServicePlanName_s, ResourceGroupName_s, SubscriptionGuid_g, Cloud_s, TenantGuid_g, SkuSize_s, NumberOfWorkers_s, Tags_s + | join kind=inner ( BilledPlans ) on InstanceId_s + | join kind=leftouter ( MemoryPerf ) on InstanceId_s + | join kind=leftouter ( ProcessorPerf ) on InstanceId_s + | project InstanceId_s, AppServicePlan = AppServicePlanName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, SkuSize_s, NumberOfWorkers_s, PMemoryPercentage, PCPUPercentage, Tags_s, Last30DaysCost, Last30DaysQuantity + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionId = SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionId + | where isnotempty(PMemoryPercentage) and isnotempty(PCPUPercentage) and PMemoryPercentage < $memoryPercentageThreshold and PCPUPercentage < $cpuPercentageThreshold +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" +let perfInterval = $($perfDaysBackwards)d; +let armId = `'$queryInstanceId`'; +let gInt = $perfTimeGrain; +let MemoryPerf = $metricsTableName +| where TimeGenerated > ago(perfInterval) +| extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) +| where ResourceId == armId +| where MetricNames_s == 'MemoryPercentage' and AggregationType_s == 'Maximum' +| extend MemoryPercentage = todouble(MetricValue_s) +| summarize percentile(MemoryPercentage, $memoryPercentile) by bin(CollectedDate, gInt); +let ProcessorPerf = $metricsTableName +| where TimeGenerated > ago(perfInterval) +| extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) +| where ResourceId == armId +| where MetricNames_s == 'CpuPercentage' and AggregationType_s == 'Maximum' +| extend ProcessorPercentage = todouble(MetricValue_s) +| summarize percentile(ProcessorPercentage, $cpuPercentile) by bin(CollectedDate, gInt); +MemoryPerf +| join kind=inner (ProcessorPerf) on CollectedDate +| render timechart +"@ + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = "$($result.SkuSize_s)" + $additionalInfoDictionary["InstanceCount"] = [int] $result.NumberOfWorkers_s + $additionalInfoDictionary["MetricCPUPercentage"] = "$($result.PCPUPercentage)" + $additionalInfoDictionary["MetricMemoryPercentage"] = "$($result.PMemoryPercentage)" + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + $additionalInfoDictionary["savingsAmount"] = ([double] $result.Last30DaysCost / 2) + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Web/serverFarms" + Impact = "High" + RecommendationType = "Saving" + RecommendationSubType = "UnderusedAppServicePlans" + RecommendationSubTypeId = "042adaca-ebdf-49b4-bc1b-2800b6e40fea" + RecommendationDescription = "Underused App Service Plans (performance capacity waste)" + RecommendationAction = "Right-size underused App Service Plans or scale it in" + InstanceId = $result.InstanceId_s + InstanceName = $result.AppServicePlan + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionId + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "appserviceplans-underused-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for performance constrained App Service Plans, with more than $cpuDegradedMaxPercentageThreshold% Max. CPU, $cpuDegradedAvgPercentageThreshold% Avg. CPU and $memoryDegradedPercentageThreshold% RAM usage..." + +$baseQuery = @" + let perfInterval = $($perfDaysBackwards)d; + + let MemoryPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where ResourceId has 'microsoft.web/serverfarms' + | where MetricNames_s == "MemoryPercentage" and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum' + | extend InstanceId_s = ResourceId + | summarize PMemoryPercentage = avg(todouble(MetricValue_s)) by InstanceId_s; + + let ProcessorMaxPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where ResourceId has 'microsoft.web/serverfarms' + | where MetricNames_s == "CpuPercentage" and AggregationType_s == 'Maximum' + | extend InstanceId_s = ResourceId + | summarize PCPUMaxPercentage = avg(todouble(MetricValue_s)) by InstanceId_s; + + let ProcessorAvgPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where ResourceId has 'microsoft.web/serverfarms' + | where MetricNames_s == "CpuPercentage" and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum' + | extend InstanceId_s = ResourceId + | summarize PCPUAvgPercentage = avg(todouble(MetricValue_s)) by InstanceId_s; + + $appServicePlansTableName + | where TimeGenerated > ago(1d) and ComputeMode_s == 'Dedicated' and SkuTier_s != 'Free' + | distinct InstanceId_s, AppServicePlanName_s, ResourceGroupName_s, SubscriptionGuid_g, Cloud_s, TenantGuid_g, SkuSize_s, NumberOfWorkers_s, Tags_s + | join kind=leftouter ( MemoryPerf ) on InstanceId_s + | join kind=leftouter ( ProcessorMaxPerf ) on InstanceId_s + | join kind=leftouter ( ProcessorAvgPerf ) on InstanceId_s + | project InstanceId_s, AppServicePlan = AppServicePlanName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, SkuSize_s, NumberOfWorkers_s, PMemoryPercentage, PCPUMaxPercentage, PCPUAvgPercentage, Tags_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionId = SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionId + | where isnotempty(PMemoryPercentage) and isnotempty(PCPUAvgPercentage) and isnotempty(PCPUMaxPercentage) and (PMemoryPercentage > $memoryDegradedPercentageThreshold or (PCPUMaxPercentage > $cpuDegradedMaxPercentageThreshold and PCPUAvgPercentage > $cpuDegradedAvgPercentageThreshold)) +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" +let perfInterval = $($perfDaysBackwards)d; +let armId = `'$queryInstanceId`'; +let gInt = $perfTimeGrain; +let MemoryPerf = $metricsTableName +| where TimeGenerated > ago(perfInterval) +| extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) +| where ResourceId == armId +| where MetricNames_s == 'MemoryPercentage' and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum' +| extend MemoryPercentage = todouble(MetricValue_s) +| summarize percentile(MemoryPercentage, $memoryPercentile) by bin(CollectedDate, gInt); +let ProcessorMaxPerf = $metricsTableName +| where TimeGenerated > ago(perfInterval) +| extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) +| where ResourceId == armId +| where MetricNames_s == 'CpuPercentage' and AggregationType_s == 'Maximum' +| extend ProcessorMaxPercentage = todouble(MetricValue_s) +| summarize percentile(ProcessorMaxPercentage, $cpuPercentile) by bin(CollectedDate, gInt); +let ProcessorAvgPerf = $metricsTableName +| where TimeGenerated > ago(perfInterval) +| extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) +| where ResourceId == armId +| where MetricNames_s == 'CpuPercentage' and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum' +| extend ProcessorAvgPercentage = todouble(MetricValue_s) +| summarize percentile(ProcessorAvgPercentage, $cpuPercentile) by bin(CollectedDate, gInt); +MemoryPerf +| join kind=inner (ProcessorMaxPerf) on CollectedDate +| join kind=inner (ProcessorAvgPerf) on CollectedDate +| render timechart +"@ + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = "$($result.SkuSize_s)" + $additionalInfoDictionary["InstanceCount"] = [int] $result.NumberOfWorkers_s + $additionalInfoDictionary["MetricCPUAvgPercentage"] = "$($result.PCPUAvgPercentage)" + $additionalInfoDictionary["MetricCPUMaxPercentage"] = "$($result.PCPUMaxPercentage)" + $additionalInfoDictionary["MetricMemoryPercentage"] = "$($result.PMemoryPercentage)" + + $fitScore = 3 # needs a more complete analysis to improve score + + if ([double] $result.PCPUMaxPercentage -gt [double] $cpuDegradedMaxPercentageThreshold -and [double] $result.PCPUAvgPercentage -gt [double] $cpuDegradedAvgPercentageThreshold) + { + $fitScore = 4 + } + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Performance" + ImpactedArea = "Microsoft.Web/serverFarms" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "PerfConstrainedAppServicePlans" + RecommendationSubTypeId = "351574cb-c105-4538-a778-11dfbe4857bf" + RecommendationDescription = "App Service Plan performance has been constrained by lack of resources" + RecommendationAction = "Resize App Service Plan to higher SKU or scale it out" + InstanceId = $result.InstanceId_s + InstanceName = $result.AppServicePlan + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionId + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "appserviceplans-perfconstrained-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for empty App Service Plans..." + +$baseQuery = @" +let interval = 30d; +let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(interval) | summarize max(todatetime(Date_s)))); +let stime = etime-interval; +$appServicePlansTableName +| where TimeGenerated > ago(1d) and ComputeMode_s == 'Dedicated' and SkuTier_s != 'Free' and toint(NumberOfSites_s) == 0 +| distinct AppServicePlanName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuSize_s, NumberOfWorkers_s, Tags_s, Cloud_s +| join kind=leftouter ( + $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s +) on InstanceId_s +| summarize Last30DaysCost=sum(todouble(CostInBillingCurrency_s)) by AppServicePlanName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuSize_s, NumberOfWorkers_s, Tags_s, Cloud_s +| join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s +) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + $appServicePlansTableName + | where InstanceId_s == '$queryInstanceId' + | where toint(NumberOfSites_s) == 0 + | distinct InstanceId_s, AppServicePlanName_s, TimeGenerated + | summarize FirstUnusedDate = min(TimeGenerated) by InstanceId_s, AppServicePlanName_s + | join kind=leftouter ( + $consumptionTableName + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize CostsSinceUnused = sumif(todouble(CostInBillingCurrency_s), todatetime(Date_s) > FirstUnusedDate) by AppServicePlanName_s, FirstUnusedDate +"@ + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $deploymentDate + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = $result.SkuSize_s + $additionalInfoDictionary["InstanceCount"] = $result.NumberOfWorkers_s + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + $additionalInfoDictionary["savingsAmount"] = [double] $result.Last30DaysCost + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Web/serverFarms" + Impact = "High" + RecommendationType = "Saving" + RecommendationSubType = "EmptyAppServicePlans" + RecommendationSubTypeId = "ef525225-8b91-47a3-81f3-e674e94564b6" + RecommendationDescription = "App Service Plans without any application incur in unnecessary costs" + RecommendationAction = "Delete the App Service Plan" + InstanceId = $result.InstanceId_s + InstanceName = $result.AppServicePlanName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "appserviceplans-empty-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..1999e2e1d --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-DiskOptimizationsToBlobStorage.ps1 @@ -0,0 +1,533 @@ +$ErrorActionPreference = "Stop" + +function Find-DiskMonthlyPrice { + param ( + [object[]] $SKUPriceSheet, + [string] $DiskSizeTier + ) + + $diskSkus = $SKUPriceSheet | Where-Object { $_.MeterName_s.Replace(" Disks","") -eq $DiskSizeTier } + $targetMonthlyPrice = [double]::MaxValue + if ($diskSkus) + { + $targetMonthlyPrice = [double] ($diskSkus | Sort-Object -Property UnitPrice_s | Select-Object -First 1).UnitPrice_s + } + return $targetMonthlyPrice +} + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +# perf thresholds variables +$iopsPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdDiskIOPSPercentage" -ErrorAction SilentlyContinue) +if (-not($iopsPercentageThreshold -gt 0)) { + $iopsPercentageThreshold = 5 +} +$mbsPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdDiskMBsPercentage" -ErrorAction SilentlyContinue) +if (-not($mbsPercentageThreshold -gt 0)) { + $mbsPercentageThreshold = 5 +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$perfDaysBackwards = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendPerfPeriodInDays" -ErrorAction SilentlyContinue) +if (-not($perfDaysBackwards -gt 0)) { + $perfDaysBackwards = 7 +} + +$perfTimeGrain = Get-AutomationVariable -Name "AzureOptimization_RecommendPerfTimeGrain" -ErrorAction SilentlyContinue +if (-not($perfTimeGrain)) { + $perfTimeGrain = "1h" +} + +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGManagedDisk','MonitorMetrics','ARGResourceContainers','AzureConsumption','Pricesheet')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$disksTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGManagedDisk' }).LogAnalyticsSuffix + "_CL" +$metricsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'MonitorMetrics' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$pricesheetTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'Pricesheet' }).LogAnalyticsSuffix + "_CL" + + +Write-Output "Will run query against tables $disksTableName, $metricsTableName, $subscriptionsTableName, $pricesheetTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +Write-Output "Getting Disks SKUs for the $referenceRegion region..." + +$skus = Get-AzComputeResourceSku -Location $referenceRegion | Where-Object { $_.ResourceType -eq "disks" } + +Write-Output "Getting the current Pricesheet..." + +if ($cloudEnvironment -eq "AzureCloud") +{ + $pricesheetRegion = "EU West" +} + +try +{ + $pricesheetEntries = @() + + $baseQuery = @" + $pricesheetTableName + | where TimeGenerated > ago(14d) + | where MeterCategory_s == 'Storage' and MeterSubCategory_s endswith "Managed Disks" and MeterName_s endswith "Disks" and MeterRegion_s == '$pricesheetRegion' and PriceType_s == 'Consumption' + | distinct MeterName_s, MeterSubCategory_s, MeterCategory_s, MeterRegion_s, UnitPrice_s, UnitOfMeasure_s +"@ + + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days 14) -Wait 600 -IncludeStatistics + $pricesheetEntries = [System.Linq.Enumerable]::ToArray($queryResults.Results) + + Write-Output "Query finished with $($pricesheetEntries.Count) results." + Write-Output "Query statistics: $($queryResults.Statistics.query)" +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + Write-Output "Consumption pricesheet not available, will estimate savings based in price difference ratio..." +} + +$skuPricesFound = @{} + +Write-Output "Looking for underutilized Disks, with less than $iopsPercentageThreshold% IOPS and $mbsPercentageThreshold% MB/s usage..." + +$baseQuery = @" + let billingInterval = 30d; + let perfInterval = $($perfDaysBackwards)d; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(billingInterval) | summarize max(todatetime(Date_s)))); + let stime = etime-billingInterval; + + let BilledDisks = $consumptionTableName + | where todatetime(Date_s) between (stime..etime) and ResourceId contains '/disks/' and MeterCategory_s == 'Storage' and MeterSubCategory_s has 'Premium' and MeterName_s has 'Disk' + | extend DiskConsumedQuantity = todouble(Quantity_s) + | extend DiskPrice = todouble(EffectivePrice_s) + | extend FinalCost = DiskPrice * DiskConsumedQuantity + | extend ResourceId = tolower(ResourceId) + | summarize Last30DaysCost = sum(FinalCost), Last30DaysQuantity = sum(DiskConsumedQuantity) by ResourceId; + + $metricsTableName + | where MetricNames_s == 'Composite Disk Read Operations/sec,Composite Disk Write Operations/sec' and TimeGenerated > ago(perfInterval) and isnotempty(MetricValue_s) + | summarize MaxIOPSMetric = max(todouble(MetricValue_s)) by ResourceId + | join kind=inner ( + $disksTableName + | where TimeGenerated > ago(1d) and DiskState_s != 'Unattached' and SKU_s startswith 'Premium' + | project ResourceId=InstanceId_s, DiskName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, Tags_s, MaxIOPSDisk=toint(DiskIOPS_s), DiskSizeGB_s, SKU_s, DiskTier_s, DiskType_s + ) on ResourceId + | project-away ResourceId1 + | extend IOPSPercentage = MaxIOPSMetric/MaxIOPSDisk*100 + | where IOPSPercentage < $iopsPercentageThreshold + | join kind=inner ( + $metricsTableName + | where MetricNames_s == 'Composite Disk Read Bytes/sec,Composite Disk Write Bytes/sec' and TimeGenerated > ago(perfInterval) and isnotempty(MetricValue_s) + | summarize MaxMBsMetric = max(todouble(MetricValue_s)/1024/1024) by ResourceId + | join kind=inner ( + $disksTableName + | where TimeGenerated > ago(1d) and DiskState_s != 'Unattached' and SKU_s startswith 'Premium' + | project ResourceId=InstanceId_s, DiskName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, Tags_s, MaxMBsDisk=toint(DiskThroughput_s), DiskSizeGB_s, SKU_s, DiskTier_s, DiskType_s + ) on ResourceId + | project-away ResourceId1 + | extend MBsPercentage = MaxMBsMetric/MaxMBsDisk*100 + | where MBsPercentage < $mbsPercentageThreshold + ) on ResourceId + | join kind=inner ( BilledDisks ) on ResourceId + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionId = SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionId +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + throw "Execution aborted" +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $targetSku = $null + $currentDiskTier = $null + + if ([string]::IsNullOrEmpty($result.DiskTier_s)) # older disks do not have Tier info in their properties + { + $currentSkuCandidates = @() + foreach ($sku in $skus) + { + $currentSkuCandidate = $null + $skuMinSizeGB = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MinSizeGiB' }).Value + $skuMaxSizeGB = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxSizeGiB' }).Value + $skuMaxIOps = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxIOps' }).Value + $skuMaxBandwidthMBps = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxBandwidthMBps' }).Value + + if ($sku.Name -eq $result.SKU_s -and $skuMinSizeGB -lt [int]$result.DiskSizeGB_s -and $skuMaxSizeGB -ge [int]$result.DiskSizeGB_s ` + -and [int]$skuMaxIOps -eq [int]$result.MaxIOPSDisk -and [int]$skuMaxBandwidthMBps -eq [int]$result.MaxMBsDisk) + { + if ($null -eq $skuPricesFound[$sku.Size]) + { + $skuPricesFound[$sku.Size] = Find-DiskMonthlyPrice -DiskSizeTier $sku.Size -SKUPriceSheet $pricesheetEntries + } + + $currentSkuCandidate = New-Object PSObject -Property @{ + Name = $sku.Size + MaxSizeGB = $skuMaxSizeGB + } + + $currentSkuCandidates += $currentSkuCandidate + } + } + $currentDiskTier = ($currentSkuCandidates | Sort-Object -Property MaxSizeGB | Select-Object -First 1).Name + } + else + { + $currentDiskTier = $result.DiskTier_s + } + + if ($null -eq $skuPricesFound[$currentDiskTier]) + { + $skuPricesFound[$currentDiskTier] = Find-DiskMonthlyPrice -DiskSizeTier $currentDiskTier -SKUPriceSheet $pricesheetEntries + } + + $targetSkuPerfTier = $result.SKU_s.Replace("Premium", "StandardSSD") + $targetSkuCandidates = @() + + foreach ($sku in $skus) + { + $targetSkuCandidate = $null + + $skuMinSizeGB = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MinSizeGiB' }).Value + $skuMaxSizeGB = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxSizeGiB' }).Value + $skuMaxIOps = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxIOps' }).Value + $skuMaxBandwidthMBps = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxBandwidthMBps' }).Value + + if ($sku.Name -eq $targetSkuPerfTier -and $skuMinSizeGB -lt [int]$result.DiskSizeGB_s -and $skuMaxSizeGB -ge [int]$result.DiskSizeGB_s ` + -and [double]$skuMaxIOps -ge [double]$result.MaxIOPSMetric -and [double]$skuMaxBandwidthMBps -ge [double]$result.MaxMBsMetric) + { + if ($null -eq $skuPricesFound[$sku.Size]) + { + $skuPricesFound[$sku.Size] = Find-DiskMonthlyPrice -DiskSizeTier $sku.Size -SKUPriceSheet $pricesheetEntries + } + + if ($skuPricesFound[$sku.Size] -lt [double]::MaxValue -and $skuPricesFound[$sku.Size] -lt $skuPricesFound[$currentDiskTier]) + { + $targetSkuCandidate = New-Object PSObject -Property @{ + Name = $sku.Size + MonthlyPrice = $skuPricesFound[$sku.Size] + MaxSizeGB = $skuMaxSizeGB + MaxIOPS = $skuMaxIOps + MaxMBps = $skuMaxBandwidthMBps + } + + $targetSkuCandidates += $targetSkuCandidate + } + } + } + + $targetSku = $targetSkuCandidates | Sort-Object -Property MonthlyPrice | Select-Object -First 1 + + if ($null -ne $targetSku) + { + $queryInstanceId = $result.ResourceId + $queryText = @" + let billingInterval = 30d; + let armId = `'$queryInstanceId`'; + let gInt = $perfTimeGrain; + let ThroughputMBsPerf = $metricsTableName + | where TimeGenerated > ago(billingInterval) + | extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) + | where ResourceId == armId + | where MetricNames_s == 'Composite Disk Read Bytes/sec,Composite Disk Write Bytes/sec' and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum' + | extend ThroughputMBs = todouble(MetricValue_s)/1024/1024 + | project CollectedDate, ThroughputMBs, InstanceId_s=ResourceId + | join kind=inner ( + $disksTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, DiskThroughput_s + ) on InstanceId_s + | extend MBsPercentage = ThroughputMBs / todouble(DiskThroughput_s) * 100 + | summarize max(MBsPercentage) by bin(CollectedDate, gInt); + let IOPSPerf = $metricsTableName + | where TimeGenerated > ago(billingInterval) + | extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) + | where ResourceId == armId + | where MetricNames_s == 'Composite Disk Read Operations/sec,Composite Disk Write Operations/sec' and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum' + | extend IOPS = todouble(MetricValue_s) + | project CollectedDate, IOPS, InstanceId_s=ResourceId + | join kind=inner ( + $disksTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, DiskIOPS_s + ) on InstanceId_s + | extend IOPSPercentage = IOPS / todouble(DiskIOPS_s) * 100 + | summarize max(IOPSPercentage) by bin(CollectedDate, gInt); + ThroughputMBsPerf + | join kind=inner (IOPSPerf) on CollectedDate + | render timechart +"@ + + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["DiskType"] = "Managed" + $additionalInfoDictionary["currentSku"] = $result.SKU_s + $additionalInfoDictionary["targetSku"] = $targetSkuPerfTier + $additionalInfoDictionary["DiskSizeGB"] = [int] $result.DiskSizeGB_s + $additionalInfoDictionary["currentTier"] = $currentDiskTier + $additionalInfoDictionary["targetTier"] = $targetSku.Name + $additionalInfoDictionary["MaxIOPSMetric"] = [double] $($result.MaxIOPSMetric) + $additionalInfoDictionary["MaxMBpsMetric"] = [double] $($result.MaxMBsMetric) + $additionalInfoDictionary["MetricIOPSPercentage"] = [double] $($result.IOPSPercentage) + $additionalInfoDictionary["MetricMBpsPercentage"] = [double] $($result.MBsPercentage) + $additionalInfoDictionary["targetMaxSizeGB"] = [int] $targetSku.MaxSizeGB + $additionalInfoDictionary["targetMaxIOPS"] = [int] $targetSku.MaxIOPS + $additionalInfoDictionary["targetMaxMBps"] =[int] $targetSku.MaxMBps + + $fitScore = 4 # needs Maximum of Maximum for metrics to have higher fit score + if ([int] $result.DiskSizeGB_s -gt 512) + { + $fitScore = 3.5 #disk will not support credit-based bursting, therefore the recommendation risk increases a bit + } + + $fitScore = [Math]::max(0.0, $fitScore) + + $savingCoefficient = 2 # Standard SSD is generally close to half the price of Premium SSD + + $targetSkuSavingsMonthly = $result.Last30DaysCost - ($result.Last30DaysCost / $savingCoefficient) + + $tentativeTargetSkuSavingsMonthly = -1 + + if ($targetSku -and $skuPricesFound[$targetSku.Name] -lt [double]::MaxValue) + { + $targetSkuPrice = $skuPricesFound[$targetSku.Name] + + if ($skuPricesFound[$currentDiskTier] -lt [double]::MaxValue) + { + $currentSkuPrice = $skuPricesFound[$currentDiskTier] + $tentativeTargetSkuSavingsMonthly = ($currentSkuPrice * [double] $result.Last30DaysQuantity) - ($targetSkuPrice * [double] $result.Last30DaysQuantity) + } + else + { + $tentativeTargetSkuSavingsMonthly = $result.Last30DaysCost - ($targetSkuPrice * [double] $result.Last30DaysQuantity) + } + } + + if ($tentativeTargetSkuSavingsMonthly -ge 0) + { + $targetSkuSavingsMonthly = $tentativeTargetSkuSavingsMonthly + } + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + if ($targetSkuSavingsMonthly -eq [double]::PositiveInfinity) + { + $targetSkuSavingsMonthly = [double] $result.Last30DaysCost / 2 + } + + $additionalInfoDictionary["savingsAmount"] = [double] $targetSkuSavingsMonthly + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Compute/disks" + Impact = "High" + RecommendationType = "Saving" + RecommendationSubType = "UnderusedPremiumSSDDisks" + RecommendationSubTypeId = "4854b5dc-4124-4ade-879e-6a7bb65350ab" + RecommendationDescription = "Premium SSD disk has been underutilized" + RecommendationAction = "Change disk tier at least to the equivalent for Standard SSD" + InstanceId = $result.ResourceId + InstanceName = $result.DiskName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionId + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation + } +} + +# Export the recommendations as JSON to blob storage + +Write-Output "Exporting final $($recommendations.Count) results as a JSON file..." + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "disks-underutilized-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..484b289e2 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1 @@ -0,0 +1,445 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$perfDaysBackwards = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendPerfPeriodInDays" -ErrorAction SilentlyContinue) +if (-not($perfDaysBackwards -gt 0)) { + $perfDaysBackwards = 7 +} + +$dtuPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileSqlDtu" -ErrorAction SilentlyContinue) +if (-not($dtuPercentile -gt 0)) { + $dtuPercentile = 99 +} +$dtuPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdDtuPercentage" -ErrorAction SilentlyContinue) +if (-not($dtuPercentageThreshold -gt 0)) { + $dtuPercentageThreshold = 40 +} +$dtuDegradedPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdDtuDegradedPercentage" -ErrorAction SilentlyContinue) +if (-not($dtuDegradedPercentageThreshold -gt 0)) { + $dtuDegradedPercentageThreshold = 75 +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGSqlDb','MonitorMetrics','AzureConsumption','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$sqlDbsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGSqlDb' }).LogAnalyticsSuffix + "_CL" +$metricsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'MonitorMetrics' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $sqlDbsTableName, $subscriptionsTableName, $metricsTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +# Execute the recommendation query against Log Analytics +Write-Output "Looking for underused SQL Databases, with less than $dtuPercentageThreshold % Max. DTU usage..." + +$baseQuery = @" + let DTUPercentageThreshold = $dtuPercentageThreshold; + let MetricsInterval = $($perfDaysBackwards)d; + let BillingInterval = 30d; + let dtuPercentPercentile = $dtuPercentile; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(BillingInterval) | summarize max(todatetime(Date_s)))); + let stime = etime-BillingInterval; + let CandidateDatabaseIds = $sqlDbsTableName + | where TimeGenerated > ago(1d) and SkuName_s in ('Standard','Premium') + | distinct InstanceId_s; + $metricsTableName + | where TimeGenerated > ago(MetricsInterval) + | where ResourceId in (CandidateDatabaseIds) and MetricNames_s == 'dtu_consumption_percent' and AggregationType_s == 'Maximum' + | summarize P99DTUPercentage = percentile(todouble(MetricValue_s), dtuPercentPercentile) by ResourceId + | where P99DTUPercentage < DTUPercentageThreshold + | join ( + $sqlDbsTableName + | where TimeGenerated > ago(1d) + | project ResourceId = InstanceId_s, DBName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, SkuName_s, ServiceObjectiveName_s, Tags_s, Cloud_s + ) on ResourceId + | join kind=leftouter ( + $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | project ResourceId=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on ResourceId + | summarize Last30DaysCost=sum(todouble(CostInBillingCurrency_s)) by DBName_s, ResourceId, TenantGuid_g, SubscriptionGuid_g, ResourceGroupName_s, SkuName_s, ServiceObjectiveName_s, Tags_s, Cloud_s, P99DTUPercentage + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.ResourceId + $queryText = @" + $metricsTableName + | where ResourceId == '$queryInstanceId' + | where MetricNames_s == 'dtu_consumption_percent' and AggregationType_s == 'Maximum' + | project TimeGenerated, DTUPercentage = toint(MetricValue_s) + | render timechart +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = "$($result.SkuName_s) $($result.ServiceObjectiveName_s)" + $additionalInfoDictionary["DTUPercentage"] = [int] $result.P99DTUPercentage + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + $additionalInfoDictionary["savingsAmount"] = ([double] $result.Last30DaysCost / 2) + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Sql/servers/databases" + Impact = "High" + RecommendationType = "Saving" + RecommendationSubType = "UnderusedSqlDatabases" + RecommendationSubTypeId = "ff68f4e5-1197-4be9-8e5f-8760d7863cb4" + RecommendationDescription = "Underused SQL Databases (performance capacity waste)" + RecommendationAction = "Right-size underused SQL Databases" + InstanceId = $result.ResourceId + InstanceName = $result.DBName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "sqldbs-underused-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for performance constrained SQL Databases, with more than $dtuDegradedPercentageThreshold % Avg. DTU usage..." + +$baseQuery = @" + let DTUPercentageThreshold = $dtuDegradedPercentageThreshold; + let MetricsInterval = $($perfDaysBackwards)d; + let CandidateDatabaseIds = $sqlDbsTableName + | where TimeGenerated > ago(1d) and SkuName_s in ('Basic','Standard','Premium') + | distinct InstanceId_s; + $metricsTableName + | where TimeGenerated > ago(MetricsInterval) + | where ResourceId in (CandidateDatabaseIds) and MetricNames_s == 'dtu_consumption_percent' and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum' + | summarize AvgDTUPercentage = avg(todouble(MetricValue_s)) by ResourceId + | where AvgDTUPercentage > DTUPercentageThreshold + | join ( + $sqlDbsTableName + | where TimeGenerated > ago(1d) + | project ResourceId = InstanceId_s, DBName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, SkuName_s, ServiceObjectiveName_s, Tags_s, Cloud_s + ) on ResourceId + | project DBName_s, ResourceId, TenantGuid_g, SubscriptionGuid_g, ResourceGroupName_s, SkuName_s, ServiceObjectiveName_s, Tags_s, Cloud_s, AvgDTUPercentage + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.ResourceId + $queryText = @" + $metricsTableName + | where ResourceId == '$queryInstanceId' + | where MetricNames_s == 'dtu_consumption_percent' and AggregationType_s == 'Average' + | project TimeGenerated, DTUPercentage = toint(MetricValue_s) + | render timechart +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = "$($result.SkuName_s) $($result.ServiceObjectiveName_s)" + $additionalInfoDictionary["DTUPercentage"] = [int] $result.AvgDTUPercentage + + $fitScore = 4 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Performance" + ImpactedArea = "Microsoft.Sql/servers/databases" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "PerfConstrainedSqlDatabases" + RecommendationSubTypeId = "724ff2f5-8c83-4105-b00d-029c4560d774" + RecommendationDescription = "SQL Database performance has been constrained by lack of resources" + RecommendationAction = "Resize SQL Database to higher SKU or scale it out" + InstanceId = $result.ResourceId + InstanceName = $result.DBName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "sqldbs-perfconstrained-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-StorageAccountOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-StorageAccountOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..e3832d8c6 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-StorageAccountOptimizationsToBlobStorage.ps1 @@ -0,0 +1,326 @@ +function ConvertTo-Hashtable { + [CmdletBinding()] + [OutputType('hashtable')] + param ( + [Parameter(ValueFromPipeline)] + $InputObject + ) + + process { + if ($null -eq $InputObject) { + return $null + } + + if ($InputObject -is [System.Collections.IEnumerable] -and $InputObject -isnot [string]) { + $collection = @( + foreach ($object in $InputObject) { + ConvertTo-Hashtable -InputObject $object + } + ) + Write-Output -NoEnumerate $collection + } elseif ($InputObject -is [psobject]) { + $hash = @{} + foreach ($property in $InputObject.PSObject.Properties) { + $hash[$property.Name] = ConvertTo-Hashtable -InputObject $property.Value + } + $hash + } else { + $InputObject + } + } +} + +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +# storage account thresholds variables +$growthPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendationStorageAcountGrowthThresholdPercentage" -ErrorAction SilentlyContinue) +if (-not($growthPercentageThreshold -gt 0)) { + $growthPercentageThreshold = 5 +} +$monthlyCostThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendationStorageAcountGrowthMonthlyCostThreshold" -ErrorAction SilentlyContinue) +if (-not($monthlyCostThreshold -gt 0)) { + $monthlyCostThreshold = 50 +} +$growthLookbackDays = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendationStorageAcountGrowthLookbackDays" -ErrorAction SilentlyContinue) +if (-not($growthLookbackDays -gt 0)) { + $growthLookbackDays = 30 +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +$tenantId = (Get-AzContext).Tenant.Id + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGResourceContainers','AzureConsumption')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $subscriptionsTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = $growthLookbackDays + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +Write-Output "Looking for ever growing Storage Accounts, with more than $monthlyCostThreshold/month costs, growing more than $growthPercentageThreshold% over the last $growthLookbackDays days..." + +$dailyCostThreshold = [Math]::Round($monthlyCostThreshold / 30) + +$baseQuery = @" +let interval = $($growthLookbackDays)d; +let etime = endofday(todatetime(toscalar($consumptionTableName | where todatetime(Date_s) > ago(interval) and todatetime(Date_s) < now() | summarize max(todatetime(Date_s))))); +let etime_subs = endofday(todatetime(toscalar($subscriptionsTableName | where TimeGenerated > ago(interval) | summarize max(TimeGenerated)))); +let stime = endofday(etime-interval); +let lastday_stime = endofday(etime-1d); +let lastday_stime_subs = endofday(etime_subs-1d); +let costThreshold = $dailyCostThreshold; +let growthPercentageThreshold = $growthPercentageThreshold; +let StorageAccountsWithLastTags = $consumptionTableName +| where todatetime(Date_s) between (lastday_stime..etime) +| where MeterCategory_s == 'Storage' and ConsumedService_s == 'Microsoft.Storage' and MeterName_s endswith 'Data Stored' and ChargeType_s == 'Usage' +| extend ResourceId = tolower(ResourceId) +| distinct ResourceId, Tags_s; +$consumptionTableName +| where todatetime(Date_s) between (stime..etime) +| where MeterCategory_s == 'Storage' and ConsumedService_s == 'Microsoft.Storage' and MeterName_s endswith 'Data Stored' and ChargeType_s == 'Usage' +| extend ResourceId = tolower(ResourceId) +| make-series CostSum=sum(todouble(CostInBillingCurrency_s)) default=0.0 on todatetime(Date_s) from stime to etime step 1d by ResourceId, ResourceGroup, SubscriptionId +| extend InitialDailyCost = todouble(CostSum[0]), CurrentDailyCost = todouble(CostSum[array_length(CostSum)-1]) +| extend GrowthPercentage = round((CurrentDailyCost-InitialDailyCost)/InitialDailyCost*100) +| where InitialDailyCost > 0 and CurrentDailyCost > costThreshold and GrowthPercentage > growthPercentageThreshold +| project ResourceId, InitialDailyCost, CurrentDailyCost, GrowthPercentage, ResourceGroup, SubscriptionId +| join kind=leftouter (StorageAccountsWithLastTags) on ResourceId +| join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > lastday_stime_subs + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionId=SubscriptionGuid_g, SubscriptionName = ContainerName_s +) on SubscriptionId +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + throw "Execution aborted" +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.ResourceId + $queryText = @" + $consumptionTableName + | where MeterCategory_s == 'Storage' and ConsumedService_s == 'Microsoft.Storage' and MeterName_s endswith 'Data Stored' and ChargeType_s == 'Usage' + | extend ResourceId = tolower(ResourceId) + | where ResourceId =~ '$queryInstanceId' + | summarize DailyCosts = sum(todouble(CostInBillingCurrency_s)) by bin(todatetime(Date_s), 1d) + | render timechart +"@ + + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-1 * $recommendationSearchTimeSpan).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $costsAmount = ([double] $result.InitialDailyCost + [double] $result.CurrentDailyCost) / 2 * 30 + + $additionalInfoDictionary["InitialDailyCost"] = $result.InitialDailyCost + $additionalInfoDictionary["CurrentDailyCost"] = $result.CurrentDailyCost + $additionalInfoDictionary["GrowthPercentage"] = $result.GrowthPercentage + $additionalInfoDictionary["CostsAmount"] = $costsAmount + $additionalInfoDictionary["savingsAmount"] = $costsAmount * 0.25 # estimated 25% savings + + $fitScore = 4 # savings are estimated with a significant error margin + + $fitScore = [Math]::max(0.0, $fitScore) + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + if (-not($result.Tags_s -like "{*")) + { + $result.Tags_s = '{' + $result.Tags_s + '}' + } + $tags = ConvertFrom-Json $result.Tags_s | ConvertTo-Hashtable + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $cloudEnvironment + Category = "Cost" + ImpactedArea = "Microsoft.Storage/storageAccounts" + Impact = "Medium" + RecommendationType = "Saving" + RecommendationSubType = "StorageAccountsGrowing" + RecommendationSubTypeId = "08e049ca-18b0-4d22-b174-131a91d0381c" + RecommendationDescription = "Storage Account without retention policy in place" + RecommendationAction = "Review whether the Storage Account has a retention policy for example via Lifecycle Management" + InstanceId = $result.ResourceId + InstanceName = $result.ResourceId.Split('/')[-1] + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionId + SubscriptionName = $result.SubscriptionName + TenantGuid = $tenantId + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +Write-Output "Exporting final $($recommendations.Count) results as a JSON file..." + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "storageaccounts-costsgrowing-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnattachedDisksToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnattachedDisksToBlobStorage.ps1 new file mode 100644 index 000000000..9dd38466b --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnattachedDisksToBlobStorage.ps1 @@ -0,0 +1,270 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGManagedDisk','AzureConsumption','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$disksTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGManagedDisk' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $disksTableName, $subscriptionsTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +# Execute the recommendation query against Log Analytics + +$baseQuery = @" + let interval = 30d; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(interval) | summarize max(todatetime(Date_s)))); + let stime = etime-interval; + $disksTableName + | where TimeGenerated > ago(1d) and isempty(OwnerVMId_s) and Tags_s !has 'ASR-ReplicaDisk' and Tags_s !has 'asrseeddisk' + | distinct DiskName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SKU_s, DiskSizeGB_s, Tags_s, Cloud_s + | join kind=leftouter ( + $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize Last30DaysCost=sum(todouble(CostInBillingCurrency_s)) by DiskName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SKU_s, DiskSizeGB_s, Tags_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + throw "Execution aborted" +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + $disksTableName + | where InstanceId_s == '$queryInstanceId' and isempty(OwnerVMId_s) + | distinct InstanceId_s, DiskName_s, DiskSizeGB_s, SKU_s, TimeGenerated + | summarize LastAttachedDate = min(TimeGenerated) by InstanceId_s, DiskName_s, DiskSizeGB_s, SKU_s + | join kind=leftouter ( + $consumptionTableName + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize CostsSinceDetached = sumif(todouble(CostInBillingCurrency_s), todatetime(Date_s) > LastAttachedDate) by DiskName_s, LastAttachedDate, DiskSizeGB_s, SKU_s +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $deploymentDate + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["DiskType"] = "Managed" + $additionalInfoDictionary["currentSku"] = $result.SKU_s + $additionalInfoDictionary["DiskSizeGB"] = [int] $result.DiskSizeGB_s + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + $additionalInfoDictionary["savingsAmount"] = [double] $result.Last30DaysCost + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Compute/disks" + Impact = "Medium" + RecommendationType = "Saving" + RecommendationSubType = "UnattachedDisks" + RecommendationSubTypeId = "c84d5e86-e2d6-4d62-be7c-cecfbd73b0db" + RecommendationDescription = "Unattached disks (without owner VM) incur in unnecessary costs" + RecommendationAction = "Delete or downgrade disk to Standard SKU" + InstanceId = $result.InstanceId_s + InstanceName = $result.DiskName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "unattacheddisks-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnusedAppGWsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnusedAppGWsToBlobStorage.ps1 new file mode 100644 index 000000000..737cb9935 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnusedAppGWsToBlobStorage.ps1 @@ -0,0 +1,271 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGAppGateway','AzureConsumption','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$appGWsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGAppGateway' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $appGWsTableName, $subscriptionsTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +# Execute the Cost recommendation query against Log Analytics + +$baseQuery = @" + let interval = 30d; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(interval) | summarize max(todatetime(Date_s)))); + let stime = etime-interval; + $appGWsTableName + | where TimeGenerated > ago(1d) + | where toint(BackendPoolsCount_s) == 0 or ((BackendIPCount_s == 0 or isempty(BackendIPCount_s)) and (BackendAddressesCount_s == 0 or isempty(BackendAddressesCount_s))) + | distinct InstanceName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuName_s, SkuCapacity_s, Tags_s, Cloud_s + | join kind=leftouter ( + $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize Last30DaysCost=sum(todouble(CostInBillingCurrency_s)) by InstanceName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuName_s, SkuCapacity_s, Tags_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + throw "Execution aborted" +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + $appGWsTableName + | where InstanceId_s == '$queryInstanceId' + | where toint(BackendPoolsCount_s) == 0 or ((toint(BackendIPCount_s) == 0 or isempty(BackendIPCount_s)) and (toint(BackendAddressesCount_s) == 0 or isempty(BackendAddressesCount_s))) + | distinct InstanceId_s, InstanceName_s, TimeGenerated + | summarize FirstUnusedDate = min(TimeGenerated) by InstanceId_s, InstanceName_s + | join kind=leftouter ( + $consumptionTableName + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize CostsSinceUnused = sumif(todouble(CostInBillingCurrency_s), todatetime(Date_s) > FirstUnusedDate) by InstanceName_s, FirstUnusedDate +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $deploymentDate + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = $result.SkuName_s + $additionalInfoDictionary["InstanceCount"] = $result.SkuCapacity_s + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + $additionalInfoDictionary["savingsAmount"] = [double] $result.Last30DaysCost + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Network/applicationGateways" + Impact = "High" + RecommendationType = "Saving" + RecommendationSubType = "UnusedAppGateways" + RecommendationSubTypeId = "dc3d2baa-26c8-435e-aa9d-edb2bfd6fff6" + RecommendationDescription = "Application Gateways without a backend pool incur in unnecessary costs" + RecommendationAction = "Delete the Application Gateway" + InstanceId = $result.InstanceId_s + InstanceName = $result.InstanceName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "unusedappgateways-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnusedLoadBalancersToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnusedLoadBalancersToBlobStorage.ps1 new file mode 100644 index 000000000..193104391 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-UnusedLoadBalancersToBlobStorage.ps1 @@ -0,0 +1,402 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGLoadBalancer','AzureConsumption','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$lbsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGLoadBalancer' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $lbsTableName, $subscriptionsTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +# Execute the Cost recommendation query against Log Analytics + +$baseQuery = @" + let interval = 30d; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(interval) | summarize max(todatetime(Date_s)))); + let stime = etime-interval; + $lbsTableName + | where TimeGenerated > ago(1d) + | where SkuName_s == 'Standard' + | where (toint(BackendPoolsCount_s) == 0 or ((toint(BackendIPCount_s) == 0 or isempty(BackendIPCount_s)) and (toint(BackendAddressesCount_s) == 0 or isempty(BackendAddressesCount_s)))) and toint(InboundNatPoolsCount_s) == 0 + | where toint(LbRulesCount_s) != 0 or toint(InboundNatRulesCount_s) != 0 or toint(OutboundRulesCount_s) != 0 + | distinct InstanceName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuName_s, Tags_s, Cloud_s + | join kind=leftouter ( + $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize Last30DaysCost=sum(todouble(CostInBillingCurrency_s)) by InstanceName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuName_s, Tags_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Costs query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + $lbsTableName + | where InstanceId_s == '$queryInstanceId' + | where SkuName_s == 'Standard' + | where (toint(BackendPoolsCount_s) == 0 or ((BackendIPCount_s == 0 or isempty(BackendIPCount_s)) and (BackendAddressesCount_s == 0 or isempty(BackendAddressesCount_s)))) and toint(InboundNatPoolsCount_s) == 0 + | where toint(LbRulesCount_s) != 0 or toint(InboundNatRulesCount_s) != 0 or toint(OutboundRulesCount_s) != 0 + | distinct InstanceId_s, InstanceName_s, SkuName_s, TimeGenerated + | summarize FirstUnusedDate = min(TimeGenerated) by InstanceId_s, InstanceName_s, SkuName_s + | join kind=leftouter ( + $consumptionTableName + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize CostsSinceUnused = sumif(todouble(CostInBillingCurrency_s), todatetime(Date_s) > FirstUnusedDate) by InstanceName_s, FirstUnusedDate, SkuName_s +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $deploymentDate + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = $result.SkuName_s + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + $additionalInfoDictionary["savingsAmount"] = [double] $result.Last30DaysCost + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Network/loadBalancers" + Impact = "Medium" + RecommendationType = "Saving" + RecommendationSubType = "UnusedStandardLoadBalancers" + RecommendationSubTypeId = "f1ed3bb2-3cb5-41e6-ba38-7001d5ff87f5" + RecommendationDescription = "Standard Load Balancers with rules defined and without a backend pool incur in unnecessary costs" + RecommendationAction = "Delete the Load Balancer" + InstanceId = $result.InstanceId_s + InstanceName = $result.InstanceName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "unusedstdloadbalancers-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + + +# Execute the Operational Excellence recommendation query against Log Analytics + +$baseQuery = @" + $lbsTableName + | where TimeGenerated > ago(1d) + | where (toint(BackendPoolsCount_s) == 0 or BackendIPCount_s == 0 or isempty(BackendIPCount_s)) and toint(InboundNatPoolsCount_s) == 0 + | distinct InstanceName_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuName_s, Tags_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days 2) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Operational Excellence query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$workspaceTenantId/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = $result.SkuName_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Network/loadBalancers" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "UnusedLoadBalancers" + RecommendationSubTypeId = "48619512-f4e6-4241-9c85-16f7c987950c" + RecommendationDescription = "Load Balancers without a backend pool are useless" + RecommendationAction = "Delete the Load Balancer" + InstanceId = $result.InstanceId_s + InstanceName = $result.InstanceName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "unusedloadbalancers-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..3165bc592 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMOptimizationsToBlobStorage.ps1 @@ -0,0 +1,459 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$deallocatedIntervalDays = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendationLongDeallocatedVmsIntervalDays") +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGManagedDisk','ARGVirtualMachine','AzureConsumption','ARGResourceContainers')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$vmsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGVirtualMachine' }).LogAnalyticsSuffix + "_CL" +$disksTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGManagedDisk' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $vmsTableName, $disksTableName, $subscriptionsTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = $deallocatedIntervalDays + $consumptionOffsetDaysStart +$offlineInterval = $deallocatedIntervalDays + $consumptionOffsetDays +$billingInterval = 30 + $consumptionOffsetDays + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +Write-Output "Looking for VMs that have been deallocated for more than 30 days..." + +# Execute the recommendation query against Log Analytics + +$baseQuery = @" + let offlineInterval = $($offlineInterval)d; + let billingInterval = $($billingInterval)d; + let billingWindowIntervalEnd = $($consumptionOffsetDays)d; + let billingWindowIntervalStart = $($consumptionOffsetDaysStart)d; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(billingInterval) | summarize max(todatetime(Date_s)))); + let stime = etime-offlineInterval; + let BilledVMs = $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | where ResourceId like 'microsoft.compute/virtualmachines/' or ResourceId like 'microsoft.classiccompute/virtualmachines/' + | extend InstanceId_s = tolower(ResourceId) + | distinct InstanceId_s; + let RunningVMs = $vmsTableName + | where TimeGenerated > ago(billingWindowIntervalStart) and TimeGenerated < ago(billingWindowIntervalEnd) + | where PowerState_s has_any ('running','starting','readyrole') + | distinct InstanceId_s; + let BilledDisks = $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | where ResourceId like 'microsoft.compute/disks/' + | extend BillingInstanceId = tolower(ResourceId) + | summarize DisksCosts = sum(todouble(CostInBillingCurrency_s)) by BillingInstanceId; + $vmsTableName + | where TimeGenerated > ago(billingWindowIntervalStart) and TimeGenerated < ago(billingWindowIntervalEnd) + | where InstanceId_s !in (RunningVMs) + | join kind=leftouter (BilledVMs) on InstanceId_s + | where isempty(InstanceId_s1) + | project InstanceId_s, VMName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s, Tags_s + | join kind=leftouter ( + $disksTableName + | where TimeGenerated > ago(1d) + | project DiskInstanceId = InstanceId_s, SKU_s, OwnerVMId_s + ) on `$left.InstanceId_s == `$right.OwnerVMId_s + | join kind=leftouter ( + BilledDisks + ) on `$left.DiskInstanceId == `$right.BillingInstanceId + | summarize TotalDisksCosts = sum(DisksCosts) by InstanceId_s, VMName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s, Tags_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + let offlineInterval = $($offlineInterval)d; + $consumptionTableName + | extend ResourceId = tolower(ResourceId) + | where ResourceId =~ '$queryInstanceId' + | where todatetime(Date_s) < now() + | join kind=inner ( + $disksTableName + | extend DiskInstanceId = InstanceId_s + ) + on `$left.ResourceId == `$right.OwnerVMId_s + | summarize DeallocatedSince = max(todatetime(Date_s)) by DiskName_s, DiskSizeGB_s, SKU_s, DiskInstanceId + | join kind=inner + ( + $consumptionTableName + | where todatetime(Date_s) > ago(offlineInterval) + | extend DiskInstanceId = tolower(ResourceId) + | summarize DiskCosts = sum(todouble(CostInBillingCurrency_s)) by DiskInstanceId + ) + on DiskInstanceId + | project DeallocatedSince, DiskName_s, DiskSizeGB_s, SKU_s, MonthlyCosts = DiskCosts +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $deploymentDate + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["LongDeallocatedThreshold"] = $deallocatedIntervalDays + $additionalInfoDictionary["CostsAmount"] = [double] $result.TotalDisksCosts + $additionalInfoDictionary["savingsAmount"] = [double] $result.TotalDisksCosts + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "Medium" + RecommendationType = "Saving" + RecommendationSubType = "LongDeallocatedVms" + RecommendationSubTypeId = "c320b790-2e58-452a-aa63-7b62c383ad8a" + RecommendationDescription = "Virtual Machine has been deallocated for long with disks still incurring costs" + RecommendationAction = "Delete Virtual Machine or downgrade its disks to Standard HDD SKU" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "longdeallocatedvms-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for VMs that are stopped (not deallocated)..." + +# Execute the recommendation query against Log Analytics + +$baseQuery = @" + $vmsTableName + | where TimeGenerated > ago(1d) + | where PowerState_s has 'stopped' + | project InstanceId_s, VMName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s, Tags_s + | join kind=leftouter ( + $consumptionTableName + | where TimeGenerated > ago(1d) and MeterCategory_s == 'Virtual Machines' + | project InstanceId_s=tolower(ResourceId), UnitPrice_s, EffectivePrice_s + | summarize arg_max(todouble(EffectivePrice_s), *) by InstanceId_s + | project InstanceId_s, MonthlyCost=24*todouble(iif(todouble(UnitPrice_s) > 0, todouble(UnitPrice_s), todouble(EffectivePrice_s)))*30 + ) on InstanceId_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + let LastNonStopped = toscalar($vmsTableName + | where InstanceId_s =~ '$queryInstanceId' + | where TimeGenerated < now() + | where PowerState_s !has 'stopped' + | summarize max(todatetime(StatusDate_s))); + $consumptionTableName + | where ResourceId =~ '$queryInstanceId' + | where todatetime(Date_s) >= LastNonStopped + | where MeterCategory_s == 'Virtual Machines' + | summarize ComputeCostsSinceStopped = sum(todouble(Quantity_s)*todouble(UnitPrice_s)) by MeterSubCategory_s, StoppedSince=LastNonStopped +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $deploymentDate + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["CostsAmount"] = [double] $result.MonthlyCost + $additionalInfoDictionary["savingsAmount"] = [double] $result.MonthlyCost + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "High" + RecommendationType = "Saving" + RecommendationSubType = "StoppedVms" + RecommendationSubTypeId = "110fea55-a9c3-480d-8248-116f61e139a8" + RecommendationDescription = "Virtual Machine is stopped (not deallocated) and still incurring costs" + RecommendationAction = "Deallocate Virtual Machine" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "stoppedvms-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMSSOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMSSOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..ddc65a5fe --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMSSOptimizationsToBlobStorage.ps1 @@ -0,0 +1,795 @@ +$ErrorActionPreference = "Stop" + +function Find-SkuHourlyPrice { + param ( + [object[]] $SKUPriceSheet, + [string] $SKUName + ) + + $skuPriceObject = $null + + if ($SKUPriceSheet) + { + $skuNameParts = $SKUName.Split('_') + + if ($skuNameParts.Count -eq 3) # e.g., Standard_D1_v2 + { + $skuNameFilter = "*" + $skuNameParts[1] + " *" + $skuVersionFilter = "*" + $skuNameParts[2] + $skuPrices = $SKUPriceSheet | Where-Object { $_.MeterName_s -like $skuNameFilter ` + -and $_.MeterName_s -notlike '*Low Priority' -and $_.MeterName_s -notlike '*Expired' ` + -and $_.MeterName_s -like $skuVersionFilter -and $_.MeterSubCategory_s -notlike '*Windows' -and $_.UnitPrice_s -ne 0 } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + if ($skuPrices.Count -gt 2) # D1-like scenarios + { + $skuFilter = "*" + $skuNameParts[1] + " " + $skuNameParts[2] + "*" + $skuPrices = $skuPrices | Where-Object { $_.MeterName_s -like $skuFilter } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + } + } + + if ($skuNameParts.Count -eq 2) # e.g., Standard_D1 + { + $skuNameFilter = "*" + $skuNameParts[1] + "*" + + $skuPrices = $SKUPriceSheet | Where-Object { $_.MeterName_s -like $skuNameFilter ` + -and $_.MeterName_s -notlike '*Low Priority' -and $_.MeterName_s -notlike '*Expired' ` + -and $_.MeterName_s -notlike '* v*' -and $_.MeterSubCategory_s -notlike '*Windows' -and $_.UnitPrice_s -ne 0 } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + if ($skuPrices.Count -gt 2) # D1-like scenarios + { + $skuFilterLeft = "*" + $skuNameParts[1] + "/*" + $skuFilterRight = "*/" + $skuNameParts[1] + "*" + $skuPrices = $skuPrices | Where-Object { $_.MeterName_s -like $skuFilterLeft -or $_.MeterName_s -like $skuFilterRight } + + if (($skuPrices -or $skuPrices.Count -ge 1) -and $skuPrices.Count -le 2) + { + $skuPriceObject = $skuPrices[0] + } + } + } + } + + $targetHourlyPrice = [double]::MaxValue + if ($null -ne $skuPriceObject) + { + $targetUnitHours = [int] (Select-String -InputObject $skuPriceObject.UnitOfMeasure_s -Pattern "^\d+").Matches[0].Value + if ($targetUnitHours -gt 0) + { + $targetHourlyPrice = [double] ($skuPriceObject.UnitPrice_s / $targetUnitHours) + } + } + + return $targetHourlyPrice +} + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +# percentiles variables +$cpuPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileCpu" -ErrorAction SilentlyContinue) +if (-not($cpuPercentile -gt 0)) { + $cpuPercentile = 99 +} +$memoryPercentile = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfPercentileMemory" -ErrorAction SilentlyContinue) +if (-not($memoryPercentile -gt 0)) { + $memoryPercentile = 99 +} + +# perf thresholds variables +$cpuPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuPercentageThreshold -gt 0)) { + $cpuPercentageThreshold = 30 +} +$memoryPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdMemoryPercentage" -ErrorAction SilentlyContinue) +if (-not($memoryPercentageThreshold -gt 0)) { + $memoryPercentageThreshold = 50 +} +$cpuDegradedMaxPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuDegradedMaxPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuDegradedMaxPercentageThreshold -gt 0)) { + $cpuDegradedMaxPercentageThreshold = 95 +} +$cpuDegradedAvgPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdCpuDegradedAvgPercentage" -ErrorAction SilentlyContinue) +if (-not($cpuDegradedAvgPercentageThreshold -gt 0)) { + $cpuDegradedAvgPercentageThreshold = 75 +} +$memoryDegradedPercentageThreshold = [int] (Get-AutomationVariable -Name "AzureOptimization_PerfThresholdMemoryDegradedPercentage" -ErrorAction SilentlyContinue) +if (-not($memoryDegradedPercentageThreshold -gt 0)) { + $memoryDegradedPercentageThreshold = 90 +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$perfDaysBackwards = [int] (Get-AutomationVariable -Name "AzureOptimization_RecommendPerfPeriodInDays" -ErrorAction SilentlyContinue) +if (-not($perfDaysBackwards -gt 0)) { + $perfDaysBackwards = 7 +} + +$perfTimeGrain = Get-AutomationVariable -Name "AzureOptimization_RecommendPerfTimeGrain" -ErrorAction SilentlyContinue +if (-not($perfTimeGrain)) { + $perfTimeGrain = "1h" +} + +$referenceRegion = Get-AutomationVariable -Name "AzureOptimization_ReferenceRegion" + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGVMSS','MonitorMetrics','ARGResourceContainers','AzureConsumption','Pricesheet')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$vmssTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGVMSS' }).LogAnalyticsSuffix + "_CL" +$metricsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'MonitorMetrics' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" +$pricesheetTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'Pricesheet' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $vmssTableName, $metricsTableName, $subscriptionsTableName, $pricesheetTableName and $consumptionTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +Write-Output "Getting Virtual Machine SKUs for the $referenceRegion region..." + +$skus = Get-AzComputeResourceSku -Location $referenceRegion | Where-Object { $_.ResourceType -eq "virtualMachines" } + +Write-Output "Getting the current Pricesheet..." + +if ($cloudEnvironment -eq "AzureCloud") +{ + $pricesheetRegion = "EU West" +} + +try +{ + $pricesheetEntries = @() + + $baseQuery = @" + $pricesheetTableName + | where TimeGenerated > ago(14d) + | where MeterCategory_s == 'Virtual Machines' and MeterRegion_s == '$pricesheetRegion' and PriceType_s == 'Consumption' + | distinct MeterName_s, MeterSubCategory_s, MeterCategory_s, MeterRegion_s, UnitPrice_s, UnitOfMeasure_s +"@ + + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days 14) -Wait 600 -IncludeStatistics + $pricesheetEntries = [System.Linq.Enumerable]::ToArray($queryResults.Results) + + Write-Output "Query finished with $($pricesheetEntries.Count) results." + Write-Output "Query statistics: $($queryResults.Statistics.query)" +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + Write-Output "Consumption pricesheet not available, will estimate savings based in cores count..." +} + +$skuPricesFound = @{} + +$recommendationsErrors = 0 + +Write-Output "Looking for underutilized Scale Sets, with less than $cpuPercentageThreshold% CPU and $memoryPercentageThreshold% RAM usage..." + +$baseQuery = @" + let billingInterval = 30d; + let perfInterval = $($perfDaysBackwards)d; + let cpuPercentileValue = $cpuPercentile; + let memoryPercentileValue = $memoryPercentile; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(30d) | summarize max(todatetime(Date_s)))); + let stime = etime-billingInterval; + + let BilledVMs = $consumptionTableName + | where todatetime(Date_s) between (stime..etime) and ResourceId contains 'virtualmachinescalesets' + | extend VMConsumedQuantity = iif(ResourceId contains 'virtualmachinescalesets' and MeterCategory_s == 'Virtual Machines', todouble(Quantity_s), 0.0) + | extend VMPrice = iif(ResourceId contains 'virtualmachinescalesets' and MeterCategory_s == 'Virtual Machines', todouble(EffectivePrice_s), 0.0) + | extend FinalCost = VMPrice * VMConsumedQuantity + | extend InstanceId_s = tolower(ResourceId) + | summarize Last30DaysCost = sum(FinalCost), Last30DaysQuantity = sum(VMConsumedQuantity) by InstanceId_s; + + let MemoryPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where MetricNames_s == "Available Memory Bytes" and AggregationType_s == "Minimum" + | extend MemoryAvailableMBs = todouble(MetricValue_s)/1024/1024 + | project TimeGenerated, MemoryAvailableMBs, InstanceId_s=ResourceId + | join kind=inner ( + $vmssTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, MemoryMB_s + ) on InstanceId_s + | extend MemoryPercentage = todouble(toint(MemoryMB_s) - toint(MemoryAvailableMBs)) / todouble(MemoryMB_s) * 100 + | summarize PMemoryPercentage = percentile(MemoryPercentage, memoryPercentileValue) by InstanceId_s; + + let ProcessorPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where MetricNames_s == "Percentage CPU" and AggregationType_s == 'Maximum' + | extend InstanceId_s = ResourceId + | summarize PCPUPercentage = percentile(todouble(MetricValue_s), cpuPercentileValue) by InstanceId_s; + + $vmssTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, VMSSName_s, ResourceGroupName_s, SubscriptionGuid_g, Cloud_s, TenantGuid_g, VMSSSize_s, NicCount_s, DataDiskCount_s, Capacity_s, Tags_s + | join kind=inner ( BilledVMs ) on InstanceId_s + | join kind=leftouter ( MemoryPerf ) on InstanceId_s + | join kind=leftouter ( ProcessorPerf ) on InstanceId_s + | project InstanceId_s, VMSSName = VMSSName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, VMSSSize_s, NicCount_s, DataDiskCount_s, Capacity_s, PMemoryPercentage, PCPUPercentage, Tags_s, Last30DaysCost, Last30DaysQuantity + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionId = SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionId + | where isnotempty(PMemoryPercentage) and isnotempty(PCPUPercentage) and PMemoryPercentage < $memoryPercentageThreshold and PCPUPercentage < $cpuPercentageThreshold +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + + $targetSku = $null + $currentSku = $skus | Where-Object { $_.Name -eq $result.VMSSSize_s } + + $currentSkuvCPUs = [int]($currentSku.Capabilities | Where-Object { $_.Name -eq 'vCPUsAvailable' }).Value + + $memoryNeeded = [double]($currentSku.Capabilities | Where-Object { $_.Name -eq 'MemoryGB' }).Value * ([double] $result.PMemoryPercentage / 100) + $cpuNeeded = [double]$currentSkuvCPUs * ([double] $result.PCPUPercentage / 100) + $currentPremiumIO = [bool] ($currentSku.Capabilities | Where-Object { $_.Name -eq 'PremiumIO' }).Value + $currentCpuArch = ($currentSku.Capabilities | Where-Object { $_.Name -eq 'CpuArchitectureType' }).Value + + if ($null -eq $skuPricesFound[$currentSku.Name]) + { + $skuPricesFound[$currentSku.Name] = Find-SkuHourlyPrice -SKUName $currentSku.Name -SKUPriceSheet $pricesheetEntries + } + + $targetSkuCandidates = @() + + foreach ($sku in $skus) + { + $targetSkuCandidate = $null + + $skuCPUs = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'vCPUsAvailable' }).Value + $skuMemory = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MemoryGB' }).Value + $skuMaxDataDisks = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxDataDiskCount' }).Value + $skuMaxNICs = [int] ($sku.Capabilities | Where-Object { $_.Name -eq 'MaxNetworkInterfaces' }).Value + $skuPremiumIO = [bool] ($sku.Capabilities | Where-Object { $_.Name -eq 'PremiumIO' }).Value + $skuCpuArch = ($sku.Capabilities | Where-Object { $_.Name -eq 'CpuArchitectureType' }).Value + + if ($currentSku.Name -ne $sku.Name -and -not($sku.Name -like "*Promo*") -and [double]$skuCPUs -ge $cpuNeeded -and [double]$skuMemory -ge $memoryNeeded ` + -and $skuMaxDataDisks -ge [int] $result.DataDiskCount_s -and $skuMaxNICs -ge [int] $result.NicCount_s ` + -and ($currentPremiumIO -eq $false -or $skuPremiumIO -eq $currentPremiumIO) -and $skuCpuArch -eq $currentCpuArch) + { + if ($null -eq $skuPricesFound[$sku.Name]) + { + $skuPricesFound[$sku.Name] = Find-SkuHourlyPrice -SKUName $sku.Name -SKUPriceSheet $pricesheetEntries + } + + if ($skuPricesFound[$currentSku.Name] -eq 0 -or $skuPricesFound[$sku.Name] -lt $skuPricesFound[$currentSku.Name]) + { + $targetSkuCandidate = New-Object PSObject -Property @{ + Name = $sku.Name + HourlyPrice = $skuPricesFound[$sku.Name] + vCPUsAvailable = $skuCPUs + MemoryGB = $skuMemory + } + + $targetSkuCandidates += $targetSkuCandidate + } + } + } + + $targetSku = $targetSkuCandidates | Sort-Object -Property HourlyPrice,MemoryGB,vCPUsAvailable | Select-Object -First 1 + + if ($null -ne $targetSku) + { + $queryInstanceId = $result.InstanceId_s + $queryText = @" + let billingInterval = 30d; + let armId = `'$queryInstanceId`'; + let gInt = $perfTimeGrain; + let MemoryPerf = $metricsTableName + | where TimeGenerated > ago(billingInterval) + | extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) + | where ResourceId == armId + | where MetricNames_s == 'Available Memory Bytes' and AggregationType_s == 'Minimum' + | extend MemoryAvailableMBs = todouble(MetricValue_s)/1024/1024 + | project CollectedDate, MemoryAvailableMBs, InstanceId_s=ResourceId + | join kind=inner ( + $vmssTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, MemoryMB_s + ) on InstanceId_s + | extend MemoryPercentage = todouble(toint(MemoryMB_s) - toint(MemoryAvailableMBs)) / todouble(MemoryMB_s) * 100 + | summarize percentile(MemoryPercentage, $memoryPercentile) by bin(CollectedDate, gInt); + let ProcessorPerf = $metricsTableName + | where TimeGenerated > ago(billingInterval) + | extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) + | where ResourceId == armId + | where MetricNames_s == 'Percentage CPU' and AggregationType_s == 'Maximum' + | extend ProcessorPercentage = todouble(MetricValue_s) + | summarize percentile(ProcessorPercentage, $cpuPercentile) by bin(CollectedDate, gInt); + MemoryPerf + | join kind=inner (ProcessorPerf) on CollectedDate + | render timechart +"@ + + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["SupportsDataDisksCount"] = "true" + $additionalInfoDictionary["SupportsNICCount"] = "true" + $additionalInfoDictionary["BelowCPUThreshold"] = "true" + $additionalInfoDictionary["BelowMemoryThreshold"] = "true" + $additionalInfoDictionary["currentSku"] = "$($result.VMSSSize_s)" + $additionalInfoDictionary["InstanceCount"] = [int] $result.Capacity_s + $additionalInfoDictionary["targetSku"] = "$($targetSku.Name)" + $additionalInfoDictionary["DataDiskCount"] = "$($result.DataDiskCount_s)" + $additionalInfoDictionary["NicCount"] = "$($result.NicCount_s)" + $additionalInfoDictionary["MetricCPUPercentage"] = "$($result.PCPUPercentage)" + $additionalInfoDictionary["MetricMemoryPercentage"] = "$($result.PMemoryPercentage)" + + $fitScore = 4 # needs disk IOPS and throughput analysis to improve score + + $fitScore = [Math]::max(0.0, $fitScore) + + $savingCoefficient = [double] $currentSkuvCPUs / [double] $targetSku.vCPUsAvailable + + if ($targetSku -and $null -eq $skuPricesFound[$targetSku.Name]) + { + $skuPricesFound[$targetSku.Name] = Find-SkuHourlyPrice -SKUName $targetSku.Name -SKUPriceSheet $pricesheetEntries + } + + $targetSkuSavingsMonthly = $result.Last30DaysCost - ($result.Last30DaysCost / $savingCoefficient) + + $tentativeTargetSkuSavingsMonthly = -1 + + if ($targetSku -and $skuPricesFound[$targetSku.Name] -lt [double]::MaxValue) + { + $targetSkuPrice = $skuPricesFound[$targetSku.Name] + + if ($null -eq $skuPricesFound[$currentSku.Name]) + { + $skuPricesFound[$currentSku.Name] = Find-SkuHourlyPrice -SKUName $currentSku.Name -SKUPriceSheet $pricesheetEntries + } + + if ($skuPricesFound[$currentSku.Name] -lt [double]::MaxValue) + { + $currentSkuPrice = $skuPricesFound[$currentSku.Name] + $tentativeTargetSkuSavingsMonthly = ($currentSkuPrice * [double] $result.Last30DaysQuantity) - ($targetSkuPrice * [double] $result.Last30DaysQuantity) + } + else + { + $tentativeTargetSkuSavingsMonthly = $result.Last30DaysCost - ($targetSkuPrice * [double] $result.Last30DaysQuantity) + } + } + + if ($tentativeTargetSkuSavingsMonthly -ge 0) + { + $targetSkuSavingsMonthly = $tentativeTargetSkuSavingsMonthly + } + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + if ($targetSkuSavingsMonthly -eq [double]::PositiveInfinity) + { + $targetSkuSavingsMonthly = [double] $result.Last30DaysCost / 2 + } + + $additionalInfoDictionary["savingsAmount"] = [double] $targetSkuSavingsMonthly + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Compute/virtualMachineScaleSets" + Impact = "High" + RecommendationType = "Saving" + RecommendationSubType = "UnderusedVMSS" + RecommendationSubTypeId = "a4955cc9-533d-46a2-8625-5c4ebd1c30d5" + RecommendationDescription = "VM Scale Set has been underutilized" + RecommendationAction = "Resize VM Scale Set to lower SKU or scale it in" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMSSName + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionId + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation + } +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "vmss-underutilized-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for performance constrained Scale Sets, with more than $cpuDegradedMaxPercentageThreshold% Max. CPU, $cpuDegradedAvgPercentageThreshold% Avg. CPU and $memoryDegradedPercentageThreshold% RAM usage..." + +$baseQuery = @" + let perfInterval = $($perfDaysBackwards)d; + + let MemoryPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where MetricNames_s == "Available Memory Bytes" and AggregationType_s == "Minimum" + | extend MemoryAvailableMBs = todouble(MetricValue_s)/1024/1024 + | project TimeGenerated, MemoryAvailableMBs, InstanceId_s=ResourceId + | join kind=inner ( + $vmssTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, MemoryMB_s + ) on InstanceId_s + | extend MemoryPercentage = todouble(toint(MemoryMB_s) - toint(MemoryAvailableMBs)) / todouble(MemoryMB_s) * 100 + | summarize PMemoryPercentage = avg(MemoryPercentage) by InstanceId_s; + + let ProcessorMaxPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where MetricNames_s == "Percentage CPU" and AggregationType_s == 'Maximum' + | extend InstanceId_s = ResourceId + | summarize PCPUMaxPercentage = avg(todouble(MetricValue_s)) by InstanceId_s; + + let ProcessorAvgPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | where MetricNames_s == "Percentage CPU" and AggregationType_s == 'Average' + | extend InstanceId_s = ResourceId + | summarize PCPUAvgPercentage = avg(todouble(MetricValue_s)) by InstanceId_s; + + $vmssTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, VMSSName_s, ResourceGroupName_s, SubscriptionGuid_g, Cloud_s, TenantGuid_g, VMSSSize_s, NicCount_s, DataDiskCount_s, Capacity_s, Tags_s + | join kind=leftouter ( MemoryPerf ) on InstanceId_s + | join kind=leftouter ( ProcessorMaxPerf ) on InstanceId_s + | join kind=leftouter ( ProcessorAvgPerf ) on InstanceId_s + | project InstanceId_s, VMSSName = VMSSName_s, ResourceGroup = ResourceGroupName_s, SubscriptionId = SubscriptionGuid_g, Cloud_s, TenantGuid_g, VMSSSize_s, NicCount_s, DataDiskCount_s, Capacity_s, PMemoryPercentage, PCPUMaxPercentage, PCPUAvgPercentage, Tags_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionId = SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionId + | where isnotempty(PMemoryPercentage) and isnotempty(PCPUAvgPercentage) and isnotempty(PCPUMaxPercentage) and (PMemoryPercentage > $memoryDegradedPercentageThreshold or (PCPUMaxPercentage > $cpuDegradedMaxPercentageThreshold and PCPUAvgPercentage > $cpuDegradedAvgPercentageThreshold)) +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + let perfInterval = $($perfDaysBackwards)d; + let armId = `'$queryInstanceId`'; + let gInt = $perfTimeGrain; + let MemoryPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) + | where ResourceId == armId + | where MetricNames_s == 'Available Memory Bytes' and AggregationType_s == 'Minimum' + | extend MemoryAvailableMBs = todouble(MetricValue_s)/1024/1024 + | project CollectedDate, MemoryAvailableMBs, InstanceId_s=ResourceId + | join kind=inner ( + $vmssTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, MemoryMB_s + ) on InstanceId_s + | extend MemoryPercentage = todouble(toint(MemoryMB_s) - toint(MemoryAvailableMBs)) / todouble(MemoryMB_s) * 100 + | summarize avg(MemoryPercentage) by bin(CollectedDate, gInt); + let ProcessorMaxPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) + | where ResourceId == armId + | where MetricNames_s == 'Percentage CPU' and AggregationType_s == 'Maximum' + | extend ProcessorMaxPercentage = todouble(MetricValue_s) + | summarize percentile(ProcessorMaxPercentage, $cpuPercentile) by bin(CollectedDate, gInt); + let ProcessorAvgPerf = $metricsTableName + | where TimeGenerated > ago(perfInterval) + | extend CollectedDate = todatetime(strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd'),'T',format_datetime(TimeGenerated, 'HH'),':00:00Z')) + | where ResourceId == armId + | where MetricNames_s == 'Percentage CPU' and AggregationType_s == 'Average' + | extend ProcessorAvgPercentage = todouble(MetricValue_s) + | summarize percentile(ProcessorAvgPercentage, $cpuPercentile) by bin(CollectedDate, gInt); + MemoryPerf + | join kind=inner (ProcessorMaxPerf) on CollectedDate + | join kind=inner (ProcessorAvgPerf) on CollectedDate + | render timechart +"@ + + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $datetime.AddDays(-30).ToString("yyyy-MM-dd") + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = "$($result.VMSSSize_s)" + $additionalInfoDictionary["InstanceCount"] = [int] $result.Capacity_s + $additionalInfoDictionary["MetricCPUAvgPercentage"] = "$($result.PCPUAvgPercentage)" + $additionalInfoDictionary["MetricCPUMaxPercentage"] = "$($result.PCPUMaxPercentage)" + $additionalInfoDictionary["MetricMemoryPercentage"] = "$($result.PMemoryPercentage)" + + $fitScore = 3 # needs disk IOPS and throughput analysis to improve score + + if ([double] $result.PCPUMaxPercentage -gt [double] $cpuDegradedMaxPercentageThreshold -and [double] $result.PCPUAvgPercentage -gt [double] $cpuDegradedAvgPercentageThreshold) + { + $fitScore = 4 + } + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Performance" + ImpactedArea = "Microsoft.Compute/virtualMachineScaleSets" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "PerfConstrainedVMSS" + RecommendationSubTypeId = "20a40c62-e5c8-4cc3-9fc2-f4ac75013182" + RecommendationDescription = "VM Scale Set performance has been constrained by lack of resources" + RecommendationAction = "Resize VM Scale Set to higher SKU or scale it out" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMSSName + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroup + SubscriptionGuid = $result.SubscriptionId + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "vmss-perfconstrained-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMsHighAvailabilityToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMsHighAvailabilityToBlobStorage.ps1 new file mode 100644 index 000000000..8e7b36368 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VMsHighAvailabilityToBlobStorage.ps1 @@ -0,0 +1,1474 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGVirtualMachine','ARGUnmanagedDisk','ARGAvailabilitySet','ARGResourceContainers','ARGVMSS')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$availSetTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGAvailabilitySet' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" +$vmsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGVirtualMachine' }).LogAnalyticsSuffix + "_CL" +$vhdsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGUnmanagedDisk' }).LogAnalyticsSuffix + "_CL" +$vmssTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGVMSS' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $availSetTableName, $vmsTableName, $vmssTableName, $vhdsTableName and $subscriptionsTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 1 + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +Write-Output "Looking for Availability Sets with a low fault domain count..." + +# Execute the recommendation query against Log Analytics + +$baseQuery = @" + $availSetTableName + | where TimeGenerated > ago(1d) and toint(FaultDomains_s) < 3 and toint(FaultDomains_s) < todouble(VmCount_s)/2 + | project TimeGenerated, InstanceId_s, InstanceName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s, Tags_s, FaultDomains_s, VmCount_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["FaultDomainCount"] = $result.FaultDomains_s + $additionalInfoDictionary["VMCount"] = $result.VmCount_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "AvailSetLowFaultDomainCount" + RecommendationSubTypeId = "255de20b-d5e4-4be5-9695-620b4a905774" + RecommendationDescription = "Availability Sets should have a fault domain count of 3 or equal or greater than half of the Virtual Machines count" + RecommendationAction = "Increase the fault domain count of your Availability Set" + InstanceId = $result.InstanceId_s + InstanceName = $result.InstanceName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "availsetsfaultdomaincount-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for Availability Sets with a low update domain count..." + +$baseQuery = @" + $availSetTableName + | where TimeGenerated > ago(1d) and toint(UpdateDomains_s) < todouble(VmCount_s)/2 + | project TimeGenerated, InstanceId_s, InstanceName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s, Tags_s, UpdateDomains_s, VmCount_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["UpdateDomainCount"] = $result.UpdateDomains_s + $additionalInfoDictionary["VMCount"] = $result.VmCount_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "AvailSetLowUpdateDomainCount" + RecommendationSubTypeId = "9764e285-2eca-46c5-b49e-649c039cf0cf" + RecommendationDescription = "Availability Sets should have an update domain count equal or greater than half of the Virtual Machines count" + RecommendationAction = "Increase the update domain count of your Availability Set" + InstanceId = $result.InstanceId_s + InstanceName = $result.InstanceName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "availsetsupdatedomaincount-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for Availability Sets with VMs sharing storage accounts..." + +$baseQuery = @" + $vhdsTableName + | where TimeGenerated > ago(1d) + | extend StorageAccountName = tostring(split(InstanceId_s, '/')[0]) + | distinct TimeGenerated, StorageAccountName, OwnerVMId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s + | join kind=inner ( + $vmsTableName + | where TimeGenerated > ago(1d) and isnotempty(AvailabilitySetId_s) + | distinct VMName_s, InstanceId_s, AvailabilitySetId_s, Cloud_s, Tags_s + ) on `$left.OwnerVMId_s == `$right.InstanceId_s + | extend AvailabilitySetName = tostring(split(AvailabilitySetId_s,'/')[8]) + | summarize TimeGenerated = any(TimeGenerated), Tags_s=any(Tags_s), VMCount = count() by AvailabilitySetName, AvailabilitySetId_s, StorageAccountName, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s + | where VMCount > 1 + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics -ErrorAction Continue + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.AvailabilitySetId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["SharedStorageAccountName"] = $result.StorageAccountName + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "AvailSetSharedStorageAccount" + RecommendationSubTypeId = "e530029f-9b6a-413a-99ed-81af54502bb9" + RecommendationDescription = "Virtual Machines in unmanaged Availability Sets should not share the same Storage Account" + RecommendationAction = "Migrate Virtual Machines disks to Managed Disks or keep the disks in a dedicated Storage Account per VM" + InstanceId = $result.AvailabilitySetId_s + InstanceName = $result.AvailabilitySetName + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "availsetsharedsa-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for Storage Accounts with multiple VMs..." + +$baseQuery = @" + $vhdsTableName + | where TimeGenerated > ago(1d) + | extend StorageAccountName = tostring(split(InstanceId_s, '/')[0]) + | distinct TimeGenerated, StorageAccountName, OwnerVMId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, Cloud_s + | join kind=inner ( + $vmsTableName + | where TimeGenerated > ago(1d) + | distinct InstanceId_s, Tags_s + ) on `$left.OwnerVMId_s == `$right.InstanceId_s + | summarize TimeGenerated = any(TimeGenerated), Tags_s=any(Tags_s), VMCount = count() by StorageAccountName, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, Cloud_s + | where VMCount > 1 + | extend StorageAccountId = strcat('/subscriptions/', SubscriptionGuid_g, '/resourcegroups/', ResourceGroupName_s, '/providers/microsoft.storage/storageaccounts/', StorageAccountName) + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics -ErrorAction Continue + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.StorageAccountId + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["VirtualMachineCount"] = $result.VMCount + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "StorageAccountsMultipleVMs" + RecommendationSubTypeId = "b70f44fa-5ef9-4180-b2f9-9cc6be07ab3e" + RecommendationDescription = "Virtual Machines with unmanaged disks should not share the same Storage Account" + RecommendationAction = "Migrate Virtual Machines disks to Managed Disks or keep the disks in a dedicated Storage Account per VM" + InstanceId = $result.StorageAccountId + InstanceName = $result.StorageAccountName + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "storageaccountsmultiplevms-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for VMs with no Availability Set..." + +$baseQuery = @" + $vmsTableName + | where TimeGenerated > ago(1d) and isempty(AvailabilitySetId_s) and isempty(Zones_s) and Tags_s !has 'databricks-instance-name' + | project TimeGenerated, VMName_s, InstanceId_s, Tags_s, TenantGuid_g, SubscriptionGuid_g, ResourceGroupName_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "VMsNoAvailSet" + RecommendationSubTypeId = "998b50d8-e654-417b-ab20-a31cb11629c0" + RecommendationDescription = "Virtual Machines should be placed in an Availability Set together with other instances with the same role" + RecommendationAction = "Add VM to an Availability Set together with other VMs of the same role" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "vmsnoavailset-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for VMs alone in an Availability Set..." + +$baseQuery = @" + $vmsTableName + | where TimeGenerated > ago(1d) and isnotempty(AvailabilitySetId_s) and isempty(Zones_s) + | distinct TimeGenerated, VMName_s, InstanceId_s, AvailabilitySetId_s, TenantGuid_g, SubscriptionGuid_g, ResourceGroupName_s, Cloud_s, Tags_s + | summarize any(TimeGenerated, VMName_s, InstanceId_s, Tags_s), VMCount = count() by AvailabilitySetId_s, TenantGuid_g, SubscriptionGuid_g, ResourceGroupName_s, Cloud_s + | where VMCount == 1 + | project TimeGenerated = any_TimeGenerated, VMName_s = any_VMName_s, InstanceId_s = any_InstanceId_s, Tags_s = any_Tags_s, TenantGuid_g, SubscriptionGuid_g, ResourceGroupName_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "VMsSingleInAvailSet" + RecommendationSubTypeId = "fe577af5-dfa2-413a-82a9-f183196c1f49" + RecommendationDescription = "Virtual Machines should not be the only instance in an Availability Set" + RecommendationAction = "Add more VMs of the same role to the Availability Set" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "vmssingleinavailset-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for VMs with disks in multiple Storage Accounts..." + +$baseQuery = @" + $vhdsTableName + | where TimeGenerated > ago(1d) + | extend StorageAccountName = tostring(split(InstanceId_s, '/')[0]) + | distinct TimeGenerated, StorageAccountName, OwnerVMId_s + | summarize TimeGenerated = any(TimeGenerated), StorageAcccountCount = count() by OwnerVMId_s + | where StorageAcccountCount > 1 + | join kind=inner ( + $vmsTableName + | where TimeGenerated > ago(1d) + | distinct VMName_s, InstanceId_s, Cloud_s, TenantGuid_g, SubscriptionGuid_g, ResourceGroupName_s, Tags_s + ) on `$left.OwnerVMId_s == `$right.InstanceId_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics -ErrorAction Continue + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["StorageAccountsUsed"] = $result.StorageAcccountCount + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "DisksMultipleStorageAccounts" + RecommendationSubTypeId = "024049e7-f63a-4e1c-b620-f011aafbc576" + RecommendationDescription = "Each Virtual Machine should have its unmanaged disks stored in a single Storage Account for higher availability and manageability" + RecommendationAction = "Migrate Virtual Machines disks to Managed Disks or move VHDs to the same Storage Account" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "disksmultiplesa-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for VMs using unmanaged disks..." + +$baseQuery = @" + $vmsTableName + | where TimeGenerated > ago(1d) and UsesManagedDisks_s == 'false' + | distinct InstanceId_s, VMName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, DeploymentModel_s, Tags_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["DeploymentModel"] = $result.DeploymentModel_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "UnmanagedDisks" + RecommendationSubTypeId = "b576a069-b1f2-43a6-9134-5ee75376402a" + RecommendationDescription = "Virtual Machines should use Managed Disks for higher availability and manageability" + RecommendationAction = "Migrate Virtual Machines disks to Managed Disks" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "unmanageddisks-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for Resource Groups with VMs not in multiple AZs..." + +$baseQuery = @" + let VMsInZones = materialize($vmsTableName + | where TimeGenerated > ago(1d) and isempty(AvailabilitySetId_s) and isnotempty(Zones_s)); + VMsInZones + | distinct ResourceGroupName_s, Zones_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s + | summarize ZonesCount=count() by ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Cloud_s + | where ZonesCount < 3 + | join kind=inner ( + VMsInZones + | where PowerState_s has 'running' + | distinct VMName_s, ResourceGroupName_s, SubscriptionGuid_g + | summarize VMCount=count() by ResourceGroupName_s, SubscriptionGuid_g + ) on ResourceGroupName_s and SubscriptionGuid_g + | where VMCount == 1 or VMCount > ZonesCount + | project-away SubscriptionGuid_g1, ResourceGroupName_s1 + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g + | extend InstanceId = strcat('/subscriptions/', SubscriptionGuid_g, '/resourcegroups/', ResourceGroupName_s) +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["ZonesCount"] = $result.ZonesCount + $additionalInfoDictionary["VMsCount"] = $result.VMCount + + $fitScore = 4 # a resource group may contain VMs from multiple applications which may lead to false negatives + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachines" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "VMsMultipleAZs" + RecommendationSubTypeId = "1a77887c-7375-434e-af19-c2543171e0b8" + RecommendationDescription = "Virtual Machines should be placed in multiple Availability Zones" + RecommendationAction = "Distribute Virtual Machines instances of the same role in multiple Availability Zones" + InstanceId = $result.InstanceId + InstanceName = $result.ResourceGroupName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "vmsmultipleazs-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for VMSS not in multiple AZs..." + +$baseQuery = @" + $vmssTableName + | where TimeGenerated > ago(1d) + | where (isempty(Zones_s) and toint(Capacity_s) > 1) or (array_length(split(Zones_s, ' ')) != 3 and toint(Capacity_s) > 2) + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["Zones"] = $result.Zones_s + $additionalInfoDictionary["VMSSCapacity"] = $result.Capacity_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachineScaleSets" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "VMSSMultipleAZs" + RecommendationSubTypeId = "47e5457c-b345-4372-b536-8887fa8f0298" + RecommendationDescription = "Virtual Machine Scale Sets should be placed in multiple Availability Zones" + RecommendationAction = "Reprovision the Scale Set leveraging enough Availability Zones" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMSSName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "vmssmultipleazs-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for VMSS using unmanaged disks..." + +$baseQuery = @" + $vmssTableName + | where TimeGenerated > ago(1d) and UsesManagedDisks_s == 'false' + | distinct InstanceId_s, VMSSName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, Tags_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "HighAvailability" + ImpactedArea = "Microsoft.Compute/virtualMachineScaleSets" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "UnmanagedDisksVMSS" + RecommendationSubTypeId = "1bf03c4a-c402-4e6c-bf20-051b18af30e2" + RecommendationDescription = "Virtual Machine Scale Sets should use Managed Disks for higher availability and manageability" + RecommendationAction = "Migrate Virtual Machine Scale Sets disks to Managed Disks" + InstanceId = $result.InstanceId_s + InstanceName = $result.VMSSName_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "unmanageddisksvmss-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VNetOptimizationsToBlobStorage.ps1 b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VNetOptimizationsToBlobStorage.ps1 new file mode 100644 index 000000000..5ccd80633 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/recommendations/Recommend-VNetOptimizationsToBlobStorage.ps1 @@ -0,0 +1,1327 @@ +$ErrorActionPreference = "Stop" + +# Collect generic and recommendation-specific variables + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" +$workspaceName = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceName" +$workspaceRG = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceRG" +$workspaceSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceSubId" +$workspaceTenantId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceTenantId" + +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "recommendationsexports" +} + +$deploymentDate = Get-AutomationVariable -Name "AzureOptimization_DeploymentDate" # yyyy-MM-dd format +$deploymentDate = $deploymentDate.Replace('"', "") + +$lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($lognamePrefix)) +{ + $lognamePrefix = "AzureOptimization" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} + +$subnetMaxUsedThresholdVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationVNetSubnetMaxUsedPercentageThreshold" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($subnetMaxUsedThresholdVar) -or $subnetMaxUsedThresholdVar -eq 0) +{ + $subnetMaxUsedThreshold = 80 +} +else +{ + $subnetMaxUsedThreshold = [int] $subnetMaxUsedThresholdVar +} + +$subnetMinUsedThresholdVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationVNetSubnetMinUsedPercentageThreshold" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($subnetMinUsedThresholdVar) -or $subnetMinUsedThresholdVar -eq 0) +{ + $subnetMinUsedThreshold = 5 +} +else +{ + $subnetMinUsedThreshold = [int] $subnetMinUsedThresholdVar +} + +# must be a comma-separated, single-quote enclosed list of subnet names, e.g., 'gatewaysubnet','azurebastionsubnet' +$subnetFreeExclusions = Get-AutomationVariable -Name "AzureOptimization_RecommendationVNetSubnetUsedPercentageExclusions" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($subnetFreeExclusions)) +{ + $subnetFreeExclusions = "'gatewaysubnet'" +} + +$subnetMinAgeVar = Get-AutomationVariable -Name "AzureOptimization_RecommendationVNetSubnetEmptyMinAgeInDays" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($subnetMinAgeVar) -or $subnetMinAgeVar -eq 0) +{ + $subnetMinAge = 30 +} +else +{ + $subnetMinAge = [int] $subnetMinAgeVar +} + +$consumptionOffsetDays = [int] (Get-AutomationVariable -Name "AzureOptimization_ConsumptionOffsetDays") +$consumptionOffsetDaysStart = $consumptionOffsetDays + 1 + +$SqlTimeout = 120 +$LogAnalyticsIngestControlTable = "LogAnalyticsIngestControl" + +# Authenticate against Azure + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +Write-Output "Finding tables where recommendations will be generated from..." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = "SELECT * FROM [dbo].[$LogAnalyticsIngestControlTable] WHERE CollectedType IN ('ARGNetworkInterface','ARGVirtualNetwork','ARGResourceContainers', 'ARGNSGRule', 'ARGPublicIP','AzureConsumption')" + + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $controlRows = New-Object System.Data.DataTable + $sqlAdapter.Fill($controlRows) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +$nicsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGNetworkInterface' }).LogAnalyticsSuffix + "_CL" +$vNetsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGVirtualNetwork' }).LogAnalyticsSuffix + "_CL" +$subscriptionsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGResourceContainers' }).LogAnalyticsSuffix + "_CL" +$nsgRulesTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGNSGRule' }).LogAnalyticsSuffix + "_CL" +$publicIpsTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'ARGPublicIP' }).LogAnalyticsSuffix + "_CL" +$consumptionTableName = $lognamePrefix + ($controlRows | Where-Object { $_.CollectedType -eq 'AzureConsumption' }).LogAnalyticsSuffix + "_CL" + +Write-Output "Will run query against tables $nicsTableName, $nsgRulesTableName, $publicIpsTableName, $subscriptionsTableName, $consumptionTableName and $vNetsTableName" + +$Conn.Close() +$Conn.Dispose() + +$recommendationSearchTimeSpan = 30 + $consumptionOffsetDaysStart + +# Grab a context reference to the Storage Account where the recommendations file will be stored + +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$sa = Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink + +if ($workspaceSubscriptionId -ne $storageAccountSinkSubscriptionId) +{ + Select-AzSubscription -SubscriptionId $workspaceSubscriptionId +} + +$recommendationsErrors = 0 + +Write-Output "Looking for subnets with free IP space less than $subnetMaxUsedThreshold%, excluding $subnetFreeExclusions..." + +$baseQuery = @" + $vNetsTableName + | where TimeGenerated > ago(1d) + | where SubnetName_s !in ($subnetFreeExclusions) + | extend FreeIPs = toint(SubnetTotalPrefixIPs_s) - toint(SubnetUsedIPs_s) + | extend UsedIPPercentage = (todouble(SubnetUsedIPs_s) / todouble(SubnetTotalPrefixIPs_s)) * 100 + | where UsedIPPercentage >= $subnetMaxUsedThreshold + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/subnets" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["subnetName"] = $result.SubnetName_s + $additionalInfoDictionary["subnetPrefix"] = $result.SubnetPrefix_s + $additionalInfoDictionary["subnetTotalIPs"] = $result.SubnetTotalPrefixIPs_s + $additionalInfoDictionary["subnetFreeIPs"] = $result.FreeIPs + $additionalInfoDictionary["subnetUsedIPPercentage"] = $result.UsedIPPercentage + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Network/virtualNetworks" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "HighSubnetIPSpaceUsage" + RecommendationSubTypeId = "5292525b-5095-4e52-803e-e17192f1d099" + RecommendationDescription = "Subnets with a high IP space usage may constrain operations" + RecommendationAction = "Move network devices to a subnet with a larger address space" + InstanceId = $result.InstanceId_s + InstanceName = "$($result.VNetName_s)/$($result.SubnetName_s)" + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "subnetshighspaceusage-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for subnets with used IP space less than $subnetMinUsedThreshold%..." + +$baseQuery = @" + $vNetsTableName + | where TimeGenerated > ago(1d) + | where SubnetName_s !in ($subnetFreeExclusions) + | extend FreeIPs = toint(SubnetTotalPrefixIPs_s) - toint(SubnetUsedIPs_s) + | extend UsedIPPercentage = (todouble(SubnetUsedIPs_s) / todouble(SubnetTotalPrefixIPs_s)) * 100 + | where UsedIPPercentage > 0 and UsedIPPercentage <= $subnetMinUsedThreshold + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/subnets" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["subnetName"] = $result.SubnetName_s + $additionalInfoDictionary["subnetPrefix"] = $result.SubnetPrefix_s + $additionalInfoDictionary["subnetTotalIPs"] = $result.SubnetTotalPrefixIPs_s + $additionalInfoDictionary["subnetUsedIPs_s"] = $result.SubnetUsedIPs_s + $additionalInfoDictionary["subnetUsedIPPercentage"] = $result.UsedIPPercentage + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Network/virtualNetworks" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "LowSubnetIPSpaceUsage" + RecommendationSubTypeId = "0f27b41c-869a-4563-86e9-d1c94232ba81" + RecommendationDescription = "Subnets with a low IP space usage are a waste of virtual network address space" + RecommendationAction = "Move network devices to a subnet with a smaller address space" + InstanceId = $result.InstanceId_s + InstanceName = "$($result.VNetName_s)/$($result.SubnetName_s)" + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "subnetslowspaceusage-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for subnets without any device..." + +$baseQuery = @" + $vNetsTableName + | where TimeGenerated > ago(1d) + | where toint(SubnetUsedIPs_s) == 0 and toint(SubnetDelegationsCount_s) == 0 + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/subnets" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["subnetName"] = $result.SubnetName_s + $additionalInfoDictionary["subnetPrefix"] = $result.SubnetPrefix_s + $additionalInfoDictionary["subnetTotalIPs"] = $result.SubnetTotalPrefixIPs_s + $additionalInfoDictionary["subnetUsedIPs_s"] = $result.SubnetUsedIPs_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Network/virtualNetworks" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "NoSubnetIPSpaceUsage" + RecommendationSubTypeId = "343bbfb7-5bec-4711-8353-398454d42b7b" + RecommendationDescription = "Subnets without any IP usage are a waste of virtual network address space" + RecommendationAction = "Delete the subnet to reclaim address space" + InstanceId = $result.InstanceId_s + InstanceName = "$($result.VNetName_s)/$($result.SubnetName_s)" + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "subnetsnospaceusage-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for orphaned NICs..." + +$baseQuery = @" + $nicsTableName + | where TimeGenerated > ago(1d) + | where isempty(OwnerVMId_s) and isempty(OwnerPEId_s) + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.InstanceId_s + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["privateIpAddress"] = $result.PrivateIPAddress_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "OperationalExcellence" + ImpactedArea = "Microsoft.Network/networkInterfaces" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "OrphanedNIC" + RecommendationSubTypeId = "4c5c2d0c-b6a4-4c59-bc18-6fff6c1f5b23" + RecommendationDescription = "Orphaned Network Interfaces (without owner VM or PE) unnecessarily consume IP address space" + RecommendationAction = "Delete the NIC to reclaim address space" + InstanceId = $result.InstanceId_s + InstanceName = $result.Name_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "orphanednics-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for NSG rules referring empty or removed subnets..." + +$baseQuery = @" + let MinimumSubnetAge = $($subnetMinAge)d; + let SubnetsToday = materialize( $vNetsTableName + | where TimeGenerated > ago(1d) + | extend SubnetId = tolower(strcat(InstanceId_s, '/subnets/', SubnetName_s)) + | distinct SubnetId, SubnetPrefix_s, SubnetUsedIPs_s, SubnetDelegationsCount_s ); + let SubnetsBefore = materialize( $vNetsTableName + | where TimeGenerated < ago(1d) + | extend SubnetId = tolower(strcat(InstanceId_s, '/subnets/', SubnetName_s)) + | summarize ExistsSince = min(todatetime(StatusDate_s)) by SubnetId, SubnetPrefix_s ); + let SubnetsExistingLongEnoughIds = SubnetsBefore | where ExistsSince < ago(MinimumSubnetAge) | distinct SubnetId; + let EmptySubnets = SubnetsToday | where SubnetId in (SubnetsExistingLongEnoughIds) and toint(SubnetUsedIPs_s) == 0 and toint(SubnetDelegationsCount_s) == 0; + let SubnetsTodayIds = SubnetsToday | distinct SubnetId; + let SubnetsTodayPrefixes = SubnetsToday | distinct SubnetPrefix_s; + let RemovedSubnets = SubnetsBefore | where SubnetId !in (SubnetsTodayIds) and SubnetPrefix_s !in (SubnetsTodayPrefixes); + let NSGRules = materialize($nsgRulesTableName + | where TimeGenerated > ago(1d) + | extend SourceAddresses = split(RuleSourceAddresses_s,',') + | mvexpand SourceAddresses + | extend SourceAddress = tostring(SourceAddresses) + | extend DestinationAddresses = split(RuleDestinationAddresses_s,',') + | mvexpand DestinationAddresses + | extend DestinationAddress = tostring(DestinationAddresses) + | project NSGId = InstanceId_s, RuleName_s, DestinationAddress, SourceAddress, SubscriptionGuid_g, Cloud_s, TenantGuid_g, ResourceGroupName_s, NSGName = NSGName_s, Tags_s); + let EmptySubnetsAsSource = EmptySubnets + | join kind=inner ( NSGRules ) on `$left.SubnetPrefix_s == `$right.SourceAddress + | extend SubnetState = 'empty'; + let EmptySubnetsAsDestination = EmptySubnets + | join kind=inner ( NSGRules ) on `$left.SubnetPrefix_s == `$right.DestinationAddress + | extend SubnetState = 'empty'; + let RemovedSubnetsAsSource = RemovedSubnets + | join kind=inner ( NSGRules ) on `$left.SubnetPrefix_s == `$right.SourceAddress + | extend SubnetState = 'unexisting'; + let RemovedSubnetsAsDestination = RemovedSubnets + | join kind=inner ( NSGRules ) on `$left.SubnetPrefix_s == `$right.DestinationAddress + | extend SubnetState = 'unexisting'; + EmptySubnetsAsSource + | union EmptySubnetsAsDestination + | union RemovedSubnetsAsSource + | union RemovedSubnetsAsDestination + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g + | where isnotempty(SubnetPrefix_s) + | distinct NSGId, NSGName, RuleName_s, SubscriptionGuid_g, SubscriptionName, ResourceGroupName_s, TenantGuid_g, Cloud_s, SubnetId, SubnetPrefix_s, SubnetState, Tags_s +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.NSGId + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["subnetId"] = $result.SubnetId + $additionalInfoDictionary["subnetPrefix"] = $result.SubnetPrefix_s + $additionalInfoDictionary["subnetState"] = $result.SubnetState + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Security" + ImpactedArea = "Microsoft.Network/networkSecurityGroups" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "NSGRuleForEmptyOrUnexistingSubnet" + RecommendationSubTypeId = "b5491cde-f76c-4423-8c4c-89e3558ff2f2" + RecommendationDescription = "NSG rules referring to empty or unexisting subnets" + RecommendationAction = "Update or remove the NSG rule to improve your network security posture" + InstanceId = $result.NSGId + InstanceName = "$($result.NSGName)/$($result.RuleName_s)" + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "nsgrules-emptyunexistingsubnets-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for NSG rules referring orphan or removed NICs..." + +$baseQuery = @" + let NICsToday = materialize( $nicsTableName + | where TimeGenerated > ago(1d) + | extend NICId = tolower(InstanceId_s) + | distinct NICId, PrivateIPAddress_s, PublicIPId_s, OwnerVMId_s, OwnerPEId_s ); + let NICsBefore = $nicsTableName + | where TimeGenerated < ago(1d) + | extend NICId = tolower(InstanceId_s) + | distinct NICId, PrivateIPAddress_s, PublicIPId_s; + let OrphanNICs = NICsToday + | where isempty(OwnerVMId_s) and isempty(OwnerPEId_s) + | extend PublicIPId_s = tolower(PublicIPId_s) + | join kind=leftouter ( + $publicIpsTableName + | where TimeGenerated > ago(1d) + | project PublicIPId_s = tolower(InstanceId_s), PublicIPAddress = IPAddress + ) on PublicIPId_s; + let NICsTodayIds = NICsToday | distinct NICId; + let NICsTodayIPs = NICsToday | distinct PrivateIPAddress_s; + let RemovedNICs = NICsBefore + | where NICId !in (NICsTodayIds) and PrivateIPAddress_s !in (NICsTodayIPs) + | extend PublicIPId_s = tolower(PublicIPId_s) + | join kind=leftouter ( + $publicIpsTableName + | where TimeGenerated < ago(1d) + | project PublicIPId_s = tolower(InstanceId_s), PublicIPAddress = IPAddress + ) on PublicIPId_s; + let NSGRules = materialize($nsgRulesTableName + | where TimeGenerated > ago(1d) + | extend SourceAddresses = split(RuleSourceAddresses_s,',') + | mvexpand SourceAddresses + | extend SourceAddress = replace('/32','',tostring(SourceAddresses)) + | extend DestinationAddresses = split(RuleDestinationAddresses_s,',') + | mvexpand DestinationAddresses + | extend DestinationAddress = replace('/32','',tostring(DestinationAddresses)) + | project NSGId = InstanceId_s, RuleName_s, DestinationAddress, SourceAddress, SubscriptionGuid_g, Cloud_s, TenantGuid_g, ResourceGroupName_s, NSGName = NSGName_s, Tags_s); + let OrphanNICsAsPrivateSource = OrphanNICs + | join kind=inner ( NSGRules ) on `$left.PrivateIPAddress_s == `$right.SourceAddress + | extend NICState = 'orphan', IPAddress = PrivateIPAddress_s; + let OrphanNICsAsPublicSource = OrphanNICs + | join kind=inner ( NSGRules ) on `$left.PublicIPAddress == `$right.SourceAddress + | extend NICState = 'orphan', IPAddress = PublicIPAddress; + let OrphanNICsAsPrivateDestination = OrphanNICs + | join kind=inner ( NSGRules ) on `$left.PrivateIPAddress_s == `$right.DestinationAddress + | extend NICState = 'orphan', IPAddress = PrivateIPAddress_s; + let OrphanNICsAsPublicDestination = OrphanNICs + | join kind=inner ( NSGRules ) on `$left.PublicIPAddress == `$right.DestinationAddress + | extend NICState = 'orphan', IPAddress = PublicIPAddress; + let RemovedNICsAsPrivateSource = RemovedNICs + | join kind=inner ( NSGRules ) on `$left.PrivateIPAddress_s == `$right.SourceAddress + | extend NICState = 'unexisting', IPAddress = PrivateIPAddress_s; + let RemovedNICsAsPublicSource = RemovedNICs + | join kind=inner ( NSGRules ) on `$left.PublicIPAddress == `$right.SourceAddress + | extend NICState = 'unexisting', IPAddress = PublicIPAddress; + let RemovedNICsAsPrivateDestination = RemovedNICs + | join kind=inner ( NSGRules ) on `$left.PrivateIPAddress_s == `$right.DestinationAddress + | extend NICState = 'unexisting', IPAddress = PrivateIPAddress_s; + let RemovedNICsAsPublicDestination = RemovedNICs + | join kind=inner ( NSGRules ) on `$left.PublicIPAddress == `$right.DestinationAddress + | extend NICState = 'unexisting', IPAddress = PublicIPAddress; + OrphanNICsAsPrivateSource + | union OrphanNICsAsPublicSource + | union OrphanNICsAsPrivateDestination + | union OrphanNICsAsPublicDestination + | union RemovedNICsAsPrivateSource + | union RemovedNICsAsPublicSource + | union RemovedNICsAsPrivateDestination + | union RemovedNICsAsPublicDestination + | where isnotempty(IPAddress) + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g + | distinct NSGId, NSGName, RuleName_s, SubscriptionGuid_g, SubscriptionName, ResourceGroupName_s, TenantGuid_g, Cloud_s, NICId, IPAddress, NICState, Tags_s +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.NSGId + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["nicId"] = $result.NICId + $additionalInfoDictionary["ipAddress"] = $result.IPAddress + $additionalInfoDictionary["nicState"] = $result.NICState + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Security" + ImpactedArea = "Microsoft.Network/networkSecurityGroups" + Impact = "Medium" + RecommendationType = "BestPractices" + RecommendationSubType = "NSGRuleForOrphanOrUnexistingNIC" + RecommendationSubTypeId = "3dc1d1f8-19ef-4572-9c9d-78d62831f55a" + RecommendationDescription = "NSG rules referring to orphan or unexisting NICs" + RecommendationAction = "Update or remove the NSG rule to improve your network security posture" + InstanceId = $result.NSGId + InstanceName = "$($result.NSGName)/$($result.RuleName_s)" + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "nsgrules-orphanunexistingnics-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for NSG rules referring orphan or removed Public IPs..." + +$baseQuery = @" + let PIPsToday = materialize( $publicIpsTableName + | where TimeGenerated > ago(1d) + | extend PublicIPId = tolower(InstanceId_s) + | distinct PublicIPId, AssociatedResourceId_s, AllocationMethod_s, IPAddress ); + let PIPsBefore = materialize( $publicIpsTableName + | where TimeGenerated < ago(1d) + | extend PublicIPId = tolower(InstanceId_s) + | distinct PublicIPId, IPAddress ); + let OrphanStaticPIPs = PIPsToday + | where isempty(AssociatedResourceId_s) and AllocationMethod_s == 'static'; + let OrphanDynamicPIPIDs = PIPsToday + | where isempty(AssociatedResourceId_s) and AllocationMethod_s == 'dynamic' + | distinct PublicIPId; + let PIPsTodayIds = PIPsToday | distinct PublicIPId; + let PIPsTodayIPs = PIPsToday | distinct IPAddress; + let OrphanDynamicPIPs = PIPsBefore + | where PublicIPId in (OrphanDynamicPIPIDs) and isnotempty(IPAddress) and IPAddress !in (PIPsTodayIPs); + let RemovedPIPs = PIPsBefore + | where PublicIPId !in (PIPsTodayIds) and isnotempty(IPAddress) and IPAddress !in (PIPsTodayIPs); + let NSGRules = materialize( $nsgRulesTableName + | where TimeGenerated > ago(1d) + | extend SourceAddresses = split(RuleSourceAddresses_s,',') + | mvexpand SourceAddresses + | extend SourceAddress = replace('/32','',tostring(SourceAddresses)) + | extend DestinationAddresses = split(RuleDestinationAddresses_s,',') + | mvexpand DestinationAddresses + | extend DestinationAddress = replace('/32','',tostring(DestinationAddresses)) + | project NSGId = InstanceId_s, RuleName_s, DestinationAddress, SourceAddress, SubscriptionGuid_g, Cloud_s, TenantGuid_g, ResourceGroupName_s, NSGName = NSGName_s, Tags_s); + let OrphanStaticPIPsAsSource = OrphanStaticPIPs + | join kind=inner ( NSGRules ) on `$left.IPAddress == `$right.SourceAddress + | extend PIPState = 'orphan'; + let OrphanStaticPIPsAsDestination = OrphanStaticPIPs + | join kind=inner ( NSGRules ) on `$left.IPAddress == `$right.DestinationAddress + | extend PIPState = 'orphan'; + let OrphanDynamicPIPsAsSource = OrphanDynamicPIPs + | join kind=inner ( NSGRules ) on `$left.IPAddress == `$right.SourceAddress + | extend PIPState = 'orphan'; + let OrphanDynamicPIPsAsDestination = OrphanDynamicPIPs + | join kind=inner ( NSGRules ) on `$left.IPAddress == `$right.DestinationAddress + | extend PIPState = 'orphan'; + let RemovedPIPsAsSource = RemovedPIPs + | join kind=inner ( NSGRules ) on `$left.IPAddress == `$right.SourceAddress + | extend PIPState = 'unexisting'; + let RemovedPIPsAsDestination = RemovedPIPs + | join kind=inner ( NSGRules ) on `$left.IPAddress == `$right.DestinationAddress + | extend PIPState = 'unexisting'; + OrphanStaticPIPsAsSource + | union OrphanDynamicPIPsAsSource + | union OrphanStaticPIPsAsDestination + | union OrphanDynamicPIPsAsDestination + | union RemovedPIPsAsSource + | union RemovedPIPsAsDestination + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g + | distinct NSGId, NSGName, RuleName_s, SubscriptionGuid_g, SubscriptionName, ResourceGroupName_s, TenantGuid_g, Cloud_s, PublicIPId, IPAddress, PIPState, AllocationMethod_s, Tags_s +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + switch ($result.Cloud_s) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + + $queryInstanceId = $result.NSGId + $detailsURL = "https://portal.azure.$azureTld/#@$($result.TenantGuid_g)/resource/$queryInstanceId/overview" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["publicIPId"] = $result.PublicIPId + $additionalInfoDictionary["ipAddress"] = $result.IPAddress + $additionalInfoDictionary["publicIPState"] = $result.PIPState + $additionalInfoDictionary["allocationMethod"] = $result.AllocationMethod_s + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Security" + ImpactedArea = "Microsoft.Network/networkSecurityGroups" + Impact = "High" + RecommendationType = "BestPractices" + RecommendationSubType = "NSGRuleForOrphanOrUnexistingPublicIP" + RecommendationSubTypeId = "fe40cbe7-bdee-4cce-b072-cf25e1247b7a" + RecommendationDescription = "NSG rules referring to orphan or unexisting Public IPs" + RecommendationAction = "Update or remove the NSG rule to improve your network security posture" + InstanceId = $result.NSGId + InstanceName = "$($result.NSGName)/$($result.RuleName_s)" + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "nsgrules-orphanunexistingpublicips-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +Write-Output "Looking for orphaned Public IPs..." + +$baseQuery = @" + let interval = 30d; + let etime = todatetime(toscalar($consumptionTableName | where todatetime(Date_s) < now() and todatetime(Date_s) > ago(30d) | summarize max(todatetime(Date_s)))); + let stime = etime-interval; + $publicIpsTableName + | where TimeGenerated > ago(1d) and isempty(AssociatedResourceId_s) + | distinct Name_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuName_s, AllocationMethod_s, Tags_s, Cloud_s + | join kind=leftouter ( + $consumptionTableName + | where todatetime(Date_s) between (stime..etime) + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize Last30DaysCost=sum(todouble(CostInBillingCurrency_s)) by Name_s, InstanceId_s, SubscriptionGuid_g, TenantGuid_g, ResourceGroupName_s, SkuName_s, AllocationMethod_s, Tags_s, Cloud_s + | join kind=leftouter ( + $subscriptionsTableName + | where TimeGenerated > ago(1d) + | where ContainerType_s =~ 'microsoft.resources/subscriptions' + | project SubscriptionGuid_g, SubscriptionName = ContainerName_s + ) on SubscriptionGuid_g +"@ + +try +{ + $queryResults = Invoke-AzOperationalInsightsQuery -WorkspaceId $workspaceId -Query $baseQuery -Timespan (New-TimeSpan -Days $recommendationSearchTimeSpan) -Wait 600 -IncludeStatistics + if ($queryResults) + { + $results = [System.Linq.Enumerable]::ToArray($queryResults.Results) + } +} +catch +{ + Write-Warning -Message "Query failed. Debug the following query in the AOE Log Analytics workspace: $baseQuery" + Write-Warning -Message $error[0] + $recommendationsErrors++ +} + +Write-Output "Query finished with $($results.Count) results." + +Write-Output "Query statistics: $($queryResults.Statistics.query)" + +# Build the recommendations objects + +$recommendations = @() +$datetime = (get-date).ToUniversalTime() +$timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") + +foreach ($result in $results) +{ + $queryInstanceId = $result.InstanceId_s + $queryText = @" + $publicIpsTableName + | where InstanceId_s == '$queryInstanceId' and isempty(AssociatedResourceId_s) + | distinct InstanceId_s, Name_s, AllocationMethod_s, SkuName_s, TimeGenerated + | summarize LastAttachedDate = min(TimeGenerated) by InstanceId_s, Name_s, AllocationMethod_s, SkuName_s + | join kind=leftouter ( + $consumptionTableName + | project InstanceId_s=tolower(ResourceId), CostInBillingCurrency_s, Date_s + ) on InstanceId_s + | summarize CostsSinceDetached = sumif(todouble(CostInBillingCurrency_s), todatetime(Date_s) > LastAttachedDate) by Name_s, LastAttachedDate, AllocationMethod_s, SkuName_s +"@ + $encodedQuery = [System.Uri]::EscapeDataString($queryText) + $detailsQueryStart = $deploymentDate + $detailsQueryEnd = $datetime.AddDays(8).ToString("yyyy-MM-dd") + switch ($cloudEnvironment) + { + "AzureCloud" { $azureTld = "com" } + "AzureChinaCloud" { $azureTld = "cn" } + "AzureUSGovernment" { $azureTld = "us" } + default { $azureTld = "com" } + } + $detailsURL = "https://portal.azure.$azureTld#@$workspaceTenantId/blade/Microsoft_Azure_Monitoring_Logs/LogsBlade/resourceId/%2Fsubscriptions%2F$workspaceSubscriptionId%2Fresourcegroups%2F$workspaceRG%2Fproviders%2Fmicrosoft.operationalinsights%2Fworkspaces%2F$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/query/$encodedQuery/timespan/$($detailsQueryStart)T00%3A00%3A00.000Z%2F$($detailsQueryEnd)T00%3A00%3A00.000Z" + + $additionalInfoDictionary = @{} + + $additionalInfoDictionary["currentSku"] = $result.SkuName_s + $additionalInfoDictionary["allocationMethod"] = $result.AllocationMethod_s + $additionalInfoDictionary["CostsAmount"] = [double] $result.Last30DaysCost + $additionalInfoDictionary["savingsAmount"] = [double] $result.Last30DaysCost + + $fitScore = 5 + + $tags = @{} + + if (-not([string]::IsNullOrEmpty($result.Tags_s))) + { + $tagPairs = $result.Tags_s.Substring(2, $result.Tags_s.Length - 3).Split(';') + foreach ($tagPairString in $tagPairs) + { + $tagPair = $tagPairString.Split('=') + if (-not([string]::IsNullOrEmpty($tagPair[0])) -and -not([string]::IsNullOrEmpty($tagPair[1]))) + { + $tagName = $tagPair[0].Trim() + $tagValue = $tagPair[1].Trim() + $tags[$tagName] = $tagValue + } + } + } + + $recommendation = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $result.Cloud_s + Category = "Cost" + ImpactedArea = "Microsoft.Network/publicIPAddresses" + Impact = "Low" + RecommendationType = "Saving" + RecommendationSubType = "OrphanedPublicIP" + RecommendationSubTypeId = "3125883f-8b9f-4bde-a0ff-6c739858c6e1" + RecommendationDescription = "Orphaned Public IP (without owner resource) incur in unnecessary costs" + RecommendationAction = "Delete the Public IP or change its configuration to dynamic allocation" + InstanceId = $result.InstanceId_s + InstanceName = $result.Name_s + AdditionalInfo = $additionalInfoDictionary + ResourceGroup = $result.ResourceGroupName_s + SubscriptionGuid = $result.SubscriptionGuid_g + SubscriptionName = $result.SubscriptionName + TenantGuid = $result.TenantGuid_g + FitScore = $fitScore + Tags = $tags + DetailsURL = $detailsURL + } + + $recommendations += $recommendation +} + +# Export the recommendations as JSON to blob storage + +$fileDate = $datetime.ToString("yyyy-MM-dd") +$jsonExportPath = "orphanedpublicips-$fileDate.json" +$recommendations | ConvertTo-Json | Out-File $jsonExportPath + +$jsonBlobName = $jsonExportPath +$jsonProperties = @{"ContentType" = "application/json"}; +Set-AzStorageBlobContent -File $jsonExportPath -Container $storageAccountSinkContainer -Properties $jsonProperties -Blob $jsonBlobName -Context $sa.Context -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Uploaded $jsonBlobName to Blob Storage..." + +Remove-Item -Path $jsonExportPath -Force + +$now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") +Write-Output "[$now] Removed $jsonExportPath from local disk..." + +if ($recommendationsErrors -gt 0) +{ + throw "Some of the recommendations queries failed. Please, review the job logs for additional information." +} \ No newline at end of file diff --git a/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-AdvisorRightSizeFiltered.ps1 b/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-AdvisorRightSizeFiltered.ps1 new file mode 100644 index 000000000..5d58fcf54 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-AdvisorRightSizeFiltered.ps1 @@ -0,0 +1,223 @@ +param( + [Parameter(Mandatory = $false)] + [bool] $Simulate = $true +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RemediationLogsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "remediationlogs" +} + +$minFitScore = [double] (Get-AutomationVariable -Name "AzureOptimization_RemediateRightSizeMinFitScore" -ErrorAction SilentlyContinue) +if (-not($minFitScore -gt 0.0)) { + $minFitScore = 5.0 +} + +$minWeeksInARow = [int] (Get-AutomationVariable -Name "AzureOptimization_RemediateRightSizeMinWeeksInARow" -ErrorAction SilentlyContinue) +if (-not($minWeeksInARow -gt 0)) { + $minWeeksInARow = 4 +} + +$tagsFilter = Get-AutomationVariable -Name "AzureOptimization_RemediateRightSizeTagsFilter" -ErrorAction SilentlyContinue +# example: '[ { "tagName": "a", "tagValue": "b" }, { "tagName": "c", "tagValue": "d" } ]' +if (-not($tagsFilter)) { + $tagsFilter = '{}' +} +$tagsFilter = $tagsFilter | ConvertFrom-Json + +$rightSizeRecommendationId = Get-AutomationVariable -Name "AzureOptimization_RecommendationAdvisorCostRightSizeId" -ErrorAction SilentlyContinue +if (-not($rightSizeRecommendationId)) { + $rightSizeRecommendationId = 'e10b1381-5f0a-47ff-8c7b-37bd13d7c974' +} + +$SqlTimeout = 0 +$recommendationsTable = "Recommendations" + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +# get reference to storage sink +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context + +Write-Output "Querying for right-size recommendations with fit score >= $minFitScore made consecutively for the last $minWeeksInARow weeks." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = @" + SELECT InstanceId, Cloud, TenantGuid, JSON_VALUE(AdditionalInfo, '`$.currentSku') AS CurrentSKU, JSON_VALUE(AdditionalInfo, '`$.targetSku') AS TargetSKU, COUNT(InstanceId) + FROM [dbo].[$recommendationsTable] + WHERE RecommendationSubTypeId = '$rightSizeRecommendationId' AND FitScore >= $minFitScore AND GeneratedDate >= GETDATE()-(7*$minWeeksInARow) + GROUP BY InstanceId, Cloud, TenantGuid, JSON_VALUE(AdditionalInfo, '`$.currentSku'), JSON_VALUE(AdditionalInfo, '`$.targetSku') + HAVING COUNT(InstanceId) >= $minWeeksInARow +"@ + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $vmsToRightSize = New-Object System.Data.DataTable + $sqlAdapter.Fill($vmsToRightSize) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +Write-Output "Found $($vmsToRightSize.Rows.Count) remediation opportunities." + +$Conn.Close() +$Conn.Dispose() + +$logEntries = @() + +$datetime = (get-date).ToUniversalTime() +$hour = $datetime.Hour +$min = $datetime.Minute +$timestamp = $datetime.ToString("yyyy-MM-ddT$($hour):$($min):00.000Z") + +$ctx = Get-AzContext + +foreach ($vm in $vmsToRightSize.Rows) +{ + $isEligible = $false + $logDetails = $null + if ([string]::IsNullOrEmpty($tagsFilter)) + { + $isEligible = $true + } + else + { + $vmTags = Get-AzTag -ResourceId $vm.InstanceId -ErrorAction SilentlyContinue + if ($vmTags) + { + foreach ($tagFilter in $tagsFilter) + { + if ($vmTags.Properties.TagsProperty.($tagFilter.tagName) -eq $tagFilter.tagValue) + { + $isEligible = $true + } + else + { + $isEligible = $false + break + } + } + } + } + + $subscriptionId = $vm.InstanceId.Split("/")[2] + $resourceGroup = $vm.InstanceId.Split("/")[4] + $instanceName = $vm.InstanceId.Split("/")[8] + + if ($isEligible) + { + Write-Output "Downsizing (SIMULATE=$Simulate) $($vm.InstanceId) to $($vm.TargetSKU)..." + if (-not($Simulate) -and $ctx.Environment.Name -eq $vm.Cloud -and $ctx.Tenant.Id -eq $vm.TenantGuid) + { + if ($ctx.Subscription.Id -ne $subscriptionId) + { + Select-AzSubscription -SubscriptionId $subscriptionId | Out-Null + $ctx = Get-AzContext + } + $vmObj = Get-AzVM -ResourceGroupName $resourceGroup -VMName $instanceName -ErrorAction SilentlyContinue + if ($vmObj) + { + $vmObj.HardwareProfile.VmSize = $vm.TargetSKU + Update-AzVM -VM $vmObj -ResourceGroupName $resourceGroup + } + else + { + Write-Output "Skipping as VM was already removed." + } + } + else + { + Write-Output "Did not apply remediation." + } + } + + $logDetails = @{ + IsEligible = $isEligible + CurrentSku = $vm.CurrentSKU + TargetSku = $vm.TargetSKU + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $vm.Cloud + TenantGuid = $vm.TenantGuid + SubscriptionGuid = $subscriptionId + ResourceGroupName = $resourceGroup.ToLower() + InstanceName = $instanceName.ToLower() + InstanceId = $vm.InstanceId.ToLower() + Simulate = $Simulate + LogDetails = $logDetails | ConvertTo-Json -Compress + RecommendationSubTypeId = $rightSizeRecommendationId + } + + $logEntries += $logentry +} + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-rightsizefiltered.csv" + +$logEntries | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force diff --git a/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-LongDeallocatedVMsFiltered.ps1 b/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-LongDeallocatedVMsFiltered.ps1 new file mode 100644 index 000000000..3fc61eb5d --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-LongDeallocatedVMsFiltered.ps1 @@ -0,0 +1,285 @@ +param( + [Parameter(Mandatory = $false)] + [bool] $Simulate = $true +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RemediationLogsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "remediationlogs" +} + +$minFitScore = [double] (Get-AutomationVariable -Name "AzureOptimization_RemediateLongDeallocatedVMsMinFitScore" -ErrorAction SilentlyContinue) +if (-not($minFitScore -gt 0.0)) { + $minFitScore = 5.0 +} + +$minWeeksInARow = [int] (Get-AutomationVariable -Name "AzureOptimization_RemediateLongDeallocatedVMsMinWeeksInARow" -ErrorAction SilentlyContinue) +if (-not($minWeeksInARow -gt 0)) { + $minWeeksInARow = 4 +} + +$tagsFilter = Get-AutomationVariable -Name "AzureOptimization_RemediateLongDeallocatedVMsTagsFilter" -ErrorAction SilentlyContinue +# example: '[ { "tagName": "a", "tagValue": "b" }, { "tagName": "c", "tagValue": "d" } ]' +if (-not($tagsFilter)) { + $tagsFilter = '{}' +} +$tagsFilter = $tagsFilter | ConvertFrom-Json + +$recommendationId = Get-AutomationVariable -Name "AzureOptimization_RecommendationLongDeallocatedVMsId" -ErrorAction SilentlyContinue +if (-not($recommendationId)) { + $recommendationId = 'c320b790-2e58-452a-aa63-7b62c383ad8a' +} + +$SqlTimeout = 0 +$recommendationsTable = "Recommendations" + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +# get reference to storage sink +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context + +Write-Output "Querying for long-deallocated recommendations with fit score >= $minFitScore made consecutively for the last $minWeeksInARow weeks." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = @" + SELECT InstanceId, Cloud, TenantGuid, COUNT(InstanceId) + FROM [dbo].[$recommendationsTable] + WHERE RecommendationSubTypeId = '$recommendationId' AND FitScore >= $minFitScore AND GeneratedDate >= GETDATE()-(7*$minWeeksInARow) + GROUP BY InstanceId, Cloud, TenantGuid + HAVING COUNT(InstanceId) >= $minWeeksInARow +"@ + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $deallocatedVMs = New-Object System.Data.DataTable + $sqlAdapter.Fill($deallocatedVMs) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +Write-Output "Found $($deallocatedVMs.Rows.Count) remediation opportunities." + +$Conn.Close() +$Conn.Dispose() + +$logEntries = @() + +$datetime = (get-date).ToUniversalTime() +$hour = $datetime.Hour +$min = $datetime.Minute +$timestamp = $datetime.ToString("yyyy-MM-ddT$($hour):$($min):00.000Z") + +$ctx = Get-AzContext + +foreach ($vm in $deallocatedVMs.Rows) +{ + $isEligible = $false + $logDetails = $null + if ([string]::IsNullOrEmpty($tagsFilter)) + { + $isEligible = $true + } + else + { + $vmTags = Get-AzTag -ResourceId $vm.InstanceId -ErrorAction SilentlyContinue + if ($vmTags) + { + foreach ($tagFilter in $tagsFilter) + { + if ($vmTags.Properties.TagsProperty.($tagFilter.tagName) -eq $tagFilter.tagValue) + { + $isEligible = $true + } + else + { + $isEligible = $false + break + } + } + } + } + + $subscriptionId = $vm.InstanceId.Split("/")[2] + $resourceGroup = $vm.InstanceId.Split("/")[4] + $instanceName = $vm.InstanceId.Split("/")[8] + + if ($isEligible) + { + $vmState = "Unknown" + $hasManagedDisks = $false + $osDiskSkuName = "Unknown" + $dataDisksSkuNames = "Unknown" + + Write-Output "Downsizing (SIMULATE=$Simulate) $($vm.InstanceId) disks to Standard_LRS..." + if ($ctx.Environment.Name -eq $vm.Cloud -and $ctx.Tenant.Id -eq $vm.TenantGuid) + { + if ($ctx.Subscription.Id -ne $subscriptionId) + { + Select-AzSubscription -SubscriptionId $subscriptionId | Out-Null + $ctx = Get-AzContext + } + $vmObj = Get-AzVM -ResourceGroupName $resourceGroup -VMName $instanceName -Status -ErrorAction SilentlyContinue + if ($vmObj.PowerState -eq 'VM deallocated') + { + $vmState = "Deallocated" + $osDiskId = $vmObj.StorageProfile.OsDisk.ManagedDisk.Id + $dataDiskIds = $vmObj.StorageProfile.DataDisks.ManagedDisk.Id + if ($osDiskId) + { + $hasManagedDisks = $true + $disk = Get-AzDisk -ResourceGroupName $osDiskId.Split("/")[4] -DiskName $osDiskId.Split("/")[8] + $osDiskSkuName = $disk.Sku.Name + if (-not($Simulate) -and $disk.Sku.Name -ne 'Standard_LRS') + { + $disk.Sku = [Microsoft.Azure.Management.Compute.Models.DiskSku]::new('Standard_LRS') + $disk | Update-AzDisk | Out-Null + } + else + { + Write-Output "Skipping as OS disk is already HDD." + } + foreach ($dataDiskId in $dataDiskIds) + { + $disk = Get-AzDisk -ResourceGroupName $dataDiskId.Split("/")[4] -DiskName $dataDiskId.Split("/")[8] + if ($dataDisksSkuNames -eq 'Unknown') + { + $dataDisksSkuNames = $disk.Sku.Name + } + else + { + if ($dataDisksSkuNames -notlike "*$($disk.Sku.Name)*") + { + $dataDisksSkuNames += ",$($disk.Sku.Name)" + } + } + + if (-not($Simulate) -and $disk.Sku.Name -ne 'Standard_LRS') + { + $disk.Sku = [Microsoft.Azure.Management.Compute.Models.DiskSku]::new('Standard_LRS') + $disk | Update-AzDisk | Out-Null + } + else + { + Write-Output "Skipping as Data disk is already HDD." + } + } + } + else + { + Write-Output "Skipping as disks are not Managed Disks." + $hasManagedDisks = $false + } + } + else + { + if ($vmObj) + { + Write-Output "Skipping as VM is not deallocated." + $vmState = "Running" + } + else + { + Write-Output "Skipping as VM was already removed." + $vmState = "Removed" + } + } + } + else + { + Write-Output "Could not apply remediation as VM is in another cloud/tenant." + } + } + + $logDetails = @{ + IsEligible = $isEligible + VMState = $vmState + HasManagedDisks = $hasManagedDisks + OsDiskSkuName = $osDiskSkuName + DataDisksSkuName = $dataDisksSkuNames + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $vm.Cloud + TenantGuid = $vm.TenantGuid + SubscriptionGuid = $subscriptionId + ResourceGroupName = $resourceGroup.ToLower() + InstanceName = $instanceName.ToLower() + InstanceId = $vm.InstanceId.ToLower() + Simulate = $Simulate + LogDetails = $logDetails | ConvertTo-Json -Compress + RecommendationSubTypeId = $recommendationId + } + + $logEntries += $logentry +} + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-longdeallocatedvmsfiltered.csv" + +$logEntries | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force diff --git a/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-UnattachedDisksFiltered.ps1 b/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-UnattachedDisksFiltered.ps1 new file mode 100644 index 000000000..68ed719d6 --- /dev/null +++ b/docs/deploy/optimization-engine/latest/runbooks/remediations/Remediate-UnattachedDisksFiltered.ps1 @@ -0,0 +1,264 @@ +param( + [Parameter(Mandatory = $false)] + [bool] $Simulate = $true +) + +$ErrorActionPreference = "Stop" + +$cloudEnvironment = Get-AutomationVariable -Name "AzureOptimization_CloudEnvironment" -ErrorAction SilentlyContinue # AzureCloud|AzureChinaCloud +if ([string]::IsNullOrEmpty($cloudEnvironment)) +{ + $cloudEnvironment = "AzureCloud" +} +$authenticationOption = Get-AutomationVariable -Name "AzureOptimization_AuthenticationOption" -ErrorAction SilentlyContinue # ManagedIdentity|UserAssignedManagedIdentity +if ([string]::IsNullOrEmpty($authenticationOption)) +{ + $authenticationOption = "ManagedIdentity" +} +if ($authenticationOption -eq "UserAssignedManagedIdentity") +{ + $uamiClientID = Get-AutomationVariable -Name "AzureOptimization_UAMIClientID" +} + +$sqlserver = Get-AutomationVariable -Name "AzureOptimization_SQLServerHostname" +$sqlserverCredential = Get-AutomationPSCredential -Name "AzureOptimization_SQLServerCredential" +$SqlUsername = $sqlserverCredential.UserName +$SqlPass = $sqlserverCredential.GetNetworkCredential().Password +$sqldatabase = Get-AutomationVariable -Name "AzureOptimization_SQLServerDatabase" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($sqldatabase)) +{ + $sqldatabase = "azureoptimization" +} +$storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSink" +$storageAccountSinkRG = Get-AutomationVariable -Name "AzureOptimization_StorageSinkRG" +$storageAccountSinkSubscriptionId = Get-AutomationVariable -Name "AzureOptimization_StorageSinkSubId" +$storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RemediationLogsContainer" -ErrorAction SilentlyContinue +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { + $storageAccountSinkContainer = "remediationlogs" +} + +$minFitScore = [double] (Get-AutomationVariable -Name "AzureOptimization_RemediateUnattachedDisksMinFitScore" -ErrorAction SilentlyContinue) +if (-not($minFitScore -gt 0.0)) { + $minFitScore = 5.0 +} + +$minWeeksInARow = [int] (Get-AutomationVariable -Name "AzureOptimization_RemediateUnattachedDisksMinWeeksInARow" -ErrorAction SilentlyContinue) +if (-not($minWeeksInARow -gt 0)) { + $minWeeksInARow = 4 +} + +$tagsFilter = Get-AutomationVariable -Name "AzureOptimization_RemediateUnattachedDisksTagsFilter" -ErrorAction SilentlyContinue +# example: '[ { "tagName": "a", "tagValue": "b" }, { "tagName": "c", "tagValue": "d" } ]' +if (-not($tagsFilter)) { + $tagsFilter = '{}' +} +$tagsFilter = $tagsFilter | ConvertFrom-Json + +$remediationAction = Get-AutomationVariable -Name "AzureOptimization_RemediateUnattachedDisksAction" -ErrorAction SilentlyContinue # Delete / Downsize +if (-not($remediationAction)) { + $remediationAction = "Delete" +} + +$recommendationId = Get-AutomationVariable -Name "AzureOptimization_RecommendationUnattachedDisksId" -ErrorAction SilentlyContinue +if (-not($recommendationId)) { + $recommendationId = 'c84d5e86-e2d6-4d62-be7c-cecfbd73b0db' +} + +$SqlTimeout = 0 +$recommendationsTable = "Recommendations" + +"Logging in to Azure with $authenticationOption..." + +switch ($authenticationOption) { + "UserAssignedManagedIdentity" { + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID + break + } + Default { #ManagedIdentity + Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment + break + } +} + +# get reference to storage sink +Select-AzSubscription -SubscriptionId $storageAccountSinkSubscriptionId +$saCtx = (Get-AzStorageAccount -ResourceGroupName $storageAccountSinkRG -Name $storageAccountSink).Context + +Write-Output "Querying for unattached disks recommendations with fit score >= $minFitScore made consecutively for the last $minWeeksInARow weeks." + +$tries = 0 +$connectionSuccess = $false +do { + $tries++ + try { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;User ID=$SqlUsername;Password=$SqlPass;Trusted_Connection=False;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.Open() + $Cmd=new-object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = @" + SELECT InstanceId, Cloud, TenantGuid, COUNT(InstanceId) + FROM [dbo].[$recommendationsTable] + WHERE RecommendationSubTypeId = '$recommendationId' AND FitScore >= $minFitScore AND GeneratedDate >= GETDATE()-(7*$minWeeksInARow) + GROUP BY InstanceId, Cloud, TenantGuid + HAVING COUNT(InstanceId) >= $minWeeksInARow +"@ + $sqlAdapter = New-Object System.Data.SqlClient.SqlDataAdapter + $sqlAdapter.SelectCommand = $Cmd + $unattachedDisks = New-Object System.Data.DataTable + $sqlAdapter.Fill($unattachedDisks) | Out-Null + $connectionSuccess = $true + } + catch { + Write-Output "Failed to contact SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($connectionSuccess) -and $tries -lt 3) + +if (-not($connectionSuccess)) +{ + throw "Could not establish connection to SQL." +} + +Write-Output "Found $($unattachedDisks.Rows.Count) remediation opportunities." + +$Conn.Close() +$Conn.Dispose() + +$logEntries = @() + +$datetime = (get-date).ToUniversalTime() +$hour = $datetime.Hour +$min = $datetime.Minute +$timestamp = $datetime.ToString("yyyy-MM-ddT$($hour):$($min):00.000Z") + +$ctx = Get-AzContext + +foreach ($disk in $unattachedDisks.Rows) +{ + $isEligible = $false + $logDetails = $null + if ([string]::IsNullOrEmpty($tagsFilter)) + { + $isEligible = $true + } + else + { + $diskTags = Get-AzTag -ResourceId $disk.InstanceId -ErrorAction SilentlyContinue + if ($diskTags) + { + foreach ($tagFilter in $tagsFilter) + { + if ($diskTags.Properties.TagsProperty.($tagFilter.tagName) -eq $tagFilter.tagValue) + { + $isEligible = $true + } + else + { + $isEligible = $false + break + } + } + } + } + + $subscriptionId = $disk.InstanceId.Split("/")[2] + $resourceGroup = $disk.InstanceId.Split("/")[4] + $instanceName = $disk.InstanceId.Split("/")[8] + + if ($isEligible) + { + $diskState = "Unknown" + $currentSku = "Unknown" + + Write-Output "Performing $remediationAction action (SIMULATE=$Simulate) on $($disk.InstanceId) disk..." + if ($ctx.Environment.Name -eq $disk.Cloud -and $ctx.Tenant.Id -eq $disk.TenantGuid) + { + if ($ctx.Subscription.Id -ne $subscriptionId) + { + Select-AzSubscription -SubscriptionId $subscriptionId | Out-Null + $ctx = Get-AzContext + } + $diskObj = Get-AzDisk -ResourceGroupName $resourceGroup -DiskName $instanceName -ErrorAction SilentlyContinue + if (-not($diskObj.ManagedBy)) + { + $diskState = "Unattached" + $currentSku = $diskObj.Sku.Name + if ($remediationAction -eq "Downsize") + { + if (-not($Simulate) -and $diskObj.Sku.Name -ne 'Standard_LRS') + { + $diskObj.Sku = [Microsoft.Azure.Management.Compute.Models.DiskSku]::new('Standard_LRS') + $diskObj | Update-AzDisk | Out-Null + } + else + { + Write-Output "Skipping as disk is already HDD." + } + } + elseif ($remediationAction -eq "Delete") + { + if (-not($Simulate)) + { + Remove-AzDisk -ResourceGroupName $resourceGroup -DiskName $instanceName -Force | Out-Null + } + } + else + { + Write-Output "Skipping as action is not supported." + } + } + else + { + if ($diskObj) + { + Write-Output "Skipping as disk is not unattached." + $diskState = "Attached" + } + else + { + Write-Output "Skipping as disk was already removed." + $diskState = "Removed" + } + } + } + else + { + Write-Output "Could not apply remediation as disk is in another cloud/tenant." + } + } + + $logDetails = @{ + IsEligible = $isEligible + RemediationAction = $remediationAction + DiskState = $diskState + CurrentSku = $currentSku + } + + $logentry = New-Object PSObject -Property @{ + Timestamp = $timestamp + Cloud = $disk.Cloud + TenantGuid = $disk.TenantGuid + SubscriptionGuid = $subscriptionId + ResourceGroupName = $resourceGroup.ToLower() + InstanceName = $instanceName.ToLower() + InstanceId = $disk.InstanceId.ToLower() + Simulate = $Simulate + LogDetails = $logDetails | ConvertTo-Json -Compress + RecommendationSubTypeId = $recommendationId + } + + $logEntries += $logentry +} + +$today = $datetime.ToString("yyyyMMdd") +$csvExportPath = "$today-unattacheddisksfiltered.csv" + +$logEntries | Export-Csv -Path $csvExportPath -NoTypeInformation + +$csvBlobName = $csvExportPath + +$csvProperties = @{"ContentType" = "text/csv"}; + +Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force diff --git a/docs/deploy/optimization-workbook-0.2.1-rc.2.json b/docs/deploy/optimization-workbook-0.2.1-rc.2.json index 93398d376..105b4e80d 100644 --- a/docs/deploy/optimization-workbook-0.2.1-rc.2.json +++ b/docs/deploy/optimization-workbook-0.2.1-rc.2.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "14764513392249211162" + "version": "0.24.24.22086", + "templateHash": "6696310657409574480" } }, "parameters": { @@ -11002,7 +11002,7 @@ "workbookJson": "[string(variables('$fxv#0'))]", "workbookId": "0b2", "telemetryId": "[format('00f120b5-2007-6120-0000-{0}30126b006', variables('workbookId'))]", - "finOpsToolkitVersion": "0.2.1-rc.2" + "finOpsToolkitVersion": "placeholder" }, "resources": [ { diff --git a/docs/deploy/optimization-workbook-latest.json b/docs/deploy/optimization-workbook-latest.json index 93398d376..105b4e80d 100644 --- a/docs/deploy/optimization-workbook-latest.json +++ b/docs/deploy/optimization-workbook-latest.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "14764513392249211162" + "version": "0.24.24.22086", + "templateHash": "6696310657409574480" } }, "parameters": { @@ -11002,7 +11002,7 @@ "workbookJson": "[string(variables('$fxv#0'))]", "workbookId": "0b2", "telemetryId": "[format('00f120b5-2007-6120-0000-{0}30126b006', variables('workbookId'))]", - "finOpsToolkitVersion": "0.2.1-rc.2" + "finOpsToolkitVersion": "placeholder" }, "resources": [ { diff --git a/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 b/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 index 16e81a6e8..b4033fa4b 100644 --- a/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 +++ b/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 @@ -252,7 +252,7 @@ if ((Test-Path -Path $lastDeploymentStatePath) -and !$silentDeploy) } } -$GitHubOriginalUri = "https://microsoft.github.io/finops-toolkit/deploy/optimization-engine/azuredeploy.bicep" +$GitHubOriginalUri = "https://microsoft.github.io/finops-toolkit/deploy/optimization-engine/latest/azuredeploy.bicep" if ([string]::IsNullOrEmpty($TemplateUri)) { $TemplateUri = $GitHubOriginalUri diff --git a/src/optimization-engine/package-manifest.json b/src/optimization-engine/package-manifest.json index 3920c3854..bc618a84e 100644 --- a/src/optimization-engine/package-manifest.json +++ b/src/optimization-engine/package-manifest.json @@ -1,21 +1,14 @@ { - "deploySubDir": "optimization-engine", + "deploySubDir": "optimization-engine/latest", + "deploySubDirVersioned": "optimization-engine/{version}", "deployFiles": [ { "source": "azuredeploy.bicep", "destination": "azuredeploy.bicep" }, - { - "source": "azuredeploy.bicep", - "destination": "azuredeploy-{version}.bicep" - }, { "source": "azuredeploy-nested.bicep", "destination": "azuredeploy-nested.bicep" - }, - { - "source": "azuredeploy-nested.bicep", - "destination": "azuredeploy-nested-{version}.bicep" } ], "deployDirectories": [ diff --git a/src/optimization-engine/views/workbooks/blockblobstorage-usage.json b/src/optimization-engine/views/workbooks/blockblobstorage-usage.json index 446668a48..ba7348853 100644 --- a/src/optimization-engine/views/workbooks/blockblobstorage-usage.json +++ b/src/optimization-engine/views/workbooks/blockblobstorage-usage.json @@ -1,718 +1,975 @@ { - "version": "Notebook/1.0", - "items": [ - { - "type": 9, - "content": { - "version": "KqlParameterItem/1.0", - "parameters": [ - { - "id": "8ef006f0-db8d-40b2-b51f-7a62b03e235e", - "version": "KqlParameterItem/1.0", - "name": "LookbackPeriod", - "label": "Lookback Period", - "type": 4, - "isRequired": true, - "typeSettings": { - "selectableValues": [ - { - "durationMs": 604800000 - }, - { - "durationMs": 1209600000 - }, - { - "durationMs": 2419200000 - }, - { - "durationMs": 2592000000 - }, - { - "durationMs": 5184000000 - }, - { - "durationMs": 7776000000 - } - ], - "allowCustom": true - }, - "timeContext": { - "durationMs": 86400000 - }, - "value": { - "durationMs": 2592000000 - } - }, - { - "id": "e560c503-9664-4cea-977e-8dbec62ddd64", - "version": "KqlParameterItem/1.0", - "name": "Subscriptions", - "type": 2, - "isRequired": true, - "multiSelect": true, - "quote": "'", - "delimiter": ",", - "query": "resourcecontainers\r\n| where type == 'microsoft.resources/subscriptions'\r\n| project subscriptionId, name\r\n| order by name asc", - "crossComponentResources": [ - "value::tenant" - ], - "typeSettings": { - "additionalResourceOptions": [ - "value::all" - ] - }, - "timeContext": { - "durationMs": 86400000 - }, - "queryType": 1, - "resourceType": "microsoft.resources/tenants", - "value": [ - "value::all" - ] - }, - { - "id": "d91ac170-ca28-403d-b06d-164b4239aaa3", - "version": "KqlParameterItem/1.0", - "name": "Currency", - "type": 1, - "timeContext": { - "durationMs": 86400000 - }, - "value": "EUR" - } - ], - "style": "above", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces" - }, - "customWidth": "50", - "name": "globalParameters" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| summarize ['Usage Start']=min(todatetime(Date_s)), ['Usage End']=max(todatetime(Date_s))\r\n| extend ['Usage Days'] = toint(format_timespan(['Usage End']-['Usage Start'],'d'))", - "size": 4, - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "table", - "tileSettings": { - "showBorder": false - } - }, - "customWidth": "50", - "name": "usagePeriod" - }, - { - "type": 1, - "content": { - "json": "Only Block Blob storage usage is presented (Files, Queues, Page Blobs, etc. are not included). Costs are presented at the on-demand price, i.e., as if no Storage reservations were applied to usage.", - "style": "info" - }, - "name": "disclaimer" - }, - { - "type": 11, - "content": { - "version": "LinkItem/1.0", - "style": "tabs", - "links": [ - { - "id": "1dcb642e-3696-4e44-9d71-00d434709cee", - "cellValue": "selectedTab", - "linkTarget": "parameter", - "linkLabel": "Overview", - "subTarget": "Overview", - "style": "link" - }, - { - "id": "980caea9-179c-4bcb-b501-bdb68d4a578c", - "cellValue": "selectedTab", - "linkTarget": "parameter", - "linkLabel": "Standard v2", - "subTarget": "StorageV2", - "style": "link" - }, - { - "id": "642cdd5a-b00a-4713-8fcf-41da789a3a22", - "cellValue": "selectedTab", - "linkTarget": "parameter", - "linkLabel": "Standard v1", - "subTarget": "StorageV1", - "style": "link" - }, - { - "id": "ae51c121-2917-43b1-bea1-9a519847c4a9", - "cellValue": "selectedTab", - "linkTarget": "parameter", - "linkLabel": "Premium", - "subTarget": "Premium", - "style": "link" - } - ] - }, - "name": "areaTabs" - }, - { - "type": 12, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by StorageOffer", - "size": 4, - "title": "Storage Offers ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" - }, - "customWidth": "33", - "name": "storageOffers" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by FileStructure", - "size": 4, - "title": "File Structure ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" - }, - "customWidth": "33", - "name": "fileStructure" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication\r\n\r\n\r\n", - "size": 4, - "title": "Replication ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" - }, - "customWidth": "33", - "name": "replication" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Data Stored'\r\n| summarize SizeGB=sum(todouble(Quantity_s)) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB) by ResourceId\r\n| project SizeGB=round(SizeGB*30), ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Operations'\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| summarize ['Cost ({Currency:value})']=round(sum(FullCost)) by ['Storage Account']=ResourceId, Offer=StorageOffer, ['File Structure']=FileStructure, Replication, ['Size (GB)']=SizeGB, Transactions\r\n| order by ['Cost ({Currency:value})']\r\n\r\n\r\n", - "size": 0, - "showAnalytics": true, - "title": "Storage Accounts List", - "showExportToExcel": true, - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "table", - "gridSettings": { - "formatters": [ + "version": "Notebook/1.0", + "items": [ + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "8ef006f0-db8d-40b2-b51f-7a62b03e235e", + "version": "KqlParameterItem/1.0", + "name": "LookbackPeriod", + "label": "Lookback Period", + "type": 4, + "isRequired": true, + "typeSettings": { + "selectableValues": [ { - "columnMatch": "Size", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } - } + "durationMs": 604800000 }, { - "columnMatch": "Transactions", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } - } + "durationMs": 1209600000 }, { - "columnMatch": "Cost (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } - } + "durationMs": 2419200000 + }, + { + "durationMs": 2592000000 + }, + { + "durationMs": 5184000000 + }, + { + "durationMs": 7776000000 } ], - "rowLimit": 10000 + "allowCustom": true + }, + "timeContext": { + "durationMs": 86400000 + }, + "value": { + "durationMs": 2592000000 } }, - "name": "storageAccountsCosts" + { + "id": "e560c503-9664-4cea-977e-8dbec62ddd64", + "version": "KqlParameterItem/1.0", + "name": "Subscriptions", + "type": 2, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "resourcecontainers\r\n| where type == 'microsoft.resources/subscriptions'\r\n| project subscriptionId, name\r\n| order by name asc", + "crossComponentResources": [ + "value::tenant" + ], + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ] + }, + "timeContext": { + "durationMs": 86400000 + }, + "queryType": 1, + "resourceType": "microsoft.resources/tenants", + "value": [ + "value::all" + ] + }, + { + "id": "d91ac170-ca28-403d-b06d-164b4239aaa3", + "version": "KqlParameterItem/1.0", + "name": "Currency", + "type": 1, + "timeContext": { + "durationMs": 86400000 + }, + "value": "EUR" + } + ], + "style": "above", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "customWidth": "50", + "name": "globalParameters" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| summarize ['Usage Start']=min(todatetime(Date_s)), ['Usage End']=max(todatetime(Date_s))\r\n| extend ['Usage Days'] = toint(format_timespan(['Usage End']-['Usage Start'],'d'))", + "size": 4, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "table", + "tileSettings": { + "showBorder": false } - ] + }, + "customWidth": "50", + "name": "usagePeriod" }, - "conditionalVisibility": { - "parameterName": "selectedTab", - "comparison": "isEqualTo", - "value": "Overview" + { + "type": 1, + "content": { + "json": "Only Block Blob storage usage is presented (Files, Queues, Page Blobs, etc. are not included). Costs are presented at the on-demand price, i.e., as if no Storage reservations were applied to usage.", + "style": "info" + }, + "name": "disclaimer" }, - "name": "overviewGroup" - }, - { - "type": 12, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by FileStructure", - "size": 4, - "title": "File Structure ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" + { + "type": 11, + "content": { + "version": "LinkItem/1.0", + "style": "tabs", + "links": [ + { + "id": "1dcb642e-3696-4e44-9d71-00d434709cee", + "cellValue": "selectedTab", + "linkTarget": "parameter", + "linkLabel": "Overview", + "subTarget": "Overview", + "style": "link" }, - "customWidth": "25", - "name": "fileStructure" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication", - "size": 4, - "title": "Replication ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" + { + "id": "d18a71d4-3494-4329-bb5c-5bc1042358b8", + "cellValue": "selectedTab", + "linkTarget": "parameter", + "linkLabel": "Data Stored Over Time", + "subTarget": "OverTime", + "style": "link" }, - "customWidth": "25", - "name": "replication" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored' and (MeterName_s has 'Hot' or MeterName_s has 'Cool' or MeterName_s has 'Archive')\r\n| extend Tier = iif(MeterName_s has 'Hot', 'Hot', iif(MeterName_s has 'Cool', 'Cool', 'Archive'))\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Tier", - "size": 4, - "title": "Data Stored Tiering ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" + { + "id": "980caea9-179c-4bcb-b501-bdb68d4a578c", + "cellValue": "selectedTab", + "linkTarget": "parameter", + "linkLabel": "Standard v2", + "subTarget": "StorageV2", + "style": "link" }, - "customWidth": "25", - "name": "dataStoredTiering" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Operations' and (MeterName_s has 'Hot' or MeterName_s has 'Cool' or MeterName_s has 'Archive')\r\n| extend Tier = iif(MeterName_s has 'Hot', 'Hot', iif(MeterName_s has 'Cool', 'Cool', 'Archive'))\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Tier", - "size": 4, - "title": "Transactions Tiering ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" + { + "id": "642cdd5a-b00a-4713-8fcf-41da789a3a22", + "cellValue": "selectedTab", + "linkTarget": "parameter", + "linkLabel": "Standard v1", + "subTarget": "StorageV1", + "style": "link" }, - "customWidth": "25", - "name": "transactionsTiering" - }, - { - "type": 9, - "content": { - "version": "KqlParameterItem/1.0", - "parameters": [ - { - "id": "aa472561-a676-4733-8e13-e9ce6366a187", - "version": "KqlParameterItem/1.0", - "name": "Hot2CoolTarget", - "label": "Hot to Cool Target (%)", - "type": 1, - "description": "The % of Hot data estimated to be moved to Cool with LCM", - "isRequired": true, - "value": "50" - }, - { - "id": "0344b31d-bfbe-4b4b-a3a4-63310cd5a719", - "version": "KqlParameterItem/1.0", - "name": "MoveToHotThreshold", - "label": "Trans. Move to Hot (%)", - "type": 1, - "description": "The max. % of transactions costs vs. total costs and of Cool transactions costs vs. total transactions costs", - "isRequired": true, - "timeContext": { - "durationMs": 86400000 - }, - "value": "50" - }, - { - "id": "de446dba-7535-4cc0-ab16-23c9ae3aa9fa", - "version": "KqlParameterItem/1.0", - "name": "ErrorMargin", - "label": "Comfort Margin (%)", - "type": 1, - "description": "The comfort margin (%) for estimation errors (difference between Enable vs. Maybe LCM)", - "isRequired": true, - "timeContext": { - "durationMs": 86400000 - }, - "value": "20" - } - ], - "style": "above", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces" + { + "id": "ae51c121-2917-43b1-bea1-9a519847c4a9", + "cellValue": "selectedTab", + "linkTarget": "parameter", + "linkLabel": "Premium", + "subTarget": "Premium", + "style": "link" + } + ] + }, + "name": "areaTabs" + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by StorageOffer", + "size": 4, + "title": "Storage Offers ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "33", + "name": "storageOffers" }, - "name": "storageV2Parameters" - }, - { - "type": 1, - "content": { - "json": "Enable Lifecycle Management recommendations may not apply when there is a very large object count to be moved to cooler tiers. Each object moved to a cooler tier will trigger a SetBlobTier transaction billed at Cool Write Operations pricing. Please review the blob count metrics and estimate Lifecycle Management operations costs before applying the recommendation. See more details [here](https://learn.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview#changing-a-blobs-access-tier). For pricing, check [here](https://azure.microsoft.com/en-us/pricing/details/storage/blobs/).", - "style": "warning" + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by FileStructure", + "size": 4, + "title": "File Structure ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "33", + "name": "fileStructure" }, - "name": "lcmWarning" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let MoveToHotThreshold = todouble({MoveToHotThreshold})/100;\r\nlet Hot2CoolPercentage = todouble({Hot2CoolTarget})/100;\r\nlet ErrorMarginPercentage = todouble({ErrorMargin})/100;\r\nlet StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize SizeGB=sum(todouble(Quantity_s)), CostStored=sum(FullCost) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB), CostStored=round(sum(CostStored)) by ResourceId\r\n| project SizeGB=round(SizeGB*30), CostStored, ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Operations'\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000), TransactionsCost=round(sum(FullCost)) by ResourceId;\r\nlet HotStorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored' and MeterName_s has 'Hot'\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize HotSizeCost=round(sum(FullCost)) by ResourceId;\r\nlet HotTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Operations' and (MeterName_s has 'Hot' or MeterName_s has 'List and Create Container' or MeterName_s has 'All Other')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize HotTransactions=round(sum(todouble(Quantity_s))*10000), HotTransactionsCost=round(sum(FullCost)) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| join kind=leftouter (HotStorageSize) on ResourceId\r\n| join kind=leftouter (HotTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| summarize TotalCost=round(sum(FullCost)) by ['Storage Account']=ResourceId, Structure=FileStructure, Replication, ['Size (GB)']=SizeGB, CostStored, HotSizeCost, Transactions, TransactionsCost, HotTransactionsCost\r\n| extend TransactionsFactor = round(TransactionsCost/TotalCost,1)\r\n| extend HotTransactionsFactor = round(HotTransactionsCost/TransactionsCost,1)\r\n| extend CostStored = iif(isnotempty(CostStored), CostStored, 0.0)\r\n| extend HotSizeCost = iif(isnotempty(HotSizeCost), HotSizeCost, 0.0)\r\n| extend TransactionsCost = iif(isnotempty(TransactionsCost), TransactionsCost, 0.0)\r\n| extend HotTransactionsCost = iif(isnotempty(HotTransactionsCost), HotTransactionsCost, 0.0)\r\n| extend EstimatedSavings = round(HotSizeCost*Hot2CoolPercentage/2-HotTransactionsCost*Hot2CoolPercentage*2)\r\n| extend EstimatedSavingsWithError = round(HotSizeCost*Hot2CoolPercentage/2*(1-ErrorMarginPercentage)-HotTransactionsCost*Hot2CoolPercentage*2)*(1+ErrorMarginPercentage)\r\n| extend Recommendation = iif(TransactionsFactor > MoveToHotThreshold and HotTransactionsFactor < MoveToHotThreshold, 'Move to Hot', iif(EstimatedSavingsWithError > 0, 'Enable LCM', iif(EstimatedSavings > 0, 'Maybe LCM', 'No tier change')))\r\n| extend EstimatedSavings = iif(Recommendation != 'Move to Hot', EstimatedSavings, round((TransactionsCost-HotTransactionsCost)/2-(CostStored-HotSizeCost)*2))\r\n| extend Recommendation = iif(Recommendation != 'No tier change' and EstimatedSavings <= 0, 'No tier change', Recommendation)\r\n| project-away HotTransactionsFactor, TransactionsFactor, EstimatedSavingsWithError\r\n| order by TotalCost\r\n| project-rename ['Total ({Currency})']=TotalCost, ['Size ({Currency})']=CostStored, ['Hot Size ({Currency})']=HotSizeCost, ['Trans. ({Currency})']=TransactionsCost, ['Hot Trans. ({Currency})']=HotTransactionsCost, ['Savings ({Currency})']=EstimatedSavings", - "size": 0, - "showAnalytics": true, - "title": "Storage Accounts List", - "showExportToExcel": true, - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "table", - "gridSettings": { - "formatters": [ - { - "columnMatch": "Storage Account", - "formatter": 0, - "formatOptions": { - "customColumnWidthSetting": "29ch" - } - }, - { - "columnMatch": "Size", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication\r\n\r\n\r\n", + "size": 4, + "title": "Replication ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "33", + "name": "replication" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Data Stored'\r\n| summarize SizeGB=sum(todouble(Quantity_s)) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB) by ResourceId\r\n| project SizeGB=round(SizeGB*30), ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterName_s has 'Operations'\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| summarize ['Cost ({Currency:value})']=round(sum(FullCost)) by ['Storage Account']=ResourceId, Offer=StorageOffer, ['File Structure']=FileStructure, Replication, ['Size (GB)']=SizeGB, Transactions\r\n| order by ['Cost ({Currency:value})']\r\n\r\n\r\n", + "size": 0, + "showAnalytics": true, + "title": "Storage Accounts List", + "showExportToExcel": true, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "table", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Size", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } } - } - }, - { - "columnMatch": "Transactions", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true + }, + { + "columnMatch": "Transactions", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } } - } - }, - { - "columnMatch": "Trans. (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true + }, + { + "columnMatch": "Cost (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } } } - }, - { - "columnMatch": "Hot Trans. (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } + ], + "rowLimit": 10000 + } + }, + "name": "storageAccountsCosts" + } + ] + }, + "conditionalVisibility": { + "parameterName": "selectedTab", + "comparison": "isEqualTo", + "value": "Overview" + }, + "name": "overviewGroup" + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize sum(FullCost) by Tier, bin(todatetime(Date_s), 1d)", + "size": 1, + "title": "Tiering cost ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart", + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Hot", + "color": "redBright" + }, + { + "seriesName": "Cool", + "color": "blue" + }, + { + "seriesName": "Archive", + "color": "gray" + }, + { + "seriesName": "Cold", + "color": "lightBlue" } - }, - { - "columnMatch": "Total Cost (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } + ] + } + }, + "name": "tieringCost" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend Month = datetime_part(\"month\", todatetime(Date_s))\r\n| extend Multiplier = case(Month in (1,3,5,7,8,10,12), 31, Month == 2, 28, 30)\r\n| extend CapacityGB = todouble(Quantity_s) * Multiplier\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| summarize sum(CapacityGB) by Tier, bin(todatetime(Date_s), 1d)", + "size": 1, + "aggregation": 5, + "title": "Tiering capacity (GiB)", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart", + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Hot", + "color": "redBright" + }, + { + "seriesName": "Cool", + "color": "blue" + }, + { + "seriesName": "Archive", + "color": "gray" + }, + { + "seriesName": "Cold", + "color": "lightBlue" } - }, - { - "columnMatch": "Est. Savings (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, + ], + "ySettings": { + "numberFormatSettings": { + "unit": 5, "options": { "style": "decimal", - "useGrouping": true + "useGrouping": true, + "maximumFractionDigits": 1 } } - }, - { - "columnMatch": "Recommendation", - "formatter": 18, - "formatOptions": { - "thresholdsOptions": "icons", - "thresholdsGrid": [ - { - "operator": "==", - "thresholdValue": "Move to Hot", - "representation": "3", - "text": "{0}{1}" - }, - { - "operator": "==", - "thresholdValue": "Enable LCM", - "representation": "success", - "text": "{0}{1}" - }, - { - "operator": "==", - "thresholdValue": "Maybe LCM", - "representation": "1", - "text": "{0}{1}" - }, - { - "operator": "Default", - "thresholdValue": null, - "representation": "stopped", - "text": "{0}{1}" - } - ] - } - }, - { - "columnMatch": "File Structure", - "formatter": 0, - "formatOptions": { - "customColumnWidthSetting": "17ch" + } + } + }, + "name": "tieringCapacity" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize sum(FullCost) by Replication, bin(todatetime(Date_s), 1d)", + "size": 1, + "title": "Replication cost ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart" + }, + "name": "replicationCost" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend Month = datetime_part(\"month\", todatetime(Date_s))\r\n| extend Multiplier = case(Month in (1,3,5,7,8,10,12), 31, Month == 2, 28, 30)\r\n| extend CapacityGB = todouble(Quantity_s) * Multiplier\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| summarize sum(CapacityGB) by Replication, bin(todatetime(Date_s), 1d)", + "size": 1, + "aggregation": 5, + "title": "Replication capacity (GiB)", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart", + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Hot", + "color": "redBright" + }, + { + "seriesName": "Cool", + "color": "blue" + }, + { + "seriesName": "Archive", + "color": "gray" + }, + { + "seriesName": "Cold", + "color": "lightBlue" } - }, - { - "columnMatch": "Cost (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, + ], + "ySettings": { + "numberFormatSettings": { + "unit": 5, "options": { "style": "decimal", - "useGrouping": true + "useGrouping": true, + "maximumFractionDigits": 1 } } } - ], - "rowLimit": 10000 - } + } + }, + "name": "replicationCapacity" }, - "name": "storageAccountsCosts" - } - ] - }, - "conditionalVisibility": { - "parameterName": "selectedTab", - "comparison": "isEqualTo", - "value": "StorageV2" - }, - "name": "storageV2Group" - }, - { - "type": 12, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication\r\n\r\n\r\n", - "size": 1, - "title": "Replication ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| extend PricingModel = iif(isnotempty(PricingModel_s), PricingModel_s, iif(isnotempty(ReservationName_s), 'Reservation', iif(MeterName_s endswith 'Spot', 'Spot', iif(isnotempty(benefitName_s), 'SavingsPlan', 'OnDemand'))))\r\n| summarize sum(FullCost) by PricingModel, bin(todatetime(Date_s), 1d)", + "size": 1, + "title": "Pricing model cost ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart", + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "OnDemand", + "color": "red" + }, + { + "seriesName": "Reservation", + "color": "green" + } + ] + } + }, + "name": "pricingModelCost" }, - "customWidth": "33", - "name": "replication" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| summarize SizeGB=sum(todouble(Quantity_s)) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB) by ResourceId\r\n| project SizeGB=round(SizeGB*30), ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Operations'\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize ['Cost ({Currency:value})']=round(sum(FullCost)) by ['Storage Account']=ResourceId, Replication, ['Size (GB)']=SizeGB, Transactions\r\n| order by ['Cost ({Currency:value})']", - "size": 0, - "showAnalytics": true, - "title": "Storage Accounts List", - "showExportToExcel": true, - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "table", - "gridSettings": { - "formatters": [ - { - "columnMatch": "Size", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend Month = datetime_part(\"month\", todatetime(Date_s))\r\n| extend Multiplier = case(Month in (1,3,5,7,8,10,12), 31, Month == 2, 28, 30)\r\n| extend CapacityGB = todouble(Quantity_s) * Multiplier\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| extend PricingModel = iif(isnotempty(PricingModel_s), PricingModel_s, iif(isnotempty(ReservationName_s), 'Reservation', iif(MeterName_s endswith 'Spot', 'Spot', iif(isnotempty(benefitName_s), 'SavingsPlan', 'OnDemand'))))\r\n| summarize sum(CapacityGB) by PricingModel, bin(todatetime(Date_s), 1d)", + "size": 1, + "aggregation": 5, + "title": "Pricing model capacity (GiB)", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart", + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Reservation", + "color": "green" + }, + { + "seriesName": "OnDemand", + "color": "red" } - }, - { - "columnMatch": "Transactions", - "formatter": 2, - "numberFormat": { - "unit": 0, + ], + "ySettings": { + "numberFormatSettings": { + "unit": 5, "options": { "style": "decimal", - "useGrouping": true + "useGrouping": true, + "maximumFractionDigits": 1 } } - }, - { - "columnMatch": "Cost (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, + } + } + }, + "name": "pricingModelCapacity" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| extend PricingModel = iif(isnotempty(PricingModel_s), PricingModel_s, iif(isnotempty(ReservationName_s), 'Reservation', iif(MeterName_s endswith 'Spot', 'Spot', iif(isnotempty(benefitName_s), 'SavingsPlan', 'OnDemand'))))\r\n| where PricingModel == 'OnDemand'\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize sum(FullCost) by Sku, bin(todatetime(Date_s), 1d)", + "size": 1, + "title": "On-demand SKU cost ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart" + }, + "name": "onDemandSkuCost" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and MeterName_s endswith \"Data Stored\" and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| extend Month = datetime_part(\"month\", todatetime(Date_s))\r\n| extend Multiplier = case(Month in (1,3,5,7,8,10,12), 31, Month == 2, 28, 30)\r\n| extend CapacityGB = todouble(Quantity_s) * Multiplier\r\n| extend StorageOffer = iif(MeterSubCategory_s == 'General Block Blob', 'Standard v1', iif(MeterSubCategory_s has 'Premium', 'Premium', 'Standard v2'))\r\n| extend Tier = iif(MeterSubCategory_s == 'General Block Blob', 'Hot', tostring(split(MeterName_s,' ')[0]))\r\n| extend Replication = tostring(split(MeterName_s,' ')[-3])\r\n| extend Sku = strcat(Tier, ' ', Replication, ' ', ResourceLocation_s)\r\n| extend PricingModel = iif(isnotempty(PricingModel_s), PricingModel_s, iif(isnotempty(ReservationName_s), 'Reservation', iif(MeterName_s endswith 'Spot', 'Spot', iif(isnotempty(benefitName_s), 'SavingsPlan', 'OnDemand'))))\r\n| where PricingModel == 'OnDemand'\r\n| summarize sum(CapacityGB) by Sku, bin(todatetime(Date_s), 1d)", + "size": 1, + "aggregation": 5, + "title": "On-demand SKU capacity (GiB)", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart", + "chartSettings": { + "ySettings": { + "numberFormatSettings": { + "unit": 5, "options": { "style": "decimal", - "useGrouping": true + "useGrouping": true, + "maximumFractionDigits": 1 } } } - ], - "rowLimit": 10000 - } - }, - "name": "storageAccountsCosts" - } - ] - }, - "conditionalVisibility": { - "parameterName": "selectedTab", - "comparison": "isEqualTo", - "value": "StorageV1" + } + }, + "name": "onDemandSkuCapacity" + } + ] + }, + "conditionalVisibility": { + "parameterName": "selectedTab", + "comparison": "isEqualTo", + "value": "OverTime" + }, + "name": "usageOverTimeGroup" }, - "name": "storageV1Group" - }, - { - "type": 12, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by FileStructure", - "size": 4, - "title": "File Structure ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by FileStructure", + "size": 4, + "title": "File Structure ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "25", + "name": "fileStructure" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication", + "size": 4, + "title": "Replication ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "25", + "name": "replication" }, - "customWidth": "33", - "name": "fileStructure" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication", - "size": 1, - "title": "Replication ({Currency:value})", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "piechart" + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored' and (MeterName_s has 'Hot' or MeterName_s has 'Cool' or MeterName_s has 'Archive')\r\n| extend Tier = iif(MeterName_s has 'Hot', 'Hot', iif(MeterName_s has 'Cool', 'Cool', 'Archive'))\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Tier", + "size": 4, + "title": "Data Stored Tiering ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "25", + "name": "dataStoredTiering" }, - "customWidth": "33", - "name": "replication" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Data Stored'\r\n| summarize SizeGB=sum(todouble(Quantity_s)) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB) by ResourceId\r\n| project SizeGB=round(SizeGB*30), ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Operations'\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| summarize ['Cost ({Currency:value})']=round(sum(FullCost)) by ['Storage Account']=ResourceId, ['File Structure']=FileStructure, Replication, ['Size (GB)']=SizeGB, Transactions\r\n| order by ['Cost ({Currency:value})']", - "size": 0, - "showAnalytics": true, - "title": "Storage Accounts List", - "showExportToExcel": true, - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "visualization": "table", - "gridSettings": { - "formatters": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Operations' and (MeterName_s has 'Hot' or MeterName_s has 'Cool' or MeterName_s has 'Archive')\r\n| extend Tier = iif(MeterName_s has 'Hot', 'Hot', iif(MeterName_s has 'Cool', 'Cool', 'Archive'))\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Tier", + "size": 4, + "title": "Transactions Tiering ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "25", + "name": "transactionsTiering" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ { - "columnMatch": "Size", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } - } + "id": "aa472561-a676-4733-8e13-e9ce6366a187", + "version": "KqlParameterItem/1.0", + "name": "Hot2CoolTarget", + "label": "Hot to Cool Target (%)", + "type": 1, + "description": "The % of Hot data estimated to be moved to Cool with LCM", + "isRequired": true, + "value": "50" }, { - "columnMatch": "Transactions", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } - } + "id": "0344b31d-bfbe-4b4b-a3a4-63310cd5a719", + "version": "KqlParameterItem/1.0", + "name": "MoveToHotThreshold", + "label": "Trans. Move to Hot (%)", + "type": 1, + "description": "The max. % of transactions costs vs. total costs and of Cool transactions costs vs. total transactions costs", + "isRequired": true, + "timeContext": { + "durationMs": 86400000 + }, + "value": "50" }, { - "columnMatch": "Cost (EUR)", - "formatter": 2, - "numberFormat": { - "unit": 0, - "options": { - "style": "decimal", - "useGrouping": true - } - } + "id": "de446dba-7535-4cc0-ab16-23c9ae3aa9fa", + "version": "KqlParameterItem/1.0", + "name": "ErrorMargin", + "label": "Comfort Margin (%)", + "type": 1, + "description": "The comfort margin (%) for estimation errors (difference between Enable vs. Maybe LCM)", + "isRequired": true, + "timeContext": { + "durationMs": 86400000 + }, + "value": "20" } ], - "rowLimit": 10000 - } + "style": "above", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "storageV2Parameters" }, - "name": "storageAccountsCosts" - } - ] + { + "type": 1, + "content": { + "json": "Enable Lifecycle Management recommendations may not apply when there is a very large object count to be moved to cooler tiers. Each object moved to a cooler tier will trigger a SetBlobTier transaction billed at Cool Write Operations pricing. Please review the blob count metrics and estimate Lifecycle Management operations costs before applying the recommendation. See more details [here](https://learn.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview#changing-a-blobs-access-tier). For pricing, check [here](https://azure.microsoft.com/en-us/pricing/details/storage/blobs/).", + "style": "warning" + }, + "name": "lcmWarning" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let MoveToHotThreshold = todouble({MoveToHotThreshold})/100;\r\nlet Hot2CoolPercentage = todouble({Hot2CoolTarget})/100;\r\nlet ErrorMarginPercentage = todouble({ErrorMargin})/100;\r\nlet StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize SizeGB=sum(todouble(Quantity_s)), CostStored=sum(FullCost) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB), CostStored=round(sum(CostStored)) by ResourceId\r\n| project SizeGB=round(SizeGB*30), CostStored, ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Operations'\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000), TransactionsCost=round(sum(FullCost)) by ResourceId;\r\nlet HotStorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Data Stored' and MeterName_s has 'Hot'\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize HotSizeCost=round(sum(FullCost)) by ResourceId;\r\nlet HotTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| where MeterName_s has 'Operations' and (MeterName_s has 'Hot' or MeterName_s has 'List and Create Container' or MeterName_s has 'All Other')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize HotTransactions=round(sum(todouble(Quantity_s))*10000), HotTransactionsCost=round(sum(FullCost)) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s !has 'Premium' and MeterSubCategory_s != 'General Block Blob'\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| join kind=leftouter (HotStorageSize) on ResourceId\r\n| join kind=leftouter (HotTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| summarize TotalCost=round(sum(FullCost)) by ['Storage Account']=ResourceId, Structure=FileStructure, Replication, ['Size (GB)']=SizeGB, CostStored, HotSizeCost, Transactions, TransactionsCost, HotTransactionsCost\r\n| extend TransactionsFactor = round(TransactionsCost/TotalCost,1)\r\n| extend HotTransactionsFactor = round(HotTransactionsCost/TransactionsCost,1)\r\n| extend CostStored = iif(isnotempty(CostStored), CostStored, 0.0)\r\n| extend HotSizeCost = iif(isnotempty(HotSizeCost), HotSizeCost, 0.0)\r\n| extend TransactionsCost = iif(isnotempty(TransactionsCost), TransactionsCost, 0.0)\r\n| extend HotTransactionsCost = iif(isnotempty(HotTransactionsCost), HotTransactionsCost, 0.0)\r\n| extend EstimatedSavings = round(HotSizeCost*Hot2CoolPercentage/2-HotTransactionsCost*Hot2CoolPercentage*2)\r\n| extend EstimatedSavingsWithError = round(HotSizeCost*Hot2CoolPercentage/2*(1-ErrorMarginPercentage)-HotTransactionsCost*Hot2CoolPercentage*2)*(1+ErrorMarginPercentage)\r\n| extend Recommendation = iif(TransactionsFactor > MoveToHotThreshold and HotTransactionsFactor < MoveToHotThreshold, 'Move to Hot', iif(EstimatedSavingsWithError > 0, 'Enable LCM', iif(EstimatedSavings > 0, 'Maybe LCM', 'No tier change')))\r\n| extend EstimatedSavings = iif(Recommendation != 'Move to Hot', EstimatedSavings, round((TransactionsCost-HotTransactionsCost)/2-(CostStored-HotSizeCost)*2))\r\n| extend Recommendation = iif(Recommendation != 'No tier change' and EstimatedSavings <= 0, 'No tier change', Recommendation)\r\n| project-away HotTransactionsFactor, TransactionsFactor, EstimatedSavingsWithError\r\n| order by TotalCost\r\n| project-rename ['Total ({Currency})']=TotalCost, ['Size ({Currency})']=CostStored, ['Hot Size ({Currency})']=HotSizeCost, ['Trans. ({Currency})']=TransactionsCost, ['Hot Trans. ({Currency})']=HotTransactionsCost, ['Savings ({Currency})']=EstimatedSavings", + "size": 0, + "showAnalytics": true, + "title": "Storage Accounts List", + "showExportToExcel": true, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "table", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Storage Account", + "formatter": 0, + "formatOptions": { + "customColumnWidthSetting": "29ch" + } + }, + { + "columnMatch": "Size", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Transactions", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Trans. (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Hot Trans. (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Total Cost (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Est. Savings (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Recommendation", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Move to Hot", + "representation": "3", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Enable LCM", + "representation": "success", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Maybe LCM", + "representation": "1", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "stopped", + "text": "{0}{1}" + } + ] + } + }, + { + "columnMatch": "File Structure", + "formatter": 0, + "formatOptions": { + "customColumnWidthSetting": "17ch" + } + }, + { + "columnMatch": "Cost (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + } + ], + "rowLimit": 10000 + } + }, + "name": "storageAccountsCosts" + } + ] + }, + "conditionalVisibility": { + "parameterName": "selectedTab", + "comparison": "isEqualTo", + "value": "StorageV2" + }, + "name": "storageV2Group" }, - "conditionalVisibility": { - "parameterName": "selectedTab", - "comparison": "isEqualTo", - "value": "Premium" + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication\r\n\r\n\r\n", + "size": 1, + "title": "Replication ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "33", + "name": "replication" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Data Stored'\r\n| summarize SizeGB=sum(todouble(Quantity_s)) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB) by ResourceId\r\n| project SizeGB=round(SizeGB*30), ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| where MeterName_s has 'Operations'\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s == 'General Block Blob'\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize ['Cost ({Currency:value})']=round(sum(FullCost)) by ['Storage Account']=ResourceId, Replication, ['Size (GB)']=SizeGB, Transactions\r\n| order by ['Cost ({Currency:value})']", + "size": 0, + "showAnalytics": true, + "title": "Storage Accounts List", + "showExportToExcel": true, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "table", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Size", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Transactions", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Cost (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + } + ], + "rowLimit": 10000 + } + }, + "name": "storageAccountsCosts" + } + ] + }, + "conditionalVisibility": { + "parameterName": "selectedTab", + "comparison": "isEqualTo", + "value": "StorageV1" + }, + "name": "storageV1Group" }, - "name": "premiumGroup" - } - ], - "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" -} \ No newline at end of file + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by FileStructure", + "size": 4, + "title": "File Structure ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "33", + "name": "fileStructure" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| join kind=inner (StorageReplication) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| summarize Cost=sum(FullCost) by Replication", + "size": 1, + "title": "Replication ({Currency:value})", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "33", + "name": "replication" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let StorageReplication = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Data Stored'\r\n| extend Replication = tostring(split(MeterName_s, ' ')[-3])\r\n| distinct ResourceId, Replication;\r\nlet StorageSize = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Data Stored'\r\n| summarize SizeGB=sum(todouble(Quantity_s)) by ResourceId, Date_s\r\n| summarize arg_max(todatetime(Date_s), SizeGB) by ResourceId\r\n| project SizeGB=round(SizeGB*30), ResourceId;\r\nlet StorageTransactions = AzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| where MeterName_s has 'Operations'\r\n| summarize Transactions=round(sum(todouble(Quantity_s))*10000) by ResourceId;\r\nAzureOptimizationConsumptionV1_CL\r\n| where todatetime(Date_s) between (todatetime('{LookbackPeriod:startISO}')..todatetime('{LookbackPeriod:endISO}'))\r\n| where SubscriptionId in ({Subscriptions:value})\r\n| where ChargeType_s == 'Usage' and MeterCategory_s == 'Storage' and (MeterSubCategory_s has 'Block Blob' or MeterSubCategory_s has 'Hierarchical Namespace' or MeterSubCategory_s has 'Blob Storage')\r\n| where ResourceId has 'microsoft.storage' or ResourceId has 'microsoft.classicstorage'\r\n| where MeterSubCategory_s has 'Premium'\r\n| join kind=leftouter (StorageReplication) on ResourceId\r\n| join kind=leftouter (StorageSize) on ResourceId\r\n| join kind=leftouter (StorageTransactions) on ResourceId\r\n| extend FullCost = todouble(Quantity_s) * todouble(iif(todouble(UnitPrice_s) > 0.0, UnitPrice_s, EffectivePrice_s))\r\n| extend FileStructure = iif(MeterSubCategory_s has 'Hierarchical Namespace', 'HNS', 'Flat')\r\n| summarize ['Cost ({Currency:value})']=round(sum(FullCost)) by ['Storage Account']=ResourceId, ['File Structure']=FileStructure, Replication, ['Size (GB)']=SizeGB, Transactions\r\n| order by ['Cost ({Currency:value})']", + "size": 0, + "showAnalytics": true, + "title": "Storage Accounts List", + "showExportToExcel": true, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "table", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Size", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Transactions", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + }, + { + "columnMatch": "Cost (EUR)", + "formatter": 2, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal", + "useGrouping": true + } + } + } + ], + "rowLimit": 10000 + } + }, + "name": "storageAccountsCosts" + } + ] + }, + "conditionalVisibility": { + "parameterName": "selectedTab", + "comparison": "isEqualTo", + "value": "Premium" + }, + "name": "premiumGroup" + } + ], + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" + } \ No newline at end of file diff --git a/src/scripts/Build-Toolkit.ps1 b/src/scripts/Build-Toolkit.ps1 index c4c4e3bdc..8cb37a2c5 100644 --- a/src/scripts/Build-Toolkit.ps1 +++ b/src/scripts/Build-Toolkit.ps1 @@ -119,6 +119,7 @@ Get-ChildItem "$PSScriptRoot/../templates/$Template*" -Directory -ErrorAction Si Write-Host '' } +# TODO: review build logic to make it more generic across all toolkit components # Package optimization engine $srcDir = "$PSScriptRoot/../optimization-engine" Write-Host "Building optimization engine..." diff --git a/src/scripts/Package-Toolkit.ps1 b/src/scripts/Package-Toolkit.ps1 index 027791646..87a6cfa5e 100644 --- a/src/scripts/Package-Toolkit.ps1 +++ b/src/scripts/Package-Toolkit.ps1 @@ -100,18 +100,33 @@ $templates = Get-ChildItem $relDir -Directory ` $docsDeployDir = "$deployDir/$($packageManifest.deploySubDir)" & "$PSScriptRoot/New-Directory" $docsDeployDir } + if ($packageManifest.deploySubDirVersioned) + { + $docsDeployDirVersioned = "$deployDir/$($packageManifest.deploySubDirVersioned.Replace('{version}', $version))" + & "$PSScriptRoot/New-Directory" $docsDeployDirVersioned + } foreach ($file in $packageManifest.deployFiles) { Copy-Item "$path/$($file.source)" "$docsDeployDir/$($file.destination.Replace('{version}', $version))" + if ($packageManifest.deploySubDirVersioned) + { + Copy-Item "$path/$($file.source)" "$docsDeployDirVersioned/$($file.destination.Replace('{version}', $version))" + } } foreach ($directory in $packageManifest.deployDirectories) { & "$PSScriptRoot/New-Directory" "$($docsDeployDir)/$($directory.destination)" Get-ChildItem "$path/$($directory.source)" | Copy-Item -Destination "$($docsDeployDir)/$($directory.destination)" -Recurse -Force + if ($packageManifest.deploySubDirVersioned) + { + & "$PSScriptRoot/New-Directory" "$($docsDeployDirVersioned)/$($directory.destination)" + Get-ChildItem "$path/$($directory.source)" | Copy-Item -Destination "$($docsDeployDirVersioned)/$($directory.destination)" -Recurse -Force + } } } else { + # TODO include this fallback logic in a fallback package-manifest.json file # Copy azuredeploy.json to docs/deploy folder Copy-Item "$path/azuredeploy.json" "$deployDir/$($path.Name)-$version.json" Copy-Item "$path/azuredeploy.json" "$deployDir/$($path.Name)-latest.json" diff --git a/src/scripts/Publish-Toolkit.ps1 b/src/scripts/Publish-Toolkit.ps1 index 5ae868b27..8f212e9fd 100644 --- a/src/scripts/Publish-Toolkit.ps1 +++ b/src/scripts/Publish-Toolkit.ps1 @@ -95,6 +95,7 @@ function Find-Repo($config, [string]$templateName) $ver = & "$PSScriptRoot/Get-Version.ps1" # Loop thru templates +# TODO remove the optimization-engine exclusion once we determine how we are going to publish it to other repos Get-ChildItem "$relDir/$Template*" -Directory -Exclude "optimization-engine" ` | ForEach-Object { $templateDir = $_