Merge pull request #163 from JulianHayward/dev

v6_major_20230106_1
JulianHayward · Jan 6, 2023 · 87e139f · 87e139f
2 parents 0ceceb5 + 97c6cfd
commit 87e139f
Show file tree

Hide file tree

Showing 10 changed files with 293 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -47,7 +47,7 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch
   * [Parameters](#parameters)
   * [API reference](#api-reference)
 * [Integrate with AzOps](#integrate-with-azops)
-* [Integrate PSRule for Azure](#integrate-psrule-for-azure)
+* [Integrate PSRule for Azure - paused](#integrate-psrule-for-azure)
 * [Stats](#stats)
 * [Security](#security)
 * [Known issues](#known-issues)
@@ -59,12 +59,15 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch
 
 ## Release history
 
-__Changes__ (2023-Jan-03 / Major)
+__Changes__ (2023-Jan-06 / Major)
 
-* Fix issue for Private Endpoints feature
-  * Subscription may not be registered for location / skip
-* Use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.1.64
-* Add ShowMemoryUsage at creation of __DefinitionInsights__
+* Fix issue PIM eligibility (do not process out-of-scope subscriptions) [issue #161](https://github.com/JulianHayward/Azure-MG-Sub-Governance-Reporting/issues/161)
+* Collect Advisor Scores foreach subscription
+* Update DailySummary
+  * Add count of subscriptions per quotaId
+  * Add 'Microsoft Defender for Cloud' Secure Score for Management Groups
+* Updated [API reference](#api-reference)
+* Use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.1.65
 
 Passed tests: Powershell Core 7.3.0 on Windows  
 Passed tests: Powershell Core 7.2.7 Azure DevOps hosted agent ubuntu-22.04  
@@ -213,6 +216,7 @@ Short presentation on AzGovViz [[download](slides/AzGovViz_intro.pdf)]
     * Summary of all UserAssigned Managed Identities assigned to Resources
     * Summary of Resources that have an UserAssigned Managed Identity assigned
   * [Integrate PSRule for Azure](#integrate-psrule-for-azure)
+    * __Pausing 'PSRule for Azure' integration__. AzGovViz leveraged the Invoke-PSRule cmdlet, but there are certain [resource types](https://github.com/Azure/PSRule.Rules.Azure/blob/ab0910359c1b9826d8134041d5ca997f6195fc58/src/PSRule.Rules.Azure/PSRule.Rules.Azure.psm1#L1582) where also child resources need to be queried to achieve full rule evaluation. 
     * Well-Architected Framework aligned best practice analysis for resources, including guidance for remediation
   * Storage Account Access Analysis
     * Provides insights on Storage Accounts with focus on anonymous access (containers/blobs and 'Static website' feature)
@@ -487,6 +491,7 @@ AzAPICall resources:
   * `-CriticalMemoryUsage` - Define at what percentage of memory usage the garbage collection should kick in (default=90)
   * `-ExcludedResourceTypesDiagnosticsCapable` - Resource Types to be excluded from processing analysis for diagnostic settings capability (default: microsoft.web/certificates)
   * PSRule for Azure
+    * __Pausing 'PSRule for Azure' integration__. AzGovViz leveraged the Invoke-PSRule cmdlet, but there are certain [resource types](https://github.com/Azure/PSRule.Rules.Azure/blob/ab0910359c1b9826d8134041d5ca997f6195fc58/src/PSRule.Rules.Azure/PSRule.Rules.Azure.psm1#L1582) where also child resources need to be queried to achieve full rule evaluation.
     * `-DoPSRule` - Execute [PSRule for Azure](https://azure.github.io/PSRule.Rules.Azure). Aggregated results are integrated in the HTML output, detailed results (per resource) are exported to CSV
     * `-PSRuleVersion` - Define the PSRule..Rules.Azure PowerShell module version, if undefined then 'latest' will be used
     * `-PSRuleFailedOnly` - PSRule for Azure will only report on failed resource (may save some space/noise). (e.g. `.\pwsh\AzGovVizParallel.ps1 -DoPSRule -PSRuleFailedOnly`)
@@ -538,6 +543,7 @@ AzGovViz polls the following APIs
 | ARM | 2020-05-01 | /providers/Microsoft.Management/managementGroups |
 | ARM | 2021-03-01 | /providers/Microsoft.ResourceGraph/resources |
 | ARM | 2020-01-01 | /subscriptions/`subscriptionId`/locations |
+| ARM | 2020-07-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Advisor/advisorScore |
 | ARM | 2016-09-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/locks |
 | ARM | 2021-06-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/policyAssignments |
 | ARM | 2021-06-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/policyDefinitions |
@@ -582,6 +588,8 @@ You can integrate AzGovViz (same project as AzOps).
 
 ## Integrate PSRule for Azure
 
+__Pausing 'PSRule for Azure' integration__. AzGovViz leveraged the Invoke-PSRule cmdlet, but there are certain [resource types](https://github.com/Azure/PSRule.Rules.Azure/blob/ab0910359c1b9826d8134041d5ca997f6195fc58/src/PSRule.Rules.Azure/PSRule.Rules.Azure.psm1#L1582) where also child resources need to be queried to achieve full rule evaluation.
+
 Let´s use [PSRule for Azure](https://azure.github.io/PSRule.Rules.Azure) and leverage over 260 pre-built rules to validate Azure resources based on the Microsoft Well-Architected Framework (WAF) principles.  
 PSRule for Azure is listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/architecture/framework/security/monitor-tools) in the Microsoft Well-Architected Framework.
 

diff --git a/history.md b/history.md
@@ -4,6 +4,16 @@
 
 ### AzGovViz version 6
 
+__Changes__ (2023-Jan-06 / Major)
+
+* Fix issue PIM eligibility (do not process out-of-scope subscriptions) [issue #161](https://github.com/JulianHayward/Azure-MG-Sub-Governance-Reporting/issues/161)
+* Collect Advisor Scores foreach subscription
+* Update DailySummary
+  * Add count of subscriptions per quotaId
+  * Add 'Microsoft Defender for Cloud' Secure Score for Management Groups
+* Updated [API reference](#api-reference)
+* Use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.1.65
+
 __Changes__ (2023-Jan-03 / Major)
 
 * Fix issue for Private Endpoints feature

diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1
@@ -359,10 +359,10 @@ Param
     $Product = 'AzGovViz',
 
     [string]
-    $AzAPICallVersion = '1.1.64',
+    $AzAPICallVersion = '1.1.65',
 
     [string]
-    $ProductVersion = 'v6_major_20230103_1',
+    $ProductVersion = 'v6_major_20230106_1',
 
     [string]
     $GithubRepository = 'aka.ms/AzGovViz',
@@ -2174,6 +2174,8 @@ function createTagList {
     Write-Host "Creating TagList array duration: $((New-TimeSpan -Start $startTagListArray -End $endTagListArray).TotalMinutes) minutes ($((New-TimeSpan -Start $startTagListArray -End $endTagListArray).TotalSeconds) seconds)"
 }
 function detailSubscriptions {
+    $start = Get-Date
+    Write-Host 'Subscription picking'
     #API in rare cases returns duplicates, therefor sorting unique (id)
     $childrenSubscriptions = $arrayEntitiesFromAPI.where( { $_.properties.parentNameChain -contains $ManagementGroupID -and $_.type -eq '/subscriptions' } ) | Sort-Object -Property id -Unique
     $script:childrenSubscriptionsCount = ($childrenSubscriptions).Count
@@ -2270,7 +2272,40 @@ function detailSubscriptions {
             }
         }
     }
+
+    if ($subsToProcessInCustomDataCollection.Count -lt $childrenSubscriptionsCount) {
+        Write-Host " $($subsToProcessInCustomDataCollection.Count) of $($childrenSubscriptionsCount) Subscriptions picked for processing" -ForegroundColor yellow
+    }
+    else {
+        Write-Host " $($subsToProcessInCustomDataCollection.Count) of $($childrenSubscriptionsCount) Subscriptions picked for processing"
+    }
+
+
+    if ($outOfScopeSubscriptions.Count -gt 0) {
+        Write-Host " $($outOfScopeSubscriptions.Count) Subscriptions excluded" -ForegroundColor yellow
+        $outOfScopeSubscriptionsGroupedByOutOfScopeReason = $outOfScopeSubscriptions | Group-Object -Property outOfScopeReason
+        foreach ($exclusionreason in $outOfScopeSubscriptionsGroupedByOutOfScopeReason) {
+            Write-Host "   $($exclusionreason.Count): $($exclusionreason.Name) ($($exclusionreason.Group.subscriptionId -join ', '))"
+        }
+
+        foreach ($outOfScopeSubscription in $outOfScopeSubscriptions) {
+            $script:htOutOfScopeSubscriptions.($outOfScopeSubscription.subscriptionId) = @{
+                subscriptionId      = $outOfScopeSubscription.subscriptionId
+                subscriptionName    = $outOfScopeSubscription.subscriptionName
+                outOfScopeReason    = $outOfScopeSubscription.outOfScopeReason
+                ManagementGroupId   = $outOfScopeSubscription.ManagementGroupId
+                ManagementGroupName = $outOfScopeSubscription.ManagementGroupName
+                Level               = $outOfScopeSubscription.Level
+            }
+        }
+    }
+    else {
+        Write-Host " $($outOfScopeSubscriptions.Count) Subscriptions excluded"
+    }
     $script:subsToProcessInCustomDataCollectionCount = ($subsToProcessInCustomDataCollection).Count
+
+    $end = Get-Date
+    Write-Host "Subscription picking duration: $((New-TimeSpan -Start $start -End $end).TotalSeconds) seconds"
 }
 function exportBaseCSV {
     Write-Host "Exporting CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName).csv'"
@@ -3184,6 +3219,7 @@ function getGroupmembers($aadGroupId, $aadGroupDisplayName) {
     }
 }
 function getMDfCSecureScoreMG {
+    $start = Get-Date
     $currentTask = 'Getting Microsoft Defender for Cloud Secure Score for Management Groups'
     Write-Host $currentTask
     #ref: https://docs.microsoft.com/en-us/azure/governance/management-groups/resource-graph-samples?tabs=azure-cli#secure-score-per-management-group
@@ -3217,15 +3253,14 @@ function getMDfCSecureScoreMG {
         }
 "@
 
-    $start = Get-Date
     $getMgAscSecureScore = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -body $body -listenOn 'Content'
-    $end = Get-Date
-    Write-Host " Getting Microsoft Defender for Cloud Secure Score for Management Groups duration: $((New-TimeSpan -Start $start -End $end).TotalSeconds) seconds"
+
     if ($getMgAscSecureScore) {
         if ($getMgAscSecureScore -eq 'capitulation') {
-            Write-Host '  Microsoft Defender for Cloud SecureScore for Management Groups will not be available' -ForegroundColor Yellow
+            Write-Host ' Microsoft Defender for Cloud SecureScore for Management Groups will not be available' -ForegroundColor Yellow
         }
         else {
+            Write-Host " Retrieved 'Microsoft Defender for Cloud' SecureScore for $($getMgAscSecureScore.Count) Management Groups"
             foreach ($entry in $getMgAscSecureScore) {
                 $script:htMgASCSecureScore.($entry.mgId) = @{}
                 if ($entry.secureScore -eq 404) {
@@ -3237,6 +3272,9 @@ function getMDfCSecureScoreMG {
             }
         }
     }
+
+    $end = Get-Date
+    Write-Host "Getting Microsoft Defender for Cloud Secure Score for Management Groups duration: $((New-TimeSpan -Start $start -End $end).TotalMinutes) minutes ($((New-TimeSpan -Start $start -End $end).TotalSeconds) seconds)"
 }
 function getOrphanedResources {
     $start = Get-Date
@@ -3442,14 +3480,24 @@ function getPIMEligible {
                     }
                     if ($entry.type -eq 'subscription') {
                         if ($htSubscriptionsMgPath.($entry.externalId -replace '.*/').ParentNameChain -contains $ManagementGroupId) {
-                            $null = $scopesToIterate.Add($entry)
+                            if ($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/')) {
+                                Write-Host "excluding subscription $($entry.externalId -replace '.*/') (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/').outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)"
+                            }
+                            else {
+                                $null = $scopesToIterate.Add($entry)
+                            }
                         }
                     }
                 }
             }
             else {
                 foreach ($entry in $res) {
-                    $null = $scopesToIterate.Add($entry)
+                    if ($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/')) {
+                        Write-Host "excluding subscription $($entry.externalId -replace '.*/') (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/').outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)"
+                    }
+                    else {
+                        $null = $scopesToIterate.Add($entry)
+                    }
                 }
             }
         }
@@ -5046,6 +5094,7 @@ function processDataCollection {
                 $arrayPrivateEndPointsFromResourceProperties = $using:arrayPrivateEndPointsFromResourceProperties
                 $htResourcePropertiesConvertfromJSONFailed = $using:htResourcePropertiesConvertfromJSONFailed
                 $htAvailablePrivateEndpointTypes = $using:htAvailablePrivateEndpointTypes
+                $arrayAdvisorScores = $using:arrayAdvisorScores
                 #$htResourcesWithProperties = $using:htResourcesWithProperties
                 #other
                 $function:addRowToTable = $using:funcAddRowToTable
@@ -5076,6 +5125,7 @@ function processDataCollection {
                 $function:dataCollectionDefenderEmailContacts = $using:funcDataCollectionDefenderEmailContacts
                 $function:dataCollectionVNets = $using:funcDataCollectionVNets
                 $function:dataCollectionPrivateEndpoints = $using:funcDataCollectionPrivateEndpoints
+                $function:dataCollectionAdvisorScores = $using:funcDataCollectionAdvisorScores
                 #endregion UsingVARs
 
                 $addRowToTableDone = $false
@@ -5129,6 +5179,9 @@ function processDataCollection {
                         #defenderEmailContacts
                         DataCollectionDefenderEmailContacts @baseParameters
 
+                        #advisorScores
+                        DataCollectionAdvisorScores @baseParameters
+
                         if (-not $azAPICallConf['htParameters'].NoNetwork) {
                             #VNets
                             DataCollectionVNets @baseParameters
@@ -28177,6 +28230,50 @@ function dataCollectionDefenderPlans {
 }
 $funcDataCollectionDefenderPlans = $function:dataCollectionDefenderPlans.ToString()
 
+
+function dataCollectionAdvisorScores {
+    [CmdletBinding()]Param(
+        [string]$scopeId,
+        [string]$scopeDisplayName,
+        $SubscriptionQuotaId
+    )
+
+    $currentTask = "Getting Advisor Scores for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$SubscriptionQuotaId']"
+    $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Advisor/advisorScore?api-version=2020-07-01-preview"
+    $method = 'GET'
+    $advisorScoreResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' -skipOnErrorCode 404
+
+    if ($advisorScoreResult -eq 'SubScriptionNotRegistered' -or $advisorScoreResult -eq 'DisallowedProvider') {
+    }
+    else {
+        if ($advisorScoreResult -like 'azgvzerrorMessage_*') {
+
+        }
+        else {
+            if ($advisorScoreResult.Count -gt 0) {
+                foreach ($entry in $advisorScoreResult) {
+                    #Write-Host ($entry | ConvertTo-Json -Depth 99)
+                    if ($entry.Name) {
+                        $objectGuid = [System.Guid]::empty
+                        if ([System.Guid]::TryParse($entry.Name, [System.Management.Automation.PSReference]$ObjectGuid)) {
+                        }
+                        else {
+                            $null = $script:arrayAdvisorScores.Add([PSCustomObject]@{
+                                    subscriptionId      = $scopeId
+                                    subscriptionName    = $scopeDisplayName
+                                    subscriptionQuotaId = $SubscriptionQuotaId
+                                    category            = $entry.Name
+                                    score               = $entry.properties.lastRefreshedScore.score
+                                })
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+$funcDataCollectionAdvisorScores = $function:dataCollectionAdvisorScores.ToString()
+
 function dataCollectionDefenderEmailContacts {
     [CmdletBinding()]Param(
         [string]$scopeId,
@@ -32753,6 +32850,7 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
     $htCachePolicyComplianceResponseTooLargeMG = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{}
     $htCachePolicyComplianceResponseTooLargeSUB = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{}
     $outOfScopeSubscriptions = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
+    $htOutOfScopeSubscriptions = @{}
     if ($azAPICallConf['htParameters'].DoAzureConsumption -eq $true) {
         $htManagementGroupsCost = @{}
         $htAzureConsumptionSubscriptions = @{}
@@ -32846,6 +32944,7 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
     #$htResourcesWithProperties = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{}
     $htResourceProvidersRef = @{}
     $htAvailablePrivateEndpointTypes = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{}
+    $arrayAdvisorScores = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
 }
 
 if (-not $HierarchyMapOnly) {
@@ -32980,6 +33079,11 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
 
     processDataCollection -mgId $ManagementGroupId
 
+    if ($arrayAdvisorScores.Count -gt 0) {
+        Write-Host "Exporting AdvisorScores CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName)_AdvisorScores.csv'"
+        $arrayAdvisorScores | Sort-Object -Property subscriptionName, subscriptionId, category | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_AdvisorScores.csv" -Delimiter "$csvDelimiter" -NoTypeInformation
+    }
+
     showMemoryUsage
 
     if (-not $NoPIMEligibility) {
@@ -33395,6 +33499,14 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
     $htDailySummary.'ManagementGroups' = $totalMgCount
     Write-Host " Total Subscriptions: $totalSubIncludedAndExcludedCount ($totalSubCount included; $totalSubOutOfScopeCount out-of-scope)"
     $htDailySummary.'Subscriptions' = $totalSubCount
+
+    $subscriptionsGroupedByQuotaId = $optimizedTableForPathQuerySub | Group-Object -Property SubscriptionQuotaId
+    if ($subscriptionsGroupedByQuotaId.Count -gt 0) {
+        foreach ($quotaId in $subscriptionsGroupedByQuotaId) {
+            $htDailySummary."Subscriptions_$($quotaId.Name)" = $quotaId.Count
+        }
+    }
+
     $htDailySummary.'SubscriptionsOutOfScope' = $totalSubOutOfScopeCount
     Write-Host " Total BuiltIn Policy definitions: $tenantBuiltInPoliciesCount"
     $htDailySummary.'PolicyDefinitionsBuiltIn' = $tenantBuiltInPoliciesCount
@@ -33465,6 +33577,20 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
         $htDailySummary.'TotalUniquePrincipalWithPermissionPIM_User' = $rbacUniqueObjectIdsPIM.where({ $_.ObjectType -like 'User*' } ).count
     }
 
+
+    # if ($arrayAdvisorScores.Count -gt 0) {
+    #     $arrayAdvisorScoresGroupedByCategory = $arrayAdvisorScores | Group-Object -Property category
+    #     foreach ($entry in $arrayAdvisorScoresGroupedByCategory) {
+    #         $htDailySummary."Advisor_$($entry.Name)" = ($entry.Group.Score | Measure-Object -Sum).Sum / $entry.Group.Count
+    #     }
+    # }
+
+    if ($htMgASCSecureScore.Keys.Count -gt 0) {
+        foreach ($mgASCSecureScore in $htMgASCSecureScore.Keys) {
+            $htDailySummary."MDfCSecureScore_$($mgASCSecureScore)" = $htMgASCSecureScore.($mgASCSecureScore).SecureScore
+        }
+    }
+
     $endSummarizeDataCollectionResults = Get-Date
     Write-Host " Summary data collection duration: $((New-TimeSpan -Start $startSummarizeDataCollectionResults -End $endSummarizeDataCollectionResults).TotalSeconds) seconds"
     showMemoryUsage