Reliability improvements (#761)

* initial commit * Updated Retry (#3) * Update Docs (#759) * Update Docs * Update Docs Wording * Update release pipeline (#756) * disable * update * Update Exemption * PolicyExemption Retry * Invoke-AzOpsRestMethod Retry * Update Retry --------- Co-authored-by: Johan Dahlbom <[email protected]> * Exemptions (#4) * Update Docs (#759) * Update Docs * Update Docs Wording * Update release pipeline (#756) * disable * update * Update Exemption * PolicyExemption Retry * Invoke-AzOpsRestMethod Retry * Update Retry * remove square brackets --------- Co-authored-by: Johan Dahlbom <[email protected]> * update throttle * ignore exit code * Update --------- Co-authored-by: Jesper Fajers <[email protected]>
Azure · Feb 24, 2023 · bc21847 · bc21847
1 parent d7133b7
commit bc21847
Show file tree

Hide file tree

Showing 12 changed files with 78 additions and 21 deletions.
diff --git a/scripts/Remove-AzOpsTestsDeployment.ps1 b/scripts/Remove-AzOpsTestsDeployment.ps1
@@ -83,13 +83,13 @@
                 $script:policySetDefinitions | Remove-AzPolicySetDefinition -Force -Confirm:$false -ErrorAction SilentlyContinue
                 # Collect and cleanup deployment jobs
                 $azTenantDeploymentJobs = Get-AzTenantDeployment
-                $azTenantDeploymentJobs | ForEach-Object -ThrottleLimit 20 -Parallel {
+                $azTenantDeploymentJobs | ForEach-Object -ThrottleLimit 10 -Parallel {
                     Write-PSFMessage -Level Verbose -Message "Executing test AzDeployment cleanup thread of $($_.DeploymentName)" -FunctionName "Remove-AzOpsTestsDeployment"
                     $_ | Remove-AzTenantDeployment -Confirm:$false
                 }
                 Get-AzManagementGroupDeployment -ManagementGroupId "cd35e23c-537f-4553-a280-f5a60033a446" | Remove-AzManagementGroupDeployment -Confirm:$false
                 $azDeploymentJobs = Get-AzDeployment
-                $azDeploymentJobs | ForEach-Object -ThrottleLimit 20 -Parallel {
+                $azDeploymentJobs | ForEach-Object -ThrottleLimit 10 -Parallel {
                     Write-PSFMessage -Level Verbose -Message "Executing test AzDeployment cleanup thread of $($_.DeploymentName)" -FunctionName "Remove-AzOpsTestsDeployment"
                     $_ | Remove-AzDeployment -Confirm:$false
                 }

diff --git a/src/functions/Initialize-AzOpsEnvironment.ps1 b/src/functions/Initialize-AzOpsEnvironment.ps1
@@ -61,6 +61,14 @@
         if (-not $IgnoreContextCheck -and $azContextTenants.Count -gt 1) {
             Stop-PSFFunction -String 'Initialize-AzOpsEnvironment.AzureContext.TooMany' -StringValues $azContextTenants.Count, ($azContextTenants -join ',') -EnableException $true -Cmdlet $PSCmdlet
         }
+
+        # Adjust ThrottleLimit from previously default 10 to 5 if system has less than 2 cores
+        $cpuCores = if ($IsWindows) { $env:NUMBER_OF_PROCESSORS } else { Invoke-AzOpsNativeCommand -ScriptBlock { nproc --all } -IgnoreExitcode }
+        $throttleLimit = (Get-PSFConfig -Module AzOps -Name Core.ThrottleLimit).Value
+        if (-not[string]::IsNullOrEmpty($cpuCores) -and $cpuCores -le 2 -and $throttleLimit -gt 5) {
+            Write-PSFMessage -Level Warning -String 'Initialize-AzOpsEnvironment.ThrottleLimit.Adjustment' -StringValues $throttleLimit, $cpuCores
+            Set-PSFConfig -Module AzOps -Name Core.ThrottleLimit -Value 5
+        }
     }
 
     process {

diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1
@@ -1,6 +1,6 @@
 Set-PSFConfig -Module AzOps -Name Core.AutoGeneratedTemplateFolderPath -Value "." -Initialize -Validation string -Description 'Auto-Generated Template Folder Path i.e. ./Az'
 Set-PSFConfig -Module AzOps -Name Core.AutoInitialize -Value $false -Initialize -Validation bool -Description '-'
-Set-PSFConfig -Module AzOps -Name Core.DeletionSupportedResourceType -Value @('Microsoft.Authorization/locks', 'locks','Microsoft.Authorization/policyAssignments','policyAssignments','Microsoft.Authorization/policyDefinitions','policyDefinitions','Microsoft.Authorization/policyExemptions','policyExemptions','Microsoft.Authorization/policySetDefinitions','policySetDefinitions','Microsoft.Authorization/roleAssignments','roleAssignments') -Initialize -Validation stringarray -Description 'Global flag declaring resource types supported for deletion by AzOps.'
+Set-PSFConfig -Module AzOps -Name Core.DeletionSupportedResourceType -Value @('Microsoft.Authorization/locks', 'locks', 'Microsoft.Authorization/policyAssignments', 'policyAssignments', 'Microsoft.Authorization/policyDefinitions', 'policyDefinitions', 'Microsoft.Authorization/policyExemptions', 'policyExemptions', 'Microsoft.Authorization/policySetDefinitions', 'policySetDefinitions', 'Microsoft.Authorization/roleAssignments', 'roleAssignments') -Initialize -Validation stringarray -Description 'Global flag declaring resource types supported for deletion by AzOps.'
 Set-PSFConfig -Module AzOps -Name Core.DefaultDeploymentRegion -Value northeurope -Initialize -Validation string -Description 'Default deployment region for state deployments (ARM region, not region where a resource is deployed)'
 Set-PSFConfig -Module AzOps -Name Core.EnrollmentAccountPrincipalName -Value '' -Initialize -Validation stringorempty -Description '-'
 Set-PSFConfig -Module AzOps -Name Core.ExcludedSubOffer -Value 'AzurePass_2014-09-01', 'FreeTrial_2014-09-01', 'AAD_2015-09-01' -Initialize -Validation stringarray -Description 'Excluded QuotaID'
@@ -19,10 +19,10 @@ Set-PSFConfig -Module AzOps -Name Core.SkipLock -Value $true -Initialize -Valida
 Set-PSFConfig -Module AzOps -Name Core.SkipPolicy -Value $false -Initialize -Validation bool -Description '-'
 Set-PSFConfig -Module AzOps -Name Core.SkipResource -Value $true -Initialize -Validation bool -Description 'Global flag to indicate whether resource should be discovered or not. Requires SkipResourceGroup to be false.'
 Set-PSFConfig -Module AzOps -Name Core.SkipResourceGroup -Value $false -Initialize -Validation bool -Description 'Global flag to indicate whether resource group should be discovered or not'
-Set-PSFConfig -Module AzOps -Name Core.SkipResourceType -Value @('Microsoft.VSOnline/plans','Microsoft.PowerPlatform/accounts','Microsoft.PowerPlatform/enterprisePolicies') -Initialize -Validation stringarray -Description 'Global flag to skip discovery of specific Resource types.'
+Set-PSFConfig -Module AzOps -Name Core.SkipResourceType -Value @('Microsoft.VSOnline/plans', 'Microsoft.PowerPlatform/accounts', 'Microsoft.PowerPlatform/enterprisePolicies') -Initialize -Validation stringarray -Description 'Global flag to skip discovery of specific Resource types.'
 Set-PSFConfig -Module AzOps -Name Core.SkipRole -Value $false -Initialize -Validation bool -Description '-'
 Set-PSFConfig -Module AzOps -Name Core.State -Value (Join-Path $pwd -ChildPath "root") -Initialize -Validation string -Description 'Folder to store AzOpsState artefact'
 Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID or Display Name that matches the filter. Powershell filter that matches with like operator is supported.'
 Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'parameter file suffix to look for'
-Set-PSFConfig -Module AzOps -Name Core.ThrottleLimit -Value 10 -Initialize -Validation integer -Description 'Throttle limit used in Foreach-Object -Parallel for resource/subscription discovery'
-Set-PSFConfig -Module AzOps -Name Core.WhatifExcludedChangeTypes -Value @('NoChange','Ignore') -Initialize -Validation stringarray -Description 'Exclude specific change types from WhatIf operations.'
+Set-PSFConfig -Module AzOps -Name Core.ThrottleLimit -Value 5 -Initialize -Validation integer -Description 'Throttle limit used in Foreach-Object -Parallel for resource/subscription discovery'
+Set-PSFConfig -Module AzOps -Name Core.WhatifExcludedChangeTypes -Value @('NoChange', 'Ignore') -Initialize -Validation stringarray -Description 'Exclude specific change types from WhatIf operations.'
diff --git a/src/internal/functions/ConvertTo-AzOpsState.ps1 b/src/internal/functions/ConvertTo-AzOpsState.ps1
@@ -98,10 +98,6 @@
         else {
             $objectFilePath = $ExportPath
         }
-        # Check for invalid characters "[" or "]"
-        if ($objectFilePath -match [regex]::Escape("[") -or $objectFilePath -match [regex]::Escape("]")) {
-            Stop-PSFFunction -String 'ConvertTo-AzOpsState.File.InvalidCharacter' -StringValues $objectFilePath -EnableException $true -Cmdlet $PSCmdlet
-        }
         # Create folder structure if it doesn't exist
         if (-not (Test-Path -Path $objectFilePath)) {
             Write-PSFMessage -Level Verbose -String 'ConvertTo-AzOpsState.File.Create' -StringValues $objectFilePath

diff --git a/src/internal/functions/Get-AzOpsPolicyExemption.ps1 b/src/internal/functions/Get-AzOpsPolicyExemption.ps1
@@ -36,7 +36,18 @@
                 Write-PSFMessage -Level Debug -String 'Get-AzOpsPolicyExemption.ResourceGroup' -StringValues $ScopeObject.ResourceGroup -Target $ScopeObject
             }
         }
-        Get-AzPolicyExemption -Scope $ScopeObject.Scope -WarningAction SilentlyContinue -ErrorAction Continue | Where-Object ResourceId -match $ScopeObject.scope -ErrorAction Continue
+        try {
+            $parameters = @{
+                Scope = $ScopeObject.Scope
+            }
+            # Gather policyExemption with retry and backoff support from Invoke-AzOpsScriptBlock
+            Invoke-AzOpsScriptBlock -ArgumentList $parameters -ScriptBlock {
+                Get-AzPolicyExemption @parameters -WarningAction SilentlyContinue -ErrorAction Stop | Where-Object ResourceId -match $parameters.Scope
+            } -RetryCount 3 -RetryWait 5 -RetryType Exponential -ErrorAction Stop
+        }
+        catch {
+            Write-PSFMessage -Level Warning -Message $_ -FunctionName "Get-AzOpsPolicyExemption"
+        }
     }
 
 }
diff --git a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 b/src/internal/functions/Get-AzOpsResourceDefinition.ps1
@@ -156,7 +156,9 @@
                                 }
                                 if (-not $using:SkipPolicy) {
                                     $policyExemptions = Get-AzOpsPolicyExemption -ScopeObject $ScopeObject
-                                    $policyExemptions | ConvertTo-AzOpsState -StatePath $runspaceData.Statepath
+                                    if ($policyExemptions) {
+                                        $policyExemptions | ConvertTo-AzOpsState -StatePath $runspaceData.Statepath
+                                    }
                                 }
                                 if (-not $using:SkipRole) {
                                     Get-AzOpsRole -ScopeObject $ScopeObject -StatePath $runspaceData.Statepath
@@ -197,7 +199,9 @@
                         }
                         if (-not $using:SkipPolicy) {
                             $policyExemptions = Get-AzOpsPolicyExemption -ScopeObject $scopeObject
-                            $policyExemptions | ConvertTo-AzOpsState -StatePath $runspaceData.Statepath
+                            if ($policyExemptions) {
+                                $policyExemptions | ConvertTo-AzOpsState -StatePath $runspaceData.Statepath
+                            }
                         }
                         if (-not $using:SkipLock) {
                             Get-AzOpsResourceLock -ScopeObject $scopeObject -StatePath $runspaceData.Statepath
@@ -262,7 +266,9 @@
                             }
                             if (-not $using:SkipPolicy) {
                                 $policyExemptions = Get-AzOpsPolicyExemption -ScopeObject $rgScopeObject
-                                $policyExemptions | ConvertTo-AzOpsState -StatePath $runspaceData.Statepath
+                                if ($policyExemptions) {
+                                    $policyExemptions | ConvertTo-AzOpsState -StatePath $runspaceData.Statepath
+                                }
                             }
                             if (-not $using:SkipRole) {
                                 Get-AzOpsRole -ScopeObject $rgScopeObject -StatePath $runspaceData.Statepath

diff --git a/src/internal/functions/Get-AzOpsResourceLock.ps1 b/src/internal/functions/Get-AzOpsResourceLock.ps1
@@ -37,8 +37,18 @@
                 Write-PSFMessage -Level Verbose -String 'Get-AzOpsResourceLock.ResourceGroup' -StringValues $ScopeObject.ResourceGroup -Target $ScopeObject
             }
         }
-        # Gather resource locks at scopeObject
-        $resourceLocks = Get-AzResourceLock -Scope $ScopeObject.Scope -AtScope -ErrorAction SilentlyContinue | Where-Object {$($_.ResourceID.Substring(0, $_.ResourceId.LastIndexOf('/'))) -Like ("$($ScopeObject.scope)/providers/Microsoft.Authorization/locks")}
+        try {
+            $parameters = @{
+                Scope = $ScopeObject.Scope
+            }
+            # Gather resource locks at scopeObject with retry and backoff support from Invoke-AzOpsScriptBlock
+            $resourceLocks = Invoke-AzOpsScriptBlock -ArgumentList $parameters -ScriptBlock {
+                Get-AzResourceLock @parameters -AtScope -ErrorAction Stop | Where-Object {$($_.ResourceID.Substring(0, $_.ResourceId.LastIndexOf('/'))) -Like ("$($parameters.Scope)/providers/Microsoft.Authorization/locks")}
+            } -RetryCount 3 -RetryWait 5 -RetryType Exponential -ErrorAction Stop
+        }
+        catch {
+            Write-PSFMessage -Level Warning -Message $_ -FunctionName "Get-AzOpsResourceLock"
+        }
         if ($resourceLocks) {
             # Process each resource lock
             foreach ($lock in $resourceLocks) {

diff --git a/src/internal/functions/Get-AzOpsRoleAssignment.ps1 b/src/internal/functions/Get-AzOpsRoleAssignment.ps1
@@ -23,7 +23,19 @@
         Write-PSFMessage -Level Debug -String 'Get-AzOpsRoleAssignment.Processing' -StringValues $ScopeObject -Target $ScopeObject
         $apiVersion = (($script:AzOpsResourceProvider | Where-Object {$_.ProviderNamespace -eq 'Microsoft.Authorization'}).ResourceTypes | Where-Object {$_.ResourceTypeName -eq 'roleAssignments'}).ApiVersions | Select-Object -First 1
         $path = "$($scopeObject.Scope)/providers/Microsoft.Authorization/roleAssignments?api-version=$apiVersion&`$filter=atScope()"
-        $roleAssignments = Invoke-AzOpsRestMethod -Path $path -Method GET
+        try {
+            $parameters = @{
+                Path = $path
+                Method = 'GET'
+            }
+            # Gather roleAssignment with retry and backoff support from Invoke-AzOpsScriptBlock
+            $roleAssignments = Invoke-AzOpsScriptBlock -ArgumentList $parameters -ScriptBlock {
+                Invoke-AzOpsRestMethod @parameters -ErrorAction Stop
+            } -RetryCount 3 -RetryWait 5 -RetryType Exponential -ErrorAction Stop
+        }
+        catch {
+            Write-PSFMessage -Level Warning -Message $_ -FunctionName "Get-AzOpsRoleAssignment"
+        }
         if ($roleAssignments) {
             $roleAssignmentMatch = @()
             foreach ($roleAssignment in $roleAssignments) {

diff --git a/src/internal/functions/Get-AzOpsRoleDefinition.ps1 b/src/internal/functions/Get-AzOpsRoleDefinition.ps1
@@ -23,7 +23,19 @@
         Write-PSFMessage -Level Debug -String 'Get-AzOpsRoleDefinition.Processing' -StringValues $ScopeObject -Target $ScopeObject
         $apiVersion = (($script:AzOpsResourceProvider | Where-Object {$_.ProviderNamespace -eq 'Microsoft.Authorization'}).ResourceTypes | Where-Object {$_.ResourceTypeName -eq 'roleDefinitions'}).ApiVersions | Select-Object -First 1
         $path = "$($scopeObject.Scope)/providers/Microsoft.Authorization/roleDefinitions?api-version=$apiVersion&`$filter=type+eq+'CustomRole'"
-        $roleDefinitions = Invoke-AzOpsRestMethod -Path $path -Method GET
+        try {
+            $parameters = @{
+                Path = $path
+                Method = 'GET'
+            }
+            # Gather roleDefinitions with retry and backoff support from Invoke-AzOpsScriptBlock
+            $roleDefinitions = Invoke-AzOpsScriptBlock -ArgumentList $parameters -ScriptBlock {
+                Invoke-AzOpsRestMethod @parameters -ErrorAction Stop
+            } -RetryCount 3 -RetryWait 5 -RetryType Exponential -ErrorAction Stop
+        }
+        catch {
+            Write-PSFMessage -Level Warning -Message $_ -FunctionName "Get-AzOpsRoleDefinition"
+        }
         if ($roleDefinitions) {
             $roleDefinitionsMatch = @()
             foreach ($roleDefinition in $roleDefinitions) {

diff --git a/src/internal/functions/Invoke-AzOpsRestMethod.ps1 b/src/internal/functions/Invoke-AzOpsRestMethod.ps1
@@ -37,7 +37,7 @@
                 }
             }
             catch {
-                Write-PSFMessage -Level Warning -String 'Invoke-AzOpsRestMethod.Processing.Warning' -StringValues $_, $Path
+                Write-PSFMessage -Level Error -String 'Invoke-AzOpsRestMethod.Processing.Error' -StringValues $_, $Path
             }
         }
         while ($path)

diff --git a/src/internal/functions/Remove-AzOpsInvalidCharacter.ps1 b/src/internal/functions/Remove-AzOpsInvalidCharacter.ps1
@@ -48,6 +48,8 @@
                 }
             }
         }
+        # Always remove square brackets
+        $String = $String -replace "(\[|\])",""
         Write-PSFMessage -Level Verbose -String 'Remove-AzOpsInvalidCharacter.Completed' -StringValues $String -FunctionName 'Remove-AzOpsInvalidCharacter'
         # Return processed string
         return $String

diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1
@@ -56,7 +56,6 @@
     'ConvertTo-AzOpsState.Exporting'                                                = 'Exporting AzOpsState to {0}' # $resourceData.ObjectFilePath
     'ConvertTo-AzOpsState.Exporting.Default'                                        = 'Exporting input resource to AzOpsState to {0}' # $resourceData.ObjectFilePath
     'ConvertTo-AzOpsState.File.Create'                                              = 'AzOpsState file not found. Creating new: {0}' # $ObjectFilePath
-    'ConvertTo-AzOpsState.File.InvalidCharacter'                                    = 'The specified AzOpsState file contains invalid characters (remove any "[" or "]" characters)! Skipping {0}' # $ObjectFilePath
     'ConvertTo-AzOpsState.File.UseExisting'                                         = 'AzOpsState file is found. Using existing file: {0}' # $ObjectFilePath
     'ConvertTo-AzOpsState.NoExportPath'                                             = 'No export path found for {0}. Ensure the original data type remains intact or specify an -ExportPath' # $Resource
     'ConvertTo-AzOpsState.Processing'                                               = 'Processing input: {0}' # $Resource
@@ -142,6 +141,7 @@
     'Initialize-AzOpsEnvironment.ManagementGroup.NoManagementGroupAccess'           = 'No management group access, discovery will happen from subscription scope(s)'
     'Initialize-AzOpsEnvironment.Processing'                                        = 'Processing AzOps environment' #
     'Initialize-AzOpsEnvironment.Processing.Completed'                              = 'AzOps environment initialization concluded' #
+    'Initialize-AzOpsEnvironment.ThrottleLimit.Adjustment'                          = 'Adjusting AzOps.Core.ThrottleLimit from {0} to 5 due to available CPU Cores ({1}) to ensure reliable and performant pipeline execution. For further details, refer to: https://github.com/azure/azops/wiki/performance-considerations' # $throttleLimit, $cpuCores
     'Initialize-AzOpsEnvironment.UsingCache'                                        = 'Using cached values for AzOpsAzManagementGroup and AzOpsSubscriptions' #
 
     'Invoke-AzOpsPull.Deleting.State'                                               = 'Removing state in {0}' # $StatePath
@@ -161,7 +161,7 @@
     'Invoke-AzOpsPull.Validating.ResourceGroupDiscovery.Failed'                     = 'SkipResource set to false or SkipChildResource set to false requires SkipResourceGroup to be set to false. Change value for SkipResourceGroup and retry operation. {0} https://github.com/azure/azops/wiki/settings' #
 
     'Invoke-AzOpsRestMethod.Processing'                                             = 'Invoke-AzRestMethod processing path: [{0}]' # $Path
-    'Invoke-AzOpsRestMethod.Processing.Warning'                                     = 'Invoke-AzRestMethod received [{0}] while processing: [{1}]' # $_, $Path
+    'Invoke-AzOpsRestMethod.Processing.Error'                                       = 'Invoke-AzRestMethod received [{0}] while processing: [{1}]' # $_, $Path
     'Invoke-AzOpsRestMethod.Processing.RateLimit'                                   = 'Invoke-AzRestMethod is throttled while processing: [{0}], going to sleep for {1} seconds' # $Path, $_.value
 
     'Invoke-AzOpsPush.Change.AddModify'                                             = 'Adding or modifying:' #
-Original file line number
+Diff line change
@@ Expand Up / @@ -37,7 +37,7 @@ @@
                     }
                 }
                 catch {
-                    Write-PSFMessage -Level Warning -String 'Invoke-AzOpsRestMethod.Processing.Warning' -StringValues $_, $Path
+                    Write-PSFMessage -Level Error -String 'Invoke-AzOpsRestMethod.Processing.Error' -StringValues $_, $Path
                 }
             }
             while ($path)
@@ Expand Down @@