From 13612fce3e4a36e37d6751496c6479dab45930e3 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 25 Jan 2024 20:20:48 +0000 Subject: [PATCH 01/47] UpdateBase --- src/functions/Invoke-AzOpsPush.ps1 | 307 ++++++++++-------- src/internal/configurations/Core.ps1 | 1 + .../functions/New-AzOpsDeployment.ps1 | 13 +- .../functions/Remove-AzOpsDeployment.ps1 | 78 ++++- .../functions/Remove-AzResourceRaw.ps1 | 71 ++++ .../Remove-AzResourceRawRecursive.ps1 | 69 ++++ src/localized/en-us/Strings.psd1 | 42 ++- 7 files changed, 427 insertions(+), 154 deletions(-) create mode 100644 src/internal/functions/Remove-AzResourceRaw.ps1 create mode 100644 src/internal/functions/Remove-AzResourceRawRecursive.ps1 diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 14a1087d..31b08f1a 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -43,6 +43,66 @@ begin { #region Utility Functions + function New-List { + [CmdletBinding()] + param ( + [string[]] + $FileSet, + [string] + $FilePath, + [string] + $AzOpsMainTemplate, + [string[]] + $ConvertedTemplate, + [string[]] + $ConvertedParameter + ) + + # Avoid duplicate entries in the deployment list + if ($FilePath.EndsWith(".parameters.json")) { + if ($FileSet -contains $FilePath.Replace(".parameters.json", ".json") -or $FileSet -contains $FilePath.Replace(".parameters.json", ".bicep")) { + continue + } + } + if ($FilePath.EndsWith(".bicepparam")) { + if ($FileSet -contains $FilePath.Replace(".bicepparam", ".bicep")) { + continue + } + } + + # Handle Bicep templates + if ($FilePath.EndsWith(".bicep")) { + $transpiledTemplatePaths = ConvertFrom-AzOpsBicepTemplate -BicepTemplatePath $FilePath -ConvertedTemplate $ConvertedTemplate -ConvertedParameter $ConvertedParameter + if ($true -eq $transpiledTemplatePaths.transpiledTemplateNew) { + $ConvertedTemplate += $transpiledTemplatePaths.transpiledTemplatePath + } + if ($true -eq $transpiledTemplatePaths.transpiledParametersNew) { + $ConvertedParameter += $transpiledTemplatePaths.transpiledParametersPath + } + $FilePath = $transpiledTemplatePaths.transpiledTemplatePath + } + + try { + $scopeObject = New-AzOpsScope -Path $FilePath -StatePath $StatePath -ErrorAction Stop + } + catch { + Write-AzOpsMessage -LogLevel Warning -LogString 'Invoke-AzOpsPush.Scope.Failed' -LogStringValues $FilePath -Target $FilePath -ErrorRecord $_ + continue + } + + $resolvedArmFileAssociation = Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $FilePath -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $ConvertedTemplate -ConvertedParameter $ConvertedParameter + if ($resolvedArmFileAssociation) { + foreach ($fileAssociation in $resolvedArmFileAssociation) { + if ($true -eq $transpiledTemplatePaths.transpiledTemplateNew -and $fileAssociation.TemplateFilePath -eq $transpiledTemplatePaths.transpiledTemplatePath) { + $fileAssociation.TranspiledTemplateNew = $true + } + if ($true -eq $transpiledTemplatePaths.TranspiledParametersNew -and $fileAssociation.TemplateParameterFilePath -eq $transpiledTemplatePaths.transpiledParametersPath) { + $fileAssociation.TranspiledParametersNew = $true + } + } + return $resolvedArmFileAssociation + } + } function Resolve-ArmFileAssociation { [CmdletBinding()] param ( @@ -347,41 +407,8 @@ #endregion Deploy State $deploymentList = foreach ($addition in $addModifySet | Where-Object { $_ -match ((Get-Item $StatePath).Name) }) { - - # Avoid duplicate entries in the deployment list - if ($addition.EndsWith(".parameters.json")) { - if ($addModifySet -contains $addition.Replace(".parameters.json", ".json") -or $addModifySet -contains $addition.Replace(".parameters.json", ".bicep")) { - continue - } - } - if ($addition.EndsWith(".bicepparam")) { - if ($addModifySet -contains $addition.Replace(".bicepparam", ".bicep")) { - continue - } - } - - # Handle Bicep templates - if ($addition.EndsWith(".bicep")) { - $transpiledTemplatePaths = ConvertFrom-AzOpsBicepTemplate -BicepTemplatePath $addition -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter - if ($true -eq $transpiledTemplatePaths.transpiledTemplateNew) { - $AzOpsTranspiledTemplate += $transpiledTemplatePaths.transpiledTemplatePath - } - if ($true -eq $transpiledTemplatePaths.transpiledParametersNew) { - $AzOpsTranspiledParameter += $transpiledTemplatePaths.transpiledParametersPath - } - $addition = $transpiledTemplatePaths.transpiledTemplatePath - } - - try { - $scopeObject = New-AzOpsScope -Path $addition -StatePath $StatePath -ErrorAction Stop - } - catch { - Write-AzOpsMessage -LogLevel Warning -LogString 'Invoke-AzOpsPush.Scope.Failed' -LogStringValues $addition -Target $addition -ErrorRecord $_ - continue - } - - $resolvedArmFileAssociation = Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $addition -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter - foreach ($fileAssociation in $resolvedArmFileAssociation) { + $deployFileAssociationList = New-List -FilePath $addition -FileSet $addModifySet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter + foreach ($fileAssociation in $deployFileAssociationList) { if ($true -eq $fileAssociation.transpiledTemplateNew) { $AzOpsTranspiledTemplate += $fileAssociation.TemplateFilePath } @@ -389,35 +416,20 @@ $AzOpsTranspiledParameter += $fileAssociation.TemplateParameterFilePath } } - $resolvedArmFileAssociation + $deployFileAssociationList } $deletionList = foreach ($deletion in $deleteSet | Where-Object { $_ -match ((Get-Item $StatePath).Name) }) { - - if ($deletion.EndsWith(".bicep")) { - continue - } - - $templateContent = Get-Content $deletion | ConvertFrom-Json -AsHashtable - $schemavalue = '$schema' - if ($templateContent.$schemavalue -like "*deploymentParameters.json#" -and (-not($templateContent.parameters.input.value.type -in $DeletionSupportedResourceType))) { - Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.SkipUnsupportedResource' -LogStringValues $deletion -Target $scopeObject - continue - } - elseif ($templateContent.$schemavalue -like "*deploymentTemplate.json#" -and (-not($templateContent.resources[0].type -in $DeletionSupportedResourceType))) { - Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.SkipUnsupportedResource' -LogStringValues $deletion -Target $scopeObject - continue - } - - try { - $scopeObject = New-AzOpsScope -Path $deletion -StatePath $StatePath -ErrorAction Stop - } - catch { - Write-AzOpsMessage -LogLevel Warning -LogString 'Invoke-AzOpsPush.Scope.Failed' -LogStringValues $deletion, $StatePath -Target $deletion -ErrorRecord $_ - continue + $deletionFileAssociationList = New-List -FilePath $deletion -FileSet $deleteSet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter + foreach ($fileAssociation in $deletionFileAssociationList) { + if ($true -eq $fileAssociation.transpiledTemplateNew) { + $AzOpsTranspiledTemplate += $fileAssociation.TemplateFilePath + } + if ($true -eq $fileAssociation.transpiledParametersNew) { + $AzOpsTranspiledParameter += $fileAssociation.TemplateParameterFilePath + } } - - Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $deletion -AzOpsMainTemplate $AzOpsMainTemplate + $deletionFileAssociationList } #Required deletion order @@ -426,7 +438,8 @@ "policyExemptions", "policyAssignments", "policySetDefinitions", - "policyDefinitions" + "policyDefinitions", + "resourceGroups" ) #Sort 'deletionList' based on 'deletionListPriority' @@ -444,87 +457,119 @@ $uniqueDeployment = $deploymentList | Select-Object $uniqueProperties -Unique $deploymentResult = @() - #Determine what deployment pattern to adopt serial or parallel - if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true -and (Get-PSFConfigValue -FullName 'AzOps.Core.ParallelDeployMultipleTemplateParameterFiles') -eq $true) { - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.ParallelCondition' - # Group deployments based on TemplateFilePath - $groups = $uniqueDeployment | Group-Object -Property TemplateFilePath | Where-Object { $_.Count -ge '2' -and $_.Name -ne $(Get-Item $AzOpsMainTemplate).FullName } - if ($groups) { - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.ParallelGroup' - $processedTargets = @() - # Process each deployment and evaluate serial or parallel deployment pattern - foreach ($deployment in $uniqueDeployment) { - if ($deployment.TemplateFilePath -in $groups.Name -and $deployment -notin $processedTargets) { - # Deployment part of group association for parallel processing, process entire group as parallel deployment - $targets = $($groups | Where-Object { $_.Name -eq $deployment.TemplateFilePath }).Group - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Parallel' -LogStringValues $deployment.TemplateFilePath, $targets.Count - # Prepare Input Data for parallel processing - $runspaceData = @{ - AzOpsPath = "$($script:ModuleRoot)\AzOps.psd1" - StatePath = $StatePath - WhatIfPreference = $WhatIfPreference - runspace_AzOpsAzManagementGroup = $script:AzOpsAzManagementGroup - runspace_AzOpsSubscriptions = $script:AzOpsSubscriptions - runspace_AzOpsPartialRoot = $script:AzOpsPartialRoot - runspace_AzOpsResourceProvider = $script:AzOpsResourceProvider - } - # Pass deployment targets for parallel processing and output deployment result for later - $deploymentResult += $targets | Foreach-Object -ThrottleLimit (Get-PSFConfigValue -FullName 'AzOps.Core.ThrottleLimit') -Parallel { - $deployment = $_ - $runspaceData = $using:runspaceData - - Import-Module "$([PSFramework.PSFCore.PSFCoreHost]::ModuleRoot)/PSFramework.psd1" - $azOps = Import-Module $runspaceData.AzOpsPath -Force -PassThru - - & $azOps { - $script:AzOpsAzManagementGroup = $runspaceData.runspace_AzOpsAzManagementGroup - $script:AzOpsSubscriptions = $runspaceData.runspace_AzOpsSubscriptions - $script:AzOpsPartialRoot = $runspaceData.runspace_AzOpsPartialRoot - $script:AzOpsResourceProvider = $runspaceData.runspace_AzOpsResourceProvider + if ($uniqueDeployment) { + #Determine what deployment pattern to adopt serial or parallel + if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true -and (Get-PSFConfigValue -FullName 'AzOps.Core.ParallelDeployMultipleTemplateParameterFiles') -eq $true) { + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.ParallelCondition' + # Group deployments based on TemplateFilePath + $groups = $uniqueDeployment | Group-Object -Property TemplateFilePath | Where-Object { $_.Count -ge '2' -and $_.Name -ne $(Get-Item $AzOpsMainTemplate).FullName } + if ($groups) { + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.ParallelGroup' + $processedTargets = @() + # Process each deployment and evaluate serial or parallel deployment pattern + foreach ($deployment in $uniqueDeployment) { + if ($deployment.TemplateFilePath -in $groups.Name -and $deployment -notin $processedTargets) { + # Deployment part of group association for parallel processing, process entire group as parallel deployment + $targets = $($groups | Where-Object { $_.Name -eq $deployment.TemplateFilePath }).Group + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Parallel' -LogStringValues $deployment.TemplateFilePath, $targets.Count + # Prepare Input Data for parallel processing + $runspaceData = @{ + AzOpsPath = "$($script:ModuleRoot)\AzOps.psd1" + StatePath = $StatePath + WhatIfPreference = $WhatIfPreference + runspace_AzOpsAzManagementGroup = $script:AzOpsAzManagementGroup + runspace_AzOpsSubscriptions = $script:AzOpsSubscriptions + runspace_AzOpsPartialRoot = $script:AzOpsPartialRoot + runspace_AzOpsResourceProvider = $script:AzOpsResourceProvider } - - & $azOps { - $deployment | New-AzOpsDeployment -WhatIf:$runspaceData.WhatIfPreference - } - } -UseNewRunspace - Clear-PSFMessage - # Add targets to processed list to avoid duplicate deployment - $processedTargets += $targets - } - elseif ($deployment -notin $processedTargets) { - # Deployment not part of group association for parallel processing, process this as serial deployment - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Serial' -LogStringValues $deployment.Count - $deploymentResult += $deployment | New-AzOpsDeployment -WhatIf:$WhatIfPreference - } - else { - # Deployment already processed by group association from parallel processing, skip this duplicate deployment - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Skip' -LogStringValues $deployment.TemplateFilePath, $deployment.TemplateParameterFilePath + # Pass deployment targets for parallel processing and output deployment result for later + $deploymentResult += $targets | Foreach-Object -ThrottleLimit (Get-PSFConfigValue -FullName 'AzOps.Core.ThrottleLimit') -Parallel { + $deployment = $_ + $runspaceData = $using:runspaceData + + Import-Module "$([PSFramework.PSFCore.PSFCoreHost]::ModuleRoot)/PSFramework.psd1" + $azOps = Import-Module $runspaceData.AzOpsPath -Force -PassThru + + & $azOps { + $script:AzOpsAzManagementGroup = $runspaceData.runspace_AzOpsAzManagementGroup + $script:AzOpsSubscriptions = $runspaceData.runspace_AzOpsSubscriptions + $script:AzOpsPartialRoot = $runspaceData.runspace_AzOpsPartialRoot + $script:AzOpsResourceProvider = $runspaceData.runspace_AzOpsResourceProvider + } + + & $azOps { + $deployment | New-AzOpsDeployment -WhatIf:$runspaceData.WhatIfPreference + } + } -UseNewRunspace + Clear-PSFMessage + # Add targets to processed list to avoid duplicate deployment + $processedTargets += $targets + } + elseif ($deployment -notin $processedTargets) { + # Deployment not part of group association for parallel processing, process this as serial deployment + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Serial' -LogStringValues $deployment.Count + $deploymentResult += $deployment | New-AzOpsDeployment -WhatIf:$WhatIfPreference + } + else { + # Deployment already processed by group association from parallel processing, skip this duplicate deployment + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Skip' -LogStringValues $deployment.TemplateFilePath, $deployment.TemplateParameterFilePath + } } } - } - else { - # No deployments with matching TemplateFilePath identified - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Serial' -LogStringValues $deployment.Count + else { + # No deployments with matching TemplateFilePath identified + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Serial' -LogStringValues $deployment.Count + $deploymentResult += $uniqueDeployment | New-AzOpsDeployment -WhatIf:$WhatIfPreference + } + } else { + # Perform serial deployment only + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Serial' -LogStringValues $uniqueDeployment.Count $deploymentResult += $uniqueDeployment | New-AzOpsDeployment -WhatIf:$WhatIfPreference } - } else { - # Perform serial deployment only - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Deployment.Serial' -LogStringValues $uniqueDeployment.Count - $deploymentResult += $uniqueDeployment | New-AzOpsDeployment -WhatIf:$WhatIfPreference - } - if ($deploymentResult) { - # Output deploymentResult outside module - $deploymentResult - #Process deploymentResult and output result - foreach ($result in $deploymentResult) { - Set-AzOpsWhatIfOutput -FilePath $result.filePath -ParameterFilePath $result.parameterFilePath -Results $result.results + if ($deploymentResult) { + # Output deploymentResult outside module + $deploymentResult + #Process deploymentResult and output result + foreach ($result in $deploymentResult) { + Set-AzOpsWhatIfOutput -FilePath $result.filePath -ParameterFilePath $result.parameterFilePath -Results $result.results + } } } #Removal of Supported resourceTypes - $uniqueProperties = 'Scope', 'TemplateFilePath', 'TemplateParameterFilePath' $removalJob = $deletionList | Select-Object $uniqueProperties -Unique | Remove-AzOpsDeployment -WhatIf:$WhatIfPreference + if ($removalJob.FullyQualifiedResourceId.Count -gt 0) { + $retry = $removalJob | Where-Object { $_.Status -eq 'failed' } + if ($retry) { + Write-AzOpsMessage -LogLevel Verbose -LogString 'Invoke-AzOpsPush.Deletion.Retry' -LogStringValues $retry.Count + Start-Sleep -Seconds 30 + foreach ($try in $retry) { $try.Status = $null } + $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry + $removeActionFail = $removeActionRecursive | Where-Object { $_.Status -eq 'failed' } + if ($removeActionFail) { + Start-Sleep -Seconds 90 + $throwFail = $false + foreach ($fail in $removeActionFail) { + $resource = $null + Set-AzOpsContext -ScopeObject $fail.ScopeObject + if ($fail.FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $fail.FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { + $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $fail.FullyQualifiedResourceId } -ErrorAction SilentlyContinue + } + else { + $resource = Get-AzResource -ResourceId $fail.FullyQualifiedResourceId -ErrorAction SilentlyContinue + } + if ($resource) { + $throwFail = $true + Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Deletion.Failed' -LogStringValues $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath + } + } + if ($throwFail) { + throw + } + } + } + } if ($removalJob.dependencyMissing -eq $true) { Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Dependency.Missing' throw diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index 12cb50a1..bf54b150 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -1,6 +1,7 @@ Set-PSFConfig -Module AzOps -Name Core.ApplicationInsights -Value $false -Initialize -Validation bool -Description 'Global flag to turn on or off logging to Application Insight' Set-PSFConfig -Module AzOps -Name Core.AutoGeneratedTemplateFolderPath -Value "." -Initialize -Validation string -Description 'Auto-Generated Template Folder Path i.e. ./Az' Set-PSFConfig -Module AzOps -Name Core.AutoInitialize -Value $false -Initialize -Validation bool -Description '-' +Set-PSFConfig -Module AzOps -Name Core.CustomTemplateResourceDeletion -Value $false -Initialize -Validation bool -Description 'Global flag declaring on or off deletion of resources in custom template.' Set-PSFConfig -Module AzOps -Name Core.DeletionSupportedResourceType -Value @('Microsoft.Authorization/locks', 'locks', 'Microsoft.Authorization/policyAssignments', 'policyAssignments', 'Microsoft.Authorization/policyDefinitions', 'policyDefinitions', 'Microsoft.Authorization/policyExemptions', 'policyExemptions', 'Microsoft.Authorization/policySetDefinitions', 'policySetDefinitions', 'Microsoft.Authorization/roleAssignments', 'roleAssignments') -Initialize -Validation stringarray -Description 'Global flag declaring resource types supported for deletion by AzOps.' Set-PSFConfig -Module AzOps -Name Core.DefaultDeploymentRegion -Value northeurope -Initialize -Validation string -Description 'Default deployment region for state deployments (ARM region, not region where a resource is deployed)' Set-PSFConfig -Module AzOps -Name Core.EnrollmentAccountPrincipalName -Value '' -Initialize -Validation stringorempty -Description '-' diff --git a/src/internal/functions/New-AzOpsDeployment.ps1 b/src/internal/functions/New-AzOpsDeployment.ps1 index 926ccdde..2d8d94e2 100644 --- a/src/internal/functions/New-AzOpsDeployment.ps1 +++ b/src/internal/functions/New-AzOpsDeployment.ps1 @@ -18,6 +18,10 @@ The root folder under which to find the resource json. .PARAMETER Confirm If this switch is enabled, you will be prompted for confirmation before executing any operations that change state. + .PARAMETER WhatifExcludedChangeTypes + Exclude specific change types from WhatIf operations. + .PARAMETER WhatIfResultFormat + Accepts ResourceIdOnly or FullResourcePayloads. .PARAMETER WhatIf If this switch is enabled, no actions are performed but informational messages will be displayed that explain what would happen if the command were to run. .EXAMPLE @@ -53,7 +57,11 @@ $StatePath = (Get-PSFConfigValue -FullName 'AzOps.Core.State'), [string[]] - $WhatifExcludedChangeTypes = (Get-PSFConfigValue -FullName 'AzOps.Core.WhatifExcludedChangeTypes') + $WhatifExcludedChangeTypes = (Get-PSFConfigValue -FullName 'AzOps.Core.WhatifExcludedChangeTypes'), + + [string] + [ValidateSet("ResourceIdOnly","FullResourcePayloads")] + $WhatIfResultFormat ) @@ -97,6 +105,9 @@ 'SkipTemplateParameterPrompt' = $true 'Location' = $defaultDeploymentRegion } + if ($WhatIfResultFormat) { + $parameters.ResultFormat = $WhatIfResultFormat + } # Resource Groups excluding Microsoft.Resources/resourceGroups that needs to be submitted at subscription scope if ($scopeObject.resourcegroup -and $templateContent.resources[0].type -ne 'Microsoft.Resources/resourceGroups') { Write-AzOpsMessage -LogLevel Verbose -LogString 'New-AzOpsDeployment.ResourceGroup.Processing' -LogStringValues $scopeObject -Target $scopeObject diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 1c144267..3932209b 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -1,9 +1,14 @@ function Remove-AzOpsDeployment { + <# .SYNOPSIS - Deletion of supported resource types from azure according to AzOps.Core.DeletionSupportedResourceType. + Deletion of supported resource types AzOps.Core.DeletionSupportedResourceType and custom templates. .DESCRIPTION - Deletion of supported resource types from azure according to AzOps.Core.DeletionSupportedResourceType. + Deletion of supported resource types AzOps.Core.DeletionSupportedResourceType and custom templates. + .PARAMETER CustomTemplateResourceDeletion + Enable or disable, deletion of resources in custom templates. + .PARAMETER DeploymentName + Dummy name used to run Azure WhatIf deployment. .PARAMETER TemplateFilePath Path where the ARM templates can be found. .PARAMETER TemplateParameterFilePath @@ -21,6 +26,13 @@ [CmdletBinding(SupportsShouldProcess = $true)] param ( + [bool] + $CustomTemplateResourceDeletion = (Get-PSFConfigValue -FullName 'AzOps.Core.CustomTemplateResourceDeletion'), + + [Parameter(ValueFromPipelineByPropertyName = $true)] + [string] + $DeploymentName = "azops-template-deployment", + [Parameter(ValueFromPipelineByPropertyName = $true)] [string] $TemplateFilePath = (Get-PSFConfigValue -FullName 'AzOps.Core.MainTemplate'), @@ -176,9 +188,10 @@ return $results } } + $dependencyMissing = $null #Adjust TemplateParameterFilePath to compensate for policyDefinitions and policySetDefinitions usage of parameters.json - if ($TemplateParameterFilePath) { + if ($TemplateParameterFilePath -and $TemplateFilePath -eq (Get-PSFConfigValue -FullName 'AzOps.Core.MainTemplate')) { $TemplateFilePath = $TemplateParameterFilePath } #Deployment Name @@ -191,8 +204,13 @@ #endregion #region Validate it is AzOpsgenerated template $schemavalue = '$schema' + $customDeletion = $false if ($templateContent.metadata._generator.name -eq "AzOps" -or $templateContent.$schemavalue -like "*deploymentParameters.json#") { - Write-AzOpsMessage -LogLevel Verbose -LogString 'Remove-AzOpsDeployment.Metadata.Success' -LogStringValues $TemplateFilePath + Write-AzOpsMessage -LogLevel Verbose -LogString 'Remove-AzOpsDeployment.Metadata.AzOps' -LogStringValues $TemplateFilePath + } + elseif ($CustomTemplateResourceDeletion) { + Write-AzOpsMessage -LogLevel Verbose -LogString 'Remove-AzOpsDeployment.Metadata.Custom' -LogStringValues $TemplateFilePath + $customDeletion = $true } else { Write-AzOpsMessage -LogLevel Error -LogString 'Remove-AzOpsDeployment.Metadata.Failed' -LogStringValues $TemplateFilePath @@ -218,7 +236,7 @@ #endregion SetContext #region remove supported resources - if ($scopeObject.Resource -in $DeletionSupportedResourceType) { + if ($customDeletion -eq $false -and $scopeObject.Resource -in $DeletionSupportedResourceType) { $dependency = @() switch ($scopeObject.Resource) { # Check resource existance through optimal path @@ -301,5 +319,55 @@ Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.SkipDueToWhatIf' } } + elseif ($customDeletion -eq $true) { + $removalJob = New-AzOpsDeployment -DeploymentName $DeploymentName -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath -WhatIfResultFormat 'ResourceIdOnly' -WhatIf:$true + if ($removalJob.results.Changes.Count -gt 0) { + $retry = @() + foreach ($change in $removalJob.results.Changes) { + $resource = $null + if ($change.RelativeResourceId.StartsWith('Microsoft.Authorization/locks/')) { + $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $change.FullyQualifiedResourceId } -ErrorAction SilentlyContinue + } + else { + $resource = Get-AzResource -ResourceId $change.FullyQualifiedResourceId -ErrorAction SilentlyContinue + } + if ($resource) { + $results = 'What if successful:{1}Performing the operation:{1}Deletion of target resource {0}.' -f $change.FullyQualifiedResourceId, [environment]::NewLine + Write-AzOpsMessage -LogLevel Verbose -LogString 'Set-AzOpsWhatIfOutput.WhatIfResults' -LogStringValues $results + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Set-AzOpsWhatIfOutput.WhatIfFile' + Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true + if ($PSCmdlet.ShouldProcess("Remove $($change.FullyQualifiedResourceId)?")) { + $removeAction = Remove-AzResourceRaw -FullyQualifiedResourceId $change.FullyQualifiedResourceId -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath + if ($removeAction.Status -eq 'failed') { + $retry += $removeAction + } + } + else { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.SkipDueToWhatIf' + } + } + else { + Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.resource, $change.FullyQualifiedResourceId + $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $change.FullyQualifiedResourceId, [environment]::NewLine + Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true + } + + } + if ($retry.Count -gt 0) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.Resource.RetryCount' -LogStringValues $retry.Count + foreach ($try in $retry) { $try.Status = $null } + $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry + $removeActionRecursiveRemaining = $removeActionRecursive | Where-Object { $_.Status -eq 'failed' } + return $removeActionRecursiveRemaining + } + } + else { + # No resource to delete was found return + Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.Resource, $scopeObject.Scope + $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $scopeObject.scope, [environment]::NewLine + Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true + return + } + } } } \ No newline at end of file diff --git a/src/internal/functions/Remove-AzResourceRaw.ps1 b/src/internal/functions/Remove-AzResourceRaw.ps1 new file mode 100644 index 00000000..17400adc --- /dev/null +++ b/src/internal/functions/Remove-AzResourceRaw.ps1 @@ -0,0 +1,71 @@ +function Remove-AzResourceRaw { + + <# + .SYNOPSIS + Performs resource deletion in Azure at any scope. + .DESCRIPTION + Performs resource deletion in Azure with FullyQualifiedResourceId and ScopeObject. + .PARAMETER FullyQualifiedResourceId + Parameter containing FullyQualifiedResourceId of resource to delete. + .PARAMETER TemplateFilePath + Path where the ARM templates can be found. + .PARAMETER TemplateParameterFilePath + Path where the parameters of the ARM templates can be found. + .PARAMETER ScopeObject + Object used to set Azure context for removal operation. + .EXAMPLE + > Remove-AzResourceRaw -FullyQualifiedResourceId '/subscriptions//resourceGroups//providers/Microsoft.KeyVault/vaults/' -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath + Name Value + ---- ----- + FullyQualifiedResourceId /subscriptions//resourceGroups//providers/Microsoft.KeyVault/vaults/ + TemplateFilePath /root/managementgroup/subscription/resourcegroup/template.json + TemplateParameterFilePath /root/managementgroup/subscription/resourcegroup/template.parameters.json + ScopeObject ScopeObject + Status success + #> + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string] + $FullyQualifiedResourceId, + [string] + $TemplateFilePath, + [string] + $TemplateParameterFilePath, + [Parameter(Mandatory = $true)] + [AzOpsScope] + $ScopeObject + ) + + process { + $result = [PSCustomObject]@{ + FullyQualifiedResourceId = $FullyQualifiedResourceId + TemplateFilePath = $TemplateFilePath + TemplateParameterFilePath = $TemplateParameterFilePath + ScopeObject = $scopeObject + Status = 'success' + } + #region SetContext + Set-AzOpsContext -ScopeObject $ScopeObject + if ($FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { + $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $FullyQualifiedResourceId } -ErrorAction SilentlyContinue + } + else { + $resource = Get-AzResource -ResourceId $FullyQualifiedResourceId -ErrorAction SilentlyContinue + } + if ($resource) { + try { + $null = Remove-AzResource -ResourceId $FullyQualifiedResourceId -Force -ErrorAction Stop + } + catch { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.Failed' -LogStringValues $ScopeObject.resource, $FullyQualifiedResourceId + $result.Status = 'failed' + } + } + else { + $result.Status = 'notfound' + } + return $result + } +} \ No newline at end of file diff --git a/src/internal/functions/Remove-AzResourceRawRecursive.ps1 b/src/internal/functions/Remove-AzResourceRawRecursive.ps1 new file mode 100644 index 00000000..8ead19d4 --- /dev/null +++ b/src/internal/functions/Remove-AzResourceRawRecursive.ps1 @@ -0,0 +1,69 @@ +function Remove-AzResourceRawRecursive { + + <# + .SYNOPSIS + Performs recursive resource deletion in Azure at any scope. + .DESCRIPTION + Takes $InputObject and performs recursive resource deletion in Azure and exhaust any permutation. + .PARAMETER InputObject + Parameter containing items for processing. + .PARAMETER CurrentOrder + Internal parameter to track recursive progress. + .PARAMETER OutputObject + Parameter to track item processing and return result. + .EXAMPLE + > $successFullItems, $failedItems = Remove-AzResourceRawRecursive -InputObject $retry + Example of a $retry array with 6 items, the number of permutations will be 6×5×4×3×2×1=720 + #> + + [CmdletBinding()] + param ( + [array] + $InputObject, + [array] + $CurrentOrder = @(), + [array] + $OutputObject = @() + ) + + process { + if ($InputObject.Count -eq 0) { + # Base case: All items have been used, perform action on the current order + foreach ($item in $CurrentOrder) { + if ($item.Status -eq 'failed' -or $null -eq $item.Status) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRawRecursive.Processing' -LogStringValues $item.ScopeObject.resource, $item.FullyQualifiedResourceId + $result = Remove-AzResourceRaw -FullyQualifiedResourceId $item.FullyQualifiedResourceId -ScopeObject $item.ScopeObject -TemplateFilePath $item.TemplateFilePath -TemplateParameterFilePath $item.TemplateParameterFilePath + if ($result.Status -eq 'failed' -and $result.FullyQualifiedResourceId -notin $OutputObject.FullyQualifiedResourceId){ + $OutputObject += $result + } + } + } + return $OutputObject + } + else { + if ($InputObject -and $OutputObject) { + $filteredOutputObject = @() + foreach ($item in $InputObject) { + if ($item.FullyQualifiedResourceId -in $OutputObject.FullyQualifiedResourceId) { + foreach ($output in $OutputObject) { + if ($output.FullyQualifiedResourceId -eq $item.FullyQualifiedResourceId -and $output.Status -eq 'failed') { + $filteredOutputObject += $output + continue + } + } + } + } + if ($filteredOutputObject) { + $InputObject = $filteredOutputObject + } + } + # Recursive case: Try each item in the current position and recurse with the remaining items + foreach ($item in $InputObject) { + $remainingItems = $InputObject -ne $item + $newOrder = $CurrentOrder + $item + $OutputObject = Remove-AzResourceRawRecursive -InputObject $remainingItems -CurrentOrder $newOrder -OutputObject $OutputObject + } + return $OutputObject + } + } +} \ No newline at end of file diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 4f93a81b..b4c45029 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -190,6 +190,8 @@ 'Invoke-AzOpsPush.Change.AddModify.File' = ' {0}' # $item 'Invoke-AzOpsPush.Change.Delete' = 'Deleting:' # 'Invoke-AzOpsPush.Change.Delete.File' = ' {0}' # $item + 'Invoke-AzOpsPush.Deletion.Failed' = 'Deletion of resources {0}, has failed using templates: {1}, {2}, this could be due to delayed deletion acceptance from Azure, please investigate and take action.' # $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath + 'Invoke-AzOpsPush.Deletion.Retry' = 'Deletion of {0} resources unsuccessful, initiate final retry combination.' # $retry.Count 'Invoke-AzOpsPush.Deploy.ProviderFeature' = 'Invoking new state deployment - *.providerfeatures.json for a file {0}' # $addition 'Invoke-AzOpsPush.Deploy.ResourceProvider' = 'Invoking new state deployment - *.resourceproviders.json for a file {0}' # $addition 'Invoke-AzOpsPush.Deploy.Subscription' = 'Invoking new state deployment - *.subscription.json for a file {0}' # $addition @@ -269,31 +271,37 @@ 'Register-AzOpsResourceProvider.Provider.Register' = 'Registering provider {0}' # $resourceprovider.ProviderNamespace 'Remove-AzOpsDeployment.Processing' = 'Processing removal {0} for template {1}' # $removeJobName, $TemplateFilePath - 'Remove-AzOpsDeployment.Metadata.Failed' = 'Detected custom template: {0} . resource Deletion is currently only supported for AzOps Generated templates' #$TemplateFilePath - 'Remove-AzOpsDeployment.Metadata.Success' = 'Processing AzOps Generated Template File {0}' # $TemplateFilePath + 'Remove-AzOpsDeployment.Metadata.AzOps' = 'Resource deletion detected with AzOps generated template file {0}' # $TemplateFilePath + 'Remove-AzOpsDeployment.Metadata.Custom' = 'Resource deletion detected with custom template file: {0}' #$TemplateFilePath + 'Remove-AzOpsDeployment.Metadata.Failed' = 'Detected custom template: {0}, and Core.CustomTemplateResourceDeletion is not set to true' #$TemplateFilePath 'Remove-AzOpsDeployment.Scope.Failed' = 'Failed to resolve the scope for template {0}' # $TemplateFilePath 'Remove-AzOpsDeployment.Scope.Empty' = 'Unable to determine the scope of template {0}' # $TemplateFilePath 'Remove-AzOpsDeployment.SkipDueToWhatIf' = 'Skipping removal of resource due to WhatIf' # 'Remove-AzOpsDeployment.ResourceDependencyNested' = 'resource dependency {0} for complete deletion of {1} is outside of supported AzOps scope. Please remove this dependency in Azure without AzOps.'# $roleAssignmentId, $policyAssignment.ResourceId 'Remove-AzOpsDeployment.ResourceDependencyNotFound' = 'Missing resource dependency {0} for successfull deletion of {1}. Please add missing resource and retry.'# $resource.ResourceId, $scopeObject.Scope + 'Remove-AzOpsDeployment.Resource.RetryCount' = 'Retry deletion of {0} resources in different order'# $retry.Count 'Remove-AzOpsDeployment.ResourceNotFound' = 'Unable to find resource of type {0} with id {1}.'# $scopeObject.resource, $scopeObject.scope, $resultsError 'Remove-AzOpsDeployment.SkipUnsupportedResource' = 'Deletion is currently only supported for policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments. Will NOT proceed with deletion of file {0}'# $templateFilePath - 'Remove-AzOpsInvalidCharacter.Completed' = 'Valid string: {0}'# $String - 'Remove-AzOpsInvalidCharacter.Invalid' = 'Invalid character detected in string: {0}, further processing initiated'# $String - 'Remove-AzOpsInvalidCharacter.Removal' = 'Removed invalid character: {0} from string: {1}'# $character, $String - - 'Save-AzOpsManagementGroupChild.Creating.Scope' = 'Creating scope object' # - 'Save-AzOpsManagementGroupChild.Data.Directory' = 'Resolved state path directory: {0}' # $statepathDirectory - 'Save-AzOpsManagementGroupChild.Data.FileName' = 'Resolved state path filename: {0}' # $statepathFileName - 'Save-AzOpsManagementGroupChild.Data.ScopeDirectory' = 'Resolved state path scope directory: {0}' # $statepathScopeDirectory - 'Save-AzOpsManagementGroupChild.Data.ScopeDirectoryParent' = 'Resolved state path scope directory parent: {0}' # $statepathScopeDirectoryParent - 'Save-AzOpsManagementGroupChild.Data.StatePath' = 'Resolved state path: {0}' # $scopeStatepath - 'Save-AzOpsManagementGroupChild.Moving.Destination' = 'Moved existing state file to: {0}' # $statepathScopeDirectoryParent - 'Save-AzOpsManagementGroupChild.Moving.Source' = 'Found existing state file in directory: {0}' # $exisitingScopePath - 'Save-AzOpsManagementGroupChild.Processing' = 'Processing Scope: {0}' # $scopeObject.Scope - 'Save-AzOpsManagementGroupChild.Starting' = 'Starting execution' # - 'Save-AzOpsManagementGroupChild.Subscription.NotFound' = 'Unable to locate subscription: {0} within AzOpsSubscriptions object' #child.Name + 'Remove-AzResourceRaw.Resource.Failed' = 'Unable to delete resource of type {0} with id {1}'# $scopeObject.scope, $FullyQualifiedResourceId + + 'Remove-AzResourceRawRecursive.Processing' = 'Recursive retry processing to delete resource of type {0} with id {1}'# $item.ScopeObject.resource, $item.FullyQualifiedResourceId + + 'Remove-AzOpsInvalidCharacter.Completed' = 'Valid string: {0}'# $String + 'Remove-AzOpsInvalidCharacter.Invalid' = 'Invalid character detected in string: {0}, further processing initiated'# $String + 'Remove-AzOpsInvalidCharacter.Removal' = 'Removed invalid character: {0} from string: {1}'# $character, $String + + 'Save-AzOpsManagementGroupChild.Creating.Scope' = 'Creating scope object' # + 'Save-AzOpsManagementGroupChild.Data.Directory' = 'Resolved state path directory: {0}' # $statepathDirectory + 'Save-AzOpsManagementGroupChild.Data.FileName' = 'Resolved state path filename: {0}' # $statepathFileName + 'Save-AzOpsManagementGroupChild.Data.ScopeDirectory' = 'Resolved state path scope directory: {0}' # $statepathScopeDirectory + 'Save-AzOpsManagementGroupChild.Data.ScopeDirectoryParent' = 'Resolved state path scope directory parent: {0}' # $statepathScopeDirectoryParent + 'Save-AzOpsManagementGroupChild.Data.StatePath' = 'Resolved state path: {0}' # $scopeStatepath + 'Save-AzOpsManagementGroupChild.Moving.Destination' = 'Moved existing state file to: {0}' # $statepathScopeDirectoryParent + 'Save-AzOpsManagementGroupChild.Moving.Source' = 'Found existing state file in directory: {0}' # $exisitingScopePath + 'Save-AzOpsManagementGroupChild.Processing' = 'Processing Scope: {0}' # $scopeObject.Scope + 'Save-AzOpsManagementGroupChild.Starting' = 'Starting execution' # + 'Save-AzOpsManagementGroupChild.Subscription.NotFound' = 'Unable to locate subscription: {0} within AzOpsSubscriptions object' #child.Name 'Search-AzOpsAzGraph.Processing' = 'AzGraph processing query: [{0}]' # $Query 'Search-AzOpsAzGraph.Processing.Done' = 'AzGraph completed processing of query: [{0}]' # $Query From 25e4a7ee18d2c5e6d7f4bce1a66534bea6372dac Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 25 Jan 2024 20:23:59 +0000 Subject: [PATCH 02/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 31b08f1a..e36d71e3 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -43,7 +43,7 @@ begin { #region Utility Functions - function New-List { + function New-AzOpsList { [CmdletBinding()] param ( [string[]] @@ -407,7 +407,7 @@ #endregion Deploy State $deploymentList = foreach ($addition in $addModifySet | Where-Object { $_ -match ((Get-Item $StatePath).Name) }) { - $deployFileAssociationList = New-List -FilePath $addition -FileSet $addModifySet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter + $deployFileAssociationList = New-AzOpsList -FilePath $addition -FileSet $addModifySet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter foreach ($fileAssociation in $deployFileAssociationList) { if ($true -eq $fileAssociation.transpiledTemplateNew) { $AzOpsTranspiledTemplate += $fileAssociation.TemplateFilePath @@ -420,7 +420,7 @@ } $deletionList = foreach ($deletion in $deleteSet | Where-Object { $_ -match ((Get-Item $StatePath).Name) }) { - $deletionFileAssociationList = New-List -FilePath $deletion -FileSet $deleteSet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter + $deletionFileAssociationList = New-AzOpsList -FilePath $deletion -FileSet $deleteSet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter foreach ($fileAssociation in $deletionFileAssociationList) { if ($true -eq $fileAssociation.transpiledTemplateNew) { $AzOpsTranspiledTemplate += $fileAssociation.TemplateFilePath From e34b4b1d3f0dd449c4ccd2dde7632adeac0848c1 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 25 Jan 2024 20:49:07 +0000 Subject: [PATCH 03/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 8 +++++++- src/internal/functions/Remove-AzResourceRaw.ps1 | 8 +++++++- src/internal/functions/Remove-AzResourceRawRecursive.ps1 | 7 +++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 3932209b..91d94f54 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -322,9 +322,11 @@ elseif ($customDeletion -eq $true) { $removalJob = New-AzOpsDeployment -DeploymentName $DeploymentName -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath -WhatIfResultFormat 'ResourceIdOnly' -WhatIf:$true if ($removalJob.results.Changes.Count -gt 0) { + # Initialize array to store items that need retry $retry = @() foreach ($change in $removalJob.results.Changes) { $resource = $null + # Check if the resource exists if ($change.RelativeResourceId.StartsWith('Microsoft.Authorization/locks/')) { $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $change.FullyQualifiedResourceId } -ErrorAction SilentlyContinue } @@ -336,8 +338,10 @@ Write-AzOpsMessage -LogLevel Verbose -LogString 'Set-AzOpsWhatIfOutput.WhatIfResults' -LogStringValues $results Write-AzOpsMessage -LogLevel InternalComment -LogString 'Set-AzOpsWhatIfOutput.WhatIfFile' Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true + # Check if the removal should be performed if ($PSCmdlet.ShouldProcess("Remove $($change.FullyQualifiedResourceId)?")) { $removeAction = Remove-AzResourceRaw -FullyQualifiedResourceId $change.FullyQualifiedResourceId -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath + # If removal failed, add to retry if ($removeAction.Status -eq 'failed') { $retry += $removeAction } @@ -347,6 +351,7 @@ } } else { + # Log warning if resource not found Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.resource, $change.FullyQualifiedResourceId $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $change.FullyQualifiedResourceId, [environment]::NewLine Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true @@ -354,6 +359,7 @@ } if ($retry.Count -gt 0) { + # Retry failed removals recursively Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.Resource.RetryCount' -LogStringValues $retry.Count foreach ($try in $retry) { $try.Status = $null } $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry @@ -362,7 +368,7 @@ } } else { - # No resource to delete was found return + # No resource to remove was found Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.Resource, $scopeObject.Scope $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $scopeObject.scope, [environment]::NewLine Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true diff --git a/src/internal/functions/Remove-AzResourceRaw.ps1 b/src/internal/functions/Remove-AzResourceRaw.ps1 index 17400adc..b51c13f1 100644 --- a/src/internal/functions/Remove-AzResourceRaw.ps1 +++ b/src/internal/functions/Remove-AzResourceRaw.ps1 @@ -39,6 +39,7 @@ ) process { + # Construct result object $result = [PSCustomObject]@{ FullyQualifiedResourceId = $FullyQualifiedResourceId TemplateFilePath = $TemplateFilePath @@ -46,26 +47,31 @@ ScopeObject = $scopeObject Status = 'success' } - #region SetContext + # Set Azure context for removal operation Set-AzOpsContext -ScopeObject $ScopeObject + # Check if the resource exists if ($FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $FullyQualifiedResourceId } -ErrorAction SilentlyContinue } else { $resource = Get-AzResource -ResourceId $FullyQualifiedResourceId -ErrorAction SilentlyContinue } + # Remove the resource if it exists if ($resource) { try { $null = Remove-AzResource -ResourceId $FullyQualifiedResourceId -Force -ErrorAction Stop } catch { + # Log failure message Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.Failed' -LogStringValues $ScopeObject.resource, $FullyQualifiedResourceId $result.Status = 'failed' } } else { + # Log not found message $result.Status = 'notfound' } + # Return result object return $result } } \ No newline at end of file diff --git a/src/internal/functions/Remove-AzResourceRawRecursive.ps1 b/src/internal/functions/Remove-AzResourceRawRecursive.ps1 index 8ead19d4..8762e7fa 100644 --- a/src/internal/functions/Remove-AzResourceRawRecursive.ps1 +++ b/src/internal/functions/Remove-AzResourceRawRecursive.ps1 @@ -32,21 +32,26 @@ foreach ($item in $CurrentOrder) { if ($item.Status -eq 'failed' -or $null -eq $item.Status) { Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRawRecursive.Processing' -LogStringValues $item.ScopeObject.resource, $item.FullyQualifiedResourceId + # Attempt to remove the resource $result = Remove-AzResourceRaw -FullyQualifiedResourceId $item.FullyQualifiedResourceId -ScopeObject $item.ScopeObject -TemplateFilePath $item.TemplateFilePath -TemplateParameterFilePath $item.TemplateParameterFilePath if ($result.Status -eq 'failed' -and $result.FullyQualifiedResourceId -notin $OutputObject.FullyQualifiedResourceId){ + # Add failed result to the output object $OutputObject += $result } } } + # Return the final result return $OutputObject } else { if ($InputObject -and $OutputObject) { + # Filter out items already processed successfully $filteredOutputObject = @() foreach ($item in $InputObject) { if ($item.FullyQualifiedResourceId -in $OutputObject.FullyQualifiedResourceId) { foreach ($output in $OutputObject) { if ($output.FullyQualifiedResourceId -eq $item.FullyQualifiedResourceId -and $output.Status -eq 'failed') { + # Add previously failed item to the filtered output $filteredOutputObject += $output continue } @@ -61,8 +66,10 @@ foreach ($item in $InputObject) { $remainingItems = $InputObject -ne $item $newOrder = $CurrentOrder + $item + # Recursively call Remove-AzResourceRawRecursive $OutputObject = Remove-AzResourceRawRecursive -InputObject $remainingItems -CurrentOrder $newOrder -OutputObject $OutputObject } + # Return the output after all permutations return $OutputObject } } From 83871d40aaac84fdfb117d738f56b7f95ed89865 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 25 Jan 2024 20:56:48 +0000 Subject: [PATCH 04/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index e36d71e3..4ff6e7c5 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -83,13 +83,16 @@ } try { + # Create scope object from the given file path $scopeObject = New-AzOpsScope -Path $FilePath -StatePath $StatePath -ErrorAction Stop } catch { + # Log a warning message if creating the scope object fails Write-AzOpsMessage -LogLevel Warning -LogString 'Invoke-AzOpsPush.Scope.Failed' -LogStringValues $FilePath -Target $FilePath -ErrorRecord $_ continue } + # Resolve ARM file association $resolvedArmFileAssociation = Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $FilePath -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $ConvertedTemplate -ConvertedParameter $ConvertedParameter if ($resolvedArmFileAssociation) { foreach ($fileAssociation in $resolvedArmFileAssociation) { From 4dbaddf96f584ceec59f60507d78d413e920692b Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 25 Jan 2024 21:27:50 +0000 Subject: [PATCH 05/47] PriorityFix --- src/functions/Invoke-AzOpsPush.ps1 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 4ff6e7c5..8250b47f 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -442,11 +442,21 @@ "policyAssignments", "policySetDefinitions", "policyDefinitions", - "resourceGroups" + "resourceGroups", + "managementGroups" ) #Sort 'deletionList' based on 'deletionListPriority' - $deletionList = $deletionList | Sort-Object -Property {$deletionListPriority.IndexOf($_.ScopeObject.Resource)} + $deletionList = $deletionList | Sort-Object -Property { + $priorityIndex = $deletionListPriority.IndexOf($_.ScopeObject.Resource) + if ($priorityIndex -eq -1) { + # Set a default priority for items not found in deletionListPriority + return [int]::MaxValue + } + else { + return $priorityIndex + } + } #If addModifySet exists and no deploymentList has been generated at the same time as the StatePath root has additional directories and AllowMultipleTemplateParameterFiles is default false, exit with terminating error if (($addModifySet -and -not $deploymentList) -and (Get-ChildItem -Path $StatePath -Directory) -and ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $false)) { From 93ef4ade4b5a7553423b86d8611e1a83e18dfbc0 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 29 Jan 2024 08:50:03 +0000 Subject: [PATCH 06/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 10 ++++++++++ src/internal/functions/Remove-AzOpsDeployment.ps1 | 15 ++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 8250b47f..96bf902b 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -553,36 +553,46 @@ #Removal of Supported resourceTypes $removalJob = $deletionList | Select-Object $uniqueProperties -Unique | Remove-AzOpsDeployment -WhatIf:$WhatIfPreference if ($removalJob.FullyQualifiedResourceId.Count -gt 0) { + Clear-PSFMessage + # Identify failed removal attempts for potential retries $retry = $removalJob | Where-Object { $_.Status -eq 'failed' } + # If there are retries, log and attempt them again if ($retry) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Invoke-AzOpsPush.Deletion.Retry' -LogStringValues $retry.Count Start-Sleep -Seconds 30 + # Reset the status of failed attempts and perform recursive removal foreach ($try in $retry) { $try.Status = $null } $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry $removeActionFail = $removeActionRecursive | Where-Object { $_.Status -eq 'failed' } + # If removal fails, log and attempt to fetch the resource causing the failure if ($removeActionFail) { Start-Sleep -Seconds 90 $throwFail = $false + # Check each failed removal and attempt to get the associated resource foreach ($fail in $removeActionFail) { $resource = $null Set-AzOpsContext -ScopeObject $fail.ScopeObject + # Determine if the resource is a lock or a regular resource if ($fail.FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $fail.FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $fail.FullyQualifiedResourceId } -ErrorAction SilentlyContinue } else { $resource = Get-AzResource -ResourceId $fail.FullyQualifiedResourceId -ErrorAction SilentlyContinue } + # If the resource is found, log the failure if ($resource) { $throwFail = $true Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Deletion.Failed' -LogStringValues $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath } } + # If any failures occurred, throw an exception if ($throwFail) { throw } } } } + # If there are missing dependencies, log the error and throw an exception if ($removalJob.dependencyMissing -eq $true) { Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Dependency.Missing' throw diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 91d94f54..6a056989 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -199,16 +199,18 @@ $removeJobName = $fileItem.BaseName -replace '\.json$' -replace ' ', '_' $removeJobName = "AzOps-RemoveResource-$removeJobName" Write-AzOpsMessage -LogLevel Important -LogString 'Remove-AzOpsDeployment.Processing' -LogStringValues $removeJobName, $TemplateFilePath + #region Parse Content $templateContent = Get-Content $TemplateFilePath | ConvertFrom-Json -AsHashtable - #endregion - #region Validate it is AzOpsgenerated template + #endregion Parse Content + + #region Validate template type AzOps generated or not $schemavalue = '$schema' $customDeletion = $false if ($templateContent.metadata._generator.name -eq "AzOps" -or $templateContent.$schemavalue -like "*deploymentParameters.json#") { Write-AzOpsMessage -LogLevel Verbose -LogString 'Remove-AzOpsDeployment.Metadata.AzOps' -LogStringValues $TemplateFilePath } - elseif ($CustomTemplateResourceDeletion) { + elseif ($CustomTemplateResourceDeletion -eq $true) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Remove-AzOpsDeployment.Metadata.Custom' -LogStringValues $TemplateFilePath $customDeletion = $true } @@ -216,7 +218,8 @@ Write-AzOpsMessage -LogLevel Error -LogString 'Remove-AzOpsDeployment.Metadata.Failed' -LogStringValues $TemplateFilePath return } - #endregion Validate it is AzOpsgenerated template + #endregion Validate template type AzOps generated or not + #region Resolve Scope try { $scopeObject = New-AzOpsScope -Path $TemplateFilePath -StatePath $StatePath -ErrorAction Stop -WhatIf:$false @@ -235,7 +238,7 @@ Set-AzOpsContext -ScopeObject $scopeObject #endregion SetContext - #region remove supported resources + #region remove resources if ($customDeletion -eq $false -and $scopeObject.Resource -in $DeletionSupportedResourceType) { $dependency = @() switch ($scopeObject.Resource) { @@ -320,6 +323,7 @@ } } elseif ($customDeletion -eq $true) { + # Perform a New-AzOpsDeployment using WhatIf with ResourceIdOnly to extrapolate resources inside template $removalJob = New-AzOpsDeployment -DeploymentName $DeploymentName -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath -WhatIfResultFormat 'ResourceIdOnly' -WhatIf:$true if ($removalJob.results.Changes.Count -gt 0) { # Initialize array to store items that need retry @@ -375,5 +379,6 @@ return } } + #endregion remove resources } } \ No newline at end of file From ca9be1be4766ca3d2c51bd3aafe2554bf8e66f8f Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 29 Jan 2024 09:38:52 +0000 Subject: [PATCH 07/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 96bf902b..8f74502d 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -409,31 +409,45 @@ $newStateDeploymentCmd.End() #endregion Deploy State + #region Create DeploymentList $deploymentList = foreach ($addition in $addModifySet | Where-Object { $_ -match ((Get-Item $StatePath).Name) }) { + # Create a list of deployment file associations using the New-AzOpsList function $deployFileAssociationList = New-AzOpsList -FilePath $addition -FileSet $addModifySet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter + # Iterate through each file association in the list foreach ($fileAssociation in $deployFileAssociationList) { + # Check if the transpiled template is new and add it to the collection if true if ($true -eq $fileAssociation.transpiledTemplateNew) { $AzOpsTranspiledTemplate += $fileAssociation.TemplateFilePath } + # Check if the transpiled parameters are new and add them to the collection if true if ($true -eq $fileAssociation.transpiledParametersNew) { $AzOpsTranspiledParameter += $fileAssociation.TemplateParameterFilePath } } + # Output the list of file associations for the current addition $deployFileAssociationList } + #endregion Create DeploymentList + #region Create DeletionList $deletionList = foreach ($deletion in $deleteSet | Where-Object { $_ -match ((Get-Item $StatePath).Name) }) { + # Create a list of deletion file associations using the New-AzOpsList function $deletionFileAssociationList = New-AzOpsList -FilePath $deletion -FileSet $deleteSet -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $AzOpsTranspiledTemplate -ConvertedParameter $AzOpsTranspiledParameter + # Iterate through each file association in the list foreach ($fileAssociation in $deletionFileAssociationList) { + # Check if the transpiled template is new and add it to the collection if true if ($true -eq $fileAssociation.transpiledTemplateNew) { $AzOpsTranspiledTemplate += $fileAssociation.TemplateFilePath } + # Check if the transpiled parameters are new and add them to the collection if true if ($true -eq $fileAssociation.transpiledParametersNew) { $AzOpsTranspiledParameter += $fileAssociation.TemplateParameterFilePath } } + # Output the list of file associations for the current deletion $deletionFileAssociationList } + #endregion Create DeletionList #Required deletion order $deletionListPriority = @( From 1e4fbc9bbfc61ed88a21e64e6cf5bcd3b09460ca Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 30 Jan 2024 18:35:38 +0000 Subject: [PATCH 08/47] Update --- scripts/Remove-AzOpsTestsDeployment.ps1 | 78 ++++++++++--------- .../functions/Remove-AzOpsDeployment.ps1 | 2 +- src/tests/integration/Repository.Tests.ps1 | 32 ++++++++ src/tests/templates/rgdualdeploy.bicep | 11 +++ .../templates/rgdualdeploy.parameters.json | 12 +++ 5 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 src/tests/templates/rgdualdeploy.bicep create mode 100644 src/tests/templates/rgdualdeploy.parameters.json diff --git a/scripts/Remove-AzOpsTestsDeployment.ps1 b/scripts/Remove-AzOpsTestsDeployment.ps1 index 6e103e9b..b19f365b 100644 --- a/scripts/Remove-AzOpsTestsDeployment.ps1 +++ b/scripts/Remove-AzOpsTestsDeployment.ps1 @@ -62,46 +62,52 @@ foreach ($script:mgclean in $script:managementGroups) { Remove-ManagementGroup -DisplayName $script:mgclean.DisplayName -Name $script:mgclean.Name -RootName (Get-AzTenant).TenantId } + $cleanupSub = @() + $cleanupSub += [PSCustomObject]@{ Id = $env:ARM_SUBSCRIPTION_ID } + $cleanupSub += (Get-AzSubscription | Where-Object { $_.Id -ne $env:ARM_SUBSCRIPTION_ID } | Sort-Object Name -Descending | Select-Object Id -First 2) # Collect resources to cleanup - Get-AzResourceLock | Remove-AzResourceLock -Force - $script:resourceGroups = Get-AzResourceGroup | Where-Object {$_.ResourceGroupName -like "*-azopsrg"} - $script:roleAssignmentsCleanBase = Get-AzRoleAssignment | Where-Object {$_.Scope -ne "/"} - $script:roleAssignments = foreach ($roleAssignment in $script:roleAssignmentsCleanBase) { - if ($roleAssignment.Scope -ne "/subscriptions/$((Get-AzContext).Subscription)") { - $roleAssignment - } - else { - if ($roleAssignment.RoleDefinitionName -ne 'Owner') { + foreach ($subscription in $cleanupSub) { + $null = Set-AzContext -SubscriptionId $subscription.Id + $null = Get-AzResourceLock | Remove-AzResourceLock -Force + $script:resourceGroups = Get-AzResourceGroup | Where-Object {$_.ResourceGroupName -like "*-azopsrg"} + $script:roleAssignmentsCleanBase = Get-AzRoleAssignment | Where-Object {$_.Scope -ne "/"} + $script:roleAssignments = foreach ($roleAssignment in $script:roleAssignmentsCleanBase) { + if ($roleAssignment.Scope -ne "/subscriptions/$((Get-AzContext).Subscription)") { $roleAssignment } + else { + if ($roleAssignment.RoleDefinitionName -ne 'Owner') { + $roleAssignment + } + } + } + $script:policyAssignments = Get-AzPolicyAssignment + $script:policyDefinitions = Get-AzPolicyDefinition -Custom + $script:policySetDefinitions = Get-AzPolicySetDefinition -Custom + $script:policyExemptions = Get-AzPolicyExemption -ErrorAction SilentlyContinue + # Cleanup resourceGroups + $script:resourceGroups | ForEach-Object -ThrottleLimit 20 -Parallel { + Write-PSFMessage -Level Verbose -Message "Executing test resourceGroups cleanup thread of $($_.ResourceGroupName)" -FunctionName "Remove-AzOpsTestsDeployment" + $script:run = $_ | Remove-AzResourceGroup -Confirm:$false -Force + } + # Cleanup roleAssignments and policyAssignments + $script:roleAssignments | Remove-AzRoleAssignment -Confirm:$false -ErrorAction SilentlyContinue + $script:policyExemptions | Remove-AzPolicyExemption -Force -Confirm:$false -ErrorAction SilentlyContinue + $script:policyAssignments | Remove-AzPolicyAssignment -Confirm:$false -ErrorAction SilentlyContinue + $script:policyDefinitions | Remove-AzPolicyDefinition -Force -Confirm:$false -ErrorAction SilentlyContinue + $script:policySetDefinitions | Remove-AzPolicySetDefinition -Force -Confirm:$false -ErrorAction SilentlyContinue + # Collect and cleanup deployment jobs + $azTenantDeploymentJobs = Get-AzTenantDeployment -ErrorAction SilentlyContinue + $azTenantDeploymentJobs | ForEach-Object -ThrottleLimit 10 -Parallel { + Write-PSFMessage -Level Verbose -Message "Executing test AzDeployment cleanup thread of $($_.DeploymentName)" -FunctionName "Remove-AzOpsTestsDeployment" + $_ | Remove-AzTenantDeployment -Confirm:$false + } + Get-AzManagementGroupDeployment -ManagementGroupId "cd35e23c-537f-4553-a280-f5a60033a446" -ErrorAction SilentlyContinue | Remove-AzManagementGroupDeployment -Confirm:$false -ErrorAction SilentlyContinue + $azDeploymentJobs = Get-AzDeployment + $azDeploymentJobs | ForEach-Object -ThrottleLimit 10 -Parallel { + Write-PSFMessage -Level Verbose -Message "Executing test AzDeployment cleanup thread of $($_.DeploymentName)" -FunctionName "Remove-AzOpsTestsDeployment" + $_ | Remove-AzDeployment -Confirm:$false } - } - $script:policyAssignments = Get-AzPolicyAssignment - $script:policyDefinitions = Get-AzPolicyDefinition -Custom - $script:policySetDefinitions = Get-AzPolicySetDefinition -Custom - $script:policyExemptions = Get-AzPolicyExemption -ErrorAction SilentlyContinue - # Cleanup resourceGroups - $script:resourceGroups | ForEach-Object -ThrottleLimit 20 -Parallel { - Write-PSFMessage -Level Verbose -Message "Executing test resourceGroups cleanup thread of $($_.ResourceGroupName)" -FunctionName "Remove-AzOpsTestsDeployment" - $script:run = $_ | Remove-AzResourceGroup -Confirm:$false -Force - } - # Cleanup roleAssignments and policyAssignments - $script:roleAssignments | Remove-AzRoleAssignment -Confirm:$false -ErrorAction SilentlyContinue - $script:policyExemptions | Remove-AzPolicyExemption -Force -Confirm:$false -ErrorAction SilentlyContinue - $script:policyAssignments | Remove-AzPolicyAssignment -Confirm:$false -ErrorAction SilentlyContinue - $script:policyDefinitions | Remove-AzPolicyDefinition -Force -Confirm:$false -ErrorAction SilentlyContinue - $script:policySetDefinitions | Remove-AzPolicySetDefinition -Force -Confirm:$false -ErrorAction SilentlyContinue - # Collect and cleanup deployment jobs - $azTenantDeploymentJobs = Get-AzTenantDeployment -ErrorAction SilentlyContinue - $azTenantDeploymentJobs | ForEach-Object -ThrottleLimit 10 -Parallel { - Write-PSFMessage -Level Verbose -Message "Executing test AzDeployment cleanup thread of $($_.DeploymentName)" -FunctionName "Remove-AzOpsTestsDeployment" - $_ | Remove-AzTenantDeployment -Confirm:$false - } - Get-AzManagementGroupDeployment -ManagementGroupId "cd35e23c-537f-4553-a280-f5a60033a446" -ErrorAction SilentlyContinue | Remove-AzManagementGroupDeployment -Confirm:$false -ErrorAction SilentlyContinue - $azDeploymentJobs = Get-AzDeployment - $azDeploymentJobs | ForEach-Object -ThrottleLimit 10 -Parallel { - Write-PSFMessage -Level Verbose -Message "Executing test AzDeployment cleanup thread of $($_.DeploymentName)" -FunctionName "Remove-AzOpsTestsDeployment" - $_ | Remove-AzDeployment -Confirm:$false } } catch { diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 6a056989..54a646b2 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -191,7 +191,7 @@ $dependencyMissing = $null #Adjust TemplateParameterFilePath to compensate for policyDefinitions and policySetDefinitions usage of parameters.json - if ($TemplateParameterFilePath -and $TemplateFilePath -eq (Get-PSFConfigValue -FullName 'AzOps.Core.MainTemplate')) { + if ($TemplateParameterFilePath -and $TemplateFilePath -eq (Resolve-Path (Get-PSFConfigValue -FullName 'AzOps.Core.MainTemplate')).Path) { $TemplateFilePath = $TemplateParameterFilePath } #Deployment Name diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 06bea35d..d314a59e 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -21,6 +21,7 @@ Describe "Repository" { $script:repositoryRoot = (Resolve-Path "$global:testroot/../..").Path $script:tenantId = $env:ARM_TENANT_ID $script:subscriptionId = $env:ARM_SUBSCRIPTION_ID + $otherSubscription = Get-AzSubscription | Where-Object { $_.Id -ne $script:subscriptionId } | Sort-Object Name -Descending | Select-Object Id -First 2 # Validate that the runtime variables are set as they are used to authenticate the Azure session. @@ -289,6 +290,16 @@ Describe "Repository" { $script:resourceGroupParallelDeployFile = ($script:resourceGroupParallelDeployPath).FullName Write-PSFMessage -Level Debug -Message "ParallelDeployResourceGroupFile: $($script:resourceGroupParallelDeployFile)" -FunctionName "BeforeAll" + $script:resourceGrouprgDualDeploy1Path = ($filePaths | Where-Object Name -eq "microsoft.subscription_subscriptions-$(($otherSubscription[0].Id).toLower()).json") + $script:resourceGrouprgDualDeploy1Directory = ($script:resourceGrouprgDualDeploy1Path).Directory + $script:resourceGrouprgDualDeploy1File = ($script:resourceGrouprgDualDeploy1Path).FullName + Write-PSFMessage -Level Debug -Message "ResourceGrouprgDualDeploy1File: $($script:resourceGrouprgDualDeploy1File)" -FunctionName "BeforeAll" + + $script:resourceGrouprgDualDeploy2Path = ($filePaths | Where-Object Name -eq "microsoft.subscription_subscriptions-$(($otherSubscription[1].Id).toLower()).json") + $script:resourceGrouprgDualDeploy2Directory = ($script:resourceGrouprgDualDeploy2Path).Directory + $script:resourceGrouprgDualDeploy2File = ($script:resourceGrouprgDualDeploy2Path).FullName + Write-PSFMessage -Level Debug -Message "ResourceGrouprgDualDeploy2File: $($script:resourceGrouprgDualDeploy2File)" -FunctionName "BeforeAll" + $script:roleAssignmentsPath = ($filePaths | Where-Object Name -eq "microsoft.authorization_roleassignments-$(($script:roleAssignments.RoleAssignmentId).toLower() -replace ".*/").json") $script:roleAssignmentsDirectory = ($script:roleAssignmentsPath).Directory $script:roleAssignmentsFile = ($script:roleAssignmentsPath).FullName @@ -1152,6 +1163,27 @@ Describe "Repository" { $timeTest | Should -Be 'good' } #endregion + + #region Deploy multiple resource group's to different subscriptions, test context switch + It "Deploy multiple resource group's to different subscriptions, test context switch" { + $script:deployRg1 = Get-ChildItem -Path "$($global:testRoot)/templates/rgdualdeploy*" | Copy-Item -Destination $script:resourceGrouprgDualDeploy1Directory -PassThru -Force + $script:deployRg2 = Get-ChildItem -Path "$($global:testRoot)/templates/rgdualdeploy*" | Copy-Item -Destination $script:resourceGrouprgDualDeploy2Directory -PassThru -Force + $changeSet = @( + "A`t$($script:deployRg1.FullName[0])", + "A`t$($script:deployRg2.FullName[0])" + ) + {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw + Start-Sleep -Seconds 5 + $null = Set-AzContext -SubscriptionId $otherSubscription[0].Id + (Get-AzResourceGroup -Name Test-azopsrg).Count | Should -Be 1 + $null = Set-AzContext -SubscriptionId $otherSubscription[1].Id + (Get-AzResourceGroup -Name Test-azopsrg).Count | Should -Be 1 + Set-AzContext -SubscriptionId $script:subscriptionId + } + #endregion + + #region Deletion of custom templates and pulled resources + #endregion } AfterAll { diff --git a/src/tests/templates/rgdualdeploy.bicep b/src/tests/templates/rgdualdeploy.bicep new file mode 100644 index 00000000..9a275846 --- /dev/null +++ b/src/tests/templates/rgdualdeploy.bicep @@ -0,0 +1,11 @@ +targetScope = 'subscription' + +param resourceGroupName string +param location string + +resource myRg 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location + tags: {} + properties: {} +} diff --git a/src/tests/templates/rgdualdeploy.parameters.json b/src/tests/templates/rgdualdeploy.parameters.json new file mode 100644 index 00000000..16c226c9 --- /dev/null +++ b/src/tests/templates/rgdualdeploy.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "resourceGroupName": { + "value": "Test-azopsrg" + }, + "location": { + "value": "northeurope" + } + } +} \ No newline at end of file From 0e1e085f4e88bae4aa7076f8f88a8b6329f119a8 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 30 Jan 2024 23:51:26 +0000 Subject: [PATCH 09/47] Update --- src/tests/integration/Repository.Tests.ps1 | 36 ++++++++++++++++++ src/tests/templates/azuredeploy.jsonc | 37 +++++++++++++++++++ src/tests/templates/customlockdelete.bicep | 9 +++++ src/tests/templates/rtcustomdelete.bicep | 32 ++++++++++++++++ .../templates/rtcustomdelete.parameters.json | 12 ++++++ 5 files changed, 126 insertions(+) create mode 100644 src/tests/templates/customlockdelete.bicep create mode 100644 src/tests/templates/rtcustomdelete.bicep create mode 100644 src/tests/templates/rtcustomdelete.parameters.json diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index d314a59e..89b7658f 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -133,10 +133,12 @@ Describe "Repository" { $script:policySetDefinitionsDep = Get-AzPolicySetDefinition -Name 'TestPolicySetDefinitionDep' -ManagementGroupName $($script:testManagementGroup.Name) $script:subscription = (Get-AzSubscription | Where-Object Id -eq $script:subscriptionId) $script:resourceGroup = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "App1-azopsrg") + $script:resourceGroupCustomDeletion = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "CustomDeletion-azopsrg") $script:resourceGroupParallelDeploy = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "ParallelDeploy-azopsrg") $script:roleAssignments = (Get-AzRoleAssignment -ObjectId "023e7c1c-1fa4-4818-bb78-0a9c5e8b0217" | Where-Object { $_.Scope -eq "/subscriptions/$script:subscriptionId" -and $_.RoleDefinitionId -eq "acdd72a7-3385-48ef-bd42-f606fba81ae7" }) $script:policyExemptions = Get-AzPolicyExemption -Name "PolicyExemptionTest" -Scope "/subscriptions/$script:subscriptionId" $script:routeTable = (Get-AzResource -Name "RouteTable" -ResourceGroupName $($script:resourceGroup).ResourceGroupName) + $script:policyAssignmentsDeletion = Get-AzPolicyAssignment -Name "TestPolicyAssignmentDeletion" -Scope "/subscriptions/$script:subscriptionId/resourceGroups/$($script:resourceGroupCustomDeletion.ResourceGroupName)" $script:ruleCollectionGroups = (Get-AzResource -ExpandProperties -Name "TestPolicy" -ResourceGroupName $($script:resourceGroup).ResourceGroupName).Properties.ruleCollectionGroups.id.split("/")[-1] $script:logAnalyticsWorkspace = (Get-AzResource -Name "thisisalongloganalyticsworkspacename123456789011121314151617181" -ResourceGroupName $($script:resourceGroup).ResourceGroupName) } @@ -226,6 +228,11 @@ Describe "Repository" { $script:policyAssignmentsDeploymentName = "AzOps-{0}-{1}" -f $($script:policyAssignmentsPath.Name.Replace(".json", '')).Substring(0, 53), $deploymentLocationId Write-PSFMessage -Level Debug -Message "PolicyAssignmentsFile: $($script:policyAssignmentsFile)" -FunctionName "BeforeAll" + $script:policyAssignmentsDeletionPath = ($filePaths | Where-Object Name -eq "microsoft.authorization_policyassignments-$(($script:policyAssignmentsDeletion.Name).toLower()).json") + $script:policyAssignmentsDeletionDirectory = ($script:policyAssignmentsDeletionPath).Directory + $script:policyAssignmentsDeletionFile = ($script:policyAssignmentsDeletionPath).FullName + Write-PSFMessage -Level Debug -Message "PolicyAssignmentsDeletionFile: $($script:policyAssignmentsDeletionFile)" -FunctionName "BeforeAll" + $script:policyAssignmentsDepPath = ($filePaths | Where-Object Name -eq "microsoft.authorization_policyassignments-$(($script:policyAssignmentsDep.Name).toLower()).json") $script:policyAssignmentsDepDirectory = ($script:policyAssignmentsDepPath).Directory $script:policyAssignmentsDepFile = ($script:policyAssignmentsDepPath).FullName @@ -290,6 +297,11 @@ Describe "Repository" { $script:resourceGroupParallelDeployFile = ($script:resourceGroupParallelDeployPath).FullName Write-PSFMessage -Level Debug -Message "ParallelDeployResourceGroupFile: $($script:resourceGroupParallelDeployFile)" -FunctionName "BeforeAll" + $script:resourceGroupCustomDeletionPath = ($filePaths | Where-Object Name -eq "microsoft.resources_resourcegroups-$(($script:resourceGroupCustomDeletion.ResourceGroupName).toLower()).json") + $script:resourceGroupCustomDeletionDirectory = ($script:resourceGroupCustomDeletionPath).Directory + $script:resourceGroupCustomDeletionFile = ($script:resourceGroupCustomDeletionPath).FullName + Write-PSFMessage -Level Debug -Message "CustomDeletionResourceGroupFile: $($script:resourceGroupCustomDeletionFile)" -FunctionName "BeforeAll" + $script:resourceGrouprgDualDeploy1Path = ($filePaths | Where-Object Name -eq "microsoft.subscription_subscriptions-$(($otherSubscription[0].Id).toLower()).json") $script:resourceGrouprgDualDeploy1Directory = ($script:resourceGrouprgDualDeploy1Path).Directory $script:resourceGrouprgDualDeploy1File = ($script:resourceGrouprgDualDeploy1Path).FullName @@ -1183,6 +1195,30 @@ Describe "Repository" { #endregion #region Deletion of custom templates and pulled resources + It "Deletion of custom templates and pulled resources" { + Set-PSFConfig -FullName AzOps.Core.CustomTemplateResourceDeletion -Value $true + $script:deployCustomRt = Get-ChildItem -Path "$($global:testRoot)/templates/rtcustomdelete*" | Copy-Item -Destination $script:resourceGroupCustomDeletionDirectory -PassThru -Force + $script:deployCustomLock = Get-ChildItem -Path "$($global:testRoot)/templates/customlockdelete*" | Copy-Item -Destination $script:subscriptionDirectory -PassThru -Force + $changeSet = @( + "A`t$($script:deployCustomRt.FullName[0])", + "A`t$($script:deployCustomLock.FullName)" + ) + {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw + Start-Sleep -Seconds 5 + $changeSet = @( + "D`t$($script:deployCustomRt.FullName[0])", + "D`t$($script:deployCustomLock.FullName)", + "D`t$script:policyAssignmentsDeletionFile" + ) + $DeleteSetContents = (Get-Content $script:deployCustomRt.FullName[0]) + $DeleteSetContents += '-- ' + $DeleteSetContents = (Get-Content $script:deployCustomLock.FullName) + $DeleteSetContents += '-- ' + $DeleteSetContents = (Get-Content $script:policyAssignmentsDeletionFile) + {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$false} | Should -Not -Throw + Set-PSFConfig -FullName AzOps.Core.CustomTemplateResourceDeletion -Value $false + Start-Sleep -Seconds 30 + } #endregion } diff --git a/src/tests/templates/azuredeploy.jsonc b/src/tests/templates/azuredeploy.jsonc index 90d16405..c6fd46b8 100644 --- a/src/tests/templates/azuredeploy.jsonc +++ b/src/tests/templates/azuredeploy.jsonc @@ -506,6 +506,12 @@ "name": "App1-azopsrg", "location": "northeurope" }, + { + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2019-10-01", + "name": "CustomDeletion-azopsrg", + "location": "northeurope" + }, { "type": "Microsoft.Resources/resourceGroups", "apiVersion": "2019-10-01", @@ -612,6 +618,37 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2019-10-01", + "name": "CustomDeletion", + "resourceGroup": "CustomDeletion-azopsrg", + "dependsOn": [ + "CustomDeletion-azopsrg" + ], + "properties": { + "mode": "Incremental", + "expressionEvaluationOptions": { + "scope": "inner" + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2021-06-01", + "name": "TestPolicyAssignmentDeletion", + "properties": { + "policyDefinitionId": "/providers/Microsoft.Authorization/policyDefinitions/0a914e76-4921-4c19-b460-a2d36003525a" + } + } + ], + "outputs": { + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2019-10-01", diff --git a/src/tests/templates/customlockdelete.bicep b/src/tests/templates/customlockdelete.bicep new file mode 100644 index 00000000..38f0fdbc --- /dev/null +++ b/src/tests/templates/customlockdelete.bicep @@ -0,0 +1,9 @@ +targetScope = 'subscription' + +resource subLock 'Microsoft.Authorization/locks@2020-05-01' = { + name: 'subscriptionLock' + properties: { + level: 'CanNotDelete' + notes: 'This subscription is locked for Delete operations.' + } +} diff --git a/src/tests/templates/rtcustomdelete.bicep b/src/tests/templates/rtcustomdelete.bicep new file mode 100644 index 00000000..a95a768b --- /dev/null +++ b/src/tests/templates/rtcustomdelete.bicep @@ -0,0 +1,32 @@ +param name string +param staName string +param location string = resourceGroup().location + +var storageName = '${toLower(staName)}${uniqueString(resourceGroup().id)}' + +resource rt 'Microsoft.Network/routeTables@2023-04-01' = { + name: name + location: location + properties: { + disableBgpRoutePropagation: false + routes: [ + ] + } +} + +resource storage_resource 'Microsoft.Storage/storageAccounts@2021-08-01' = { + name: storageName + location: location + kind: 'StorageV2' + sku: { + name: 'Standard_GZRS' + } + properties: { + minimumTlsVersion: 'TLS1_2' + networkAcls: { + bypass: 'None' + defaultAction: 'Deny' + } + supportsHttpsTrafficOnly: true + } +} diff --git a/src/tests/templates/rtcustomdelete.parameters.json b/src/tests/templates/rtcustomdelete.parameters.json new file mode 100644 index 00000000..fc60cff9 --- /dev/null +++ b/src/tests/templates/rtcustomdelete.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "value": "CustomRouteTable" + }, + "staName": { + "value": "deleteazops" + } + } +} \ No newline at end of file From ade1452140071e51437eaaf2a8e9dd285d15c799 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 31 Jan 2024 00:16:32 +0000 Subject: [PATCH 10/47] Update --- src/tests/integration/Repository.Tests.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 89b7658f..12d79ec5 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -1218,6 +1218,8 @@ Describe "Repository" { {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$false} | Should -Not -Throw Set-PSFConfig -FullName AzOps.Core.CustomTemplateResourceDeletion -Value $false Start-Sleep -Seconds 30 + (Get-AzResource -ResourceGroupName $script:resourceGroupCustomDeletion.ResourceGroupName).Count | Should -Be 0 + Get-AzPolicyAssignment -Id $script:policyAssignmentsDeletion.ResourceId -ErrorAction SilentlyContinue | Should -BeNullOrEmpty } #endregion } From ce40bfc12af1d8c5848352613b8f55fafeb6b58c Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 31 Jan 2024 13:26:34 +0000 Subject: [PATCH 11/47] Update --- docs/wiki/ResourceDeletion.md | 36 +++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/wiki/ResourceDeletion.md b/docs/wiki/ResourceDeletion.md index a6c2a028..44131d1f 100644 --- a/docs/wiki/ResourceDeletion.md +++ b/docs/wiki/ResourceDeletion.md @@ -1,16 +1,25 @@ # AzOps Resource Deletion - [Introduction](#introduction) -- [Deletion dependency validation](#deletion-dependency-validation) +- [Deletion of AzOps generated File](#deletion-of-azops-generated-file) + - [Deletion dependency validation](#deletion-dependency-validation) - [Deletion dependency validation scenario](#deletion-dependency-validation-scenario) +- [Deletion of Custom Template](#deletion-of-custom-template) + - [Enable Deletion of Custom Template](#enable-deletion-of-custom-template) - [Integration with AzOps Accelerator](#integration-with-azops-accelerator) - [How to Add AzOps Resource Deletion to existing AzOps Push](#how-to-add-azops-resource-deletion-to-existing-azops-push-and-validate-pipelines) ## Introduction -**AzOps Resource Deletion** performs deletion of locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments in Azure, based on `AzOps - Pull` generated templates at all Azure scope levels `(Management Group/Subscription/Resource Group)`. +**AzOps Resource Deletion** at a high level enables two scenarios. +1. [Deletion of AzOps generated File](#deletion-of-azops-generated-file) of a supported resource type, resulting in AzOps removes the corresponding resource in Azure. +2. [Deletion of Custom Template](#deletion-of-custom-template), resulting in AzOps removes the corresponding resource in Azure. -- For any other resource type **deletion** is **not** supported by AzOps at this time. +## Deletion of AzOps generated File + +Default AzOps behaviour when performing deletion of locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments in Azure, based on `AzOps - Pull` generated templates at all Azure scope levels `(Management Group/Subscription/Resource Group)`. + +- For any other `AzOps - Pull` generated resource **deletion** is **not** supported by AzOps at this time. By removing a AzOps generated file of a supported resource type AzOps removes the corresponding resource in Azure. @@ -68,7 +77,7 @@ By removing a AzOps generated file of a supported resource type AzOps removes th OR Microsoft.Authorization/roleAssignments/* ``` -## Deletion dependency validation +### Deletion dependency validation When deletion of a supported object is sent to AzOps it evaluates to ensure resource dependencies are included in the deletion job. If a dependency is missing the module will throw (exit with error) and post the result of missing dependencies to the pull request conversation asking you to add it and try again. **_Please Note: For the validation pipeline to fail in the manner intended (applicable to implementations created prior to AzOps release v1.9.0)_** @@ -91,6 +100,25 @@ Scenario: Deletion of a policy definition and policy assignment where the assign - a) In the branch delete the dependent file corresponding to the resulting error. - b) Delete the dependency in Azure and re-run validation. +## Deletion of Custom Template +Deletion of custom templates is a opt-in feature that you need to enable [see](#enable-deletion-of-custom-template). + +AzOps attempts deletion of custom templates according to these steps. +1. Validate template. +2. Resolve template parameter file, depending on module settings and possible multiple parameter file scenario. +3. Sort templates for deletion based (attempt locks before other resources). +4. Process templates for deletion in series. +5. Identify resources within template by attempting a WhatIf deployment and gather returned resource ids. +6. Attempt resource deletion for each resource id. +7. If resource fails deletion, recursively retry deletion in different order. +8. For resources still failing deletion, collect them for a last deletion attempt, once all other templates are processed. +9. If resource deletion still fails, module will log error and throw. + +### Enable Deletion of Custom Template +Set the `Core.CustomTemplateResourceDeletion` value in `settings.json` to `true`. + +`AzOps - Push` will now evaluate and attempt deletion of corresponding resource (*from template*) in Azure when a custom template is deleted. + ## Integration with AzOps Accelerator The [AzOps Accelerator pipelines](https://github.com/azure/azops-accelerator) (including `Git Hub Actions` & `Azure Pipelines`) incorporates the execution of resource deletion. From c95bff1b56c806a6150493e305e2206147713c19 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 2 Feb 2024 17:33:02 +0000 Subject: [PATCH 12/47] Update --- docs/wiki/ResourceDeletion.md | 8 ++++---- src/functions/Invoke-AzOpsPush.ps1 | 2 -- src/internal/functions/Remove-AzOpsDeployment.ps1 | 4 ++++ src/localized/en-us/Strings.psd1 | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/wiki/ResourceDeletion.md b/docs/wiki/ResourceDeletion.md index 44131d1f..6f99844b 100644 --- a/docs/wiki/ResourceDeletion.md +++ b/docs/wiki/ResourceDeletion.md @@ -17,11 +17,11 @@ ## Deletion of AzOps generated File -Default AzOps behaviour when performing deletion of locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments in Azure, based on `AzOps - Pull` generated templates at all Azure scope levels `(Management Group/Subscription/Resource Group)`. +By removing a AzOps generated file of a supported resource type AzOps removes the corresponding resource in Azure. -- For any other `AzOps - Pull` generated resource **deletion** is **not** supported by AzOps at this time. +_Supported resource types include: locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments in Azure._ -By removing a AzOps generated file of a supported resource type AzOps removes the corresponding resource in Azure. +- For any other `AzOps - Pull` generated resource **deletion** is **not** supported by AzOps at this time. **_Please Note_** @@ -106,7 +106,7 @@ Deletion of custom templates is a opt-in feature that you need to enable [see](# AzOps attempts deletion of custom templates according to these steps. 1. Validate template. 2. Resolve template parameter file, depending on module settings and possible multiple parameter file scenario. -3. Sort templates for deletion based (attempt locks before other resources). +3. Sort templates for deletion (attempt locks before other resources). 4. Process templates for deletion in series. 5. Identify resources within template by attempting a WhatIf deployment and gather returned resource ids. 6. Attempt resource deletion for each resource id. diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 8f74502d..c70bd2a7 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -325,8 +325,6 @@ process { if (-not $ChangeSet) { return } Assert-AzOpsInitialization -Cmdlet $PSCmdlet -StatePath $StatePath - #Supported resource types for deletion - $DeletionSupportedResourceType = (Get-PSFConfigValue -FullName 'AzOps.Core.DeletionSupportedResourceType') #region Categorize Input Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Deployment.Required' $deleteSet = @() diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 54a646b2..fb438520 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -322,6 +322,10 @@ Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.SkipDueToWhatIf' } } + elseif ($customDeletion -eq $false -and $scopeObject.Resource -notin $DeletionSupportedResourceType) { + Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.SkipUnsupportedResource' -LogStringValues $TemplateFilePath -Target $scopeObject + return + } elseif ($customDeletion -eq $true) { # Perform a New-AzOpsDeployment using WhatIf with ResourceIdOnly to extrapolate resources inside template $removalJob = New-AzOpsDeployment -DeploymentName $DeploymentName -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath -WhatIfResultFormat 'ResourceIdOnly' -WhatIf:$true diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index b4c45029..0eb86170 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -281,7 +281,7 @@ 'Remove-AzOpsDeployment.ResourceDependencyNotFound' = 'Missing resource dependency {0} for successfull deletion of {1}. Please add missing resource and retry.'# $resource.ResourceId, $scopeObject.Scope 'Remove-AzOpsDeployment.Resource.RetryCount' = 'Retry deletion of {0} resources in different order'# $retry.Count 'Remove-AzOpsDeployment.ResourceNotFound' = 'Unable to find resource of type {0} with id {1}.'# $scopeObject.resource, $scopeObject.scope, $resultsError - 'Remove-AzOpsDeployment.SkipUnsupportedResource' = 'Deletion is currently only supported for policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments. Will NOT proceed with deletion of file {0}'# $templateFilePath + 'Remove-AzOpsDeployment.SkipUnsupportedResource' = 'Deletion of AzOps generated file resources is only supported for locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments. Will NOT proceed with deletion of resource in file {0}'# $TemplateFilePath 'Remove-AzResourceRaw.Resource.Failed' = 'Unable to delete resource of type {0} with id {1}'# $scopeObject.scope, $FullyQualifiedResourceId From 376d8380b888b7718c2b61ef56d7174acf40dd5e Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 5 Feb 2024 11:53:18 +0000 Subject: [PATCH 13/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 108 +++++++----------- .../functions/Remove-AzOpsDeployment.ps1 | 3 +- .../functions/Set-AzOpsRemoveOrder.ps1 | 53 +++++++++ .../functions/Set-AzOpsRemoveOrder.Tests.ps1 | 51 +++++++++ 4 files changed, 150 insertions(+), 65 deletions(-) create mode 100644 src/internal/functions/Set-AzOpsRemoveOrder.ps1 create mode 100644 src/tests/functions/Set-AzOpsRemoveOrder.Tests.ps1 diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index c70bd2a7..4c629f5b 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -447,29 +447,6 @@ } #endregion Create DeletionList - #Required deletion order - $deletionListPriority = @( - "locks", - "policyExemptions", - "policyAssignments", - "policySetDefinitions", - "policyDefinitions", - "resourceGroups", - "managementGroups" - ) - - #Sort 'deletionList' based on 'deletionListPriority' - $deletionList = $deletionList | Sort-Object -Property { - $priorityIndex = $deletionListPriority.IndexOf($_.ScopeObject.Resource) - if ($priorityIndex -eq -1) { - # Set a default priority for items not found in deletionListPriority - return [int]::MaxValue - } - else { - return $priorityIndex - } - } - #If addModifySet exists and no deploymentList has been generated at the same time as the StatePath root has additional directories and AllowMultipleTemplateParameterFiles is default false, exit with terminating error if (($addModifySet -and -not $deploymentList) -and (Get-ChildItem -Path $StatePath -Directory) -and ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $false)) { Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.DeploymentList.NotFound' @@ -562,52 +539,55 @@ } } - #Removal of Supported resourceTypes - $removalJob = $deletionList | Select-Object $uniqueProperties -Unique | Remove-AzOpsDeployment -WhatIf:$WhatIfPreference - if ($removalJob.FullyQualifiedResourceId.Count -gt 0) { - Clear-PSFMessage - # Identify failed removal attempts for potential retries - $retry = $removalJob | Where-Object { $_.Status -eq 'failed' } - # If there are retries, log and attempt them again - if ($retry) { - Write-AzOpsMessage -LogLevel Verbose -LogString 'Invoke-AzOpsPush.Deletion.Retry' -LogStringValues $retry.Count - Start-Sleep -Seconds 30 - # Reset the status of failed attempts and perform recursive removal - foreach ($try in $retry) { $try.Status = $null } - $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry - $removeActionFail = $removeActionRecursive | Where-Object { $_.Status -eq 'failed' } - # If removal fails, log and attempt to fetch the resource causing the failure - if ($removeActionFail) { - Start-Sleep -Seconds 90 - $throwFail = $false - # Check each failed removal and attempt to get the associated resource - foreach ($fail in $removeActionFail) { - $resource = $null - Set-AzOpsContext -ScopeObject $fail.ScopeObject - # Determine if the resource is a lock or a regular resource - if ($fail.FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $fail.FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { - $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $fail.FullyQualifiedResourceId } -ErrorAction SilentlyContinue - } - else { - $resource = Get-AzResource -ResourceId $fail.FullyQualifiedResourceId -ErrorAction SilentlyContinue + if ($deletionList) { + #Removal of Supported resourceTypes and Custom Templates + $deletionList = Set-AzOpsRemoveOrder -DeletionList $deletionList -Index { $_.ScopeObject.Resource } + $removalJob = $deletionList | Select-Object $uniqueProperties -Unique | Remove-AzOpsDeployment -WhatIf:$WhatIfPreference + if ($removalJob.FullyQualifiedResourceId.Count -gt 0) { + Clear-PSFMessage + # Identify failed removal attempts for potential retries + $retry = $removalJob | Where-Object { $_.Status -eq 'failed' } + # If there are retries, log and attempt them again + if ($retry) { + Write-AzOpsMessage -LogLevel Verbose -LogString 'Invoke-AzOpsPush.Deletion.Retry' -LogStringValues $retry.Count + Start-Sleep -Seconds 30 + # Reset the status of failed attempts and perform recursive removal + foreach ($try in $retry) { $try.Status = $null } + $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry + $removeActionFail = $removeActionRecursive | Where-Object { $_.Status -eq 'failed' } + # If removal fails, log and attempt to fetch the resource causing the failure + if ($removeActionFail) { + Start-Sleep -Seconds 90 + $throwFail = $false + # Check each failed removal and attempt to get the associated resource + foreach ($fail in $removeActionFail) { + $resource = $null + Set-AzOpsContext -ScopeObject $fail.ScopeObject + # Determine if the resource is a lock or a regular resource + if ($fail.FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $fail.FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { + $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $fail.FullyQualifiedResourceId } -ErrorAction SilentlyContinue + } + else { + $resource = Get-AzResource -ResourceId $fail.FullyQualifiedResourceId -ErrorAction SilentlyContinue + } + # If the resource is found, log the failure + if ($resource) { + $throwFail = $true + Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Deletion.Failed' -LogStringValues $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath + } } - # If the resource is found, log the failure - if ($resource) { - $throwFail = $true - Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Deletion.Failed' -LogStringValues $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath + # If any failures occurred, throw an exception + if ($throwFail) { + throw } } - # If any failures occurred, throw an exception - if ($throwFail) { - throw - } } } - } - # If there are missing dependencies, log the error and throw an exception - if ($removalJob.dependencyMissing -eq $true) { - Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Dependency.Missing' - throw + # If there are missing dependencies, log the error and throw an exception + if ($removalJob.dependencyMissing -eq $true) { + Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Dependency.Missing' + throw + } } $stopWatch.Stop() Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Duration' -LogStringValues $stopWatch.Elapsed -Metric $stopWatch.Elapsed.TotalSeconds -MetricName 'AzOpsPush Time' diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index fb438520..5839b742 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -16,7 +16,7 @@ .PARAMETER StatePath The root folder under which to find the resource json. .PARAMETER DeletionSupportedResourceType - Supported resource types for deletion. + Supported resource types for deletion of AzOps generated file. .PARAMETER WhatIf If this switch is enabled, no actions are performed but informational messages will be displayed that explain what would happen if the command were to run. .EXAMPLE @@ -332,6 +332,7 @@ if ($removalJob.results.Changes.Count -gt 0) { # Initialize array to store items that need retry $retry = @() + $removalJob = Set-AzOpsRemoveOrder -DeletionList $removalJob -Index { (New-AzOpsScope -Scope $_.results.Changes.FullyQualifiedResourceId).Resource } foreach ($change in $removalJob.results.Changes) { $resource = $null # Check if the resource exists diff --git a/src/internal/functions/Set-AzOpsRemoveOrder.ps1 b/src/internal/functions/Set-AzOpsRemoveOrder.ps1 new file mode 100644 index 00000000..1975c752 --- /dev/null +++ b/src/internal/functions/Set-AzOpsRemoveOrder.ps1 @@ -0,0 +1,53 @@ +function Set-AzOpsRemoveOrder { + + <# + .SYNOPSIS + Sorts a custom object list based on a specified priority order using a user-defined index. + .DESCRIPTION + Used to sort deletion priority, aka locks are removed prior to resource deletion attempts. + .PARAMETER DeletionList + Custom object list to be sorted based on the defined priority. + .PARAMETER Index + Script block that determines the index used for sorting the deletion list. + .PARAMETER Priority + Optional array of strings representing the priority order. Defaults to a predefined order if not provided. + .EXAMPLE + > $sortedList = Set-AzOpsRemoveOrder -DeletionList $myCustomObjectList -Index { $_.SomeProperty } + #> + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] + $DeletionList, + [Parameter(Mandatory = $true)] + [scriptblock] + $Index, + [string[]] + $Priority = @( + "locks", + "policyExemptions", + "policyAssignments", + "policySetDefinitions", + "policyDefinitions", + "resourceGroups", + "managementGroups" + ) + ) + + process { + #Sort 'DeletionList' based on 'Priority' + $deletionListSorted = $DeletionList | Sort-Object -Property { + $resolvedIndex = & $Index + $priorityIndex = $Priority.IndexOf($resolvedIndex) + if ($priorityIndex -eq -1) { + # Set a default priority for items not found in Priority + return [int]::MaxValue + } + else { + return $priorityIndex + } + } + # Return processed list + return $deletionListSorted + } +} \ No newline at end of file diff --git a/src/tests/functions/Set-AzOpsRemoveOrder.Tests.ps1 b/src/tests/functions/Set-AzOpsRemoveOrder.Tests.ps1 new file mode 100644 index 00000000..e7952b3a --- /dev/null +++ b/src/tests/functions/Set-AzOpsRemoveOrder.Tests.ps1 @@ -0,0 +1,51 @@ +Describe "Function Test - Set-AzOpsRemoveOrder" { + + BeforeAll { + + } + + Context "Test: Set-AzOpsRemoveOrder Sorts as Expected" { + It 'Sort based on priority' { + InModuleScope AzOps { + $storageAccount = 'storageAccount' + $resourceGroups = 'resourceGroups' + $locks = 'locks' + $managementGroups = 'managementGroups' + $routeTables = 'routeTables' + $testList = @( + [PSCustomObject]@{ + Name = 'Item1' + Type = $storageAccount + }, + [PSCustomObject]@{ + Name = 'Item2' + Type = $resourceGroups + }, + [PSCustomObject]@{ + Name = 'Item3' + Type = $locks + }, + [PSCustomObject]@{ + Name = 'Item4' + Type = $managementGroups + }, + [PSCustomObject]@{ + Name = 'Item5' + Type = $routeTables + } + ) + $returnList = Set-AzOpsRemoveOrder -DeletionList $testList -Index { $_.Type } + $returnList[0].Type | Should -Be $locks + $returnList[1].Type | Should -Be $resourceGroups + $returnList[2].Type | Should -Be $managementGroups + $returnList[3].Type | Should -BeIn $storageAccount,$routeTables + $returnList[4].Type | Should -BeIn $storageAccount,$routeTables + } + } + } + + AfterAll { + + } + +} \ No newline at end of file From d7fc6a8d0b4b41135f1047c40d3bd75af8cd67c6 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 12 Feb 2024 12:01:41 +0000 Subject: [PATCH 14/47] Update --- docs/wiki/ResourceDeletion.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/wiki/ResourceDeletion.md b/docs/wiki/ResourceDeletion.md index 6f99844b..bc206a2d 100644 --- a/docs/wiki/ResourceDeletion.md +++ b/docs/wiki/ResourceDeletion.md @@ -103,7 +103,8 @@ Scenario: Deletion of a policy definition and policy assignment where the assign ## Deletion of Custom Template Deletion of custom templates is a opt-in feature that you need to enable [see](#enable-deletion-of-custom-template). -AzOps attempts deletion of custom templates according to these steps. +How does AzOps attempt deletion of custom template? + 1. Validate template. 2. Resolve template parameter file, depending on module settings and possible multiple parameter file scenario. 3. Sort templates for deletion (attempt locks before other resources). From a39cd00c85dea4202dccedc8fcb9abf3823a9a90 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 14 Feb 2024 11:00:19 +0000 Subject: [PATCH 15/47] Update --- docs/wiki/ResourceDeletion.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/wiki/ResourceDeletion.md b/docs/wiki/ResourceDeletion.md index bc206a2d..c890fcf0 100644 --- a/docs/wiki/ResourceDeletion.md +++ b/docs/wiki/ResourceDeletion.md @@ -12,7 +12,7 @@ ## Introduction **AzOps Resource Deletion** at a high level enables two scenarios. -1. [Deletion of AzOps generated File](#deletion-of-azops-generated-file) of a supported resource type, resulting in AzOps removes the corresponding resource in Azure. +1. [Deletion of AzOps generated File](#deletion-of-azops-generated-file) with a supported resource type, resulting in AzOps removes the corresponding resource in Azure. 2. [Deletion of Custom Template](#deletion-of-custom-template), resulting in AzOps removes the corresponding resource in Azure. ## Deletion of AzOps generated File @@ -103,6 +103,8 @@ Scenario: Deletion of a policy definition and policy assignment where the assign ## Deletion of Custom Template Deletion of custom templates is a opt-in feature that you need to enable [see](#enable-deletion-of-custom-template). +Once enabled, deletion of `yourCustomTemplate.bicep`, `yourCustomTemplate.bicepparam`, `yourCustomTemplate.json` or `yourCustomTemplate.parameters.json` results in AzOps attempting deletion of the resolved Azure resources. + How does AzOps attempt deletion of custom template? 1. Validate template. @@ -110,7 +112,7 @@ How does AzOps attempt deletion of custom template? 3. Sort templates for deletion (attempt locks before other resources). 4. Process templates for deletion in series. 5. Identify resources within template by attempting a WhatIf deployment and gather returned resource ids. -6. Attempt resource deletion for each resource id. +6. Attempt resource deletion for each identified resource id. 7. If resource fails deletion, recursively retry deletion in different order. 8. For resources still failing deletion, collect them for a last deletion attempt, once all other templates are processed. 9. If resource deletion still fails, module will log error and throw. From cdb551e47e2d2b24de8dcf9591c9e02863ebb348 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 20 Feb 2024 09:05:35 +0000 Subject: [PATCH 16/47] Update --- src/AzOps.psd1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AzOps.psd1 b/src/AzOps.psd1 index 6e3c335e..2d6fbbd3 100644 --- a/src/AzOps.psd1 +++ b/src/AzOps.psd1 @@ -3,7 +3,7 @@ # # Generated by: Customer Architecture Team (CAT) # -# Generated on: 02/07/2024 +# Generated on: 2/20/2024 # @{ @@ -55,7 +55,7 @@ RequiredModules = @(@{ModuleName = 'PSFramework'; RequiredVersion = '1.10.318'; @{ModuleName = 'Az.Accounts'; RequiredVersion = '2.15.1'; }, @{ModuleName = 'Az.Billing'; RequiredVersion = '2.0.3'; }, @{ModuleName = 'Az.ResourceGraph'; RequiredVersion = '0.13.0'; }, - @{ModuleName = 'Az.Resources'; RequiredVersion = '6.15.0'; }) + @{ModuleName = 'Az.Resources'; RequiredVersion = '6.15.1'; }) # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @() From 74bd6262bb368d722094cb55f0f9d8c618237e96 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 20 Feb 2024 14:13:50 +0000 Subject: [PATCH 17/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 4c629f5b..be3b34dd 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -122,12 +122,6 @@ ) #region Initialization Prep - $common = @{ - Level = 'Host' - Tag = 'pwsh' - FunctionName = 'Invoke-AzOpsPush' - Target = $ScopeObject - } $result = [PSCustomObject] @{ TemplateFilePath = $null @@ -300,11 +294,6 @@ } #endregion Utility Functions - $common = @{ - Level = 'Host' - Tag = 'git' - } - $WhatIfPreferenceState = $WhatIfPreference $WhatIfPreference = $false @@ -385,7 +374,7 @@ #endregion Categorize Input #region Deploy State - $common.Tag = 'pwsh' + # Nested Pipeline allows economizing on New-AzOpsStateDeployment having to run its "begin" block once only $newStateDeploymentCmd = { New-AzOpsStateDeployment -StatePath $StatePath }.GetSteppablePipeline() $newStateDeploymentCmd.Begin($true) From b631cee49c8a3d95876bbd99258fa36bef9ef08b Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 22 Feb 2024 20:42:10 +0000 Subject: [PATCH 18/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 2 ++ src/localized/en-us/Strings.psd1 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index be3b34dd..ca8cc4d4 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -363,9 +363,11 @@ } # When processed as designed there is no file present in the running branch. To run a removal AzOps re-creates the file and content based on $DeleteSetContents momentarily for processing, it is disregarded afterwards. if (-not(Test-Path -Path (Split-Path -Path $item))) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.TempFile' -LogStringValues $item New-Item -Path (Split-Path -Path $item) -ItemType Directory | Out-Null } # Update item + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' -LogStringValues $item, $jsonValue Set-Content -Path $item -Value $jsonValue } } diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 00db7e6f..1dd0083a 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -192,6 +192,8 @@ 'Invoke-AzOpsPush.Change.AddModify.File' = ' {0}' # $item 'Invoke-AzOpsPush.Change.Delete' = 'Deleting:' # 'Invoke-AzOpsPush.Change.Delete.File' = ' {0}' # $item + 'Invoke-AzOpsPush.Change.Delete.TempFile' = 'Creating temporary file for deletion processing: {0}' # $item + 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' = 'Set temporary file content: {1}, in {0}' # $item, $jsonValue 'Invoke-AzOpsPush.Deletion.Failed' = 'Deletion of resources {0}, has failed using templates: {1}, {2}, this could be due to delayed deletion acceptance from Azure, please investigate and take action.' # $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath 'Invoke-AzOpsPush.Deletion.Retry' = 'Deletion of {0} resources unsuccessful, initiate final retry combination.' # $retry.Count 'Invoke-AzOpsPush.Deploy.ProviderFeature' = 'Invoking new state deployment - *.providerfeatures.json for a file {0}' # $addition From bbadd479d76832ea6a6bc1c42ad22d70a888774c Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 22 Feb 2024 20:46:27 +0000 Subject: [PATCH 19/47] Update --- src/localized/en-us/Strings.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 1dd0083a..a07bc7a7 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -193,7 +193,7 @@ 'Invoke-AzOpsPush.Change.Delete' = 'Deleting:' # 'Invoke-AzOpsPush.Change.Delete.File' = ' {0}' # $item 'Invoke-AzOpsPush.Change.Delete.TempFile' = 'Creating temporary file for deletion processing: {0}' # $item - 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' = 'Set temporary file content: {1}, in {0}' # $item, $jsonValue + 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' = 'Set temporary file content: {{1}}, in {{0}}' # $item, $jsonValue 'Invoke-AzOpsPush.Deletion.Failed' = 'Deletion of resources {0}, has failed using templates: {1}, {2}, this could be due to delayed deletion acceptance from Azure, please investigate and take action.' # $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath 'Invoke-AzOpsPush.Deletion.Retry' = 'Deletion of {0} resources unsuccessful, initiate final retry combination.' # $retry.Count 'Invoke-AzOpsPush.Deploy.ProviderFeature' = 'Invoking new state deployment - *.providerfeatures.json for a file {0}' # $addition From e03dc3e6bf7cea33c4ec39393d42a7bbef348307 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 23 Feb 2024 08:33:03 +0000 Subject: [PATCH 20/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 56 +++++++++++++++++------------- src/localized/en-us/Strings.psd1 | 7 ++-- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index ca8cc4d4..77cda027 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -343,32 +343,40 @@ } if ($DeleteSetContents -and $deleteSet) { Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Change.Delete' - # Unique delimiter used to join, split and replace data in DeleteSetContents - $delimiter = (New-Guid).Guid - # Transform $DeleteSetContents for further processing - $DeleteSetContents = $DeleteSetContents -join $delimiter -split "$delimiter-- " -replace $delimiter,"" - # Process each $deleteSet $item - foreach ($item in $deleteSet) { - Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Change.Delete.File' -LogStringValues $item - # Processing each $deleteSet, compare it to each $DeleteSetContents - foreach ($content in $DeleteSetContents) { - if ($content.Contains($item)) { - # Transform original first line in content with missing delimiter - if ($content.StartsWith("-- ")) { - $jsonValue = $content.replace("-- $item", "") - } - # Transform remaining content - else { - $jsonValue = $content.replace($item, "") + # Iterate through each line in $DeleteSetContents + for ($i = 0; $i -lt $DeleteSetContents.Count; $i++) { + $line = $DeleteSetContents[$i].Trim() + # Check if the line starts with '-- ' and matches any filename in $deleteSet + if ($line -match '^-- (.+)$') { + $fileName = $matches[1] + if ($deleteSet -contains $fileName) { + Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Change.Delete.File' -LogStringValues $fileName + # Collect lines until the next line starting with '--' + $objectLines = @($line) + $i++ + while ($i -lt $DeleteSetContents.Count) { + $currentLine = $DeleteSetContents[$i].Trim() + # Check if the line starts with '-- ' followed by any filename in $deleteSet + if ($currentLine -match '^-- (.+)$' -and $deleteSet -contains $matches[1]) { + $i-- + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.NextTempFile' -LogStringValues $currentLine + break # Exit the loop if the line starts with '-- ' and matches a filename in $deleteSet + } + $objectLines += $currentLine + $i++ } - # When processed as designed there is no file present in the running branch. To run a removal AzOps re-creates the file and content based on $DeleteSetContents momentarily for processing, it is disregarded afterwards. - if (-not(Test-Path -Path (Split-Path -Path $item))) { - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.TempFile' -LogStringValues $item - New-Item -Path (Split-Path -Path $item) -ItemType Directory | Out-Null + # When processed as designed there is no file present in the running branch. + # To run a removal AzOps re-creates the file and content based on $DeleteSetContents momentarily for processing, it is disregarded afterwards. + if (-not(Test-Path -Path (Split-Path -Path $fileName))) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.TempFile' -LogStringValues $fileName + New-Item -Path (Split-Path -Path $fileName) -ItemType Directory | Out-Null } - # Update item - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' -LogStringValues $item, $jsonValue - Set-Content -Path $item -Value $jsonValue + # Create $fileName and set $content + $objectLines = $objectLines[1..$objectLines.Count] + $content = $objectLines.replace("-- $fileName", "") -join "`r`n" + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' -LogStringValues $fileName, $content + Set-Content -Path $fileName -Value $content + $i-- # Move back one step to process the next line properly } } } diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index a07bc7a7..30c6191b 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -191,9 +191,10 @@ 'Invoke-AzOpsPush.Change.AddModify' = 'Adding or modifying:' # 'Invoke-AzOpsPush.Change.AddModify.File' = ' {0}' # $item 'Invoke-AzOpsPush.Change.Delete' = 'Deleting:' # - 'Invoke-AzOpsPush.Change.Delete.File' = ' {0}' # $item - 'Invoke-AzOpsPush.Change.Delete.TempFile' = 'Creating temporary file for deletion processing: {0}' # $item - 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' = 'Set temporary file content: {{1}}, in {{0}}' # $item, $jsonValue + 'Invoke-AzOpsPush.Change.Delete.File' = ' {0}' # $fileName + 'Invoke-AzOpsPush.Change.Delete.TempFile' = 'Creating temporary file dir for deletion processing: {0}' # $fileName + 'Invoke-AzOpsPush.Change.Delete.NextTempFile' = 'Exiting while loop, next file detected in $DeleteSetContents for deletion processing based on this content line: [{0}]' # $currentLine + 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' = 'Set temporary file content: [{1}], in [{0}]' # $fileName, $jsonValue 'Invoke-AzOpsPush.Deletion.Failed' = 'Deletion of resources {0}, has failed using templates: {1}, {2}, this could be due to delayed deletion acceptance from Azure, please investigate and take action.' # $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath 'Invoke-AzOpsPush.Deletion.Retry' = 'Deletion of {0} resources unsuccessful, initiate final retry combination.' # $retry.Count 'Invoke-AzOpsPush.Deploy.ProviderFeature' = 'Invoking new state deployment - *.providerfeatures.json for a file {0}' # $addition From fa691b15520f6ccaf507b00be8c7ebfc9893cea2 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 23 Feb 2024 08:58:53 +0000 Subject: [PATCH 21/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 5839b742..683eed45 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -346,7 +346,7 @@ $results = 'What if successful:{1}Performing the operation:{1}Deletion of target resource {0}.' -f $change.FullyQualifiedResourceId, [environment]::NewLine Write-AzOpsMessage -LogLevel Verbose -LogString 'Set-AzOpsWhatIfOutput.WhatIfResults' -LogStringValues $results Write-AzOpsMessage -LogLevel InternalComment -LogString 'Set-AzOpsWhatIfOutput.WhatIfFile' - Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true + Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $results -RemoveAzOpsFlag $true # Check if the removal should be performed if ($PSCmdlet.ShouldProcess("Remove $($change.FullyQualifiedResourceId)?")) { $removeAction = Remove-AzResourceRaw -FullyQualifiedResourceId $change.FullyQualifiedResourceId -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath @@ -363,7 +363,7 @@ # Log warning if resource not found Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.resource, $change.FullyQualifiedResourceId $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $change.FullyQualifiedResourceId, [environment]::NewLine - Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true + Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $results -RemoveAzOpsFlag $true } } @@ -380,7 +380,7 @@ # No resource to remove was found Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.Resource, $scopeObject.Scope $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $scopeObject.scope, [environment]::NewLine - Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true + Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $results -RemoveAzOpsFlag $true return } } From bc1e7ab27308d16a13b74a851ca33461dbda7a3b Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 23 Feb 2024 14:10:39 +0000 Subject: [PATCH 22/47] Update --- src/localized/en-us/Strings.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 30c6191b..6f3a38b9 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -193,7 +193,7 @@ 'Invoke-AzOpsPush.Change.Delete' = 'Deleting:' # 'Invoke-AzOpsPush.Change.Delete.File' = ' {0}' # $fileName 'Invoke-AzOpsPush.Change.Delete.TempFile' = 'Creating temporary file dir for deletion processing: {0}' # $fileName - 'Invoke-AzOpsPush.Change.Delete.NextTempFile' = 'Exiting while loop, next file detected in $DeleteSetContents for deletion processing based on this content line: [{0}]' # $currentLine + 'Invoke-AzOpsPush.Change.Delete.NextTempFile' = 'Exiting while loop, file detected in $DeleteSetContents for deletion processing based on this content line: [{0}]' # $currentLine 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' = 'Set temporary file content: [{1}], in [{0}]' # $fileName, $jsonValue 'Invoke-AzOpsPush.Deletion.Failed' = 'Deletion of resources {0}, has failed using templates: {1}, {2}, this could be due to delayed deletion acceptance from Azure, please investigate and take action.' # $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath 'Invoke-AzOpsPush.Deletion.Retry' = 'Deletion of {0} resources unsuccessful, initiate final retry combination.' # $retry.Count From f33a738b883d64ed315cf4d4081361e455ea9890 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 23 Feb 2024 16:13:18 +0000 Subject: [PATCH 23/47] Update --- src/internal/functions/New-AzOpsScope.ps1 | 1 - src/internal/functions/Remove-AzOpsDeployment.ps1 | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/internal/functions/New-AzOpsScope.ps1 b/src/internal/functions/New-AzOpsScope.ps1 index 13802fe6..c1a0be23 100644 --- a/src/internal/functions/New-AzOpsScope.ps1 +++ b/src/internal/functions/New-AzOpsScope.ps1 @@ -42,7 +42,6 @@ [AzOpsScope] #> - #[OutputType([AzOpsScope])] [CmdletBinding(SupportsShouldProcess = $true)] param ( [Parameter(ParameterSetName = "scope")] diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 683eed45..a517e854 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -332,8 +332,8 @@ if ($removalJob.results.Changes.Count -gt 0) { # Initialize array to store items that need retry $retry = @() - $removalJob = Set-AzOpsRemoveOrder -DeletionList $removalJob -Index { (New-AzOpsScope -Scope $_.results.Changes.FullyQualifiedResourceId).Resource } - foreach ($change in $removalJob.results.Changes) { + $removalJobChanges = Set-AzOpsRemoveOrder -DeletionList $removalJob.results.Changes -Index { (New-AzOpsScope -Scope $_.FullyQualifiedResourceId -WhatIf:$false).Resource } + foreach ($change in $removalJobChanges) { $resource = $null # Check if the resource exists if ($change.RelativeResourceId.StartsWith('Microsoft.Authorization/locks/')) { From ea833ad17b150b39b6bfcb941c55c810aefd4a8b Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Sat, 24 Feb 2024 13:39:45 +0000 Subject: [PATCH 24/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index a517e854..f5dde2d2 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -333,6 +333,7 @@ # Initialize array to store items that need retry $retry = @() $removalJobChanges = Set-AzOpsRemoveOrder -DeletionList $removalJob.results.Changes -Index { (New-AzOpsScope -Scope $_.FullyQualifiedResourceId -WhatIf:$false).Resource } + $allResults = @() foreach ($change in $removalJobChanges) { $resource = $null # Check if the resource exists @@ -344,9 +345,9 @@ } if ($resource) { $results = 'What if successful:{1}Performing the operation:{1}Deletion of target resource {0}.' -f $change.FullyQualifiedResourceId, [environment]::NewLine + $allResults += $results Write-AzOpsMessage -LogLevel Verbose -LogString 'Set-AzOpsWhatIfOutput.WhatIfResults' -LogStringValues $results Write-AzOpsMessage -LogLevel InternalComment -LogString 'Set-AzOpsWhatIfOutput.WhatIfFile' - Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $results -RemoveAzOpsFlag $true # Check if the removal should be performed if ($PSCmdlet.ShouldProcess("Remove $($change.FullyQualifiedResourceId)?")) { $removeAction = Remove-AzResourceRaw -FullyQualifiedResourceId $change.FullyQualifiedResourceId -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath @@ -363,10 +364,12 @@ # Log warning if resource not found Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.resource, $change.FullyQualifiedResourceId $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $change.FullyQualifiedResourceId, [environment]::NewLine - Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $results -RemoveAzOpsFlag $true + $allResults += $results } } + # Log WhatIf Output once for all resources in template + Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $allResults -RemoveAzOpsFlag $true if ($retry.Count -gt 0) { # Retry failed removals recursively Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.Resource.RetryCount' -LogStringValues $retry.Count From a19f5dfda535a86db6c83221dc33250a17f575cd Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Sat, 24 Feb 2024 14:25:26 +0000 Subject: [PATCH 25/47] Update --- src/internal/functions/Set-AzOpsWhatIfOutput.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 index 45f481c3..25ca963e 100644 --- a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 +++ b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 @@ -91,13 +91,13 @@ else { if ($RemoveAzOpsFlag) { if ($Results -match 'Missing resource dependency' ) { - $mdOutput = ':x: **Action Required**{0}WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $Results, $resultHeadline + $mdOutput = ':x: **Action Required**{0}WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $resultString, $resultHeadline } elseif ($Results -match 'What if operation failed') { - $mdOutput = ':warning: WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $Results, $resultHeadline + $mdOutput = ':warning: WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $resultString, $resultHeadline } else { - $mdOutput = ':white_check_mark: WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $Results, $resultHeadline + $mdOutput = ':white_check_mark: WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $resultString, $resultHeadline } } else { From 4a62fb5f700a58ba6e076977f30823db72b1e8ea Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Sun, 25 Feb 2024 12:23:25 +0000 Subject: [PATCH 26/47] Update --- docs/wiki/ResourceDeletion.md | 52 ++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/wiki/ResourceDeletion.md b/docs/wiki/ResourceDeletion.md index c890fcf0..52b6dd56 100644 --- a/docs/wiki/ResourceDeletion.md +++ b/docs/wiki/ResourceDeletion.md @@ -12,9 +12,28 @@ ## Introduction **AzOps Resource Deletion** at a high level enables two scenarios. -1. [Deletion of AzOps generated File](#deletion-of-azops-generated-file) with a supported resource type, resulting in AzOps removes the corresponding resource in Azure. +1. [Deletion of AzOps generated File](#deletion-of-azops-generated-file) of supported resource type, resulting in AzOps removes the corresponding resource in Azure. 2. [Deletion of Custom Template](#deletion-of-custom-template), resulting in AzOps removes the corresponding resource in Azure. +```mermaid +flowchart TD + A[(Main Branch)] --> B[(1.Delete Branch)] + B -- Remove Template Files --> C([2. filename.json]) + C --> D([3. Commit]) + D --> B + B -- Pull Request to Main ----> E(((4. AzOps - Validate + '/tmp/diff.txt' + '/tmp/diffdeletedfiles.txt'))) + E -- git diff --- A + E ---> F[5. Invoke-AzOpsPush -WhatIf:$true] + E -- Merge ---> A + A -- Automated trigger----> G(((6. AzOps - Push + '/tmp/diff.txt' + '/tmp/diffdeletedfiles.txt'))) + G -- git diff --- A + G --> H[7. Invoke-AzOpsPush -WhatIf:$false] +``` + ## Deletion of AzOps generated File By removing a AzOps generated file of a supported resource type AzOps removes the corresponding resource in Azure. @@ -107,15 +126,28 @@ Once enabled, deletion of `yourCustomTemplate.bicep`, `yourCustomTemplate.bicepp How does AzOps attempt deletion of custom template? -1. Validate template. -2. Resolve template parameter file, depending on module settings and possible multiple parameter file scenario. -3. Sort templates for deletion (attempt locks before other resources). -4. Process templates for deletion in series. -5. Identify resources within template by attempting a WhatIf deployment and gather returned resource ids. -6. Attempt resource deletion for each identified resource id. -7. If resource fails deletion, recursively retry deletion in different order. -8. For resources still failing deletion, collect them for a last deletion attempt, once all other templates are processed. -9. If resource deletion still fails, module will log error and throw. +```mermaid +flowchart TD + A(((Invoke-AzOpsPush))) --> B[Validate Template + filename.parameters.json] + B -- Failed --> K[Skip] + B -- Success --> C[Resolve template files] + C -- No template found --> D + C -- Found template + filename.json --> D[Sort Templates + Attempt locks before other resources] + D --> E[(Process templates for deletion in series)] + E -- Success ---> A + E --> F([Identify resources within template by attempting a WhatIf deployment and gather returned resource ids]) + F --> G([Attempt resource deletion for each identified resource id]) + G -- Success ---> E + G -- Fail --> H([If resource fails deletion, recursively retry deletion in different order]) + H -- Success ---> E + H -- Fail --> I([For resources still failing deletion, collect them for a last deletion attempt, once all other templates are processed]) + I -- Success ---> A + I --Fail --> J([If resource deletion still fails, module will log error and throw]) + J --Fail ---> A +``` ### Enable Deletion of Custom Template Set the `Core.CustomTemplateResourceDeletion` value in `settings.json` to `true`. From 5f01ea7c339f286180edf828f3c40d7eed1e7e3b Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 27 Feb 2024 07:44:16 +0000 Subject: [PATCH 27/47] Update --- src/tests/integration/Repository.Tests.ps1 | 58 +++++++++++++++++++--- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 12d79ec5..4e6773da 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -381,16 +381,34 @@ Describe "Repository" { "D`t$script:roleAssignmentsFile", "D`t$script:locksFile" ) + $DeleteSetContents = '-- ' + $DeleteSetContents += $script:policyAssignmentsFile + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:policyAssignmentsFile) + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += '-- ' + $DeleteSetContents += $Script:policyDefinitionsFile + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:policyDefinitionsFile) + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += '-- ' + $DeleteSetContents += $Script:policySetDefinitionsFile + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:policySetDefinitionsFile) + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += '-- ' + $DeleteSetContents += $Script:policyExemptionsFile + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:policyExemptionsFile) + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += '-- ' + $DeleteSetContents += $Script:roleAssignmentsFile + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:roleAssignmentsFile) + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += '-- ' + $DeleteSetContents += $Script:locksFile + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:locksFile) Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents } @@ -1042,35 +1060,50 @@ Describe "Repository" { $changeSet = @( "D`t$script:policyDefinitionsDepFile" ) - $DeleteSetContents = (Get-Content $Script:policyDefinitionsDepFile) + $DeleteSetContents += '-- ' + $DeleteSetContents += $Script:policyDefinitionsDepFile + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $Script:policyDefinitionsDepFile) {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policySetDefinitionsFile with assignment dependency should fail" { $changeSet = @( "D`t$script:policySetDefinitionsDepFile" ) - $DeleteSetContents = (Get-Content $Script:policySetDefinitionsDepFile) + $DeleteSetContents = '-- ' + $DeleteSetContents += $Script:policySetDefinitionsDepFile + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $Script:policySetDefinitionsDepFile) {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policyDefinitionsFile with setDefinition dependency should fail" { $changeSet = @( "D`t$script:policyDefinitionsDep2File" ) - $DeleteSetContents = (Get-Content $script:policyDefinitionsDep2File) + $DeleteSetContents = '-- ' + $DeleteSetContents += $script:policyDefinitionsDep2File + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $script:policyDefinitionsDep2File) {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policyAssignmentFile with role assignment dependency should fail" { $changeSet = @( "D`t$script:policyAssignmentsDepFile" ) - $DeleteSetContents = (Get-Content $script:policyAssignmentsDepFile) + $DeleteSetContents = '-- ' + $DeleteSetContents += $script:policyAssignmentsDepFile + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $script:policyAssignmentsDepFile) {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policyAssignmentFile with lock dependency should fail" { $changeSet = @( "D`t$script:policyAssignmentsDep2File" ) - $DeleteSetContents = (Get-Content $script:policyAssignmentsDep2File) + $DeleteSetContents = '-- ' + $DeleteSetContents += $script:policyAssignmentsDep2File + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $script:policyAssignmentsDep2File) {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } #endregion @@ -1210,11 +1243,20 @@ Describe "Repository" { "D`t$($script:deployCustomLock.FullName)", "D`t$script:policyAssignmentsDeletionFile" ) - $DeleteSetContents = (Get-Content $script:deployCustomRt.FullName[0]) + $DeleteSetContents = '-- ' + $DeleteSetContents += $script:deployCustomRt.FullName[0] + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $script:deployCustomRt.FullName[0]) + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += '-- ' - $DeleteSetContents = (Get-Content $script:deployCustomLock.FullName) + $DeleteSetContents += $script:deployCustomLock.FullName + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $script:deployCustomLock.FullName) + $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += '-- ' - $DeleteSetContents = (Get-Content $script:policyAssignmentsDeletionFile) + $DeleteSetContents += $script:policyAssignmentsDeletionFile + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $script:policyAssignmentsDeletionFile) {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$false} | Should -Not -Throw Set-PSFConfig -FullName AzOps.Core.CustomTemplateResourceDeletion -Value $false Start-Sleep -Seconds 30 From 5ca96026c976234c842ada884ff5591996be0b64 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 27 Feb 2024 08:49:52 +0000 Subject: [PATCH 28/47] Update --- .../Microsoft.Authorization/policyAssignments/scenario.ps1 | 3 +++ .../Microsoft.Authorization/roleAssignments/scenario.ps1 | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 b/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 index 5e3458c1..92a3f0db 100644 --- a/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 +++ b/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 @@ -40,6 +40,9 @@ Describe "Scenario - policyAssignments" { $changeSet = @( "D`t$script:file" ) + $deleteSetContents = '-- ' + $deleteSetContents += $script:file + $deleteSetContents += [Environment]::NewLine $deleteSetContents += (Get-Content $script:file) try { Write-PSFMessage -Level Debug -Message "Deletion Scenario $script:resourceType starting: $script:file" -FunctionName "Functional Tests" diff --git a/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 b/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 index a30d2316..52cb7087 100644 --- a/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 +++ b/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 @@ -40,6 +40,9 @@ Describe "Scenario - roleAssignments" { $changeSet = @( "D`t$script:file" ) + $deleteSetContents = '-- ' + $deleteSetContents += $script:file + $deleteSetContents += [Environment]::NewLine $deleteSetContents += (Get-Content $script:file) try { Write-PSFMessage -Level Debug -Message "Deletion Scenario $script:resourceType starting: $script:file" -FunctionName "Functional Tests" From f7a87caf44f34e4510a20cb6185ffe5e79231e78 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 27 Feb 2024 09:16:22 +0000 Subject: [PATCH 29/47] Update --- .../policyAssignments/scenario.ps1 | 1 + .../roleAssignments/scenario.ps1 | 1 + src/tests/integration/Repository.Tests.ps1 | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 b/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 index 92a3f0db..3380a61f 100644 --- a/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 +++ b/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 @@ -44,6 +44,7 @@ Describe "Scenario - policyAssignments" { $deleteSetContents += $script:file $deleteSetContents += [Environment]::NewLine $deleteSetContents += (Get-Content $script:file) + Remove-Item -Path $script:file -Force try { Write-PSFMessage -Level Debug -Message "Deletion Scenario $script:resourceType starting: $script:file" -FunctionName "Functional Tests" $script:deletion = Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents diff --git a/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 b/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 index 52cb7087..fdc23fa7 100644 --- a/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 +++ b/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 @@ -44,6 +44,7 @@ Describe "Scenario - roleAssignments" { $deleteSetContents += $script:file $deleteSetContents += [Environment]::NewLine $deleteSetContents += (Get-Content $script:file) + Remove-Item -Path $script:file -Force try { Write-PSFMessage -Level Debug -Message "Deletion Scenario $script:resourceType starting: $script:file" -FunctionName "Functional Tests" $script:deletion = Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 4e6773da..5ff4e715 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -410,6 +410,12 @@ Describe "Repository" { $DeleteSetContents += $Script:locksFile $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:locksFile) + Remove-Item -Path $script:policyAssignmentsFile -Force + Remove-Item -Path $Script:policyDefinitionsFile -Force + Remove-Item -Path $Script:policySetDefinitionsFile -Force + Remove-Item -Path $Script:policyExemptionsFile -Force + Remove-Item -Path $Script:roleAssignmentsFile -Force + Remove-Item -Path $Script:locksFile -Force Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents } @@ -1064,6 +1070,7 @@ Describe "Repository" { $DeleteSetContents += $Script:policyDefinitionsDepFile $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:policyDefinitionsDepFile) + Remove-Item -Path $Script:policyDefinitionsDepFile -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policySetDefinitionsFile with assignment dependency should fail" { @@ -1074,6 +1081,7 @@ Describe "Repository" { $DeleteSetContents += $Script:policySetDefinitionsDepFile $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:policySetDefinitionsDepFile) + Remove-Item -Path $Script:policySetDefinitionsDepFile -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policyDefinitionsFile with setDefinition dependency should fail" { @@ -1084,6 +1092,7 @@ Describe "Repository" { $DeleteSetContents += $script:policyDefinitionsDep2File $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $script:policyDefinitionsDep2File) + Remove-Item -Path $script:policyDefinitionsDep2File -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policyAssignmentFile with role assignment dependency should fail" { @@ -1094,6 +1103,7 @@ Describe "Repository" { $DeleteSetContents += $script:policyAssignmentsDepFile $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $script:policyAssignmentsDepFile) + Remove-Item -Path $script:policyAssignmentsDepFile -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } It "Deletion of policyAssignmentFile with lock dependency should fail" { @@ -1104,6 +1114,7 @@ Describe "Repository" { $DeleteSetContents += $script:policyAssignmentsDep2File $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $script:policyAssignmentsDep2File) + Remove-Item -Path $script:policyAssignmentsDep2File -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } #endregion @@ -1257,6 +1268,9 @@ Describe "Repository" { $DeleteSetContents += $script:policyAssignmentsDeletionFile $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $script:policyAssignmentsDeletionFile) + Remove-Item -Path $script:deployCustomRt.FullName[0] -Force + Remove-Item -Path $script:deployCustomLock.FullName -Force + Remove-Item -Path $script:policyAssignmentsDeletionFile -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$false} | Should -Not -Throw Set-PSFConfig -FullName AzOps.Core.CustomTemplateResourceDeletion -Value $false Start-Sleep -Seconds 30 From 78c64172fb3f1318f8fb6be2d124d65223d90693 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 27 Feb 2024 17:37:04 +0000 Subject: [PATCH 30/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 2 +- .../functions/Remove-AzOpsDeployment.ps1 | 31 +++++++++++++++++-- .../functions/Set-AzOpsWhatIfOutput.ps1 | 2 +- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 77cda027..9bdc870f 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -541,7 +541,7 @@ if ($deletionList) { #Removal of Supported resourceTypes and Custom Templates $deletionList = Set-AzOpsRemoveOrder -DeletionList $deletionList -Index { $_.ScopeObject.Resource } - $removalJob = $deletionList | Select-Object $uniqueProperties -Unique | Remove-AzOpsDeployment -WhatIf:$WhatIfPreference + $removalJob = $deletionList | Select-Object $uniqueProperties -Unique | Remove-AzOpsDeployment -WhatIf:$WhatIfPreference -DeleteSet (Resolve-Path -Path $deleteSet).Path if ($removalJob.FullyQualifiedResourceId.Count -gt 0) { Clear-PSFMessage # Identify failed removal attempts for potential retries diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index f5dde2d2..d3647c8a 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -17,6 +17,8 @@ The root folder under which to find the resource json. .PARAMETER DeletionSupportedResourceType Supported resource types for deletion of AzOps generated file. + .PARAMETER DeleteSet + String of file names to validate deletion. .PARAMETER WhatIf If this switch is enabled, no actions are performed but informational messages will be displayed that explain what would happen if the command were to run. .EXAMPLE @@ -45,7 +47,10 @@ $StatePath = (Get-PSFConfigValue -FullName 'AzOps.Core.State'), [object[]] - $DeletionSupportedResourceType = (Get-PSFConfigValue -FullName 'AzOps.Core.DeletionSupportedResourceType') + $DeletionSupportedResourceType = (Get-PSFConfigValue -FullName 'AzOps.Core.DeletionSupportedResourceType'), + + [string[]] + $DeleteSet ) process { @@ -368,7 +373,29 @@ } } - # Log WhatIf Output once for all resources in template + $baseTemplateCheck = $TemplateFilePath -replace '\.bicep$', '.json' + if ($TemplateParameterFilePath) { + $baseParameterCheck = $TemplateParameterFilePath -replace '\.bicepparam$', 'parameters.json' + } + if ($DeleteSet) { + $deleteSetCheck = $DeleteSet -replace '\.bicep$', '.json' + $deleteSetCheck = $deleteSetCheck -replace '\.bicepparam$', '.parameters.json' + $resultsFileAssociation = switch ($null) { + { $baseTemplateCheck -notin $deleteSetCheck } { + 'Missing template file association:{1}{0} for deletion:{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion. If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateFilePath, [environment]::NewLine + } + { $baseParameterCheck -notin $deleteSetCheck } { + 'Missing parameter file association:{1}{0} for deletion:{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion. If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateParameterFilePath, [environment]::NewLine + } + } + if ($resultsFileAssociation) { + $finallResults = @() + $finallResults += $resultsFileAssociation + $finallResults += $allResults + $allResults = $finallResults + Write-AzOpsMessage -LogLevel Warning -LogString 'Set-AzOpsWhatIfOutput.WhatIfResults' -LogStringValues $allResults + } + } Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $allResults -RemoveAzOpsFlag $true if ($retry.Count -gt 0) { # Retry failed removals recursively diff --git a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 index 25ca963e..debcdaff 100644 --- a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 +++ b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 @@ -93,7 +93,7 @@ if ($Results -match 'Missing resource dependency' ) { $mdOutput = ':x: **Action Required**{0}WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $resultString, $resultHeadline } - elseif ($Results -match 'What if operation failed') { + elseif ($Results -match 'What if operation failed' -or $Results -match 'Missing template file association' -or $Results -match 'Missing parameter file association') { $mdOutput = ':warning: WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $resultString, $resultHeadline } else { From 0f885eae77cb648e36ac875b7cfc308bfb87b05d Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 27 Feb 2024 17:44:25 +0000 Subject: [PATCH 31/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index d3647c8a..9b72d029 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -382,10 +382,10 @@ $deleteSetCheck = $deleteSetCheck -replace '\.bicepparam$', '.parameters.json' $resultsFileAssociation = switch ($null) { { $baseTemplateCheck -notin $deleteSetCheck } { - 'Missing template file association:{1}{0} for deletion:{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion. If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateFilePath, [environment]::NewLine + 'Missing template file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateFilePath, [environment]::NewLine } { $baseParameterCheck -notin $deleteSetCheck } { - 'Missing parameter file association:{1}{0} for deletion:{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion. If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateParameterFilePath, [environment]::NewLine + 'Missing parameter file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateParameterFilePath, [environment]::NewLine } } if ($resultsFileAssociation) { From 1ceac1b1da2fd2135ebfa1299204ff555b46fe65 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 27 Feb 2024 18:13:08 +0000 Subject: [PATCH 32/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 9 +++++++-- src/internal/functions/Set-AzOpsWhatIfOutput.ps1 | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 9b72d029..29e89b42 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -380,14 +380,19 @@ if ($DeleteSet) { $deleteSetCheck = $DeleteSet -replace '\.bicep$', '.json' $deleteSetCheck = $deleteSetCheck -replace '\.bicepparam$', '.parameters.json' + # Check if template and parameter file exist in $DeleteSet, example AzOps has been instructed to remove template.json but not the associated parameter.json $resultsFileAssociation = switch ($null) { + { $baseTemplateCheck -notin $deleteSetCheck -and $baseParameterCheck -notin $deleteSetCheck } { + 'Missing template and parameter file association:{2}{0} and {1} for deletion.{2}{2}Ensure that you have reviewed and confirmed the necessity of each deletion.{2}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json or .parameters.json for deletion processing and outputs the results from the converted files here.{2}' -f $TemplateFilePath, $TemplateParameterFilePath, [environment]::NewLine + } { $baseTemplateCheck -notin $deleteSetCheck } { - 'Missing template file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateFilePath, [environment]::NewLine + 'Missing template file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json or deletion processing and outputs the results from the converted files here.{1}' -f $TemplateFilePath, [environment]::NewLine } { $baseParameterCheck -notin $deleteSetCheck } { - 'Missing parameter file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with the extensions .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateParameterFilePath, [environment]::NewLine + 'Missing parameter file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json or deletion processing and outputs the results from the converted files here.{1}' -f $TemplateParameterFilePath, [environment]::NewLine } } + # If there are $resultsFileAssociation, combine them with existing results and log a warning if ($resultsFileAssociation) { $finallResults = @() $finallResults += $resultsFileAssociation diff --git a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 index debcdaff..171fa372 100644 --- a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 +++ b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 @@ -93,7 +93,7 @@ if ($Results -match 'Missing resource dependency' ) { $mdOutput = ':x: **Action Required**{0}WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $resultString, $resultHeadline } - elseif ($Results -match 'What if operation failed' -or $Results -match 'Missing template file association' -or $Results -match 'Missing parameter file association') { + elseif ($Results -match 'What if operation failed' -or $Results -match 'Missing template and parameter file association' -or $Results -match 'Missing template file association' -or $Results -match 'Missing parameter file association') { $mdOutput = ':warning: WhatIf Results for Resource Deletion of {2}:{0}```{0}{1}{0}```' -f [environment]::NewLine, $resultString, $resultHeadline } else { From 1ee69fa5f8bf52dfed7f01dfd72ca1eed1c7bc04 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 27 Feb 2024 18:18:36 +0000 Subject: [PATCH 33/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 29e89b42..30492dbd 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -386,10 +386,10 @@ 'Missing template and parameter file association:{2}{0} and {1} for deletion.{2}{2}Ensure that you have reviewed and confirmed the necessity of each deletion.{2}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json or .parameters.json for deletion processing and outputs the results from the converted files here.{2}' -f $TemplateFilePath, $TemplateParameterFilePath, [environment]::NewLine } { $baseTemplateCheck -notin $deleteSetCheck } { - 'Missing template file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json or deletion processing and outputs the results from the converted files here.{1}' -f $TemplateFilePath, [environment]::NewLine + 'Missing template file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json or .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateFilePath, [environment]::NewLine } { $baseParameterCheck -notin $deleteSetCheck } { - 'Missing parameter file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json and .parameters.json or deletion processing and outputs the results from the converted files here.{1}' -f $TemplateParameterFilePath, [environment]::NewLine + 'Missing parameter file association:{1}{0} for deletion.{1}{1}Ensure that you have reviewed and confirmed the necessity of each deletion.{1}If you are deleting files with extension .bicep or .bicepparam, keep in mind that AzOps converts them to .json or .parameters.json for deletion processing and outputs the results from the converted files here.{1}' -f $TemplateParameterFilePath, [environment]::NewLine } } # If there are $resultsFileAssociation, combine them with existing results and log a warning From 34dd50dbb91b5cf880f8beadf31ecb2810fe2090 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 28 Feb 2024 09:47:56 +0000 Subject: [PATCH 34/47] Update --- src/internal/configurations/Core.ps1 | 2 +- src/internal/functions/Remove-AzOpsDeployment.ps1 | 10 +++++++++- src/localized/en-us/Strings.psd1 | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index bf54b150..ad5a98ce 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -2,7 +2,7 @@ Set-PSFConfig -Module AzOps -Name Core.AutoGeneratedTemplateFolderPath -Value "." -Initialize -Validation string -Description 'Auto-Generated Template Folder Path i.e. ./Az' Set-PSFConfig -Module AzOps -Name Core.AutoInitialize -Value $false -Initialize -Validation bool -Description '-' Set-PSFConfig -Module AzOps -Name Core.CustomTemplateResourceDeletion -Value $false -Initialize -Validation bool -Description 'Global flag declaring on or off deletion of resources in custom template.' -Set-PSFConfig -Module AzOps -Name Core.DeletionSupportedResourceType -Value @('Microsoft.Authorization/locks', 'locks', 'Microsoft.Authorization/policyAssignments', 'policyAssignments', 'Microsoft.Authorization/policyDefinitions', 'policyDefinitions', 'Microsoft.Authorization/policyExemptions', 'policyExemptions', 'Microsoft.Authorization/policySetDefinitions', 'policySetDefinitions', 'Microsoft.Authorization/roleAssignments', 'roleAssignments') -Initialize -Validation stringarray -Description 'Global flag declaring resource types supported for deletion by AzOps.' +Set-PSFConfig -Module AzOps -Name Core.DeletionSupportedResourceType -Value @('Microsoft.Authorization/locks', 'locks', 'Microsoft.Authorization/policyAssignments', 'policyAssignments', 'Microsoft.Authorization/policyDefinitions', 'policyDefinitions', 'Microsoft.Authorization/policyExemptions', 'policyExemptions', 'Microsoft.Authorization/policySetDefinitions', 'policySetDefinitions', 'Microsoft.Authorization/roleAssignments', 'roleAssignments', 'Microsoft.Resources/resourceGroups', 'microsoft.resources/subscriptions/resourcegroups', 'resourceGroups') -Initialize -Validation stringarray -Description 'Global flag declaring resource types supported for deletion by AzOps.' Set-PSFConfig -Module AzOps -Name Core.DefaultDeploymentRegion -Value northeurope -Initialize -Validation string -Description 'Default deployment region for state deployments (ARM region, not region where a resource is deployed)' Set-PSFConfig -Module AzOps -Name Core.EnrollmentAccountPrincipalName -Value '' -Initialize -Validation stringorempty -Description '-' Set-PSFConfig -Module AzOps -Name Core.ExcludedSubOffer -Value 'AzurePass_2014-09-01', 'FreeTrial_2014-09-01', 'AAD_2015-09-01' -Initialize -Validation stringarray -Description 'Excluded QuotaID' diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 30492dbd..401c08c2 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -280,11 +280,19 @@ } } 'roleAssignments' { - $resourceToDelete = Invoke-AzRestMethod -Path "$($scopeObject.scope)?api-version=2022-01-01-preview" | Where-Object { $_.StatusCode -eq 200 } + $resourceToDelete = Invoke-AzRestMethod -Path "$($scopeObject.scope)?api-version=2022-04-01" | Where-Object { $_.StatusCode -eq 200 } if ($resourceToDelete) { $dependency += Get-AzLocksDeletionDependency -resourceToDelete $resourceToDelete } } + 'resourceGroups' { + $resourceToDelete = Get-AzResourceGroup -Id $scopeObject.Scope -ErrorAction SilentlyContinue + if ($resourceToDelete) { + $resourceToDelete | Add-Member -MemberType NoteProperty -Name "ResourceType" -Value "$($scopeObject.Type)" + $resourceToDelete | Add-Member -MemberType NoteProperty -Name "SubscriptionId" -Value "$($scopeObject.Subscription)" + $dependency += Get-AzLocksDeletionDependency -resourceToDelete $resourceToDelete + } + } } # If no resource to delete was found return if (-not $resourceToDelete) { diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 6f3a38b9..e7d15d54 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -218,7 +218,7 @@ 'Invoke-AzOpsPush.Resolve.NoJson' = 'The specified file is not a json or bicep file! Skipping {0}' # $fileItem.FullName 'Invoke-AzOpsPush.Resolve.NotFoundTemplate' = 'Did NOT find template {1} for parameters {0}' # $FilePath, $templatePath 'Invoke-AzOpsPush.Resolve.ParameterFound' = 'Found parameter file for template {0} : {1}' # $FilePath, $parameterPath - 'Invoke-AzOpsPush.Resolve.ParameterNotFound' = 'No parameter file found for template {0} : {1}' # $FilePath, $parameterPath + 'Invoke-AzOpsPush.Resolve.ParameterNotFound' = 'No parameter file found for template: {0}, at: {1}' # $FilePath, $parameterPath 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' = 'Template {0} with parameter: {1} missing defaultValue and no parameter file found, skip deployment' # $FilePath, $missingString 'Invoke-AzOpsPush.Scope.Failed' = 'Failed to read {0} as part of {1}' # $addition, $StatePath From 10c98b44ac93cea5ceae00acc2253dd6713302af Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 28 Feb 2024 10:13:48 +0000 Subject: [PATCH 35/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 1 + src/internal/functions/Remove-AzResourceRaw.ps1 | 1 + 2 files changed, 2 insertions(+) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 401c08c2..78a34c0c 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -330,6 +330,7 @@ } if ($PSCmdlet.ShouldProcess("Remove $($scopeObject.Scope)?")) { $null = Remove-AzResource -ResourceId $scopeObject.Scope -Force + Start-Sleep -Seconds 5 } else { Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.SkipDueToWhatIf' diff --git a/src/internal/functions/Remove-AzResourceRaw.ps1 b/src/internal/functions/Remove-AzResourceRaw.ps1 index b51c13f1..c6b7a1aa 100644 --- a/src/internal/functions/Remove-AzResourceRaw.ps1 +++ b/src/internal/functions/Remove-AzResourceRaw.ps1 @@ -60,6 +60,7 @@ if ($resource) { try { $null = Remove-AzResource -ResourceId $FullyQualifiedResourceId -Force -ErrorAction Stop + Start-Sleep -Seconds 5 } catch { # Log failure message From dd6c50a42dfe00546d335308c340fc447d2e936d Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 28 Feb 2024 10:36:08 +0000 Subject: [PATCH 36/47] Update --- docs/wiki/ResourceDeletion.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/wiki/ResourceDeletion.md b/docs/wiki/ResourceDeletion.md index 52b6dd56..c0a2c51a 100644 --- a/docs/wiki/ResourceDeletion.md +++ b/docs/wiki/ResourceDeletion.md @@ -38,7 +38,7 @@ flowchart TD By removing a AzOps generated file of a supported resource type AzOps removes the corresponding resource in Azure. -_Supported resource types include: locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments in Azure._ +_Supported resource types include: locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions, roleAssignments and resourceGroups in Azure._ - For any other `AzOps - Pull` generated resource **deletion** is **not** supported by AzOps at this time. @@ -96,6 +96,14 @@ _Supported resource types include: locks, policyAssignments, policyDefinitions, OR Microsoft.Authorization/roleAssignments/* ``` + +- For Azure Resource group removal + +```bash + Microsoft.Resources/subscriptions/resourceGroups/delete + OR + Microsoft.Resources/subscriptions/resourceGroups/* +``` ### Deletion dependency validation When deletion of a supported object is sent to AzOps it evaluates to ensure resource dependencies are included in the deletion job. If a dependency is missing the module will throw (exit with error) and post the result of missing dependencies to the pull request conversation asking you to add it and try again. From 95e539cc6da3a14cab24a570da4b175289025b14 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 28 Feb 2024 11:05:19 +0000 Subject: [PATCH 37/47] Update --- src/tests/integration/Repository.Tests.ps1 | 48 +++++++++++++++++++++- src/tests/templates/azuredeploy.jsonc | 38 +++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 5ff4e715..eafb9f73 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -133,6 +133,7 @@ Describe "Repository" { $script:policySetDefinitionsDep = Get-AzPolicySetDefinition -Name 'TestPolicySetDefinitionDep' -ManagementGroupName $($script:testManagementGroup.Name) $script:subscription = (Get-AzSubscription | Where-Object Id -eq $script:subscriptionId) $script:resourceGroup = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "App1-azopsrg") + $script:resourceGroupRemovalSupport = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "RemovalSupport-azopsrg") $script:resourceGroupCustomDeletion = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "CustomDeletion-azopsrg") $script:resourceGroupParallelDeploy = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "ParallelDeploy-azopsrg") $script:roleAssignments = (Get-AzRoleAssignment -ObjectId "023e7c1c-1fa4-4818-bb78-0a9c5e8b0217" | Where-Object { $_.Scope -eq "/subscriptions/$script:subscriptionId" -and $_.RoleDefinitionId -eq "acdd72a7-3385-48ef-bd42-f606fba81ae7" }) @@ -292,6 +293,11 @@ Describe "Repository" { $script:resourceGroupDeploymentName = "AzOps-{0}-{1}" -f $($script:resourceGroupPath.Name.Replace(".json", '')), $deploymentLocationId Write-PSFMessage -Level Debug -Message "ResourceGroupFile: $($script:resourceGroupFile)" -FunctionName "BeforeAll" + $script:resourceGroupRemovalSupportPath = ($filePaths | Where-Object Name -eq "microsoft.resources_resourcegroups-$(($script:resourceGroupRemovalSupport.ResourceGroupName).toLower()).json") + $script:resourceGroupRemovalSupportDirectory = ($script:resourceGroupRemovalSupportPath).Directory + $script:resourceGroupRemovalSupportFile = ($script:resourceGroupRemovalSupportPath).FullName + Write-PSFMessage -Level Debug -Message "ResourceGroupFile: $($script:resourceGroupRemovalSupportFile)" -FunctionName "BeforeAll" + $script:resourceGroupParallelDeployPath = ($filePaths | Where-Object Name -eq "microsoft.resources_resourcegroups-$(($script:resourceGroupParallelDeploy.ResourceGroupName).toLower()).json") $script:resourceGroupParallelDeployDirectory = ($script:resourceGroupParallelDeployPath).Directory $script:resourceGroupParallelDeployFile = ($script:resourceGroupParallelDeployPath).FullName @@ -379,7 +385,8 @@ Describe "Repository" { "D`t$script:policySetDefinitionsFile", "D`t$script:policyExemptionsFile", "D`t$script:roleAssignmentsFile", - "D`t$script:locksFile" + "D`t$script:locksFile", + "D`t$script:resourceGroupRemovalSupportFile" ) $DeleteSetContents = '-- ' $DeleteSetContents += $script:policyAssignmentsFile @@ -410,12 +417,18 @@ Describe "Repository" { $DeleteSetContents += $Script:locksFile $DeleteSetContents += [Environment]::NewLine $DeleteSetContents += (Get-Content $Script:locksFile) + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += '-- ' + $DeleteSetContents += $script:resourceGroupRemovalSupportFile + $DeleteSetContents += [Environment]::NewLine + $DeleteSetContents += (Get-Content $script:resourceGroupRemovalSupportFile) Remove-Item -Path $script:policyAssignmentsFile -Force Remove-Item -Path $Script:policyDefinitionsFile -Force Remove-Item -Path $Script:policySetDefinitionsFile -Force Remove-Item -Path $Script:policyExemptionsFile -Force Remove-Item -Path $Script:roleAssignmentsFile -Force Remove-Item -Path $Script:locksFile -Force + Remove-Item -Path $script:resourceGroupRemovalSupportFile -Force Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents } @@ -856,6 +869,39 @@ Describe "Repository" { } #endregion + #region Scope - Resource Group (./root/tenant root group/test/platform/management/subscription-0/RemovalSupport-azopsrg) + It "Resource Group directory should exist" { + Test-Path -Path $script:resourceGroupRemovalSupportDirectory | Should -BeTrue + } + It "Resource Group file should exist" { + Test-Path -Path $script:resourceGroupRemovalSupportFile | Should -BeTrue + } + It "Resource Group resource type should exist" { + $fileContents = Get-Content -Path $script:resourceGroupRemovalSupportFile -Raw | ConvertFrom-Json -Depth 25 + $fileContents.resources[0].type | Should -BeTrue + } + It "Resource Group resource name should exist" { + $fileContents = Get-Content -Path $script:resourceGroupRemovalSupportFile -Raw | ConvertFrom-Json -Depth 25 + $fileContents.resources[0].name | Should -BeTrue + } + It "Resource Group resource apiVersion should exist" { + $fileContents = Get-Content -Path $script:resourceGroupRemovalSupportFile -Raw | ConvertFrom-Json -Depth 25 + $fileContents.resources[0].apiVersion | Should -BeTrue + } + It "Resource Group resource properties should exist" { + $fileContents = Get-Content -Path $script:resourceGroupRemovalSupportFile -Raw | ConvertFrom-Json -Depth 25 + $fileContents.resources[0].properties | Should -BeTrue + } + It "Resource Group resource type should match" { + $fileContents = Get-Content -Path $script:resourceGroupRemovalSupportFile -Raw | ConvertFrom-Json -Depth 25 + $fileContents.resources[0].type | Should -Be "Microsoft.Resources/resourceGroups" + } + It "Resource Group deletion should be successful" { + $rgDeletion = Get-AzResourceGroup -Id $script:resourceGroupRemovalSupport.ResourceId -ErrorAction SilentlyContinue + $rgDeletion | Should -Be $Null + } + #endregion + #region Deploy Resource Group via bicep It "Bicep deployment should be successful" { $script:bicepDeployment = Get-AzSubscriptionDeployment -Name $script:bicepDeploymentName diff --git a/src/tests/templates/azuredeploy.jsonc b/src/tests/templates/azuredeploy.jsonc index c6fd46b8..5d64d707 100644 --- a/src/tests/templates/azuredeploy.jsonc +++ b/src/tests/templates/azuredeploy.jsonc @@ -506,6 +506,12 @@ "name": "App1-azopsrg", "location": "northeurope" }, + { + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2019-10-01", + "name": "RemovalSupport-azopsrg", + "location": "northeurope" + }, { "type": "Microsoft.Resources/resourceGroups", "apiVersion": "2019-10-01", @@ -618,6 +624,38 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2019-10-01", + "name": "RemovalRGSupportDeploy", + "resourceGroup": "RemovalSupport-azopsrg", + "dependsOn": [ + "RemovalSupport-azopsrg" + ], + "properties": { + "mode": "Incremental", + "expressionEvaluationOptions": { + "scope": "inner" + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "apiVersion": "2019-02-01", + "type": "Microsoft.Network/routeTables", + "name": "ForgetMeRouteTable", + "location": "northeurope", + "properties": { + "disableBgpRoutePropagation": "false" + } + } + ], + "outputs": { + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2019-10-01", From 2d111c48a7b1f846d13f67727cd6c4737606f50c Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 28 Feb 2024 14:38:50 +0000 Subject: [PATCH 38/47] Update --- src/tests/integration/Repository.Tests.ps1 | 68 +++++++++++----------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index eafb9f73..4786f450 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -388,40 +388,40 @@ Describe "Repository" { "D`t$script:locksFile", "D`t$script:resourceGroupRemovalSupportFile" ) - $DeleteSetContents = '-- ' - $DeleteSetContents += $script:policyAssignmentsFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:policyAssignmentsFile) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $Script:policyDefinitionsFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:policyDefinitionsFile) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $Script:policySetDefinitionsFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:policySetDefinitionsFile) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $Script:policyExemptionsFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:policyExemptionsFile) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $Script:roleAssignmentsFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:roleAssignmentsFile) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $Script:locksFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:locksFile) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $script:resourceGroupRemovalSupportFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $script:resourceGroupRemovalSupportFile) + $deleteSetContents = '-- ' + $deleteSetContents += $script:policyAssignmentsFile + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += (Get-Content $Script:policyAssignmentsFile) + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += '-- ' + $deleteSetContents += $Script:policyDefinitionsFile + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += (Get-Content $Script:policyDefinitionsFile) + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += '-- ' + $deleteSetContents += $Script:policySetDefinitionsFile + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += (Get-Content $Script:policySetDefinitionsFile) + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += '-- ' + $deleteSetContents += $Script:policyExemptionsFile + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += (Get-Content $Script:policyExemptionsFile) + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += '-- ' + $deleteSetContents += $Script:roleAssignmentsFile + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += (Get-Content $Script:roleAssignmentsFile) + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += '-- ' + $deleteSetContents += $Script:locksFile + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += (Get-Content $Script:locksFile) + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += '-- ' + $deleteSetContents += $script:resourceGroupRemovalSupportFile + $deleteSetContents += [Environment]::NewLine + $deleteSetContents += (Get-Content $script:resourceGroupRemovalSupportFile) Remove-Item -Path $script:policyAssignmentsFile -Force Remove-Item -Path $Script:policyDefinitionsFile -Force Remove-Item -Path $Script:policySetDefinitionsFile -Force From d4d309304846dd4dcdd7d1785af2ac42f606f808 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 29 Feb 2024 10:55:11 +0000 Subject: [PATCH 39/47] Update --- .../policyAssignments/scenario.ps1 | 6 +- .../roleAssignments/scenario.ps1 | 6 +- src/tests/integration/Repository.Tests.ps1 | 109 +++++------------- 3 files changed, 36 insertions(+), 85 deletions(-) diff --git a/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 b/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 index 3380a61f..969dcd8b 100644 --- a/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 +++ b/src/tests/functional/Microsoft.Authorization/policyAssignments/scenario.ps1 @@ -40,10 +40,8 @@ Describe "Scenario - policyAssignments" { $changeSet = @( "D`t$script:file" ) - $deleteSetContents = '-- ' - $deleteSetContents += $script:file - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $script:file) + [string[]]$deleteSetContents = "-- $script:file" + [string[]]$deleteSetContents += (Get-Content $script:file) Remove-Item -Path $script:file -Force try { Write-PSFMessage -Level Debug -Message "Deletion Scenario $script:resourceType starting: $script:file" -FunctionName "Functional Tests" diff --git a/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 b/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 index fdc23fa7..dab1c6d3 100644 --- a/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 +++ b/src/tests/functional/Microsoft.Authorization/roleAssignments/scenario.ps1 @@ -40,10 +40,8 @@ Describe "Scenario - roleAssignments" { $changeSet = @( "D`t$script:file" ) - $deleteSetContents = '-- ' - $deleteSetContents += $script:file - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $script:file) + [string[]]$deleteSetContents = "-- $script:file" + [string[]]$deleteSetContents += (Get-Content $script:file) Remove-Item -Path $script:file -Force try { Write-PSFMessage -Level Debug -Message "Deletion Scenario $script:resourceType starting: $script:file" -FunctionName "Functional Tests" diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 4786f450..1fb24ba0 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -388,47 +388,20 @@ Describe "Repository" { "D`t$script:locksFile", "D`t$script:resourceGroupRemovalSupportFile" ) - $deleteSetContents = '-- ' - $deleteSetContents += $script:policyAssignmentsFile - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $Script:policyAssignmentsFile) - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += '-- ' - $deleteSetContents += $Script:policyDefinitionsFile - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $Script:policyDefinitionsFile) - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += '-- ' - $deleteSetContents += $Script:policySetDefinitionsFile - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $Script:policySetDefinitionsFile) - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += '-- ' - $deleteSetContents += $Script:policyExemptionsFile - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $Script:policyExemptionsFile) - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += '-- ' - $deleteSetContents += $Script:roleAssignmentsFile - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $Script:roleAssignmentsFile) - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += '-- ' - $deleteSetContents += $Script:locksFile - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $Script:locksFile) - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += '-- ' - $deleteSetContents += $script:resourceGroupRemovalSupportFile - $deleteSetContents += [Environment]::NewLine - $deleteSetContents += (Get-Content $script:resourceGroupRemovalSupportFile) - Remove-Item -Path $script:policyAssignmentsFile -Force - Remove-Item -Path $Script:policyDefinitionsFile -Force - Remove-Item -Path $Script:policySetDefinitionsFile -Force - Remove-Item -Path $Script:policyExemptionsFile -Force - Remove-Item -Path $Script:roleAssignmentsFile -Force - Remove-Item -Path $Script:locksFile -Force - Remove-Item -Path $script:resourceGroupRemovalSupportFile -Force + [string[]]$deleteSetContents = "-- $script:policyAssignmentsFile" + [string[]]$deleteSetContents += (Get-Content $Script:policyAssignmentsFile) + [string[]]$deleteSetContents = "-- $Script:policyDefinitionsFile" + [string[]]$deleteSetContents += (Get-Content $Script:policyDefinitionsFile) + [string[]]$deleteSetContents = "-- $Script:policySetDefinitionsFile" + [string[]]$deleteSetContents += (Get-Content $Script:policySetDefinitionsFile) + [string[]]$deleteSetContents = "-- $Script:policyExemptionsFile" + [string[]]$deleteSetContents += (Get-Content $Script:policyExemptionsFile) + [string[]]$deleteSetContents = "-- $Script:roleAssignmentsFile" + [string[]]$deleteSetContents += (Get-Content $Script:roleAssignmentsFile) + [string[]]$deleteSetContents = "-- $Script:locksFile" + [string[]]$deleteSetContents += (Get-Content $Script:locksFile) + [string[]]$deleteSetContents = "-- $script:resourceGroupRemovalSupportFile" + [string[]]$deleteSetContents += (Get-Content $script:resourceGroupRemovalSupportFile) Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents } @@ -1112,10 +1085,8 @@ Describe "Repository" { $changeSet = @( "D`t$script:policyDefinitionsDepFile" ) - $DeleteSetContents += '-- ' - $DeleteSetContents += $Script:policyDefinitionsDepFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:policyDefinitionsDepFile) + [string[]]$deleteSetContents = "-- $Script:policyDefinitionsDepFile" + [string[]]$deleteSetContents += (Get-Content $Script:policyDefinitionsDepFile) Remove-Item -Path $Script:policyDefinitionsDepFile -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } @@ -1123,10 +1094,8 @@ Describe "Repository" { $changeSet = @( "D`t$script:policySetDefinitionsDepFile" ) - $DeleteSetContents = '-- ' - $DeleteSetContents += $Script:policySetDefinitionsDepFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $Script:policySetDefinitionsDepFile) + [string[]]$deleteSetContents = "-- $Script:policySetDefinitionsDepFile" + [string[]]$deleteSetContents += (Get-Content $Script:policySetDefinitionsDepFile) Remove-Item -Path $Script:policySetDefinitionsDepFile -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } @@ -1134,10 +1103,8 @@ Describe "Repository" { $changeSet = @( "D`t$script:policyDefinitionsDep2File" ) - $DeleteSetContents = '-- ' - $DeleteSetContents += $script:policyDefinitionsDep2File - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $script:policyDefinitionsDep2File) + [string[]]$deleteSetContents = "-- $script:policyDefinitionsDep2File" + [string[]]$deleteSetContents += (Get-Content $script:policyDefinitionsDep2File) Remove-Item -Path $script:policyDefinitionsDep2File -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } @@ -1145,10 +1112,8 @@ Describe "Repository" { $changeSet = @( "D`t$script:policyAssignmentsDepFile" ) - $DeleteSetContents = '-- ' - $DeleteSetContents += $script:policyAssignmentsDepFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $script:policyAssignmentsDepFile) + [string[]]$deleteSetContents = "-- $script:policyAssignmentsDepFile" + [string[]]$deleteSetContents += (Get-Content $script:policyAssignmentsDepFile) Remove-Item -Path $script:policyAssignmentsDepFile -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } @@ -1156,10 +1121,8 @@ Describe "Repository" { $changeSet = @( "D`t$script:policyAssignmentsDep2File" ) - $DeleteSetContents = '-- ' - $DeleteSetContents += $script:policyAssignmentsDep2File - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $script:policyAssignmentsDep2File) + [string[]]$deleteSetContents = "-- $script:policyAssignmentsDep2File" + [string[]]$deleteSetContents += (Get-Content $script:policyAssignmentsDep2File) Remove-Item -Path $script:policyAssignmentsDep2File -Force {Invoke-AzOpsPush -ChangeSet $changeSet -DeleteSetContents $deleteSetContents -WhatIf:$true} | Should -Throw } @@ -1294,26 +1257,18 @@ Describe "Repository" { "A`t$($script:deployCustomLock.FullName)" ) {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw - Start-Sleep -Seconds 5 + Start-Sleep -Seconds 10 $changeSet = @( "D`t$($script:deployCustomRt.FullName[0])", "D`t$($script:deployCustomLock.FullName)", "D`t$script:policyAssignmentsDeletionFile" ) - $DeleteSetContents = '-- ' - $DeleteSetContents += $script:deployCustomRt.FullName[0] - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $script:deployCustomRt.FullName[0]) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $script:deployCustomLock.FullName - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $script:deployCustomLock.FullName) - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += '-- ' - $DeleteSetContents += $script:policyAssignmentsDeletionFile - $DeleteSetContents += [Environment]::NewLine - $DeleteSetContents += (Get-Content $script:policyAssignmentsDeletionFile) + [string[]]$deleteSetContents = "-- $($script:deployCustomRt.FullName[0])" + [string[]]$deleteSetContents += (Get-Content $script:deployCustomRt.FullName[0]) + [string[]]$deleteSetContents += "-- $($script:deployCustomLock.FullName)" + [string[]]$deleteSetContents += (Get-Content $script:deployCustomLock.FullName) + [string[]]$deleteSetContents += "-- $script:policyAssignmentsDeletionFile" + [string[]]$deleteSetContents += (Get-Content $script:policyAssignmentsDeletionFile) Remove-Item -Path $script:deployCustomRt.FullName[0] -Force Remove-Item -Path $script:deployCustomLock.FullName -Force Remove-Item -Path $script:policyAssignmentsDeletionFile -Force @@ -1321,7 +1276,7 @@ Describe "Repository" { Set-PSFConfig -FullName AzOps.Core.CustomTemplateResourceDeletion -Value $false Start-Sleep -Seconds 30 (Get-AzResource -ResourceGroupName $script:resourceGroupCustomDeletion.ResourceGroupName).Count | Should -Be 0 - Get-AzPolicyAssignment -Id $script:policyAssignmentsDeletion.ResourceId -ErrorAction SilentlyContinue | Should -BeNullOrEmpty + Get-AzPolicyAssignment -Id $script:policyAssignmentsDeletion.ResourceId -ErrorAction SilentlyContinue | Should -Be $Null } #endregion } From a1f782906ca0026bbb2094775da8bbb41ea5b821 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 29 Feb 2024 13:45:00 +0000 Subject: [PATCH 40/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 71 ++++++++++++++++-------------- src/localized/en-us/Strings.psd1 | 1 + 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 9bdc870f..3ae67f90 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -343,40 +343,47 @@ } if ($DeleteSetContents -and $deleteSet) { Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Change.Delete' - # Iterate through each line in $DeleteSetContents - for ($i = 0; $i -lt $DeleteSetContents.Count; $i++) { - $line = $DeleteSetContents[$i].Trim() - # Check if the line starts with '-- ' and matches any filename in $deleteSet - if ($line -match '^-- (.+)$') { - $fileName = $matches[1] - if ($deleteSet -contains $fileName) { - Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Change.Delete.File' -LogStringValues $fileName - # Collect lines until the next line starting with '--' - $objectLines = @($line) - $i++ - while ($i -lt $DeleteSetContents.Count) { - $currentLine = $DeleteSetContents[$i].Trim() - # Check if the line starts with '-- ' followed by any filename in $deleteSet - if ($currentLine -match '^-- (.+)$' -and $deleteSet -contains $matches[1]) { - $i-- - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.NextTempFile' -LogStringValues $currentLine - break # Exit the loop if the line starts with '-- ' and matches a filename in $deleteSet - } - $objectLines += $currentLine + # Count if $DeleteSetContents contains 1 or less + if ($DeleteSetContents.Count -le 1) { + # DeleteSetContents has no file content or is malformed + Write-AzOpsMessage -LogLevel Error -LogString 'Invoke-AzOpsPush.Change.Delete.DeleteSetContents' -LogStringValues $DeleteSetContents + } + else { + # Iterate through each line in $DeleteSetContents + for ($i = 0; $i -lt $DeleteSetContents.Count; $i++) { + $line = $DeleteSetContents[$i].Trim() + # Check if the line starts with '-- ' and matches any filename in $deleteSet + if ($line -match '^-- (.+)$') { + $fileName = $matches[1] + if ($deleteSet -contains $fileName) { + Write-AzOpsMessage -LogLevel Important -LogString 'Invoke-AzOpsPush.Change.Delete.File' -LogStringValues $fileName + # Collect lines until the next line starting with '--' + $objectLines = @($line) $i++ + while ($i -lt $DeleteSetContents.Count) { + $currentLine = $DeleteSetContents[$i].Trim() + # Check if the line starts with '-- ' followed by any filename in $deleteSet + if ($currentLine -match '^-- (.+)$' -and $deleteSet -contains $matches[1]) { + $i-- + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.NextTempFile' -LogStringValues $currentLine + break # Exit the loop if the line starts with '-- ' and matches a filename in $deleteSet + } + $objectLines += $currentLine + $i++ + } + # When processed as designed there is no file present in the running branch. + # To run a removal AzOps re-creates the file and content based on $DeleteSetContents momentarily for processing, it is disregarded afterwards. + if (-not(Test-Path -Path (Split-Path -Path $fileName))) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.TempFile' -LogStringValues $fileName + New-Item -Path (Split-Path -Path $fileName) -ItemType Directory | Out-Null + } + # Create $fileName and set $content + $objectLines = $objectLines[1..$objectLines.Count] + $content = $objectLines.replace("-- $fileName", "") -join "`r`n" + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' -LogStringValues $fileName, $content + Set-Content -Path $fileName -Value $content + $i-- # Move back one step to process the next line properly } - # When processed as designed there is no file present in the running branch. - # To run a removal AzOps re-creates the file and content based on $DeleteSetContents momentarily for processing, it is disregarded afterwards. - if (-not(Test-Path -Path (Split-Path -Path $fileName))) { - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.TempFile' -LogStringValues $fileName - New-Item -Path (Split-Path -Path $fileName) -ItemType Directory | Out-Null - } - # Create $fileName and set $content - $objectLines = $objectLines[1..$objectLines.Count] - $content = $objectLines.replace("-- $fileName", "") -join "`r`n" - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' -LogStringValues $fileName, $content - Set-Content -Path $fileName -Value $content - $i-- # Move back one step to process the next line properly } } } diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index e7d15d54..b04f2851 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -191,6 +191,7 @@ 'Invoke-AzOpsPush.Change.AddModify' = 'Adding or modifying:' # 'Invoke-AzOpsPush.Change.AddModify.File' = ' {0}' # $item 'Invoke-AzOpsPush.Change.Delete' = 'Deleting:' # + 'Invoke-AzOpsPush.Change.Delete.DeleteSetContents' = '[DeleteSetContents] has no file content or is malformed: {0}' # $DeleteSetContents 'Invoke-AzOpsPush.Change.Delete.File' = ' {0}' # $fileName 'Invoke-AzOpsPush.Change.Delete.TempFile' = 'Creating temporary file dir for deletion processing: {0}' # $fileName 'Invoke-AzOpsPush.Change.Delete.NextTempFile' = 'Exiting while loop, file detected in $DeleteSetContents for deletion processing based on this content line: [{0}]' # $currentLine From 735b620f5a6e057d154f1db5b709a1062e6b2497 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 29 Feb 2024 18:59:04 +0000 Subject: [PATCH 41/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 9 +---- src/internal/functions/Get-AzOpsResource.ps1 | 39 +++++++++++++++++++ .../functions/Remove-AzOpsDeployment.ps1 | 3 +- .../functions/Remove-AzResourceRaw.ps1 | 24 +++++++----- src/localized/en-us/Strings.psd1 | 2 +- 5 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 src/internal/functions/Get-AzOpsResource.ps1 diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 3ae67f90..47443ec6 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -568,14 +568,7 @@ # Check each failed removal and attempt to get the associated resource foreach ($fail in $removeActionFail) { $resource = $null - Set-AzOpsContext -ScopeObject $fail.ScopeObject - # Determine if the resource is a lock or a regular resource - if ($fail.FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $fail.FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { - $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $fail.FullyQualifiedResourceId } -ErrorAction SilentlyContinue - } - else { - $resource = Get-AzResource -ResourceId $fail.FullyQualifiedResourceId -ErrorAction SilentlyContinue - } + $resource = Get-AzOpsResource -ResourceId $fail.FullyQualifiedResourceId -ScopeObject $fail.ScopeObject # If the resource is found, log the failure if ($resource) { $throwFail = $true diff --git a/src/internal/functions/Get-AzOpsResource.ps1 b/src/internal/functions/Get-AzOpsResource.ps1 new file mode 100644 index 00000000..d9b33d20 --- /dev/null +++ b/src/internal/functions/Get-AzOpsResource.ps1 @@ -0,0 +1,39 @@ +function Get-AzOpsResource { + + <# + .SYNOPSIS + Check if the Azure resource exists. + .DESCRIPTION + Check if the Azure resource exists. + .PARAMETER ResourceId + The ResourceId to check. + .PARAMETER ScopeObject + Object used to set Azure context for operation. + .EXAMPLE + > Get-AzOpsResource -ResourceId /subscriptions/6a35d1cc-ae17-4c0c-9a66-1d9a25647b19/resourceGroups/NetworkWatcherRG -ScopeObject $ScopeObject + #> + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string] + $ResourceId, + [Parameter(Mandatory = $true)] + [AzOpsScope] + $ScopeObject + ) + + process { + Set-AzOpsContext -ScopeObject $ScopeObject + # Check if the resource exists + if ($ResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $ResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { + $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $ResourceId } -ErrorAction SilentlyContinue + } + else { + $resource = Get-AzResource -ResourceId $ResourceId -ErrorAction SilentlyContinue + } + if ($resource) { + return $resource + } + } +} \ No newline at end of file diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 78a34c0c..dde41045 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -329,8 +329,7 @@ Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true } if ($PSCmdlet.ShouldProcess("Remove $($scopeObject.Scope)?")) { - $null = Remove-AzResource -ResourceId $scopeObject.Scope -Force - Start-Sleep -Seconds 5 + $null = Remove-AzResourceRaw -FullyQualifiedResourceId $scopeObject.Scope -ScopeObject $scopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath } else { Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.SkipDueToWhatIf' diff --git a/src/internal/functions/Remove-AzResourceRaw.ps1 b/src/internal/functions/Remove-AzResourceRaw.ps1 index c6b7a1aa..e04cd252 100644 --- a/src/internal/functions/Remove-AzResourceRaw.ps1 +++ b/src/internal/functions/Remove-AzResourceRaw.ps1 @@ -47,20 +47,26 @@ ScopeObject = $scopeObject Status = 'success' } - # Set Azure context for removal operation - Set-AzOpsContext -ScopeObject $ScopeObject # Check if the resource exists - if ($FullyQualifiedResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $FullyQualifiedResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { - $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $FullyQualifiedResourceId } -ErrorAction SilentlyContinue - } - else { - $resource = Get-AzResource -ResourceId $FullyQualifiedResourceId -ErrorAction SilentlyContinue - } + $resource = Get-AzOpsResource -ResourceId $FullyQualifiedResourceId -ScopeObject $ScopeObject # Remove the resource if it exists if ($resource) { try { + # Set Azure context for removal operation + Set-AzOpsContext -ScopeObject $ScopeObject $null = Remove-AzResource -ResourceId $FullyQualifiedResourceId -Force -ErrorAction Stop - Start-Sleep -Seconds 5 + $maxAttempts = 4 + $attempt = 1 + $gone = $false + while ($gone -eq $false -and $attempt -le $maxAttempts) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.CheckExistence' -LogStringValues $FullyQualifiedResourceId + Start-Sleep -Seconds 10 + $tryResource = Get-AzOpsResource -ResourceId $FullyQualifiedResourceId -ScopeObject $ScopeObject + if (-not $tryResource) { + $gone = $true + } + $attempt++ + } } catch { # Log failure message diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index b04f2851..2cab349f 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -289,8 +289,8 @@ 'Remove-AzOpsDeployment.ResourceNotFound' = 'Unable to find resource of type {0} with id {1}.'# $scopeObject.resource, $scopeObject.scope, $resultsError 'Remove-AzOpsDeployment.SkipUnsupportedResource' = 'Deletion of AzOps generated file resources is only supported for locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments. Will NOT proceed with deletion of resource in file {0}'# $TemplateFilePath + 'Remove-AzResourceRaw.Resource.CheckExistence' = 'Checking existence after deletion of: [{0}]'# $FullyQualifiedResourceId 'Remove-AzResourceRaw.Resource.Failed' = 'Unable to delete resource of type {0} with id {1}'# $scopeObject.scope, $FullyQualifiedResourceId - 'Remove-AzResourceRawRecursive.Processing' = 'Recursive retry processing to delete resource of type {0} with id {1}'# $item.ScopeObject.resource, $item.FullyQualifiedResourceId 'Remove-AzOpsInvalidCharacter.Completed' = 'Valid string: {0}'# $String From b56ba7b889a8601d80ae4ded7def13bc37533801 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 29 Feb 2024 22:08:15 +0000 Subject: [PATCH 42/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 6 +-- src/internal/functions/Get-AzOpsResource.ps1 | 41 +++++++++++++------ .../functions/Remove-AzOpsDeployment.ps1 | 33 +++++++-------- .../functions/Remove-AzResourceRaw.ps1 | 23 ++++------- .../Remove-AzResourceRawRecursive.ps1 | 10 ++--- src/localized/en-us/Strings.psd1 | 8 ++-- 6 files changed, 63 insertions(+), 58 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 47443ec6..d48c044d 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -549,7 +549,7 @@ #Removal of Supported resourceTypes and Custom Templates $deletionList = Set-AzOpsRemoveOrder -DeletionList $deletionList -Index { $_.ScopeObject.Resource } $removalJob = $deletionList | Select-Object $uniqueProperties -Unique | Remove-AzOpsDeployment -WhatIf:$WhatIfPreference -DeleteSet (Resolve-Path -Path $deleteSet).Path - if ($removalJob.FullyQualifiedResourceId.Count -gt 0) { + if ($removalJob.ScopeObject.Scope.Count -gt 0) { Clear-PSFMessage # Identify failed removal attempts for potential retries $retry = $removalJob | Where-Object { $_.Status -eq 'failed' } @@ -568,11 +568,11 @@ # Check each failed removal and attempt to get the associated resource foreach ($fail in $removeActionFail) { $resource = $null - $resource = Get-AzOpsResource -ResourceId $fail.FullyQualifiedResourceId -ScopeObject $fail.ScopeObject + $resource = Get-AzOpsResource -ScopeObject $fail.ScopeObject # If the resource is found, log the failure if ($resource) { $throwFail = $true - Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Deletion.Failed' -LogStringValues $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath + Write-AzOpsMessage -LogLevel Critical -LogString 'Invoke-AzOpsPush.Deletion.Failed' -LogStringValues $fail.ScopeObject.Scope, $fail.TemplateFilePath, $fail.TemplateParameterFilePath } } # If any failures occurred, throw an exception diff --git a/src/internal/functions/Get-AzOpsResource.ps1 b/src/internal/functions/Get-AzOpsResource.ps1 index d9b33d20..d1e26963 100644 --- a/src/internal/functions/Get-AzOpsResource.ps1 +++ b/src/internal/functions/Get-AzOpsResource.ps1 @@ -5,19 +5,14 @@ Check if the Azure resource exists. .DESCRIPTION Check if the Azure resource exists. - .PARAMETER ResourceId - The ResourceId to check. .PARAMETER ScopeObject - Object used to set Azure context for operation. + The Resource to check. .EXAMPLE - > Get-AzOpsResource -ResourceId /subscriptions/6a35d1cc-ae17-4c0c-9a66-1d9a25647b19/resourceGroups/NetworkWatcherRG -ScopeObject $ScopeObject + > Get-AzOpsResource -ScopeObject $ScopeObject #> [CmdletBinding()] param ( - [Parameter(Mandatory = $true)] - [string] - $ResourceId, [Parameter(Mandatory = $true)] [AzOpsScope] $ScopeObject @@ -25,12 +20,32 @@ process { Set-AzOpsContext -ScopeObject $ScopeObject - # Check if the resource exists - if ($ResourceId -match '^/subscriptions/.*/providers/Microsoft.Authorization/locks' -or $ResourceId -match '^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Authorization/locks') { - $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $ResourceId } -ErrorAction SilentlyContinue - } - else { - $resource = Get-AzResource -ResourceId $ResourceId -ErrorAction SilentlyContinue + switch ($ScopeObject.Resource) { + # Check if the resource exist + 'locks' { + $resource = Get-AzResourceLock -Scope "/subscriptions/$($ScopeObject.Subscription)" -ErrorAction SilentlyContinue | Where-Object { $_.ResourceID -eq $ScopeObject.Scope } + } + 'policyAssignments' { + $resource = Get-AzPolicyAssignment -Id $scopeObject.Scope -ErrorAction SilentlyContinue + } + 'policyDefinitions' { + $resource = Get-AzPolicyDefinition -Id $scopeObject.Scope -ErrorAction SilentlyContinue + } + 'policyExemptions' { + $resource = Get-AzPolicyExemption -Id $scopeObject.Scope -ErrorAction SilentlyContinue + } + 'policySetDefinitions' { + $resource = Get-AzPolicySetDefinition -Id $scopeObject.Scope -ErrorAction SilentlyContinue + } + 'roleAssignments' { + $resource = Invoke-AzRestMethod -Path "$($scopeObject.Scope)?api-version=2022-04-01" | Where-Object { $_.StatusCode -eq 200 } + } + 'resourceGroups' { + $resource = Get-AzResourceGroup -Id $scopeObject.Scope -ErrorAction SilentlyContinue + } + default { + $resource = Get-AzResource -ResourceId $ScopeObject.Scope -ErrorAction SilentlyContinue + } } if ($resource) { return $resource diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index dde41045..82639a1a 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -249,17 +249,17 @@ switch ($scopeObject.Resource) { # Check resource existance through optimal path 'locks' { - $resourceToDelete = Get-AzResourceLock -Scope "/subscriptions/$($ScopeObject.Subscription)" -ErrorAction SilentlyContinue | Where-Object {$_.ResourceID -eq $ScopeObject.scope} + $resourceToDelete = Get-AzResourceLock -Scope "/subscriptions/$($ScopeObject.Subscription)" -ErrorAction SilentlyContinue | Where-Object { $_.ResourceID -eq $ScopeObject.Scope } } 'policyAssignments' { - $resourceToDelete = Get-AzPolicyAssignment -Id $scopeObject.scope -ErrorAction SilentlyContinue + $resourceToDelete = Get-AzPolicyAssignment -Id $scopeObject.Scope -ErrorAction SilentlyContinue if ($resourceToDelete) { $dependency += Get-AzPolicyAssignmentDeletionDependency -resourceToDelete $resourceToDelete $dependency += Get-AzLocksDeletionDependency -resourceToDelete $resourceToDelete } } 'policyDefinitions' { - $resourceToDelete = Get-AzPolicyDefinition -Id $scopeObject.scope -ErrorAction SilentlyContinue + $resourceToDelete = Get-AzPolicyDefinition -Id $scopeObject.Scope -ErrorAction SilentlyContinue if ($resourceToDelete) { $dependency += Get-AzPolicyAssignmentDeletionDependency -resourceToDelete $resourceToDelete $dependency += Get-AzPolicyDefinitionDeletionDependency -resourceToDelete $resourceToDelete @@ -267,20 +267,20 @@ } } 'policyExemptions' { - $resourceToDelete = Get-AzPolicyExemption -Id $scopeObject.scope -ErrorAction SilentlyContinue + $resourceToDelete = Get-AzPolicyExemption -Id $scopeObject.Scope -ErrorAction SilentlyContinue if ($resourceToDelete) { $dependency += Get-AzLocksDeletionDependency -resourceToDelete $resourceToDelete } } 'policySetDefinitions' { - $resourceToDelete = Get-AzPolicySetDefinition -Id $scopeObject.scope -ErrorAction SilentlyContinue + $resourceToDelete = Get-AzPolicySetDefinition -Id $scopeObject.Scope -ErrorAction SilentlyContinue if ($resourceToDelete) { $dependency += Get-AzPolicyAssignmentDeletionDependency -resourceToDelete $resourceToDelete $dependency += Get-AzLocksDeletionDependency -resourceToDelete $resourceToDelete } } 'roleAssignments' { - $resourceToDelete = Invoke-AzRestMethod -Path "$($scopeObject.scope)?api-version=2022-04-01" | Where-Object { $_.StatusCode -eq 200 } + $resourceToDelete = Invoke-AzRestMethod -Path "$($scopeObject.Scope)?api-version=2022-04-01" | Where-Object { $_.StatusCode -eq 200 } if ($resourceToDelete) { $dependency += Get-AzLocksDeletionDependency -resourceToDelete $resourceToDelete } @@ -329,7 +329,7 @@ Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -Results $results -RemoveAzOpsFlag $true } if ($PSCmdlet.ShouldProcess("Remove $($scopeObject.Scope)?")) { - $null = Remove-AzResourceRaw -FullyQualifiedResourceId $scopeObject.Scope -ScopeObject $scopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath + $null = Remove-AzResourceRaw -ScopeObject $scopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath } else { Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.SkipDueToWhatIf' @@ -349,21 +349,18 @@ $allResults = @() foreach ($change in $removalJobChanges) { $resource = $null + $resourceScopeObject = $null # Check if the resource exists - if ($change.RelativeResourceId.StartsWith('Microsoft.Authorization/locks/')) { - $resource = Get-AzResourceLock | Where-Object { $_.ResourceId -eq $change.FullyQualifiedResourceId } -ErrorAction SilentlyContinue - } - else { - $resource = Get-AzResource -ResourceId $change.FullyQualifiedResourceId -ErrorAction SilentlyContinue - } + $resourceScopeObject = New-AzOpsScope -Scope $change.FullyQualifiedResourceId -WhatIf:$false + $resource = Get-AzOpsResource -ScopeObject $resourceScopeObject if ($resource) { - $results = 'What if successful:{1}Performing the operation:{1}Deletion of target resource {0}.' -f $change.FullyQualifiedResourceId, [environment]::NewLine + $results = 'What if successful:{1}Performing the operation:{1}Deletion of target resource {0}.' -f $resourceScopeObject.Scope, [environment]::NewLine $allResults += $results Write-AzOpsMessage -LogLevel Verbose -LogString 'Set-AzOpsWhatIfOutput.WhatIfResults' -LogStringValues $results Write-AzOpsMessage -LogLevel InternalComment -LogString 'Set-AzOpsWhatIfOutput.WhatIfFile' # Check if the removal should be performed - if ($PSCmdlet.ShouldProcess("Remove $($change.FullyQualifiedResourceId)?")) { - $removeAction = Remove-AzResourceRaw -FullyQualifiedResourceId $change.FullyQualifiedResourceId -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath + if ($PSCmdlet.ShouldProcess("Remove $($resourceScopeObject.Scope)?")) { + $removeAction = Remove-AzResourceRaw -ScopeObject $resourceScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath # If removal failed, add to retry if ($removeAction.Status -eq 'failed') { $retry += $removeAction @@ -375,7 +372,7 @@ } else { # Log warning if resource not found - Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.resource, $change.FullyQualifiedResourceId + Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $ScopeObject.Resource, $change.FullyQualifiedResourceId $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $change.FullyQualifiedResourceId, [environment]::NewLine $allResults += $results } @@ -422,7 +419,7 @@ else { # No resource to remove was found Write-AzOpsMessage -LogLevel Warning -LogString 'Remove-AzOpsDeployment.ResourceNotFound' -LogStringValues $scopeObject.Resource, $scopeObject.Scope - $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $scopeObject.scope, [environment]::NewLine + $results = 'What if operation failed:{1}Deletion of target resource {0}.{1}Resource could not be found' -f $scopeObject.Scope, [environment]::NewLine Set-AzOpsWhatIfOutput -FilePath $TemplateFilePath -ParameterFilePath $TemplateParameterFilePath -Results $results -RemoveAzOpsFlag $true return } diff --git a/src/internal/functions/Remove-AzResourceRaw.ps1 b/src/internal/functions/Remove-AzResourceRaw.ps1 index e04cd252..1a8304ea 100644 --- a/src/internal/functions/Remove-AzResourceRaw.ps1 +++ b/src/internal/functions/Remove-AzResourceRaw.ps1 @@ -5,19 +5,16 @@ Performs resource deletion in Azure at any scope. .DESCRIPTION Performs resource deletion in Azure with FullyQualifiedResourceId and ScopeObject. - .PARAMETER FullyQualifiedResourceId - Parameter containing FullyQualifiedResourceId of resource to delete. .PARAMETER TemplateFilePath Path where the ARM templates can be found. .PARAMETER TemplateParameterFilePath Path where the parameters of the ARM templates can be found. .PARAMETER ScopeObject - Object used to set Azure context for removal operation. + Resource to delete. .EXAMPLE - > Remove-AzResourceRaw -FullyQualifiedResourceId '/subscriptions//resourceGroups//providers/Microsoft.KeyVault/vaults/' -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath + > Remove-AzResourceRaw -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath Name Value ---- ----- - FullyQualifiedResourceId /subscriptions//resourceGroups//providers/Microsoft.KeyVault/vaults/ TemplateFilePath /root/managementgroup/subscription/resourcegroup/template.json TemplateParameterFilePath /root/managementgroup/subscription/resourcegroup/template.parameters.json ScopeObject ScopeObject @@ -26,9 +23,6 @@ [CmdletBinding()] param ( - [Parameter(Mandatory = $true)] - [string] - $FullyQualifiedResourceId, [string] $TemplateFilePath, [string] @@ -41,27 +35,26 @@ process { # Construct result object $result = [PSCustomObject]@{ - FullyQualifiedResourceId = $FullyQualifiedResourceId TemplateFilePath = $TemplateFilePath TemplateParameterFilePath = $TemplateParameterFilePath - ScopeObject = $scopeObject + ScopeObject = $ScopeObject Status = 'success' } # Check if the resource exists - $resource = Get-AzOpsResource -ResourceId $FullyQualifiedResourceId -ScopeObject $ScopeObject + $resource = Get-AzOpsResource -ScopeObject $ScopeObject # Remove the resource if it exists if ($resource) { try { # Set Azure context for removal operation Set-AzOpsContext -ScopeObject $ScopeObject - $null = Remove-AzResource -ResourceId $FullyQualifiedResourceId -Force -ErrorAction Stop + $null = Remove-AzResource -ResourceId $ScopeObject.Scope -Force -ErrorAction Stop $maxAttempts = 4 $attempt = 1 $gone = $false while ($gone -eq $false -and $attempt -le $maxAttempts) { - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.CheckExistence' -LogStringValues $FullyQualifiedResourceId + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.CheckExistence' -LogStringValues $ScopeObject.Scope Start-Sleep -Seconds 10 - $tryResource = Get-AzOpsResource -ResourceId $FullyQualifiedResourceId -ScopeObject $ScopeObject + $tryResource = Get-AzOpsResource -ScopeObject $ScopeObject if (-not $tryResource) { $gone = $true } @@ -70,7 +63,7 @@ } catch { # Log failure message - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.Failed' -LogStringValues $ScopeObject.resource, $FullyQualifiedResourceId + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.Failed' -LogStringValues $ScopeObject.Resource, $ScopeObject.Scope $result.Status = 'failed' } } diff --git a/src/internal/functions/Remove-AzResourceRawRecursive.ps1 b/src/internal/functions/Remove-AzResourceRawRecursive.ps1 index 8762e7fa..aaaacb1b 100644 --- a/src/internal/functions/Remove-AzResourceRawRecursive.ps1 +++ b/src/internal/functions/Remove-AzResourceRawRecursive.ps1 @@ -31,10 +31,10 @@ # Base case: All items have been used, perform action on the current order foreach ($item in $CurrentOrder) { if ($item.Status -eq 'failed' -or $null -eq $item.Status) { - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRawRecursive.Processing' -LogStringValues $item.ScopeObject.resource, $item.FullyQualifiedResourceId + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRawRecursive.Processing' -LogStringValues $item.ScopeObject.Resource, $item.ScopeObject.Scope # Attempt to remove the resource - $result = Remove-AzResourceRaw -FullyQualifiedResourceId $item.FullyQualifiedResourceId -ScopeObject $item.ScopeObject -TemplateFilePath $item.TemplateFilePath -TemplateParameterFilePath $item.TemplateParameterFilePath - if ($result.Status -eq 'failed' -and $result.FullyQualifiedResourceId -notin $OutputObject.FullyQualifiedResourceId){ + $result = Remove-AzResourceRaw -ScopeObject $item.ScopeObject -TemplateFilePath $item.TemplateFilePath -TemplateParameterFilePath $item.TemplateParameterFilePath + if ($result.Status -eq 'failed' -and $result.ScopeObject.Scope -notin $OutputObject.ScopeObject.Scope){ # Add failed result to the output object $OutputObject += $result } @@ -48,9 +48,9 @@ # Filter out items already processed successfully $filteredOutputObject = @() foreach ($item in $InputObject) { - if ($item.FullyQualifiedResourceId -in $OutputObject.FullyQualifiedResourceId) { + if ($item.ScopeObject.Scope -in $OutputObject.ScopeObject.Scope) { foreach ($output in $OutputObject) { - if ($output.FullyQualifiedResourceId -eq $item.FullyQualifiedResourceId -and $output.Status -eq 'failed') { + if ($output.ScopeObject.Scope -eq $item.ScopeObject.Scope -and $output.Status -eq 'failed') { # Add previously failed item to the filtered output $filteredOutputObject += $output continue diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 2cab349f..61cba749 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -196,7 +196,7 @@ 'Invoke-AzOpsPush.Change.Delete.TempFile' = 'Creating temporary file dir for deletion processing: {0}' # $fileName 'Invoke-AzOpsPush.Change.Delete.NextTempFile' = 'Exiting while loop, file detected in $DeleteSetContents for deletion processing based on this content line: [{0}]' # $currentLine 'Invoke-AzOpsPush.Change.Delete.SetTempFileContent' = 'Set temporary file content: [{1}], in [{0}]' # $fileName, $jsonValue - 'Invoke-AzOpsPush.Deletion.Failed' = 'Deletion of resources {0}, has failed using templates: {1}, {2}, this could be due to delayed deletion acceptance from Azure, please investigate and take action.' # $fail.FullyQualifiedResourceId, $fail.TemplateFilePath, $fail.TemplateParameterFilePath + 'Invoke-AzOpsPush.Deletion.Failed' = 'Deletion of resources {0}, has failed using templates: {1}, {2}, this could be due to delayed deletion acceptance from Azure, please investigate and take action.' # $fail.ScopeObject.Scope, $fail.TemplateFilePath, $fail.TemplateParameterFilePath 'Invoke-AzOpsPush.Deletion.Retry' = 'Deletion of {0} resources unsuccessful, initiate final retry combination.' # $retry.Count 'Invoke-AzOpsPush.Deploy.ProviderFeature' = 'Invoking new state deployment - *.providerfeatures.json for a file {0}' # $addition 'Invoke-AzOpsPush.Deploy.ResourceProvider' = 'Invoking new state deployment - *.resourceproviders.json for a file {0}' # $addition @@ -286,12 +286,12 @@ 'Remove-AzOpsDeployment.ResourceDependencyNested' = 'resource dependency {0} for complete deletion of {1} is outside of supported AzOps scope. Please remove this dependency in Azure without AzOps.'# $roleAssignmentId, $policyAssignment.ResourceId 'Remove-AzOpsDeployment.ResourceDependencyNotFound' = 'Missing resource dependency {0} for successfull deletion of {1}. Please add missing resource and retry.'# $resource.ResourceId, $scopeObject.Scope 'Remove-AzOpsDeployment.Resource.RetryCount' = 'Retry deletion of {0} resources in different order'# $retry.Count - 'Remove-AzOpsDeployment.ResourceNotFound' = 'Unable to find resource of type {0} with id {1}.'# $scopeObject.resource, $scopeObject.scope, $resultsError + 'Remove-AzOpsDeployment.ResourceNotFound' = 'Unable to find resource of type {0} with id {1}.'# $scopeObject.Resource, $scopeObject.Scope, $resultsError 'Remove-AzOpsDeployment.SkipUnsupportedResource' = 'Deletion of AzOps generated file resources is only supported for locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments. Will NOT proceed with deletion of resource in file {0}'# $TemplateFilePath 'Remove-AzResourceRaw.Resource.CheckExistence' = 'Checking existence after deletion of: [{0}]'# $FullyQualifiedResourceId - 'Remove-AzResourceRaw.Resource.Failed' = 'Unable to delete resource of type {0} with id {1}'# $scopeObject.scope, $FullyQualifiedResourceId - 'Remove-AzResourceRawRecursive.Processing' = 'Recursive retry processing to delete resource of type {0} with id {1}'# $item.ScopeObject.resource, $item.FullyQualifiedResourceId + 'Remove-AzResourceRaw.Resource.Failed' = 'Unable to delete resource of type {0} with id {1}'# $ScopeObject.Resource, $ScopeObject.Scope + 'Remove-AzResourceRawRecursive.Processing' = 'Recursive retry processing to delete resource of type {0} with id {1}'# $item.ScopeObject.Resource, $item.ScopeObject.Scope 'Remove-AzOpsInvalidCharacter.Completed' = 'Valid string: {0}'# $String 'Remove-AzOpsInvalidCharacter.Invalid' = 'Invalid character detected in string: {0}, further processing initiated'# $String From 86d2303e8b4a35e404574863f28cb59d7f3f1440 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 1 Mar 2024 06:38:31 +0000 Subject: [PATCH 43/47] Update --- src/functions/Invoke-AzOpsPush.ps1 | 2 +- src/internal/functions/Remove-AzOpsDeployment.ps1 | 2 +- src/internal/functions/Remove-AzResourceRaw.ps1 | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index d48c044d..5525c61f 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -568,7 +568,7 @@ # Check each failed removal and attempt to get the associated resource foreach ($fail in $removeActionFail) { $resource = $null - $resource = Get-AzOpsResource -ScopeObject $fail.ScopeObject + $resource = Get-AzOpsResource -ScopeObject $fail.ScopeObject -ErrorAction SilentlyContinue # If the resource is found, log the failure if ($resource) { $throwFail = $true diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 82639a1a..1d71ad54 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -352,7 +352,7 @@ $resourceScopeObject = $null # Check if the resource exists $resourceScopeObject = New-AzOpsScope -Scope $change.FullyQualifiedResourceId -WhatIf:$false - $resource = Get-AzOpsResource -ScopeObject $resourceScopeObject + $resource = Get-AzOpsResource -ScopeObject $resourceScopeObject -ErrorAction SilentlyContinue if ($resource) { $results = 'What if successful:{1}Performing the operation:{1}Deletion of target resource {0}.' -f $resourceScopeObject.Scope, [environment]::NewLine $allResults += $results diff --git a/src/internal/functions/Remove-AzResourceRaw.ps1 b/src/internal/functions/Remove-AzResourceRaw.ps1 index 1a8304ea..74c3253d 100644 --- a/src/internal/functions/Remove-AzResourceRaw.ps1 +++ b/src/internal/functions/Remove-AzResourceRaw.ps1 @@ -41,7 +41,7 @@ Status = 'success' } # Check if the resource exists - $resource = Get-AzOpsResource -ScopeObject $ScopeObject + $resource = Get-AzOpsResource -ScopeObject $ScopeObject -ErrorAction SilentlyContinue # Remove the resource if it exists if ($resource) { try { @@ -54,7 +54,7 @@ while ($gone -eq $false -and $attempt -le $maxAttempts) { Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.CheckExistence' -LogStringValues $ScopeObject.Scope Start-Sleep -Seconds 10 - $tryResource = Get-AzOpsResource -ScopeObject $ScopeObject + $tryResource = Get-AzOpsResource -ScopeObject $ScopeObject -ErrorAction SilentlyContinue if (-not $tryResource) { $gone = $true } From 41c12c0e5375566f9776b5b5bde367d9e4314e17 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 1 Mar 2024 08:32:05 +0000 Subject: [PATCH 44/47] Update --- docs/wiki/Settings.md | 59 ++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/docs/wiki/Settings.md b/docs/wiki/Settings.md index ed5716ef..17ab7e8b 100644 --- a/docs/wiki/Settings.md +++ b/docs/wiki/Settings.md @@ -13,35 +13,36 @@ The following configuration values can be modified within the `settings.json` fi | 01 | ApplicationInsights | Turn on or off logging to Application Insight | `"Core.ApplicationInsights": true` | | 02 | AutoGeneratedTemplateFolderPath | Generate sub folder for composite resources (/.az) | `"Core.AutoGeneratedTemplateFolderPath": ".az"`
root
└── tenant root group (e42bc18f)
        ├── .az
        │      └── microsoft.management_managementgroups.json
        └── mymanagementgroup (mymanagementgroup)
                └── .az
                        ├── microsoft.authorization_policyassignments.json
                        ├── microsoft.authorization_policydefinitions.json
                        ├── microsoft.authorization_roleassignments.json
                        └── microsoft.management_managementgroups.json | | 03 | AutoInitialize | Run Initialize-AzOpsEnvironment when module is loaded. *Not recommended to change* | `"Core.AutoInitialize": true` | -| 04 | CustomJqTemplatePath | Folder where custom Jq templates are located. | `"Core.CustomJqTemplatePath": ".customtemplates"` | -| 05 | SkipCustomJqTemplate | Do not use custom Jq templates, controls if AzOps looks for custom templates at `CustomJqTemplatePath`. [Read more](https://github.com/azure/azops/wiki/custom-jq-templates) | `"Core.SkipCustomJqTemplate": true` | -| 06 | DefaultDeploymentRegion | Default region for deployments | `"Core.DefaultDeploymentRegion": "northeurope"` | -| 07 | EnrollmentAccountPrincipalName | Default enrollment account for Subscription creation | `"Core.EnrollmentAccountPrincipalName": ""` | -| 08 | ExcludedSubOffer | Exclude specific Subscription offer types from pull | `"Core.ExcludedSubOffer": ["AzurePass_2014-09-01","FreeTrial_2014-09-01","AAD_2015-09-01"]` | -| 09 | ExcludedSubState | Exclude specific states of Subscription from pull | `"Core.ExcludedSubState": ["Disabled","Deleted","Warned","Expired"]` | -| 10 | IgnoreContextCheck | Skip Azure PowerShell context validation. *Not recommended to change* | `"Core.IgnoreContextCheck": false` | -| 11 | IncludeResourcesInResourceGroup | Discover only resources in these resource groups | `"Core.IncludeResourcesInResourceGroup": ["rg1","rg2"]` | -| 12 | IncludeResourceType | Discover only specific resource types [Resource Types](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types) (only targets Resource Group scoped resources) | `"Core.IncludeResourceType": ["Microsoft.Network/privateDnsZones","Microsoft.Network/firewallPolicies"]` | -| 13 | InvalidateCache | Invalidate cached Subscriptions and Management Groups and do a full discovery. *Not recommended to change* | `"Core.InvalidateCache": false` | -| 14 | OfferType | Default offer type for Subscription creation | `"Core.OfferType": "MS-AZR-0017P"` | -| 15 | PartialMgDiscoveryRoot | Generate folder hierachy for specific Management Groups IDs | `"Core.PartialMgDiscoveryRoot": []` | -| 16 | SkipPim | Do not include Privileged Identity Management resources in pull | `"Core.SkipPim": true` | -| 17 | SkipLock | Do not include ResourceLock resources in pull | `"Core.SkipLock": true` | -| 18 | SkipPolicy | Do not include Azure Policy state in pull | `"Core.SkipPolicy": false` | -| 19 | SkipResource | Do not include Resources within Resource Groups | `"Core.SkipResource": false` | -| 20 | SkipChildResource | Do not include Azure child resources | `"Core.SkipChildResource": false` | -| 21 | SkipResourceGroup | Do not include Resource Groups in pull | `"Core.SkipResourceGroup": false` | -| 22 | SkipResourceType | Skip specific [Resource Types](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types) (only targets Resource Group scoped resources) | `"Core.SkipResourceType": ["Microsoft.VSOnline/plans"]` | -| 23 | SkipRole | Do not include Role types in pull | `"Core.SkipRole": false` | -| 24 | State | Folder to store AzOpsState artefact, defaults to `root` | `"Core.State: "/root"` | -| 25 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | -| 26 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | -| 27 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | -| 28 | DeployAllMultipleTemplateParameterFiles | Control base template deployment behaviour with changes and un-changed multiple corresponding parameter files. | `"Core.DeployAllMultipleTemplateParameterFiles": false` | -| 29 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | -| 30 | ParallelDeployMultipleTemplateParameterFiles | Control parallel deployment of MultipleTemplateParameterFiles behaviour | `"Core.ParallelDeployMultipleTemplateParameterFiles": false` | -| 31 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | -| 32 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | +| 04 | CustomTemplateResourceDeletion | Turn on or off deletion of custom ARM and Bicep template resources | `"Core.CustomTemplateResourceDeletion": true` | +| 05 | CustomJqTemplatePath | Folder where custom Jq templates are located. | `"Core.CustomJqTemplatePath": ".customtemplates"` | +| 06 | SkipCustomJqTemplate | Do not use custom Jq templates, controls if AzOps looks for custom templates at `CustomJqTemplatePath`. [Read more](https://github.com/azure/azops/wiki/custom-jq-templates) | `"Core.SkipCustomJqTemplate": true` | +| 07 | DefaultDeploymentRegion | Default region for deployments | `"Core.DefaultDeploymentRegion": "northeurope"` | +| 08 | EnrollmentAccountPrincipalName | Default enrollment account for Subscription creation | `"Core.EnrollmentAccountPrincipalName": ""` | +| 09 | ExcludedSubOffer | Exclude specific Subscription offer types from pull | `"Core.ExcludedSubOffer": ["AzurePass_2014-09-01","FreeTrial_2014-09-01","AAD_2015-09-01"]` | +| 10 | ExcludedSubState | Exclude specific states of Subscription from pull | `"Core.ExcludedSubState": ["Disabled","Deleted","Warned","Expired"]` | +| 11 | IgnoreContextCheck | Skip Azure PowerShell context validation. *Not recommended to change* | `"Core.IgnoreContextCheck": false` | +| 12 | IncludeResourcesInResourceGroup | Discover only resources in these resource groups | `"Core.IncludeResourcesInResourceGroup": ["rg1","rg2"]` | +| 13 | IncludeResourceType | Discover only specific resource types [Resource Types](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types) (only targets Resource Group scoped resources) | `"Core.IncludeResourceType": ["Microsoft.Network/privateDnsZones","Microsoft.Network/firewallPolicies"]` | +| 14 | InvalidateCache | Invalidate cached Subscriptions and Management Groups and do a full discovery. *Not recommended to change* | `"Core.InvalidateCache": false` | +| 15 | OfferType | Default offer type for Subscription creation | `"Core.OfferType": "MS-AZR-0017P"` | +| 16 | PartialMgDiscoveryRoot | Generate folder hierachy for specific Management Groups IDs | `"Core.PartialMgDiscoveryRoot": []` | +| 17 | SkipPim | Do not include Privileged Identity Management resources in pull | `"Core.SkipPim": true` | +| 18 | SkipLock | Do not include ResourceLock resources in pull | `"Core.SkipLock": true` | +| 19 | SkipPolicy | Do not include Azure Policy state in pull | `"Core.SkipPolicy": false` | +| 20 | SkipResource | Do not include Resources within Resource Groups | `"Core.SkipResource": false` | +| 21 | SkipChildResource | Do not include Azure child resources | `"Core.SkipChildResource": false` | +| 22 | SkipResourceGroup | Do not include Resource Groups in pull | `"Core.SkipResourceGroup": false` | +| 23 | SkipResourceType | Skip specific [Resource Types](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types) (only targets Resource Group scoped resources) | `"Core.SkipResourceType": ["Microsoft.VSOnline/plans"]` | +| 24 | SkipRole | Do not include Role types in pull | `"Core.SkipRole": false` | +| 25 | State | Folder to store AzOpsState artefact, defaults to `root` | `"Core.State: "/root"` | +| 26 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | +| 27 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | +| 28 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | +| 29 | DeployAllMultipleTemplateParameterFiles | Control base template deployment behaviour with changes and un-changed multiple corresponding parameter files. | `"Core.DeployAllMultipleTemplateParameterFiles": false` | +| 30 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | +| 31 | ParallelDeployMultipleTemplateParameterFiles | Control parallel deployment of MultipleTemplateParameterFiles behaviour | `"Core.ParallelDeployMultipleTemplateParameterFiles": false` | +| 32 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | +| 33 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | ## Workflow / Pipeline Settings From 9951e2fba834c8226b8fa9e68b20a3e56049dd29 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 4 Mar 2024 18:04:31 +0000 Subject: [PATCH 45/47] Update --- docs/wiki/ResourceDeletion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wiki/ResourceDeletion.md b/docs/wiki/ResourceDeletion.md index c0a2c51a..022167e7 100644 --- a/docs/wiki/ResourceDeletion.md +++ b/docs/wiki/ResourceDeletion.md @@ -160,7 +160,7 @@ flowchart TD ### Enable Deletion of Custom Template Set the `Core.CustomTemplateResourceDeletion` value in `settings.json` to `true`. -`AzOps - Push` will now evaluate and attempt deletion of corresponding resource (*from template*) in Azure when a custom template is deleted. +`AzOps - Push` will now evaluate and attempt deletion of corresponding resource (_from template_) in Azure when a custom template is deleted. ## Integration with AzOps Accelerator From 7191fee19dffe1bfa61c621fd1090a0c85711f2c Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 5 Mar 2024 14:11:27 +0000 Subject: [PATCH 46/47] Update --- src/AzOps.psd1 | 6 +- src/functions/Invoke-AzOpsPush.ps1 | 2 +- .../functions/Remove-AzOpsDeployment.ps1 | 3 +- .../functions/Remove-AzResourceRaw.ps1 | 183 ++++++++++++++---- .../Remove-AzResourceRawRecursive.ps1 | 76 -------- src/localized/en-us/Strings.psd1 | 2 + 6 files changed, 156 insertions(+), 116 deletions(-) delete mode 100644 src/internal/functions/Remove-AzResourceRawRecursive.ps1 diff --git a/src/AzOps.psd1 b/src/AzOps.psd1 index 2d6fbbd3..1197070d 100644 --- a/src/AzOps.psd1 +++ b/src/AzOps.psd1 @@ -3,7 +3,7 @@ # # Generated by: Customer Architecture Team (CAT) # -# Generated on: 2/20/2024 +# Generated on: 3/5/2024 # @{ @@ -52,10 +52,10 @@ PowerShellVersion = '7.2' # Modules that must be imported into the global environment prior to importing this module RequiredModules = @(@{ModuleName = 'PSFramework'; RequiredVersion = '1.10.318'; }, - @{ModuleName = 'Az.Accounts'; RequiredVersion = '2.15.1'; }, + @{ModuleName = 'Az.Accounts'; RequiredVersion = '2.16.0'; }, @{ModuleName = 'Az.Billing'; RequiredVersion = '2.0.3'; }, @{ModuleName = 'Az.ResourceGraph'; RequiredVersion = '0.13.0'; }, - @{ModuleName = 'Az.Resources'; RequiredVersion = '6.15.1'; }) + @{ModuleName = 'Az.Resources'; RequiredVersion = '6.16.0'; }) # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @() diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 5525c61f..a75c1866 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -559,7 +559,7 @@ Start-Sleep -Seconds 30 # Reset the status of failed attempts and perform recursive removal foreach ($try in $retry) { $try.Status = $null } - $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry + $removeActionRecursive = Remove-AzResourceRaw -InputObject $retry -Recursive $removeActionFail = $removeActionRecursive | Where-Object { $_.Status -eq 'failed' } # If removal fails, log and attempt to fetch the resource causing the failure if ($removeActionFail) { diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index 1d71ad54..c27d209d 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -350,6 +350,7 @@ foreach ($change in $removalJobChanges) { $resource = $null $resourceScopeObject = $null + $removeAction = $null # Check if the resource exists $resourceScopeObject = New-AzOpsScope -Scope $change.FullyQualifiedResourceId -WhatIf:$false $resource = Get-AzOpsResource -ScopeObject $resourceScopeObject -ErrorAction SilentlyContinue @@ -411,7 +412,7 @@ # Retry failed removals recursively Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzOpsDeployment.Resource.RetryCount' -LogStringValues $retry.Count foreach ($try in $retry) { $try.Status = $null } - $removeActionRecursive = Remove-AzResourceRawRecursive -InputObject $retry + $removeActionRecursive = Remove-AzResourceRaw -InputObject $retry -Recursive $removeActionRecursiveRemaining = $removeActionRecursive | Where-Object { $_.Status -eq 'failed' } return $removeActionRecursiveRemaining } diff --git a/src/internal/functions/Remove-AzResourceRaw.ps1 b/src/internal/functions/Remove-AzResourceRaw.ps1 index 74c3253d..e47e42c9 100644 --- a/src/internal/functions/Remove-AzResourceRaw.ps1 +++ b/src/internal/functions/Remove-AzResourceRaw.ps1 @@ -11,6 +11,10 @@ Path where the parameters of the ARM templates can be found. .PARAMETER ScopeObject Resource to delete. + .PARAMETER InputObject + Object containing items for processing, used in combination with parameter Recursive. + .PARAMETER Recursive + If specified, performs recursive resource deletion and requires use of parameter InputObject. .EXAMPLE > Remove-AzResourceRaw -ScopeObject $ScopeObject -TemplateFilePath $TemplateFilePath -TemplateParameterFilePath $TemplateParameterFilePath Name Value @@ -19,6 +23,14 @@ TemplateParameterFilePath /root/managementgroup/subscription/resourcegroup/template.parameters.json ScopeObject ScopeObject Status success + + > Remove-AzResourceRaw -InputObject $retry -Recursive + Name Value + ---- ----- + TemplateFilePath /root/managementgroup/subscription/resourcegroup/template.json + TemplateParameterFilePath /root/managementgroup/subscription/resourcegroup/template.parameters.json + ScopeObject ScopeObject + Status success #> [CmdletBinding()] @@ -27,51 +39,152 @@ $TemplateFilePath, [string] $TemplateParameterFilePath, - [Parameter(Mandatory = $true)] [AzOpsScope] - $ScopeObject + $ScopeObject, + [array] + $InputObject, + [switch] + $Recursive ) process { - # Construct result object - $result = [PSCustomObject]@{ - TemplateFilePath = $TemplateFilePath - TemplateParameterFilePath = $TemplateParameterFilePath - ScopeObject = $ScopeObject - Status = 'success' - } - # Check if the resource exists - $resource = Get-AzOpsResource -ScopeObject $ScopeObject -ErrorAction SilentlyContinue - # Remove the resource if it exists - if ($resource) { - try { - # Set Azure context for removal operation - Set-AzOpsContext -ScopeObject $ScopeObject - $null = Remove-AzResource -ResourceId $ScopeObject.Scope -Force -ErrorAction Stop - $maxAttempts = 4 - $attempt = 1 - $gone = $false - while ($gone -eq $false -and $attempt -le $maxAttempts) { - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.CheckExistence' -LogStringValues $ScopeObject.Scope - Start-Sleep -Seconds 10 - $tryResource = Get-AzOpsResource -ScopeObject $ScopeObject -ErrorAction SilentlyContinue - if (-not $tryResource) { - $gone = $true + function Remove-AzResourceRawRecursive { + + <# + .SYNOPSIS + Performs recursive resource deletion in Azure at any scope. + .DESCRIPTION + Takes $InputObject and performs recursive resource deletion in Azure and exhaust any permutation. + .PARAMETER InputObject + Parameter containing items for processing. + .PARAMETER CurrentOrder + Internal parameter to track recursive progress. + .PARAMETER OutputObject + Track item processing and return result. + .EXAMPLE + > $successFullItems, $failedItems = Remove-AzResourceRawRecursive -InputObject $retry + Example of a $retry array with 6 items, the number of permutations will be 6×5×4×3×2×1=720 + #> + + [CmdletBinding()] + param ( + [array] + $InputObject, + [array] + $CurrentOrder = @(), + [array] + $OutputObject = @() + ) + + process { + if ($InputObject.Count -eq 0) { + # Base case: All items have been used, perform action on the current order + foreach ($item in $CurrentOrder) { + if ($item.Status -eq 'failed' -or $null -eq $item.Status) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRawRecursive.Processing' -LogStringValues $item.ScopeObject.Resource, $item.ScopeObject.Scope + # Attempt to remove the resource + $result = Remove-AzResourceRaw -ScopeObject $item.ScopeObject -TemplateFilePath $item.TemplateFilePath -TemplateParameterFilePath $item.TemplateParameterFilePath + if ($result.Status -eq 'failed' -and $result.ScopeObject.Scope -notin $OutputObject.ScopeObject.Scope){ + # Add failed result to the output object + $OutputObject += $result + } + } + } + # Return the final result + return $OutputObject + } + else { + if ($InputObject -and $OutputObject) { + # Filter out items already processed successfully + $filteredOutputObject = @() + foreach ($item in $InputObject) { + if ($item.ScopeObject.Scope -in $OutputObject.ScopeObject.Scope) { + foreach ($output in $OutputObject) { + if ($output.ScopeObject.Scope -eq $item.ScopeObject.Scope -and $output.Status -eq 'failed') { + # Add previously failed item to the filtered output + $filteredOutputObject += $output + continue + } + } + } + } + if ($filteredOutputObject) { + $InputObject = $filteredOutputObject + } + } + # Recursive case: Try each item in the current position and recurse with the remaining items + foreach ($item in $InputObject) { + $remainingItems = $InputObject -ne $item + $newOrder = $CurrentOrder + $item + # Recursively call Remove-AzResourceRawRecursive + $OutputObject = Remove-AzResourceRawRecursive -InputObject $remainingItems -CurrentOrder $newOrder -OutputObject $OutputObject } - $attempt++ + # Return the output after all permutations + return $OutputObject } } - catch { - # Log failure message - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.Failed' -LogStringValues $ScopeObject.Resource, $ScopeObject.Scope - $result.Status = 'failed' + } + if ($null -ne $InputObject -and $Recursive) { + # Perform recursive resource deletion + $result = Remove-AzResourceRawRecursive -InputObject $InputObject + if ($result) { + return $result + } + else { + return } } + elseif ($null -eq $InputObject -and $Recursive) { + # Recursive resource deletion missing input + Write-AzOpsMessage -LogLevel Error -LogString 'Remove-AzResourceRaw.Resource.Recursive.Missing' + return + } else { - # Log not found message - $result.Status = 'notfound' + if (-not $ScopeObject) { + # Resource deletion missing input + Write-AzOpsMessage -LogLevel Error -LogString 'Remove-AzResourceRaw.Resource.Missing' + return + } + # Construct result object + $result = [PSCustomObject]@{ + TemplateFilePath = $TemplateFilePath + TemplateParameterFilePath = $TemplateParameterFilePath + ScopeObject = $ScopeObject + Status = 'success' + } + # Check if the resource exists + $resource = Get-AzOpsResource -ScopeObject $ScopeObject -ErrorAction SilentlyContinue + # Remove the resource if it exists + if ($resource) { + try { + # Set Azure context for removal operation + Set-AzOpsContext -ScopeObject $ScopeObject + $null = Remove-AzResource -ResourceId $ScopeObject.Scope -Force -ErrorAction Stop + $maxAttempts = 4 + $attempt = 1 + $gone = $false + while ($gone -eq $false -and $attempt -le $maxAttempts) { + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.CheckExistence' -LogStringValues $ScopeObject.Scope + Start-Sleep -Seconds 10 + $tryResource = Get-AzOpsResource -ScopeObject $ScopeObject -ErrorAction SilentlyContinue + if (-not $tryResource) { + $gone = $true + } + $attempt++ + } + } + catch { + # Log failure message + Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRaw.Resource.Failed' -LogStringValues $ScopeObject.Resource, $ScopeObject.Scope + $result.Status = 'failed' + } + } + else { + # Log not found message + $result.Status = 'notfound' + } + # Return result object + return $result } - # Return result object - return $result } } \ No newline at end of file diff --git a/src/internal/functions/Remove-AzResourceRawRecursive.ps1 b/src/internal/functions/Remove-AzResourceRawRecursive.ps1 deleted file mode 100644 index aaaacb1b..00000000 --- a/src/internal/functions/Remove-AzResourceRawRecursive.ps1 +++ /dev/null @@ -1,76 +0,0 @@ -function Remove-AzResourceRawRecursive { - - <# - .SYNOPSIS - Performs recursive resource deletion in Azure at any scope. - .DESCRIPTION - Takes $InputObject and performs recursive resource deletion in Azure and exhaust any permutation. - .PARAMETER InputObject - Parameter containing items for processing. - .PARAMETER CurrentOrder - Internal parameter to track recursive progress. - .PARAMETER OutputObject - Parameter to track item processing and return result. - .EXAMPLE - > $successFullItems, $failedItems = Remove-AzResourceRawRecursive -InputObject $retry - Example of a $retry array with 6 items, the number of permutations will be 6×5×4×3×2×1=720 - #> - - [CmdletBinding()] - param ( - [array] - $InputObject, - [array] - $CurrentOrder = @(), - [array] - $OutputObject = @() - ) - - process { - if ($InputObject.Count -eq 0) { - # Base case: All items have been used, perform action on the current order - foreach ($item in $CurrentOrder) { - if ($item.Status -eq 'failed' -or $null -eq $item.Status) { - Write-AzOpsMessage -LogLevel InternalComment -LogString 'Remove-AzResourceRawRecursive.Processing' -LogStringValues $item.ScopeObject.Resource, $item.ScopeObject.Scope - # Attempt to remove the resource - $result = Remove-AzResourceRaw -ScopeObject $item.ScopeObject -TemplateFilePath $item.TemplateFilePath -TemplateParameterFilePath $item.TemplateParameterFilePath - if ($result.Status -eq 'failed' -and $result.ScopeObject.Scope -notin $OutputObject.ScopeObject.Scope){ - # Add failed result to the output object - $OutputObject += $result - } - } - } - # Return the final result - return $OutputObject - } - else { - if ($InputObject -and $OutputObject) { - # Filter out items already processed successfully - $filteredOutputObject = @() - foreach ($item in $InputObject) { - if ($item.ScopeObject.Scope -in $OutputObject.ScopeObject.Scope) { - foreach ($output in $OutputObject) { - if ($output.ScopeObject.Scope -eq $item.ScopeObject.Scope -and $output.Status -eq 'failed') { - # Add previously failed item to the filtered output - $filteredOutputObject += $output - continue - } - } - } - } - if ($filteredOutputObject) { - $InputObject = $filteredOutputObject - } - } - # Recursive case: Try each item in the current position and recurse with the remaining items - foreach ($item in $InputObject) { - $remainingItems = $InputObject -ne $item - $newOrder = $CurrentOrder + $item - # Recursively call Remove-AzResourceRawRecursive - $OutputObject = Remove-AzResourceRawRecursive -InputObject $remainingItems -CurrentOrder $newOrder -OutputObject $OutputObject - } - # Return the output after all permutations - return $OutputObject - } - } -} \ No newline at end of file diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 61cba749..9eac2e32 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -289,6 +289,8 @@ 'Remove-AzOpsDeployment.ResourceNotFound' = 'Unable to find resource of type {0} with id {1}.'# $scopeObject.Resource, $scopeObject.Scope, $resultsError 'Remove-AzOpsDeployment.SkipUnsupportedResource' = 'Deletion of AzOps generated file resources is only supported for locks, policyAssignments, policyDefinitions, policyExemptions, policySetDefinitions and roleAssignments. Will NOT proceed with deletion of resource in file {0}'# $TemplateFilePath + 'Remove-AzResourceRaw.Resource.Recursive.Missing' = 'Missing required parameter InputObject, when running Recursive'# + 'Remove-AzResourceRaw.Resource.Missing' = 'Missing required parameter ScopeObject'# 'Remove-AzResourceRaw.Resource.CheckExistence' = 'Checking existence after deletion of: [{0}]'# $FullyQualifiedResourceId 'Remove-AzResourceRaw.Resource.Failed' = 'Unable to delete resource of type {0} with id {1}'# $ScopeObject.Resource, $ScopeObject.Scope 'Remove-AzResourceRawRecursive.Processing' = 'Recursive retry processing to delete resource of type {0} with id {1}'# $item.ScopeObject.Resource, $item.ScopeObject.Scope From c9d34b29a1e2fb9ec088320263294d6696b700d4 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 11 Mar 2024 15:45:09 +0000 Subject: [PATCH 47/47] Update --- src/internal/functions/Remove-AzOpsDeployment.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/internal/functions/Remove-AzOpsDeployment.ps1 b/src/internal/functions/Remove-AzOpsDeployment.ps1 index c27d209d..b969fd6b 100644 --- a/src/internal/functions/Remove-AzOpsDeployment.ps1 +++ b/src/internal/functions/Remove-AzOpsDeployment.ps1 @@ -215,7 +215,7 @@ if ($templateContent.metadata._generator.name -eq "AzOps" -or $templateContent.$schemavalue -like "*deploymentParameters.json#") { Write-AzOpsMessage -LogLevel Verbose -LogString 'Remove-AzOpsDeployment.Metadata.AzOps' -LogStringValues $TemplateFilePath } - elseif ($CustomTemplateResourceDeletion -eq $true) { + elseif ($true -eq $CustomTemplateResourceDeletion) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Remove-AzOpsDeployment.Metadata.Custom' -LogStringValues $TemplateFilePath $customDeletion = $true } @@ -400,10 +400,10 @@ } # If there are $resultsFileAssociation, combine them with existing results and log a warning if ($resultsFileAssociation) { - $finallResults = @() - $finallResults += $resultsFileAssociation - $finallResults += $allResults - $allResults = $finallResults + $finalResults = @() + $finalResults += $resultsFileAssociation + $finalResults += $allResults + $allResults = $finalResults Write-AzOpsMessage -LogLevel Warning -LogString 'Set-AzOpsWhatIfOutput.WhatIfResults' -LogStringValues $allResults } }