From c0342fb1418c5d3bf8e1fa2270dc3fbf64130894 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 7 Dec 2023 14:48:51 +0000 Subject: [PATCH 01/14] BaseLogicUpdate --- src/functions/Invoke-AzOpsPush.ps1 | 31 +++++++++-- src/internal/configurations/Core.ps1 | 4 +- .../ConvertFrom-AzOpsBicepTemplate.ps1 | 53 ++++++++++++------- src/localized/en-us/Strings.psd1 | 2 + 4 files changed, 66 insertions(+), 24 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 3958466e..4a4a96f0 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -92,8 +92,15 @@ #region Directly Associated Template file exists switch ($fileItem.Name) { { $_.EndsWith('.parameters.json') } { - $templatePath = $fileItem.FullName -replace '\.parameters.json', (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix') - $bicepTemplatePath = $fileItem.FullName -replace '.parameters.json', '.bicep' + if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true -and $fileItem.FullName.Split('.')[-3] -match $(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix').Replace('.','')) { + Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -StringValues $FilePath + $templatePath = $fileItem.FullName -replace (".$($fileItem.FullName.Split('.')[-3])"), '' -replace '\.parameters.json', '.json' + $bicepTemplatePath = $fileItem.FullName -replace (".$($fileItem.FullName.Split('.')[-3])"), '' -replace '.parameters.json', '.bicep' + } + else { + $templatePath = $fileItem.FullName -replace '\.parameters.json', (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix') + $bicepTemplatePath = $fileItem.FullName -replace '.parameters.json', '.bicep' + } if (Test-Path $templatePath) { Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.FoundTemplate' -StringValues $FilePath, $templatePath $result.TemplateFilePath = $templatePath @@ -101,16 +108,22 @@ } elseif (Test-Path $bicepTemplatePath) { Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.FoundBicepTemplate' -StringValues $FilePath, $bicepTemplatePath - $transpiledTemplatePaths = ConvertFrom-AzOpsBicepTemplate -BicepTemplatePath $bicepTemplatePath + $transpiledTemplatePaths = ConvertFrom-AzOpsBicepTemplate -BicepTemplatePath $bicepTemplatePath -SkipParam $result.TemplateFilePath = $transpiledTemplatePaths.transpiledTemplatePath return $result } } { $_.EndsWith('.bicepparam') } { - $bicepTemplatePath = $fileItem.FullName -replace '\.bicepparam', '.bicep' + if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true -and $fileItem.FullName.Split('.')[-2] -match $(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix').Replace('.','')) { + Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -StringValues $FilePath + $bicepTemplatePath = $fileItem.FullName -replace (".$($fileItem.FullName.Split('.')[-2])"), '' -replace '\.bicepparam', '.bicep' + } + else { + $bicepTemplatePath = $fileItem.FullName -replace '\.bicepparam', '.bicep' + } if (Test-Path $bicepTemplatePath) { Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.FoundBicepTemplate' -StringValues $FilePath, $bicepTemplatePath - $transpiledTemplatePaths = ConvertFrom-AzOpsBicepTemplate -BicepTemplatePath $bicepTemplatePath + $transpiledTemplatePaths = ConvertFrom-AzOpsBicepTemplate -BicepTemplatePath $bicepTemplatePath -BicepParamTemplatePath $fileItem.FullName $result.TemplateFilePath = $transpiledTemplatePaths.transpiledTemplatePath $result.TemplateParameterFilePath = $transpiledTemplatePaths.transpiledParametersPath return $result @@ -164,6 +177,14 @@ } else { Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -StringValues $FilePath, $parameterPath + # Check for template parameters without defaultValue + $defaultValueContent = Get-Content $FilePath + $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' + if ($missingDefaultParam) { + # Skip template deployment when template parameters without defaultValue are found and no parameter file identified + Write-PSFMessage -Level Warning -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingDefaultParam | ConvertFrom-Json) + continue + } } $deploymentName = $fileItem.BaseName -replace '\.json$' -replace ' ', '_' diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index a7bad44f..c57f1266 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -25,6 +25,8 @@ Set-PSFConfig -Module AzOps -Name Core.SkipResourceType -Value @('Microsoft.VSOn Set-PSFConfig -Module AzOps -Name Core.SkipRole -Value $false -Initialize -Validation bool -Description '-' Set-PSFConfig -Module AzOps -Name Core.State -Value (Join-Path $pwd -ChildPath "root") -Initialize -Validation string -Description 'Folder to store AzOpsState artefact' Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID or Display Name that matches the filter. Powershell filter that matches with like operator is supported.' -Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'parameter file suffix to look for' +Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'parameter file suffix identifier' +Set-PSFConfig -Module AzOps -Name Core.AllowMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control multiple parameter file behaviour.' +Set-PSFConfig -Module AzOps -Name Core.MultipleTemplateParameterFileSuffix -Value '.x' -Initialize -Validation string -Description 'Multiple parameter file suffix identifier' Set-PSFConfig -Module AzOps -Name Core.ThrottleLimit -Value 5 -Initialize -Validation integer -Description 'Throttle limit used in Foreach-Object -Parallel for resource/subscription discovery' Set-PSFConfig -Module AzOps -Name Core.WhatifExcludedChangeTypes -Value @('NoChange', 'Ignore') -Initialize -Validation stringarray -Description 'Exclude specific change types from WhatIf operations.' \ No newline at end of file diff --git a/src/internal/functions/ConvertFrom-AzOpsBicepTemplate.ps1 b/src/internal/functions/ConvertFrom-AzOpsBicepTemplate.ps1 index 24bbd77e..27e66a97 100644 --- a/src/internal/functions/ConvertFrom-AzOpsBicepTemplate.ps1 +++ b/src/internal/functions/ConvertFrom-AzOpsBicepTemplate.ps1 @@ -4,7 +4,11 @@ Transpiles bicep template and associated bicepparam to Azure Resource Manager (ARM) template. The json file will be created in the same folder as the bicep file. .PARAMETER BicepTemplatePath - BicepTemplatePath + BicepTemplatePath. + .PARAMETER BicepParamTemplatePath + BicepParamTemplatePath, when provided function does not attempt default parameter file discovery. + .PARAMETER SkipParam + Switch when set will avoid parameter file discovery. .EXAMPLE ConvertFrom-AzOpsBicepTemplate -BicepTemplatePath "root/tenant root group (xxxx-xxxx-xxxx-xxxx-xxxx)/es (es)/subscription (xxxx-xxxx-xxxx-xxxx)/resource-rg/main.bicep" transpiledTemplatePath : root/tenant root group (xxxx-xxxx-xxxx-xxxx-xxxx)/es (es)/subscription (xxxx-xxxx-xxxx-xxxx)/resource-rg/main.json @@ -15,7 +19,11 @@ param ( [Parameter(Mandatory = $true)] [string] - $BicepTemplatePath + $BicepTemplatePath, + [string] + $BicepParamTemplatePath, + [switch] + $SkipParam ) begin { @@ -33,23 +41,32 @@ Write-PSFMessage -Level Error -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.ConvertBicepTemplate.Error' -StringValues $BicepTemplatePath throw } - # Check if bicep template has associated bicepparam file - $bicepParametersPath = $BicepTemplatePath -replace '\.bicep', '.bicepparam' - Write-PSFMessage -Level Verbose -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.BicepParam' -StringValues $BicepTemplatePath, $bicepParametersPath - if (Test-Path $bicepParametersPath) { - # Convert bicepparam to ARM parameter file - $transpiledParametersPath = $bicepParametersPath -replace '\.bicepparam', ('.parameters' + (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix')) - Write-PSFMessage -Level Verbose -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.ConvertBicepParam' -StringValues $bicepParametersPath, $transpiledParametersPath - Invoke-AzOpsNativeCommand -ScriptBlock { bicep build-params $bicepParametersPath --outfile $transpiledParametersPath } - # Check if bicep build-params created (ARM) parameters - if (-not (Test-Path $transpiledParametersPath)) { - # If bicep build-params did not produce file exit with error - Write-PSFMessage -Level Error -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.ConvertBicepParam.Error' -StringValues $bicepParametersPath - throw + if (-not $SkipParam) { + if (-not $BicepParamTemplatePath) { + # Check if bicep template has associated bicepparam file + $bicepParametersPath = $BicepTemplatePath -replace '\.bicep', '.bicepparam' + Write-PSFMessage -Level Verbose -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.BicepParam' -StringValues $BicepTemplatePath, $bicepParametersPath + } + elseif ($BicepParamTemplatePath) { + # BicepParamTemplatePath path provided as input + $bicepParametersPath = $BicepParamTemplatePath + Write-PSFMessage -Level Verbose -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.BicepParam' -StringValues $BicepTemplatePath, $bicepParametersPath + } + if ($bicepParametersPath -and (Test-Path $bicepParametersPath)) { + # Convert bicepparam to ARM parameter file + $transpiledParametersPath = $bicepParametersPath -replace '\.bicepparam', '.parameters.json' + Write-PSFMessage -Level Verbose -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.ConvertBicepParam' -StringValues $bicepParametersPath, $transpiledParametersPath + Invoke-AzOpsNativeCommand -ScriptBlock { bicep build-params $bicepParametersPath --outfile $transpiledParametersPath } + # Check if bicep build-params created (ARM) parameters + if (-not (Test-Path $transpiledParametersPath)) { + # If bicep build-params did not produce file exit with error + Write-PSFMessage -Level Error -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.ConvertBicepParam.Error' -StringValues $bicepParametersPath + throw + } + } + else { + Write-PSFMessage -Level Verbose -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.BicepParam.NotFound' -StringValues $BicepTemplatePath } - } - else { - Write-PSFMessage -Level Verbose -String 'ConvertFrom-AzOpsBicepTemplate.Resolve.BicepParam.NotFound' -StringValues $BicepTemplatePath } # Return transpiled (ARM) template paths $return = [PSCustomObject]@{ diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index aa07491c..949fc86f 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -194,11 +194,13 @@ 'Invoke-AzOpsPush.Resolve.FoundBicepTemplate' = 'Found Bicep template {1} for parameters {0}' # $FilePath, $bicepTemplatePath 'Invoke-AzOpsPush.Resolve.FromMainTemplate' = 'Determining template from main template file: {0}' # $mainTemplateItem.FullName 'Invoke-AzOpsPush.Resolve.MainTemplate.NotSupported' = 'effectiveResourceType: {0} AzOpsMainTemplate does NOT supports resource type {0} in {1}. Deployment will be ignored' # $effectiveResourceType, $AzOpsMainTemplate.FullName + 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' = 'Found AllowMultipleTemplateParameterFile {0}' # $FilePath 'Invoke-AzOpsPush.Resolve.MainTemplate.Supported' = 'effectiveResourceType: {0} - AzOpsMainTemplate supports resource type {0} in {1}' # $effectiveResourceType, $AzOpsMainTemplate.FullName 'Invoke-AzOpsPush.Resolve.NoJson' = 'The specified file is not a json or bicep file! Skipping {0}' # $fileItem.FullName 'Invoke-AzOpsPush.Resolve.NotFoundTemplate' = 'Did NOT find template {1} for parameters {0}' # $FilePath, $templatePath 'Invoke-AzOpsPush.Resolve.ParameterFound' = 'Found parameter file for template {0} : {1}' # $FilePath, $parameterPath 'Invoke-AzOpsPush.Resolve.ParameterNotFound' = 'No parameter file found for template {0} : {1}' # $FilePath, $parameterPath + 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' = 'Template {0} with parameter {1} missing defaultValue and no parameter file found, skip deployment' # $FilePath, $missingDefaultParam 'Invoke-AzOpsPush.Scope.Failed' = 'Failed to read {0} as part of {1}' # $addition, $StatePath 'Invoke-AzOpsNativeCommand.Failed.NoCallstack' = 'Execution of {{{0}}} failed with exit code {1}' # $ScriptBlock, $LASTEXITCODE From bdd651de42656eae0d8c6fefcb356839c814b5c0 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 7 Dec 2023 16:33:22 +0000 Subject: [PATCH 02/14] Update --- src/functions/Invoke-AzOpsPush.ps1 | 7 ++++--- src/internal/configurations/Core.ps1 | 2 +- src/localized/en-us/Strings.psd1 | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 4a4a96f0..2a4642e3 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -179,10 +179,11 @@ Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -StringValues $FilePath, $parameterPath # Check for template parameters without defaultValue $defaultValueContent = Get-Content $FilePath - $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' - if ($missingDefaultParam) { + $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable + if ($missingDefaultParam.Count -ge 1) { # Skip template deployment when template parameters without defaultValue are found and no parameter file identified - Write-PSFMessage -Level Warning -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingDefaultParam | ConvertFrom-Json) + $missingString = ForEach($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} + Write-PSFMessage -Level Warning -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingString | Out-String -NoNewline) continue } } diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index c57f1266..cb9f6103 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -25,7 +25,7 @@ Set-PSFConfig -Module AzOps -Name Core.SkipResourceType -Value @('Microsoft.VSOn Set-PSFConfig -Module AzOps -Name Core.SkipRole -Value $false -Initialize -Validation bool -Description '-' Set-PSFConfig -Module AzOps -Name Core.State -Value (Join-Path $pwd -ChildPath "root") -Initialize -Validation string -Description 'Folder to store AzOpsState artefact' Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID or Display Name that matches the filter. Powershell filter that matches with like operator is supported.' -Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'parameter file suffix identifier' +Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'Parameter file suffix identifier' Set-PSFConfig -Module AzOps -Name Core.AllowMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control multiple parameter file behaviour.' Set-PSFConfig -Module AzOps -Name Core.MultipleTemplateParameterFileSuffix -Value '.x' -Initialize -Validation string -Description 'Multiple parameter file suffix identifier' Set-PSFConfig -Module AzOps -Name Core.ThrottleLimit -Value 5 -Initialize -Validation integer -Description 'Throttle limit used in Foreach-Object -Parallel for resource/subscription discovery' diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 949fc86f..7cc94419 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -200,7 +200,7 @@ 'Invoke-AzOpsPush.Resolve.NotFoundTemplate' = 'Did NOT find template {1} for parameters {0}' # $FilePath, $templatePath 'Invoke-AzOpsPush.Resolve.ParameterFound' = 'Found parameter file for template {0} : {1}' # $FilePath, $parameterPath 'Invoke-AzOpsPush.Resolve.ParameterNotFound' = 'No parameter file found for template {0} : {1}' # $FilePath, $parameterPath - 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' = 'Template {0} with parameter {1} missing defaultValue and no parameter file found, skip deployment' # $FilePath, $missingDefaultParam + 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' = 'Template {0} with parameter: {1} missing defaultValue and no parameter file found, skip deployment' # $FilePath, $missingDefaultParam 'Invoke-AzOpsPush.Scope.Failed' = 'Failed to read {0} as part of {1}' # $addition, $StatePath 'Invoke-AzOpsNativeCommand.Failed.NoCallstack' = 'Execution of {{{0}}} failed with exit code {1}' # $ScriptBlock, $LASTEXITCODE From ac57fe2bf1ddbe9c0ee4b57d4e636662fc9a24ed Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 7 Dec 2023 16:36:18 +0000 Subject: [PATCH 03/14] Update --- src/localized/en-us/Strings.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 7cc94419..d40edc55 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -200,7 +200,7 @@ 'Invoke-AzOpsPush.Resolve.NotFoundTemplate' = 'Did NOT find template {1} for parameters {0}' # $FilePath, $templatePath 'Invoke-AzOpsPush.Resolve.ParameterFound' = 'Found parameter file for template {0} : {1}' # $FilePath, $parameterPath 'Invoke-AzOpsPush.Resolve.ParameterNotFound' = 'No parameter file found for template {0} : {1}' # $FilePath, $parameterPath - 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' = 'Template {0} with parameter: {1} missing defaultValue and no parameter file found, skip deployment' # $FilePath, $missingDefaultParam + 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' = 'Template {0} with parameter: {1} missing defaultValue and no parameter file found, skip deployment' # $FilePath, $missingString 'Invoke-AzOpsPush.Scope.Failed' = 'Failed to read {0} as part of {1}' # $addition, $StatePath 'Invoke-AzOpsNativeCommand.Failed.NoCallstack' = 'Execution of {{{0}}} failed with exit code {1}' # $ScriptBlock, $LASTEXITCODE From fd6b995d335009875fc7452bf9e4292da2c1ffe8 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 7 Dec 2023 16:45:20 +0000 Subject: [PATCH 04/14] Update --- src/functions/Invoke-AzOpsPush.ps1 | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 2a4642e3..626b9c76 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -177,14 +177,16 @@ } else { Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -StringValues $FilePath, $parameterPath - # Check for template parameters without defaultValue - $defaultValueContent = Get-Content $FilePath - $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable - if ($missingDefaultParam.Count -ge 1) { - # Skip template deployment when template parameters without defaultValue are found and no parameter file identified - $missingString = ForEach($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} - Write-PSFMessage -Level Warning -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingString | Out-String -NoNewline) - continue + if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true) { + # Check for template parameters without defaultValue + $defaultValueContent = Get-Content $FilePath + $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable + if ($missingDefaultParam.Count -ge 1) { + # Skip template deployment when template parameters without defaultValue are found and no parameter file identified + $missingString = ForEach($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} + Write-PSFMessage -Level Warning -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingString | Out-String -NoNewline) + continue + } } } From 0ca8c9cf3a25ee86ce80ecc7bdab2eca08a06163 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 7 Dec 2023 22:25:04 +0000 Subject: [PATCH 05/14] Update --- docs/wiki/Frequently-Asked-Questions.md | 56 ++++++++++++++++++++++++- docs/wiki/Settings.md | 6 ++- src/functions/Invoke-AzOpsPush.ps1 | 2 +- src/internal/configurations/Core.ps1 | 2 +- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/docs/wiki/Frequently-Asked-Questions.md b/docs/wiki/Frequently-Asked-Questions.md index b4c39c99..c48b8da5 100644 --- a/docs/wiki/Frequently-Asked-Questions.md +++ b/docs/wiki/Frequently-Asked-Questions.md @@ -15,6 +15,9 @@ This article answers frequently asked questions relating to AzOps. - [**I want to discover all resources in specific resource groups in one specific subscription**](#i-want-to-discover-all-resources-in-specific-resource-groups-in-one-specific-subscription) - [**I want to discover a specific resource type in specific resource group in one specific subscription**](#i-want-to-discover-a-specific-resource-type-in-specific-resource-group-in-one-specific-subscription) - [**I want to discover and manage several Azure Firewall Policy's and rule collections spread out across several resource groups and subscriptions**](#i-want-to-discover-and-manage-several-azure-firewall-policys-and-rule-collections-spread-out-across-several-resource-groups-and-subscriptions) + - [Push scenarios and settings](#push-scenarios-and-settings) + - [**I want to have multiple different deployments at scope using the same template file but different parameter files**](#i-want-to-have-multiple-different-deployments-at-scope-using-the-same-template-file-but-different-parameter-files) + - [**I am getting: Missing defaultValue and no parameter file found, skip deployment**](#i-am-getting-missing-defaultvalue-and-no-parameter-file-found-skip-deployment) ## Subscriptions or resources not showing up in repository @@ -145,5 +148,56 @@ Yes, ensure the following setting combinations are applied (replace `rgname1`, ` Can AzOps settings be configured to enable this? -Yes, ensure that the variable `AZOPS_CUSTOM_SORT_ORDER` is set to `true` and create a file named `.order` in the same folder as your template files. +Yes, ensure that the variable `AZOPS_CUSTOM_SORT_ORDER` is set to `true` and create a file named `.order` in the same folder as your template files. Template files listed in the order file will be deployed in the order specified in the file and before any other templates. + +## Push scenarios and settings + +### **I want to have multiple different deployments at scope using the same template file but different parameter files** + +When using custom deployment templates, can I avoid the pattern of duplicating the `.bicep` file for each `parameter` file below? +``` +scope/ +├── template-a.bicep +├── template-a.bicepparam +├── template-b.bicep +├── template-b.bicepparam +├── template-c.bicep +└── template-c.parameters.json +``` +Yes, ensure the following setting combinations are applied (replace `x` with your specific pattern identifier) + +```bash + "Core.AllowMultipleTemplateParameterFiles": true + + "Core.MultipleTemplateParameterFileSuffix": ".x" +``` +AzOps module will evaluate each parameter file individually and try to find matching template by removing parameter file identifiers. +``` +scope/ +├── template.x1.bicepparam +├── template.x2.bicepparam +├── template.x3.parameters.json +└── template.bicep +``` +> Note: To avoid having AzOps deploy the base `template.bicep` unintentionally, ensure you have at least one parameter without default value in `template.bicep` and no lingering 1:1 matching parameter file. + +### **I am getting: Missing defaultValue and no parameter file found, skip deployment** + +To confirm if this applies to you, check the pipeline logs for the following message: + +```powershell +[Resolve-ArmFileAssociation] Template with parameter: , missing defaultValue and no parameter file found, skip deployment +``` + +What does this mean? + +AzOps have detected that parameters used in the template do not have defaultValues, no 1:1 parameter file mapped and that `Core.AllowMultipleTemplateParameterFiles` is set to `true`. + +To avoid exiting with error or attempt to deploy the updated base template unintentionally AzOps skips the file and logs it. + +The following must be true for this to happen: +- `Core.AllowMultipleTemplateParameterFiles` is set to `true` +- A template file is a part of the changeset sent to AzOps +- Template file contains parameters with no defaultValue +- Template file does not have 1:1 mapping to parameter file \ No newline at end of file diff --git a/docs/wiki/Settings.md b/docs/wiki/Settings.md index 9c804f0f..c4776255 100644 --- a/docs/wiki/Settings.md +++ b/docs/wiki/Settings.md @@ -35,8 +35,10 @@ The following configuration values can be modified within the `settings.json` fi | 23 | State | Folder to store AzOpsState artefact, defaults to `root` | `"Core.State: "/root"` | | 24 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | | 25 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | -| 26 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | -| 27 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | +| 26 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": "false"` | +| 27 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | +| 28 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | +| 29 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | ## Workflow / Pipeline Settings diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 626b9c76..f7d21b80 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -184,7 +184,7 @@ if ($missingDefaultParam.Count -ge 1) { # Skip template deployment when template parameters without defaultValue are found and no parameter file identified $missingString = ForEach($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} - Write-PSFMessage -Level Warning -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingString | Out-String -NoNewline) + Write-PSFMessage -Level Verbose -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingString | Out-String -NoNewline) continue } } diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index cb9f6103..490a8146 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -26,7 +26,7 @@ Set-PSFConfig -Module AzOps -Name Core.SkipRole -Value $false -Initialize -Valid Set-PSFConfig -Module AzOps -Name Core.State -Value (Join-Path $pwd -ChildPath "root") -Initialize -Validation string -Description 'Folder to store AzOpsState artefact' Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID or Display Name that matches the filter. Powershell filter that matches with like operator is supported.' Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'Parameter file suffix identifier' -Set-PSFConfig -Module AzOps -Name Core.AllowMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control multiple parameter file behaviour.' +Set-PSFConfig -Module AzOps -Name Core.AllowMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control multiple parameter file behaviour' Set-PSFConfig -Module AzOps -Name Core.MultipleTemplateParameterFileSuffix -Value '.x' -Initialize -Validation string -Description 'Multiple parameter file suffix identifier' Set-PSFConfig -Module AzOps -Name Core.ThrottleLimit -Value 5 -Initialize -Validation integer -Description 'Throttle limit used in Foreach-Object -Parallel for resource/subscription discovery' Set-PSFConfig -Module AzOps -Name Core.WhatifExcludedChangeTypes -Value @('NoChange', 'Ignore') -Initialize -Validation stringarray -Description 'Exclude specific change types from WhatIf operations.' \ No newline at end of file From b15bf754115d9cc0fe6e1fa2541c7177aeddae34 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 8 Dec 2023 07:23:39 +0000 Subject: [PATCH 06/14] Update --- docs/wiki/Frequently-Asked-Questions.md | 2 +- docs/wiki/Settings.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/wiki/Frequently-Asked-Questions.md b/docs/wiki/Frequently-Asked-Questions.md index c48b8da5..8822704c 100644 --- a/docs/wiki/Frequently-Asked-Questions.md +++ b/docs/wiki/Frequently-Asked-Questions.md @@ -172,7 +172,7 @@ Yes, ensure the following setting combinations are applied (replace `x` with you "Core.MultipleTemplateParameterFileSuffix": ".x" ``` -AzOps module will evaluate each parameter file individually and try to find matching template by removing parameter file identifiers. +AzOps module will evaluate each parameter file individually and try to find base template by matching (*regex*) according to `MultipleTemplateParameterFileSuffix` pattern identifier. ``` scope/ ├── template.x1.bicepparam diff --git a/docs/wiki/Settings.md b/docs/wiki/Settings.md index c4776255..aa58160a 100644 --- a/docs/wiki/Settings.md +++ b/docs/wiki/Settings.md @@ -35,7 +35,7 @@ The following configuration values can be modified within the `settings.json` fi | 23 | State | Folder to store AzOpsState artefact, defaults to `root` | `"Core.State: "/root"` | | 24 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | | 25 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | -| 26 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": "false"` | +| 26 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | | 27 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | | 28 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | | 29 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | From b075c0a2d0d2ac6f92cc81e4c0fe1a7294c88a25 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 8 Dec 2023 07:29:52 +0000 Subject: [PATCH 07/14] Update --- src/functions/Invoke-AzOpsPush.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index f7d21b80..f729f988 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -183,7 +183,7 @@ $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable if ($missingDefaultParam.Count -ge 1) { # Skip template deployment when template parameters without defaultValue are found and no parameter file identified - $missingString = ForEach($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} + $missingString = foreach ($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} Write-PSFMessage -Level Verbose -String 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -StringValues $FilePath, ($missingString | Out-String -NoNewline) continue } From 85e82c32413d84a0dbd7df85a121efa0d23c929e Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 8 Dec 2023 14:37:16 +0000 Subject: [PATCH 08/14] Update --- src/functions/Invoke-AzOpsPush.ps1 | 4 +-- src/tests/integration/Repository.Tests.ps1 | 34 +++++++++++++++++++ src/tests/templates/rtmultibase.bicep | 12 +++++++ .../rtmultibase.x123.parameters.json | 9 +++++ .../templates/rtmultibase.xabc.bicepparam | 3 ++ 5 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 src/tests/templates/rtmultibase.bicep create mode 100644 src/tests/templates/rtmultibase.x123.parameters.json create mode 100644 src/tests/templates/rtmultibase.xabc.bicepparam diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index f729f988..32966d52 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -374,8 +374,8 @@ #Sort 'deletionList' based on 'deletionListPriority' $deletionList = $deletionList | Sort-Object -Property {$deletionListPriority.IndexOf($_.ScopeObject.Resource)} - #If addModifySet exists and no deploymentList has been generated at the same time as the StatePath root has additional directories, exit with terminating error - if (($addModifySet -and -not $deploymentList) -and (Get-ChildItem -Path $StatePath -Directory)) { + #If addModifySet exists and no deploymentList has been generated at the same time as the StatePath root has additional directories and AllowMultipleTemplateParameterFiles is default false, exit with terminating error + if (($addModifySet -and -not $deploymentList) -and (Get-ChildItem -Path $StatePath -Directory) -and ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $false)) { Write-PSFMessage -Level Critical @common -String 'Invoke-AzOpsPush.DeploymentList.NotFound' throw } diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 6abc42aa..6314c3df 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -1064,6 +1064,40 @@ Describe "Repository" { {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw } #endregion + + #region Bicep multiple parameter file Test + It "Deploy Bicep base template with multiple parameter files (bicepparam, parameters.json)" { + Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $true + $script:bicepMultiParamPath = Get-ChildItem -Path "$($global:testRoot)/templates/rtmultibase*" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force + $changeSet = @( + "A`t$($script:bicepMultiParamPath.FullName[1])", + "A`t$($script:bicepMultiParamPath.FullName[2])" + ) + {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw + $script:bicepMultiParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "rtmultibasex*"} + $script:bicepMultiParamPathDeployment.Count | Should -Be 2 + } + #endregion + + #region Bicep base template with no 1-1 parameter file and AllowMultipleTemplateParameterFile set to true Test should not deploy + It "Try deployment of Bicep base template with missing defaultValue parameter with no 1-1 parameter file and AllowMultipleTemplateParameterFile set to true, Test should not deploy and exit gracefully" { + Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $true + $changeSet = @( + "A`t$($script:bicepMultiParamPath.FullName[0])" + ) + {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw + } + #endregion + + #region Bicep base template with no 1-1 parameter file and AllowMultipleTemplateParameterFile set to false Test should throw + It "Try deployment of Bicep base template with missing defaultValue parameter with no 1-1 parameter file and AllowMultipleTemplateParameterFile set to false, Test should not deploy and throw" { + Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $false + $changeSet = @( + "A`t$($script:bicepMultiParamPath.FullName[0])" + ) + {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Throw + } + #endregion } AfterAll { diff --git a/src/tests/templates/rtmultibase.bicep b/src/tests/templates/rtmultibase.bicep new file mode 100644 index 00000000..f790cbf7 --- /dev/null +++ b/src/tests/templates/rtmultibase.bicep @@ -0,0 +1,12 @@ +param name string +param location string = resourceGroup().location + +resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = { + name: name + location: location + properties: { + disableBgpRoutePropagation: false + routes: [ + ] + } +} diff --git a/src/tests/templates/rtmultibase.x123.parameters.json b/src/tests/templates/rtmultibase.x123.parameters.json new file mode 100644 index 00000000..e5672485 --- /dev/null +++ b/src/tests/templates/rtmultibase.x123.parameters.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "value": "rtmultibasex123" + } + } +} \ No newline at end of file diff --git a/src/tests/templates/rtmultibase.xabc.bicepparam b/src/tests/templates/rtmultibase.xabc.bicepparam new file mode 100644 index 00000000..d167da67 --- /dev/null +++ b/src/tests/templates/rtmultibase.xabc.bicepparam @@ -0,0 +1,3 @@ +using './rtmultibase.bicep' + +param name = toLower('rtmultibasexabc') From 7fb1beb5b97066467d9c24c2a219e23082e8d532 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 8 Dec 2023 15:59:26 +0000 Subject: [PATCH 09/14] Update --- src/internal/functions/New-AzOpsDeployment.ps1 | 14 ++++++++++++-- src/internal/functions/Set-AzOpsWhatIfOutput.ps1 | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/internal/functions/New-AzOpsDeployment.ps1 b/src/internal/functions/New-AzOpsDeployment.ps1 index ddc25680..b0e084de 100644 --- a/src/internal/functions/New-AzOpsDeployment.ps1 +++ b/src/internal/functions/New-AzOpsDeployment.ps1 @@ -175,7 +175,12 @@ # Handle WhatIf prediction errors elseif ($resultsErrorMessage -match 'DeploymentWhatIfResourceError' -and $resultsErrorMessage -match "The request to predict template deployment") { Write-PSFMessage -Level Warning -String 'New-AzOpsDeployment.WhatIfWarning' -Target $scopeObject -Tag Error -StringValues $resultsErrorMessage - Set-AzOpsWhatIfOutput -FilePath $parameters.TemplateFile -Results ('{0}WhatIf prediction failed with error - validate changes manually before merging:{0}{1}' -f [environment]::NewLine, $resultsErrorMessage) + if ($parameters.TemplateParameterFile) { + Set-AzOpsWhatIfOutput -FilePath $parameters.TemplateFile -ParameterFilePath $parameters.TemplateParameterFile -Results ('{0}WhatIf prediction failed with error - validate changes manually before merging:{0}{1}' -f [environment]::NewLine, $resultsErrorMessage) + } + else { + Set-AzOpsWhatIfOutput -FilePath $parameters.TemplateFile -Results ('{0}WhatIf prediction failed with error - validate changes manually before merging:{0}{1}' -f [environment]::NewLine, $resultsErrorMessage) + } } else { Write-PSFMessage -Level Warning -String 'New-AzOpsDeployment.WhatIfWarning' -Target $scopeObject -Tag Error -StringValues $resultsErrorMessage @@ -189,7 +194,12 @@ else { Write-PSFMessage -Level Verbose -String 'New-AzOpsDeployment.WhatIfResults' -StringValues ($results | Out-String) -Target $scopeObject Write-PSFMessage -Level Verbose -String 'New-AzOpsDeployment.WhatIfFile' -Target $scopeObject - Set-AzOpsWhatIfOutput -FilePath $parameters.TemplateFile -Results $results + if ($parameters.TemplateParameterFile) { + Set-AzOpsWhatIfOutput -FilePath $parameters.TemplateFile -ParameterFilePath $parameters.TemplateParameterFile -Results $results + } + else { + Set-AzOpsWhatIfOutput -FilePath $parameters.TemplateFile -Results $results + } } # Remove ExcludeChangeType parameter as it doesn't exist for deployment cmdlets if ($parameters.ExcludeChangeType) { diff --git a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 index 4cdf174e..c9260b3d 100644 --- a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 +++ b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 @@ -14,7 +14,9 @@ .PARAMETER ResultSizeMaxLimit The maximum upper character limit allowed for comments 64,600 .PARAMETER FilePath - File in scope of WhatIf + Template File in scope of WhatIf + .PARAMETER ParameterFilePath + Parameter File in scope of WhatIf .EXAMPLE > Set-AzOpsWhatIfOutput -Results $results > Set-AzOpsWhatIfOutput -Results $results -RemoveAzOpsFlag $true @@ -35,8 +37,11 @@ [Parameter(Mandatory = $false)] $ResultSizeMaxLimit = "64600", + [Parameter(Mandatory = $true)] + $FilePath, + [Parameter(Mandatory = $false)] - $FilePath + $ParameterFilePath ) process { @@ -47,7 +52,12 @@ New-Item -Path ($tempPath + 'OUTPUT.json') -WhatIf:$false } - $resultHeadline = $FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1] + if ($ParameterFilePath) { + $resultHeadline = "$($FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1]), $($ParameterFilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1])" + } + else { + $resultHeadline = $resultHeadline = $FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1] + } # Measure input $Results.Changes content $resultString = $Results | Out-String From cf512f2571bad7d994e90748b2510230c850290c Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 8 Dec 2023 16:04:38 +0000 Subject: [PATCH 10/14] Update --- src/internal/functions/Set-AzOpsWhatIfOutput.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 index c9260b3d..a0be73ed 100644 --- a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 +++ b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 @@ -53,7 +53,7 @@ } if ($ParameterFilePath) { - $resultHeadline = "$($FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1]), $($ParameterFilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1])" + $resultHeadline = "$($FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1]) with $($ParameterFilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1])" } else { $resultHeadline = $resultHeadline = $FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1] From a8fc2ef861e7d71fd24c644df85c2cd27d32d6a0 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 8 Dec 2023 17:47:23 +0000 Subject: [PATCH 11/14] Update --- src/functions/Initialize-AzOpsEnvironment.ps1 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/functions/Initialize-AzOpsEnvironment.ps1 b/src/functions/Initialize-AzOpsEnvironment.ps1 index 39bc7a65..33cbc621 100644 --- a/src/functions/Initialize-AzOpsEnvironment.ps1 +++ b/src/functions/Initialize-AzOpsEnvironment.ps1 @@ -62,6 +62,13 @@ Stop-PSFFunction -String 'Initialize-AzOpsEnvironment.AzureContext.TooMany' -StringValues $azContextTenants.Count, ($azContextTenants -join ',') -EnableException $true -Cmdlet $PSCmdlet } + # Adjust MultipleTemplateParameterFileSuffix if incorrect MultipleTemplateParameterFileSuffix is set and log warning + if (-not $(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix').StartsWith('.')) { + $updateMultipleTemplateParameterFileSuffix = ".$(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix')" + Write-PSFMessage -Level Warning -String 'Initialize-AzOpsEnvironment.MultipleTemplateParameterFileSuffix.Adjustment' -StringValues (Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix'), $updateMultipleTemplateParameterFileSuffix + Set-PSFConfig -Module AzOps -Name Core.MultipleTemplateParameterFileSuffix -Value $updateMultipleTemplateParameterFileSuffix + } + # Adjust ThrottleLimit from previously default 10 to 5 if system has less than 2 cores [int]$cpuCores = if ($IsWindows) { $env:NUMBER_OF_PROCESSORS } else { Invoke-AzOpsNativeCommand -ScriptBlock { nproc --all } -IgnoreExitcode } $throttleLimit = (Get-PSFConfig -Module AzOps -Name Core.ThrottleLimit).Value From 208f4261de481565239f0389c2bc16579c7d1f68 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 8 Dec 2023 20:35:28 +0000 Subject: [PATCH 12/14] UpdateStringAndMd --- docs/wiki/Frequently-Asked-Questions.md | 10 +++++----- src/localized/en-us/Strings.psd1 | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/wiki/Frequently-Asked-Questions.md b/docs/wiki/Frequently-Asked-Questions.md index 8822704c..9ff9702b 100644 --- a/docs/wiki/Frequently-Asked-Questions.md +++ b/docs/wiki/Frequently-Asked-Questions.md @@ -16,8 +16,8 @@ This article answers frequently asked questions relating to AzOps. - [**I want to discover a specific resource type in specific resource group in one specific subscription**](#i-want-to-discover-a-specific-resource-type-in-specific-resource-group-in-one-specific-subscription) - [**I want to discover and manage several Azure Firewall Policy's and rule collections spread out across several resource groups and subscriptions**](#i-want-to-discover-and-manage-several-azure-firewall-policys-and-rule-collections-spread-out-across-several-resource-groups-and-subscriptions) - [Push scenarios and settings](#push-scenarios-and-settings) - - [**I want to have multiple different deployments at scope using the same template file but different parameter files**](#i-want-to-have-multiple-different-deployments-at-scope-using-the-same-template-file-but-different-parameter-files) - - [**I am getting: Missing defaultValue and no parameter file found, skip deployment**](#i-am-getting-missing-defaultvalue-and-no-parameter-file-found-skip-deployment) + - [**I want to have multiple different deployments at scope using the same template file but different parameter files**](#i-want-to-have-multiple-different-deployments-at-scope-using-the-same-template-file-but-different-parameter-files) + - [**I am getting: Missing defaultValue and no parameter file found, skip deployment**](#i-am-getting-missing-defaultvalue-and-no-parameter-file-found-skip-deployment) ## Subscriptions or resources not showing up in repository @@ -156,7 +156,7 @@ Template files listed in the order file will be deployed in the order specified ### **I want to have multiple different deployments at scope using the same template file but different parameter files** When using custom deployment templates, can I avoid the pattern of duplicating the `.bicep` file for each `parameter` file below? -``` +```bash scope/ ├── template-a.bicep ├── template-a.bicepparam @@ -172,8 +172,8 @@ Yes, ensure the following setting combinations are applied (replace `x` with you "Core.MultipleTemplateParameterFileSuffix": ".x" ``` -AzOps module will evaluate each parameter file individually and try to find base template by matching (*regex*) according to `MultipleTemplateParameterFileSuffix` pattern identifier. -``` +AzOps module will evaluate each parameter file individually and try to find base template by matching (*regular expression*) according to `MultipleTemplateParameterFileSuffix` pattern identifier. +```bash scope/ ├── template.x1.bicepparam ├── template.x2.bicepparam diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index d40edc55..390e59b5 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -155,6 +155,7 @@ 'Initialize-AzOpsEnvironment.Processing' = 'Processing AzOps environment' # 'Initialize-AzOpsEnvironment.Processing.Completed' = 'AzOps environment initialization concluded' # 'Initialize-AzOpsEnvironment.ThrottleLimit.Adjustment' = 'Adjusting AzOps.Core.ThrottleLimit from {0} to 5 due to available CPU Cores ({1}) to ensure reliable and performant pipeline execution. For further details, refer to: https://github.com/azure/azops/wiki/performance-considerations' # $throttleLimit, $cpuCores + 'Initialize-AzOpsEnvironment.MultipleTemplateParameterFileSuffix.Adjustment' = 'Adjusting AzOps.Core.MultipleTemplateParameterFileSuffix from ({0}) to ({1}) to ensure reliable file matching. To avoid this warning update your MultipleTemplateParameterFileSuffix setting to startwith a [.]' # AzOps.Core.MultipleTemplateParameterFileSuffix, $updateMultipleTemplateParameterFileSuffix 'Initialize-AzOpsEnvironment.SkipCustomJqTemplate.True' = 'AzOps.Core.SkipCustomJqTemplate is true, using module defaults' # 'Initialize-AzOpsEnvironment.CustomJqTemplatePath' = 'AzOps.Core.CustomJqTemplatePath {0}' # $customJqTemplatePath 'Initialize-AzOpsEnvironment.CustomJqTemplatePath.PathNotFound' = 'The path specified in AzOps.Core.CustomJqTemplatePath {0} was not found, reverting to module defaults' # $customJqTemplatePath From 283053290b887b6366f256099121690157aaa740 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 11 Dec 2023 07:37:07 +0000 Subject: [PATCH 13/14] Update --- src/internal/functions/Set-AzOpsWhatIfOutput.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 index a0be73ed..5768394a 100644 --- a/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 +++ b/src/internal/functions/Set-AzOpsWhatIfOutput.ps1 @@ -56,7 +56,7 @@ $resultHeadline = "$($FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1]) with $($ParameterFilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1])" } else { - $resultHeadline = $resultHeadline = $FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1] + $resultHeadline = $FilePath.split([System.IO.Path]::DirectorySeparatorChar)[-1] } # Measure input $Results.Changes content From 150d23d3c58246d123b6445d317774faad105936 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 13 Dec 2023 17:23:26 +0000 Subject: [PATCH 14/14] UpdateLogic --- docs/wiki/Frequently-Asked-Questions.md | 9 ++++++++ docs/wiki/Settings.md | 7 +++--- src/functions/Invoke-AzOpsPush.ps1 | 22 +++++++++++++++++++ src/internal/configurations/Core.ps1 | 1 + src/tests/integration/Repository.Tests.ps1 | 16 ++++++++++++++ src/tests/templates/deployallrtbase.bicep | 12 ++++++++++ .../deployallrtbase.x123.parameters.json | 9 ++++++++ .../templates/deployallrtbase.xabc.bicepparam | 3 +++ 8 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 src/tests/templates/deployallrtbase.bicep create mode 100644 src/tests/templates/deployallrtbase.x123.parameters.json create mode 100644 src/tests/templates/deployallrtbase.xabc.bicepparam diff --git a/docs/wiki/Frequently-Asked-Questions.md b/docs/wiki/Frequently-Asked-Questions.md index 9ff9702b..ecc1f7b4 100644 --- a/docs/wiki/Frequently-Asked-Questions.md +++ b/docs/wiki/Frequently-Asked-Questions.md @@ -17,6 +17,7 @@ This article answers frequently asked questions relating to AzOps. - [**I want to discover and manage several Azure Firewall Policy's and rule collections spread out across several resource groups and subscriptions**](#i-want-to-discover-and-manage-several-azure-firewall-policys-and-rule-collections-spread-out-across-several-resource-groups-and-subscriptions) - [Push scenarios and settings](#push-scenarios-and-settings) - [**I want to have multiple different deployments at scope using the same template file but different parameter files**](#i-want-to-have-multiple-different-deployments-at-scope-using-the-same-template-file-but-different-parameter-files) + - [**I have AllowMultipleTemplateParameterFiles set to true and when changes are made to a template no deployment is performed**](#i-have-allowmultipletemplateparameterfiles-set-to-true-and-when-changes-are-made-to-a-template-no-deployment-is-performed) - [**I am getting: Missing defaultValue and no parameter file found, skip deployment**](#i-am-getting-missing-defaultvalue-and-no-parameter-file-found-skip-deployment) ## Subscriptions or resources not showing up in repository @@ -182,6 +183,14 @@ scope/ ``` > Note: To avoid having AzOps deploy the base `template.bicep` unintentionally, ensure you have at least one parameter without default value in `template.bicep` and no lingering 1:1 matching parameter file. +### **I have AllowMultipleTemplateParameterFiles set to true and when changes are made to a template no deployment is performed** + +When using a custom deployment templates with multiple corresponding parameter files, can I ensure that changes made to the template triggers AzOps to create separate deployments for each corresponding parameter file? + +Yes, ensure the following setting `Core.DeployAllMultipleTemplateParameterFiles` is set to `true`. + +> Note: By default, AzOps does not try to identify and deploy files that have not changed, by changing this setting AzOps will attempt to resolve matching parameter files for deployment based on deployment template. + ### **I am getting: Missing defaultValue and no parameter file found, skip deployment** To confirm if this applies to you, check the pipeline logs for the following message: diff --git a/docs/wiki/Settings.md b/docs/wiki/Settings.md index aa58160a..d37fa2ad 100644 --- a/docs/wiki/Settings.md +++ b/docs/wiki/Settings.md @@ -36,9 +36,10 @@ The following configuration values can be modified within the `settings.json` fi | 24 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | | 25 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | | 26 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | -| 27 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | -| 28 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | -| 29 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | +| 27 | DeployAllMultipleTemplateParameterFiles | Control base template deployment behaviour with changes and un-changed multiple corresponding parameter files. | `"Core.DeployAllMultipleTemplateParameterFiles": false` | +| 28 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | +| 29 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | +| 30 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | ## Workflow / Pipeline Settings diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 32966d52..90105c4c 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -175,6 +175,28 @@ Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.ParameterFound' -StringValues $FilePath, $parameterPath $result.TemplateParameterFilePath = $parameterPath } + elseif ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true -and (Get-PSFConfigValue -FullName 'AzOps.Core.DeployAllMultipleTemplateParameterFiles') -eq $true) { + # Check for multiple associated template parameter files + $paramFileList = Get-ChildItem -Path $fileItem.Directory | Where-Object { ($_.Name.Split('.')[-3] -match $(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix').Replace('.','')) -or ($_.Name.Split('.')[-2] -match $(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix').Replace('.','')) } + if ($paramFileList) { + $multiResult = @() + foreach ($paramFile in $paramFileList) { + # Process possible parameter files for template equivalent + if (($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-3]) -or ($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-4])) { + Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -StringValues $paramFile.FullName + $multiResult += Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $paramFile -AzOpsMainTemplate $AzOpsMainTemplate + } + } + if ($multiResult) { + # Return completed object + return $multiResult + } + else { + Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -StringValues $FilePath, $parameterPath + } + + } + } else { Write-PSFMessage -Level Verbose @common -String 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -StringValues $FilePath, $parameterPath if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true) { diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index 490a8146..8eebec11 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -27,6 +27,7 @@ Set-PSFConfig -Module AzOps -Name Core.State -Value (Join-Path $pwd -ChildPath " Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID or Display Name that matches the filter. Powershell filter that matches with like operator is supported.' Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'Parameter file suffix identifier' Set-PSFConfig -Module AzOps -Name Core.AllowMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control multiple parameter file behaviour' +Set-PSFConfig -Module AzOps -Name Core.DeployAllMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control base template deployment behaviour with changes and un-changed multiple corresponding parameter files' Set-PSFConfig -Module AzOps -Name Core.MultipleTemplateParameterFileSuffix -Value '.x' -Initialize -Validation string -Description 'Multiple parameter file suffix identifier' Set-PSFConfig -Module AzOps -Name Core.ThrottleLimit -Value 5 -Initialize -Validation integer -Description 'Throttle limit used in Foreach-Object -Parallel for resource/subscription discovery' Set-PSFConfig -Module AzOps -Name Core.WhatifExcludedChangeTypes -Value @('NoChange', 'Ignore') -Initialize -Validation stringarray -Description 'Exclude specific change types from WhatIf operations.' \ No newline at end of file diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 6314c3df..8182511e 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -1074,6 +1074,7 @@ Describe "Repository" { "A`t$($script:bicepMultiParamPath.FullName[2])" ) {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw + Start-Sleep -Seconds 5 $script:bicepMultiParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "rtmultibasex*"} $script:bicepMultiParamPathDeployment.Count | Should -Be 2 } @@ -1098,6 +1099,21 @@ Describe "Repository" { {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Throw } #endregion + + #region Bicep template with change, AzOps set to resolve corresponding parameter files and create multiple deployments + It "Deploy Bicep template with change, AzOps set to resolve corresponding parameter files and create multiple deployments" { + Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $true + Set-PSFConfig -FullName AzOps.Core.DeployAllMultipleTemplateParameterFiles -Value $true + $script:deployAllRtParamPath = Get-ChildItem -Path "$($global:testRoot)/templates/deployallrtbase*" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force + $changeSet = @( + "A`t$($script:deployAllRtParamPath.FullName[0])" + ) + {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw + Start-Sleep -Seconds 5 + $script:deployAllRtParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "deployallrtbasex*"} + $script:deployAllRtParamPathDeployment.Count | Should -Be 2 + } + #endregion } AfterAll { diff --git a/src/tests/templates/deployallrtbase.bicep b/src/tests/templates/deployallrtbase.bicep new file mode 100644 index 00000000..f790cbf7 --- /dev/null +++ b/src/tests/templates/deployallrtbase.bicep @@ -0,0 +1,12 @@ +param name string +param location string = resourceGroup().location + +resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = { + name: name + location: location + properties: { + disableBgpRoutePropagation: false + routes: [ + ] + } +} diff --git a/src/tests/templates/deployallrtbase.x123.parameters.json b/src/tests/templates/deployallrtbase.x123.parameters.json new file mode 100644 index 00000000..cf4c480b --- /dev/null +++ b/src/tests/templates/deployallrtbase.x123.parameters.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "value": "deployallrtbasex123" + } + } +} \ No newline at end of file diff --git a/src/tests/templates/deployallrtbase.xabc.bicepparam b/src/tests/templates/deployallrtbase.xabc.bicepparam new file mode 100644 index 00000000..2db0b566 --- /dev/null +++ b/src/tests/templates/deployallrtbase.xabc.bicepparam @@ -0,0 +1,3 @@ +using './deployallrtbase.bicep' + +param name = toLower('deployallrtbasexabc')