From d7b8781398df834a915ffb86d12851d786b93434 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:29:59 -0500 Subject: [PATCH 1/4] Update stop script to support multiple App GWs, fixes #3643 Adding PowerShell 7.2 as a runbook option Linting PowerShell Adding/expanding comments --- docs/tre-admins/start-stop.md | 93 ++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/docs/tre-admins/start-stop.md b/docs/tre-admins/start-stop.md index d3bc77289a..55f57acd3d 100644 --- a/docs/tre-admins/start-stop.md +++ b/docs/tre-admins/start-stop.md @@ -1,11 +1,11 @@ # Start/Stop Azure TRE -Once you've provisioned an Azure TRE instance it will begin to incurr running costs of the underlying Azure services. +Once you've provisioned an Azure TRE instance it will begin to incur running costs of the underlying Azure services. -Within evaluation or development, you may want to "pause" the TRE environment during out of hours or weekends, to reduce costs without having to completely destroy the environment. The following `make targets` provide a simple way to start and stop both the Azure Firewall and Azure Application Gateway instances, considerably reducing the Azure TRE instance running costs. +Within evaluation or development, you may want to "pause" the TRE environment during out of office hours or weekends, to reduce costs without having to completely destroy the environment. The following `make` targets provide a simple way to start and stop both the Azure Firewall and Azure Application Gateway instances, considerably reducing the Azure TRE instance running costs. !!! info - After running `make all` underlying Azure TRE services are automatically started, billing will start. + After running `make all` underlying Azure TRE services are automatically started and billing will start. ## Start Azure TRE @@ -32,7 +32,7 @@ We have this procedure setup in our development subscriptions where each night w We use [Azure Automation](https://docs.microsoft.com/en-us/azure/automation/overview) to run this procedure. -Be sure to create a runbook with Powershell 7.1 enabled and an identity with contributor permissions on the subscription. Note that the script below uses a system managed identity and if you use something different then you might need to update the authentication part. +Be sure to create a runbook with PowerShell 7.1 or PowerShell 7.2 enabled and an identity with contributor permissions on the subscription. Note that the script below uses a system managed identity and if you use something different then you might need to update the authentication part. If you create a new Automation account, you will have the required modules preinstalled. @@ -41,14 +41,13 @@ Finally, schedule it to run when it makes sense for you. ### Runbook Script ```powershell -try -{ - "Logging in to Azure..." - Connect-AzAccount -Identity +try { + "Logging in to Azure..." + Connect-AzAccount -Identity } catch { - Write-Error -Message $_.Exception - throw $_.Exception + Write-Error -Message $_.Exception + throw $_.Exception } $azContext = Get-AzContext @@ -57,62 +56,64 @@ $profileClient = New-Object -TypeName Microsoft.Azure.Commands.ResourceManager.C $token = $profileClient.AcquireAccessToken($azContext.Subscription.TenantId) $authHeader = @{ - 'Content-Type'='application/json' - 'Authorization'='Bearer ' + $token.AccessToken + 'Content-Type' = 'application/json' + 'Authorization' = 'Bearer ' + $token.AccessToken } +# Get all resource groups that have the default Azure TRE project tag value +$ResourceGroups = Get-AzResourceGroup -Tag @{'project' = 'Azure Trusted Research Environment' } +foreach ($Group in $ResourceGroups) { + if ($Group.ResourceGroupName -like '*-ws-*') { + # Deal with the workspace resource groups separately (below) + continue + } -$ResourceGroups = Get-AzResourceGroup -Tag @{'project'='Azure Trusted Research Environment'} -foreach ($Group in $ResourceGroups) -{ - if ($Group.ResourceGroupName -like '*-ws-*') { - # we deal with the workspace resource groups separately. - continue - } - - $Firewall = Get-AzFirewall -ResourceGroupName $Group.ResourceGroupName - if ($Firewall -ne $null) { - $Firewall.Deallocate() - Write-Output "Deallocating $($Firewall.Name)" - Set-AzFirewall -AzureFirewall $Firewall - } + # Deallocate the Azure Firewall (expecting only one per TRE instance) + $Firewall = Get-AzFirewall -ResourceGroupName $Group.ResourceGroupName + if ($null -ne $Firewall) { + $Firewall.Deallocate() + Write-Output "Deallocating Firewall '$($Firewall.Name)'" + Set-AzFirewall -AzureFirewall $Firewall + } - $Gateway = Get-AzApplicationGateway -ResourceGroupName $Group.ResourceGroupName - if ($Gateway -ne $null) { - Write-Output "Stopping $($Gateway.Name)" - Stop-AzApplicationGateway -ApplicationGateway $Gateway - } + # Stop the Application Gateway(s) + # Multiple Application Gateways may exist if the certs shared service is installed + $Gateways = Get-AzApplicationGateway -ResourceGroupName $Group.ResourceGroupName + foreach ($Gateway in $Gateways) { + Write-Output "Stopping Application Gateway '$($Gateway.Name)'" + Stop-AzApplicationGateway -ApplicationGateway $Gateway + } + # Stop the MySQL servers $MySQLServers = Get-AzResource -ResourceGroupName $Group.ResourceGroupName -ResourceType "Microsoft.DBforMySQL/servers" - foreach ($Server in $MySQLServers) - { + foreach ($Server in $MySQLServers) { # Invoke the REST API - Write-Output "Stopping $($Server.Name)" - $restUri='https://management.azure.com/subscriptions/'+$azContext.Subscription.Id+'/resourceGroups/'+$Group.ResourceGroupName+'/providers/Microsoft.DBForMySQL/servers/'+$Server.Name+'/stop?api-version=2020-01-01' + Write-Output "Stopping MySQL '$($Server.Name)'" + $restUri = 'https://management.azure.com/subscriptions/' + $azContext.Subscription.Id + '/resourceGroups/' + $Group.ResourceGroupName + '/providers/Microsoft.DBForMySQL/servers/' + $Server.Name + '/stop?api-version=2020-01-01' $response = Invoke-RestMethod -Uri $restUri -Method POST -Headers $authHeader } + # Deallocate all the virtual machine scale sets (resource processor) $VMSS = Get-AzVMSS -ResourceGroupName $Group.ResourceGroupName - foreach ($item in $VMSS) - { - Write-Output "Stopping $($item.Name)" + foreach ($item in $VMSS) { + Write-Output "Stopping VMSS '$($item.Name)'" Stop-AzVmss -ResourceGroupName $item.ResourceGroupName -VMScaleSetName $item.Name -Force } + # Deallocate all the VMs $VM = Get-AzVM -ResourceGroupName $Group.ResourceGroupName - foreach ($item in $VM) - { - Write-Output "Stopping $($item.Name)" + foreach ($item in $VM) { + Write-Output "Stopping VM '$($item.Name)'" Stop-AzVm -ResourceGroupName $item.ResourceGroupName -Name $item.Name -Force } + # Process all the workspace resource groups for this TRE instance $WorkspaceResourceGroups = Get-AzResourceGroup -Name "$($Group.ResourceGroupName)-ws-*" - foreach ($wsrg in $WorkspaceResourceGroups) - { + foreach ($wsrg in $WorkspaceResourceGroups) { + # Deallocate all the VMs $VM = Get-AzVM -ResourceGroupName $wsrg.ResourceGroupName - foreach ($item in $VM) - { - Write-Output "Stopping $($item.Name)" + foreach ($item in $VM) { + Write-Output "Stopping workspace VM '$($item.Name)'" Stop-AzVm -ResourceGroupName $item.ResourceGroupName -Name $item.Name -Force } } From 7692b1d02b5d289933824f87fad284457b223182 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:35:33 -0500 Subject: [PATCH 2/4] Provide PS script to start TRE core #3644 --- docs/tre-admins/start-stop.md | 72 ++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/tre-admins/start-stop.md b/docs/tre-admins/start-stop.md index 55f57acd3d..1943bd90d2 100644 --- a/docs/tre-admins/start-stop.md +++ b/docs/tre-admins/start-stop.md @@ -38,7 +38,7 @@ If you create a new Automation account, you will have the required modules prein Finally, schedule it to run when it makes sense for you. -### Runbook Script +### Stop Runbook Script ```powershell try { @@ -119,3 +119,73 @@ foreach ($Group in $ResourceGroups) { } } ``` + +### Automating `start` + +To restart the TRE core services (Firewall, Application Gateway(s), Virtual Machine Scale Sets, and MySQL), you can use `make tre-start`. Depending on your workflow, you might not be able to easily execute the `make` target. Alternatively, you can create a second Runbook and execute it manually. The PowerShell code to start TRE core services is below: + +```powershell +try { + "Logging in to Azure..." + Connect-AzAccount -Identity +} +catch { + Write-Error -Message $_.Exception + throw $_.Exception +} + +$azContext = Get-AzContext +$azProfile = [Microsoft.Azure.Commands.Common.Authentication.Abstractions.AzureRmProfileProvider]::Instance.Profile +$profileClient = New-Object -TypeName Microsoft.Azure.Commands.ResourceManager.Common.RMProfileClient -ArgumentList ($azProfile) +$token = $profileClient.AcquireAccessToken($azContext.Subscription.TenantId) + +$authHeader = @{ + 'Content-Type' = 'application/json' + 'Authorization' = 'Bearer ' + $token.AccessToken +} + +# Get all resource groups that have the default Azure TRE project tag value +$ResourceGroups = Get-AzResourceGroup -Tag @{'project' = 'Azure Trusted Research Environment' } +foreach ($Group in $ResourceGroups) { + if ($Group.ResourceGroupName -like '*-ws-*') { + # Don't deal with the workspace resource groups + continue + } + + $azureTreId = 'actredemo01' + # Allocate the Azure Firewall (expecting only one per TRE instance) + $Firewall = Get-AzFirewall -ResourceGroupName $Group.ResourceGroupName + if ($null -ne $Firewall) { + # Find the firewall's public IP and virtual network + $pip = Get-AzPublicIpAddress -ResourceGroupName $Group.ResourceGroupName -Name "pip-fw-$azureTreId" + $vnet = Get-AzVirtualNetwork -ResourceGroupName $Group.ResourceGroupName -Name "vnet-$azureTreId" + $Firewall.Allocate($vnet, $pip) + Write-Output "Allocating Firewall '$($Firewall.Name)'" + Set-AzFirewall -AzureFirewall $Firewall + } + + # Start the Application Gateway(s) + # Multiple Application Gateways may exist if the certs shared service is installed + $Gateways = Get-AzApplicationGateway -ResourceGroupName $Group.ResourceGroupName + foreach ($Gateway in $Gateways) { + Write-Output "Starting Application Gateway '$($Gateway.Name)'" + Start-AzApplicationGateway -ApplicationGateway $Gateway + } + + # Start the MySQL servers + $MySQLServers = Get-AzResource -ResourceGroupName $Group.ResourceGroupName -ResourceType "Microsoft.DBforMySQL/servers" + foreach ($Server in $MySQLServers) { + # Invoke the REST API + Write-Output "Starting MySQL '$($Server.Name)'" + $restUri = 'https://management.azure.com/subscriptions/' + $azContext.Subscription.Id + '/resourceGroups/' + $Group.ResourceGroupName + '/providers/Microsoft.DBForMySQL/servers/' + $Server.Name + '/start?api-version=2020-01-01' + $response = Invoke-RestMethod -Uri $restUri -Method POST -Headers $authHeader + } + + # Allocate all the virtual machine scale sets (resource processor) + $VMSS = Get-AzVMSS -ResourceGroupName $Group.ResourceGroupName + foreach ($item in $VMSS) { + Write-Output "Starting VMSS '$($item.Name)'" + Start-AzVmss -ResourceGroupName $item.ResourceGroupName -VMScaleSetName $item.Name + } +} +``` From c0e076ac0b85aaf98eba1d726457e41bc95cec47 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Tue, 1 Aug 2023 21:26:28 -0500 Subject: [PATCH 3/4] Fix hardcoded TRE ID --- docs/tre-admins/start-stop.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/tre-admins/start-stop.md b/docs/tre-admins/start-stop.md index 1943bd90d2..36f41643a5 100644 --- a/docs/tre-admins/start-stop.md +++ b/docs/tre-admins/start-stop.md @@ -152,7 +152,9 @@ foreach ($Group in $ResourceGroups) { continue } - $azureTreId = 'actredemo01' + $azureTreId = $Group.Tags['tre_id'] + Write-Output "Starting TRE core resources for '$azureTreId'" + # Allocate the Azure Firewall (expecting only one per TRE instance) $Firewall = Get-AzFirewall -ResourceGroupName $Group.ResourceGroupName if ($null -ne $Firewall) { @@ -160,7 +162,7 @@ foreach ($Group in $ResourceGroups) { $pip = Get-AzPublicIpAddress -ResourceGroupName $Group.ResourceGroupName -Name "pip-fw-$azureTreId" $vnet = Get-AzVirtualNetwork -ResourceGroupName $Group.ResourceGroupName -Name "vnet-$azureTreId" $Firewall.Allocate($vnet, $pip) - Write-Output "Allocating Firewall '$($Firewall.Name)'" + Write-Output "Allocating Firewall '$($Firewall.Name)' with public IP '$($pip.Name)'" Set-AzFirewall -AzureFirewall $Firewall } From cd41c87ae4ab723454ee84faafbfe88e601d398a Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Wed, 2 Aug 2023 06:16:04 -0500 Subject: [PATCH 4/4] Update Automation link to Learn --- docs/tre-admins/start-stop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-admins/start-stop.md b/docs/tre-admins/start-stop.md index 36f41643a5..f52a528b6e 100644 --- a/docs/tre-admins/start-stop.md +++ b/docs/tre-admins/start-stop.md @@ -30,7 +30,7 @@ We have this procedure setup in our development subscriptions where each night w ### Requirements -We use [Azure Automation](https://docs.microsoft.com/en-us/azure/automation/overview) to run this procedure. +We use [Azure Automation](https://learn.microsoft.com/azure/automation/overview) to run this procedure. Be sure to create a runbook with PowerShell 7.1 or PowerShell 7.2 enabled and an identity with contributor permissions on the subscription. Note that the script below uses a system managed identity and if you use something different then you might need to update the authentication part.