diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index c146e84fd..4707f55bd 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -19,7 +19,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","GitHubAppClientId") $runAlPipelineOverrides = @( "DockerPull" @@ -1315,12 +1315,13 @@ function CloneIntoNewFolder { $baseFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) New-Item $baseFolder -ItemType Directory | Out-Null Set-Location $baseFolder - $serverUri = [Uri]::new($env:GITHUB_SERVER_URL) - $serverUrl = "$($serverUri.Scheme)://$($actor):$($token)@$($serverUri.Host)/$($env:GITHUB_REPOSITORY)" # Environment variables for hub commands $env:GITHUB_USER = $actor - $env:GITHUB_TOKEN = $token + $env:GITHUB_TOKEN = GetRealToken -token $token + + $serverUri = [Uri]::new($env:GITHUB_SERVER_URL) + $serverUrl = "$($serverUri.Scheme)://$($env:GITHUB_USER):$($env:GITHUB_TOKEN)@$($serverUri.Host)/$($env:GITHUB_REPOSITORY)" # Configure git invoke-git config --global user.email "$actor@users.noreply.github.com" diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 766d7ee8b..583afee2f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -51,15 +51,22 @@ function GetLatestTemplateSha { try { $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry - $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch } } catch { if ($_.Exception.Message -like "*401*") { - throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))" - } else { + try { + $headers.Remove('Authorization') + $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry + } + catch { + throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))" + } + } + else { throw $_.Exception.Message } } + $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch } if (!$branchInfo) { throw "$templateUrl doesn't exist" } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 3a9454008..e47f611f9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -27,15 +27,12 @@ if ($update -eq 'Y') { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } else { - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $token = GetRealToken -token ([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token))) } } # Use Authenticated API request to avoid the 60 API calls per hour limit -$headers = @{ - "Accept" = "application/vnd.github.baptiste-preview+json" - "Authorization" = "Bearer $token" -} +$headers = GetHeaders -token $token if (-not $templateUrl.Contains('@')) { $templateUrl += "@main" diff --git a/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 b/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 index 6d7870a68..f699120cf 100644 --- a/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 +++ b/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 @@ -7,7 +7,7 @@ ) function IsGitHubPagesAvailable() { - $headers = GetHeader -token $env:GITHUB_TOKEN + $headers = GetHeaders -token $env:GITHUB_TOKEN $url = "$($ENV:GITHUB_API_URL)/repos/$($ENV:GITHUB_REPOSITORY)/pages" try { Write-Host "Requesting GitHub Pages settings from GitHub" @@ -20,7 +20,7 @@ function IsGitHubPagesAvailable() { } function GetGitHubEnvironments() { - $headers = GetHeader -token $env:GITHUB_TOKEN + $headers = GetHeaders -token $env:GITHUB_TOKEN $url = "$($ENV:GITHUB_API_URL)/repos/$($ENV:GITHUB_REPOSITORY)/environments" try { Write-Host "Requesting environments from GitHub" @@ -36,7 +36,7 @@ function GetGitHubEnvironments() { function Get-BranchesFromPolicy($ghEnvironment) { if ($ghEnvironment) { # Environment is defined in GitHub - check protection rules - $headers = GetHeader -token $env:GITHUB_TOKEN + $headers = GetHeaders -token $env:GITHUB_TOKEN $branchPolicy = ($ghEnvironment.protection_rules | Where-Object { $_.type -eq "branch_policy" }) if ($branchPolicy) { if ($ghEnvironment.deployment_branch_policy.protected_branches) { diff --git a/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 b/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 index 42f12ecb8..08be4fc6e 100644 --- a/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 +++ b/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 @@ -25,10 +25,7 @@ function Get-ModifiedFiles { $url = "$($env:GITHUB_API_URL)/repos/$($env:GITHUB_REPOSITORY)/compare/$($ghEvent.pull_request.base.sha)...$($ghEvent.pull_request.head.sha)" - $headers = @{ - "Authorization" = "token $token" - "Accept" = "application/vnd.github.baptiste-preview+json" - } + $headers = GetHeaders -token $token $response = (InvokeWebRequest -Headers $headers -Uri $url).Content | ConvertFrom-Json diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index c4439d7f4..8e499a974 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -1,3 +1,41 @@ +$script:escchars = @(' ','!','\"','#','$','%','\u0026','\u0027','(',')','*','+',',','-','.','/','0','1','2','3','4','5','6','7','8','9',':',';','\u003c','=','\u003e','?','@','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','[','\\',']','^','_',[char]96,'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','{','|','}','~') +$script:realTokenCache = @{ + "token" = '' + "repository" = '' + "realToken" = '' + "expires" = [datetime]::Now +} + +function MaskValue { + Param( + [string] $key, + [string] $value + ) + + Write-Host "Masking value for $key" + $value.Split("`n") | ForEach-Object { + Write-Host "::add-mask::$_" + } + + $val2 = "" + $value.ToCharArray() | ForEach-Object { + $chint = [int]$_ + if ($chint -lt 32 -or $chint -gt 126 ) { + $val2 += $_ + } + else { + $val2 += $script:escchars[$chint-32] + } + } + + if ($val2 -ne $value) { + $val2.Split("`n") | ForEach-Object { + Write-Host "::add-mask::$_" + } + } + Write-Host "::add-mask::$([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($value)))" +} + function GetExtendedErrorMessage { [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSAvoidUsingEmptyCatchBlock", "", Justification="We want to ignore errors")] Param( @@ -501,7 +539,7 @@ function GetReleases { ) Write-Host "Analyzing releases $api_url/repos/$repository/releases" - $releases = (InvokeWebRequest -Headers (GetHeader -token $token) -Uri "$api_url/repos/$repository/releases").Content | ConvertFrom-Json + $releases = (InvokeWebRequest -Headers (GetHeaders -token $token) -Uri "$api_url/repos/$repository/releases").Content | ConvertFrom-Json if ($releases.Count -gt 1) { # Sort by SemVer tag try { @@ -556,20 +594,70 @@ function GetLatestRelease { $latestRelease } -function GetHeader { +function GetRealToken { + Param( + [string] $token, + [string] $api_url = $ENV:GITHUB_API_URL, + [string] $repository = $ENV:GITHUB_REPOSITORY + ) + + if (!($token.StartsWith("{"))) { + # not a json token + return $token + } + elseif ($script:realTokenCache.token -eq $token -and $script:realTokenCache.repository -eq $repository -and $script:realTokenCache.expires -gt [datetime]::Now.AddMinutes(10)) { + # Same token request and cached token won't expire in 10 minutes + Write-Host "return cached token" + return $script:realTokenCache.realToken + } + else { + try { + $json = $token | ConvertFrom-Json + $gitHubAppClientId = $json.GitHubAppClientId + $privateKey = $json.PrivateKey + Write-Host "Using GitHub App with ClientId $gitHubAppClientId for authentication" + $jwt = GenerateJwtForTokenRequest -gitHubAppClientId $gitHubAppClientId -privateKey $privateKey + $headers = @{ + "Accept" = "application/vnd.github+json" + "Authorization" = "Bearer $jwt" + "X-GitHub-Api-Version" = "2022-11-28" + } + Write-Host "Get App Info $api_url/repos/$repository/installation" + $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$api_url/repos/$repository/installation" + Write-Host "Get Token Response $($appInfo.access_tokens_url)" + $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url + Write-Host "return token" + $script:realTokenCache = @{ + "token" = $token + "repository" = $repository + "realToken" = $tokenResponse.token + "expires" = [datetime]::Now.AddSeconds($tokenResponse.expires_in) + } + return $tokenResponse.token + } + catch { + # Not a json token + return $token + } + } +} + +function GetHeaders { param ( [string] $token, [string] $accept = "application/vnd.github+json", - [string] $apiVersion = "2022-11-28" + [string] $apiVersion = "2022-11-28", + [string] $api_url = $ENV:GITHUB_API_URL, + [string] $repository = $ENV:GITHUB_REPOSITORY ) $headers = @{ "Accept" = $accept "X-GitHub-Api-Version" = $apiVersion } if (![string]::IsNullOrEmpty($token)) { - $headers["Authorization"] = "token $token" + $realToken = GetRealToken -token $token -api_url $api_url -repository $repository + $headers["Authorization"] = "token $realToken" } - return $headers } @@ -616,7 +704,7 @@ function GetReleaseNotes { $postParams["target_commitish"] = $target_commitish } - InvokeWebRequest -Headers (GetHeader -token $token) -Method POST -Body ($postParams | ConvertTo-Json) -Uri "$api_url/repos/$repository/releases/generate-notes" + InvokeWebRequest -Headers (GetHeaders -token $token) -Method POST -Body ($postParams | ConvertTo-Json) -Uri "$api_url/repos/$repository/releases/generate-notes" } function DownloadRelease { @@ -636,7 +724,7 @@ function DownloadRelease { if ([string]::IsNullOrEmpty($token)) { $token = invoke-gh -silent -returnValue auth token } - $headers = GetHeader -token $token -accept "application/octet-stream" + $headers = GetHeaders -token $token -accept "application/octet-stream" foreach($project in $projects.Split(',')) { # GitHub replaces series of special characters with a single dot when uploading release assets $project = [Uri]::EscapeDataString($project.Replace('\','_').Replace('/','_').Replace(' ','.')).Replace('%2A','*').Replace('%3F','?').Replace('%','') @@ -669,25 +757,6 @@ function DownloadRelease { } } -function CheckRateLimit { - Param( - [string] $token = '' - ) - - $headers = GetHeader -token $token - $rate = (InvokeWebRequest -Headers $headers -Uri "https://api.github.com/rate_limit").Content | ConvertFrom-Json - $rate | ConvertTo-Json -Depth 99 | Out-Host - $rate = $rate.rate - $percent = [int]($rate.remaining*100/$rate.limit) - Write-Host "$($rate.remaining) API calls remaining out of $($rate.limit) ($percent%)" - if ($percent -lt 10) { - $resetTimeStamp = ([datetime] '1970-01-01Z').AddSeconds($rate.reset) - $waitTime = $resetTimeStamp.Subtract([datetime]::Now) - Write-Host "Less than 10% API calls left, waiting for $($waitTime.TotalSeconds) seconds for limits to reset." - Start-Sleep -seconds ($waitTime.TotalSeconds+1) - } -} - # Get Content of UTF8 encoded file as a string with LF line endings # No empty line at the end of the file function Get-ContentLF { @@ -771,7 +840,7 @@ function CheckBuildJobsInWorkflowRun { [string] $workflowRunId ) - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $per_page = 100 $page = 1 @@ -822,7 +891,7 @@ function FindLatestSuccessfulCICDRun { [string] $token ) - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $lastSuccessfulCICDRun = 0 $per_page = 100 $page = 1 @@ -899,7 +968,7 @@ function GetArtifactsFromWorkflowRun { Write-Host "Getting artifacts for workflow run $workflowRun, mask $mask, projects $projects and version $version" - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $foundArtifacts = @() $per_page = 100 @@ -975,7 +1044,7 @@ function GetArtifacts { ) $refname = $branch.Replace('/','_') - $headers = GetHeader -token $token + $headers = GetHeaders -token $token if ($version -eq 'latest') { $version = '*' } # For latest version, use the artifacts from the last successful CICD run @@ -1082,7 +1151,7 @@ function DownloadArtifact { if ([string]::IsNullOrEmpty($token)) { $token = invoke-gh -silent -returnValue auth token } - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $foldername = Join-Path $path $artifact.Name $filename = "$foldername.zip" InvokeWebRequest -Headers $headers -Uri $artifact.archive_download_url -OutFile $filename @@ -1098,3 +1167,29 @@ function DownloadArtifact { return $filename } } + +function GenerateJwtForTokenRequest { + Param( + [string] $gitHubAppClientId, + [string] $privateKey + ) + + $header = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + alg = "RS256" + typ = "JWT" + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + $payload = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + iat = [System.DateTimeOffset]::UtcNow.AddSeconds(-10).ToUnixTimeSeconds() + exp = [System.DateTimeOffset]::UtcNow.AddMinutes(10).ToUnixTimeSeconds() + iss = $gitHubAppClientId + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + $signature = pwsh -command { + $rsa = [System.Security.Cryptography.RSA]::Create() + $privateKey = "$($args[1])" + $rsa.ImportFromPem($privateKey) + $signature = [Convert]::ToBase64String($rsa.SignData([System.Text.Encoding]::UTF8.GetBytes($args[0]), [System.Security.Cryptography.HashAlgorithmName]::SHA256, [System.Security.Cryptography.RSASignaturePadding]::Pkcs1)).TrimEnd('=').Replace('+', '-').Replace('/', '_') + Write-OutPut $signature + } -args "$header.$payload", $privateKey + return "$header.$payload.$signature" +} diff --git a/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 b/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 index 27775ceee..2f14aa256 100644 --- a/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 +++ b/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 @@ -24,7 +24,7 @@ Set-Location -Path $location # Environment variables for hub commands $env:GITHUB_USER = $actor -$env:GITHUB_TOKEN = $token +$env:GITHUB_TOKEN = GetRealToken -token $token # Commit from the new folder Write-Host "Committing changes from the new folder $Location\$PowerPlatformSolutionName to branch $gitHubBranch" diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index 4f7b48044..f6b0533da 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -4,7 +4,6 @@ Param( ) $script:gitHubSecrets = $_gitHubSecrets | ConvertFrom-Json -$script:escchars = @(' ','!','\"','#','$','%','\u0026','\u0027','(',')','*','+',',','-','.','/','0','1','2','3','4','5','6','7','8','9',':',';','\u003c','=','\u003e','?','@','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','[','\\',']','^','_',[char]96,'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','{','|','}','~') . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) @@ -26,32 +25,6 @@ function GetAzureCredentials { return $null } -function MaskValue { - Param( - [string] $key, - [string] $value - ) - - Write-Host "Masking value for $key" - Write-Host "::add-mask::$value" - - $val2 = "" - $value.ToCharArray() | ForEach-Object { - $chint = [int]$_ - if ($chint -lt 32 -or $chint -gt 126 ) { - $val2 += $_ - } - else { - $val2 += $script:escchars[$chint-32] - } - } - - if ($val2 -ne $value) { - Write-Host "::add-mask::$val2" - } - Write-Host "::add-mask::$([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($value)))" -} - function GetGithubSecret { param ( [string] $secretName, diff --git a/Internal/Collect.ps1 b/Internal/Collect.ps1 deleted file mode 100644 index add9c467e..000000000 --- a/Internal/Collect.ps1 +++ /dev/null @@ -1,187 +0,0 @@ -Param( - [string] $configName = "", - [string] $githubOwner, - [string] $token, - [string] $algoBranch, - [switch] $directCommit -) - -Import-Module (Join-Path $PSScriptRoot "..\Actions\Github-Helper.psm1" -Resolve) -DisableNameChecking - -$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - -Write-Host "::WARNING::The collect mechanism is deprecated. Use the new direct AL-Go development mechanism instead." - -$oldPath = Get-Location -try { - invoke-git config --global user.email "$githubOwner@users.noreply.github.com" - invoke-git config --global user.name "$githubOwner" - invoke-git config --global hub.protocol https - invoke-git config --global core.autocrlf false - $ENV:GITHUB_TOKEN = '' - - Write-Host "Authenticating with GitHub using token" - $token | invoke-gh auth login --with-token - $ENV:GITHUB_TOKEN = $token - - $originalOwnerAndRepo = @{ - "actionsRepo" = "microsoft/AL-Go-Actions" - "perTenantExtensionRepo" = "microsoft/AL-Go-PTE" - "appSourceAppRepo" = "microsoft/AL-Go-AppSource" - } - $originalBranch = "main" - - Set-Location $PSScriptRoot - $baseRepoPath = invoke-git -returnValue rev-parse --show-toplevel - Write-Host "Base repo path: $baseRepoPath" - $user = invoke-gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" user -silent -returnValue | ConvertFrom-Json - Write-Host "GitHub user: $($user.login)" - - if ($configName -eq "") { $configName = $user.login } - if ([System.IO.Path]::GetExtension($configName) -eq "") { $configName += ".json" } - $config = Get-Content $configName -Encoding UTF8 | ConvertFrom-Json - - Write-Host "Using config file: $configName" - $config | ConvertTo-Json | Out-Host - - Set-Location $baseRepoPath - - if ($algoBranch) { - invoke-git checkout $algoBranch - } - else { - $algoBranch = invoke-git -returnValue branch --show-current - Write-Host "Source branch: $algoBranch" - } - $status = invoke-git -returnValue status --porcelain=v1 | Where-Object { ($_) -and ($_.SubString(3) -notlike "Internal/*") } - if ($status) { - throw "Destination repo is not clean, cannot collect changes into dirty repo" - } - - $srcUrl = invoke-git -returnValue config --get remote.origin.url - if ($srcUrl.EndsWith('.git')) { $srcUrl = $srcUrl.Substring(0, $srcUrl.Length - 4) } - $uri = [Uri]::new($srcUrl) - $srcOwnerAndRepo = $uri.LocalPath.Trim('/') - Write-Host "Source Owner+Repo: $srcOwnerAndRepo" - - if (($config.PSObject.Properties.Name -eq "baseFolder") -and ($config.baseFolder)) { - $baseFolder = Join-Path $config.baseFolder $config.localFolder - } - else { - $baseFolder = Join-Path ([Environment]::GetFolderPath("MyDocuments")) $config.localFolder - } - - if (!(Test-Path $baseFolder)) { - New-Item $baseFolder -ItemType Directory | Out-Null - } - Set-Location $baseFolder - - $config.actionsRepo, $config.perTenantExtensionRepo, $config.appSourceAppRepo | ForEach-Object { - if (Test-Path $_) { - Set-Location $_ - $expectedUrl = "https://github.com/$($config.githubOwner)/$_.git" - $actualUrl = invoke-git -returnValue config --get remote.origin.url - if ($expectedUrl -ne $actualUrl) { - throw "unexpected git repo - was $actualUrl, expected $expectedUrl" - } - Set-Location $baseFolder - } - } - - $actionsRepoPath = Join-Path $baseFolder $config.actionsRepo - $appSourceAppRepoPath = Join-Path $baseFolder $config.appSourceAppRepo - $perTenantExtensionRepoPath = Join-Path $baseFolder $config.perTenantExtensionRepo - - Write-Host "This script will collect the changes in $($config.branch) from three repositories:" - Write-Host "- https://github.com/$($config.githubOwner)/$($config.actionsRepo) (folder $actionsRepoPath)" - Write-Host "- https://github.com/$($config.githubOwner)/$($config.perTenantExtensionRepo) (folder $perTenantExtensionRepoPath)" - Write-Host "- https://github.com/$($config.githubOwner)/$($config.appSourceAppRepo) (folder $appSourceAppRepoPath)" - Write-Host - Write-Host "To the $algoBranch branch from $srcOwnerAndRepo (folder $baseRepoPath)" - Write-Host - - $config.actionsRepo, $config.perTenantExtensionRepo, $config.appSourceAppRepo | ForEach-Object { - if (Test-Path $_) { - Set-Location $_ - invoke-git pull - } - else { - $serverUrl = "https://github.com/$($config.githubOwner)/$_.git" - invoke-git clone --quiet $serverUrl - Set-Location $_ - } - invoke-git checkout $config.branch - Set-Location $baseFolder - } - - $repos = @( - @{ "repo" = $config.actionsRepo; "srcPath" = Join-Path $baseRepoPath "Actions"; "dstPath" = $actionsRepoPath; "branch" = $config.branch } - @{ "repo" = $config.perTenantExtensionRepo; "srcPath" = Join-Path $baseRepoPath "Templates\Per Tenant Extension"; "dstPath" = $perTenantExtensionRepoPath; "branch" = $config.branch } - @{ "repo" = $config.appSourceAppRepo; "srcPath" = Join-Path $baseRepoPath "Templates\AppSource App"; "dstPath" = $appSourceAppRepoPath; "branch" = $config.branch } - ) - - $baseRepoBranch = '' - if (!$directCommit) { - $baseRepoBranch = "collect-from-$($config.branch)/$alGoBranch/$((Get-Date).ToUniversalTime().ToString(`"yyMMddHHmmss`"))" # e.g. collect-from-nopr/main/210101120000 - Set-Location $baseRepoPath - invoke-git checkout -b $baseRepoBranch - } - - $repos | ForEach-Object { - Set-Location $baseFolder - $repo = $_.repo - $srcPath = $_.srcPath - $dstPath = $_.dstPath - - Write-Host "Removing $srcPath content" - Get-ChildItem -Path $srcPath -Force | Where-Object { !($_.PSIsContainer -and $_.Name -eq ".git") } | ForEach-Object { - $name = $_.FullName - Write-Host "Remove $name" - if ($_.PSIsContainer) { - Remove-Item $name -Force -Recurse - } - else { - Remove-Item $name -Force - } - } - - Write-Host -ForegroundColor Yellow "Collecting from $repo" - Get-ChildItem -Path $dstPath -Recurse -File -Force | Where-Object { $_.name -notlike '*.copy.md' } | ForEach-Object { - $dstFile = $_.FullName - $srcFile = $srcPath + $dstFile.Substring($dstPath.Length) - $srcFilePath = [System.IO.Path]::GetDirectoryName($srcFile) - if (!(Test-Path $srcFilePath)) { - New-Item $srcFilePath -ItemType Directory | Out-Null - } - Write-Host "$dstFile -> $srcFile" - $lines = ([string](Get-ContentLF -path $dstFile)).Split("`n") - "actionsRepo", "perTenantExtensionRepo", "appSourceAppRepo" | ForEach-Object { - $regex = "^(.*)$($config.githubOwner)\/$($config."$_")(.*)$($config.branch)(.*)$" - $replace = "`${1}$($originalOwnerAndRepo."$_")`${2}$originalBranch`${3}" - $lines = $lines | ForEach-Object { $_ -replace $regex, $replace } - } - if ($_.Name -eq "AL-Go-Helper.ps1") { - $lines = $lines | ForEach-Object { $_ -replace '^(\s*)\$defaultBcContainerHelperVersion(\s*)=(\s*)"(.*)"(.*)$', "`${1}`$defaultBcContainerHelperVersion`${2}=`${3}""""`${5}" } - } - [System.IO.File]::WriteAllText($srcFile, "$($lines -join "`n")`n") - } - } - Set-Location $baseRepoPath - - $serverUrl = "https://$($user.login):$token@github.com/$($srcOwnerAndRepo).git" - - $commitMessage = "Collect changes from $($config.githubOwner)/*@$($config.branch)" - invoke-git add * - invoke-git commit --allow-empty -m "'$commitMessage'" - if ($baseRepoBranch) { - invoke-git push -u $serverUrl $baseRepoBranch - invoke-gh pr create --fill --head $baseRepoBranch --repo $srcOwnerAndRepo --base $ENV:GITHUB_REF_NAME - invoke-git checkout $algoBranch - } - else { - invoke-git push $serverUrl - } -} -finally { - set-location $oldPath -} diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 62eba2435..c7547a5b9 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -37,7 +37,6 @@ function PushChanges } $branchName = "deploy/$BaseBranch/$((Get-Date).ToUniversalTime().ToString(`"yyMMddHHmmss`"))" - invoke-git checkout -b $branchName origin/$BaseBranch invoke-git commit --allow-empty -m $CommitMessage invoke-git push origin $branchName @@ -45,6 +44,7 @@ function PushChanges } } +$token = GetRealToken -token $token -repository "$($config.githubOwner)/.github" $oldPath = Get-Location try { @@ -73,8 +73,8 @@ try { Set-Location $baseRepoPath # Whoami - $user = invoke-gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" user -silent -returnValue | ConvertFrom-Json - Write-Host "GitHub user: $($user.login)" + #$user = invoke-gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" user -silent -returnValue | ConvertFrom-Json + #Write-Host "GitHub user: $($user.login)" # Dump configuration Write-Host "Configuration:" @@ -162,7 +162,7 @@ try { Write-Host -ForegroundColor Yellow "Deploying to $repo" try { - $serverUrl = "https://$($user.login):$token@github.com/$($config.githubOwner)/$repo.git" + $serverUrl = "https://$($config.githubOwner):$token@github.com/$($config.githubOwner)/$repo.git" if (Test-Path $repo) { Remove-Item $repo -Recurse -Force } diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 3b78df681..f09ca0f7b 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -20,9 +20,11 @@ function SetTokenAndRepository { [switch] $github ) + MaskValue -key "token" -value $token $script:githubOwner = $githubOwner $script:token = $token $script:defaultRepository = $repository + $realToken = GetRealToken -token $script:token -repository "$githubOwner/.github" if ($github) { invoke-git config --global user.email "$githubOwner@users.noreply.github.com" @@ -32,9 +34,9 @@ function SetTokenAndRepository { $ENV:GITHUB_TOKEN = '' } Write-Host "Authenticating with GitHub using token" - $token | invoke-gh auth login --with-token + $realToken | invoke-gh auth login --with-token if ($github) { - $ENV:GITHUB_TOKEN = $token + $ENV:GITHUB_TOKEN = $realToken } } @@ -63,7 +65,7 @@ function Add-PropertiesToJsonFile { ) if ($wait -and $commit) { - $headers = GetHeader -token $token + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" Write-Host "Get Previous runs" $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id @@ -123,7 +125,7 @@ function Remove-PropertiesFromJsonFile { } function DisplayTokenAndRepository { - Write-Host "Token: $token" + Write-Host "Token: $($script:token)" Write-Host "Repo: $defaultRepository" } @@ -145,7 +147,7 @@ function RunWorkflow { Write-Host ($parameters | ConvertTo-Json) } - $headers = GetHeader -token $token + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" WaitForRateLimit -headers $headers -displayStatus Write-Host "Get Workflows" @@ -210,7 +212,7 @@ function DownloadWorkflowLog { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeader -token $token + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" $url = "https://api.github.com/repos/$repository/actions/runs/$runid" $run = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url).Content | ConvertFrom-Json) $log = InvokeWebRequest -Method Get -Headers $headers -Uri $run.logs_url @@ -266,9 +268,13 @@ function WaitWorkflow { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeader -token $token + $count = 0 $status = "" do { + if ($count % 45 -eq 0) { + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" + $count++ + } if ($delay) { Start-Sleep -Seconds 60 } @@ -301,7 +307,8 @@ function SetRepositorySecret { $repository = $defaultRepository } Write-Host -ForegroundColor Yellow "`nSet Secret $name in $repository" - invoke-gh secret set $name -b $value --repo $repository + $value = $value.Replace("`r", '').Replace("`n", '') + gh secret set $name -b $value --repo $repository } function CreateNewAppInFolder { @@ -485,8 +492,9 @@ function CreateAlGoRepository { invoke-git add * invoke-git commit --allow-empty -m 'init' invoke-git branch -M $branch - if ($githubOwner -and $token) { - invoke-git remote set-url origin "https://$($githubOwner):$token@github.com/$repository.git" + if ($githubOwner -and $script:token) { + $realToken = GetRealToken -token $script:token -repository "$githubOwner/.github" + invoke-git remote set-url origin "https://$($githubOwner):$($realtoken)@github.com/$repository.git" } invoke-git push --set-upstream origin $branch if (!$github) { @@ -525,7 +533,7 @@ function MergePRandPull { } Write-Host "Get Previous runs" - $headers = GetHeader -token $token + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id if ($previousrunids) { diff --git a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 index 0a0d2a627..279ea1abb 100644 --- a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 +++ b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 @@ -58,11 +58,8 @@ $template = "https://github.com/$appSourceTemplate" $repository = 'microsoft/bcsamples-bingmaps.appsource' SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository -$headers = @{ - "Authorization" = "token $token" - "X-GitHub-Api-Version" = "2022-11-28" - "Accept" = "application/vnd.github+json" -} +$headers = GetHeaders $token -repository "$githubOwner/.github" + $existingBranch = gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/$repository/branches/$branch 2> $null | ConvertFrom-Json if ($existingBranch.PSObject.Properties.Name -eq 'Name' -and $existingBranch.Name -eq $branch) { Write-Host "Removing existing branch $branch" diff --git a/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 b/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 index 8aa200e14..3c86b7332 100644 --- a/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 +++ b/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 @@ -81,6 +81,7 @@ WaitWorkflow -repository $repository -runid $run.id Test-ArtifactsFromRun -runid $run.id -folder 'artifacts' -expectedArtifacts @{"Apps"=1;"TestApps"=0;"Dependencies"=0;"github-pages"=1} -repoVersion '1.0' -appVersion '1.0' # Set GitHub Pages in repository to GitHub Actions +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository gh api --method POST /repos/$repository/pages -f build_type=workflow | Out-Null # Add setting to deploy to GitHub Pages @@ -92,6 +93,7 @@ CommitAndPush -commitMessage 'DeployToGitHubPages' RunDeployReferenceDocumentation -repository $repository -wait | Out-Null # Get Pages URL and read the content +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository $pagesInfo = gh api /repos/$repository/pages | ConvertFrom-Json $html = (Invoke-WebRequest -Uri $pagesInfo.html_url -UseBasicParsing).Content $html | Should -belike "*Documentation for $repository*" @@ -108,6 +110,7 @@ CommitAndPush -commitMessage 'Continuous Deployment of ALDoc' WaitAllWorkflows -repository $repository -noError # Get Pages URL and read the content +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository $pagesInfo = gh api /repos/$repository/pages | ConvertFrom-Json $html = (Invoke-WebRequest -Uri $pagesInfo.html_url -UseBasicParsing).Content $html | Should -belike "*Documentazione per $repository*"