From a091a352b698de490fa79755aa0eb3bf2496240c Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 17 Nov 2024 06:37:11 +0100 Subject: [PATCH 01/34] Support GitHub App Auth --- Actions/AL-Go-Helper.ps1 | 2 +- Actions/ReadSecrets/ReadSecrets.ps1 | 18 +++++++++++++++ Actions/ReadSecrets/ReadSecretsHelper.psm1 | 26 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index c146e84fd..acea5e37c 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -19,7 +19,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","GitHubAppClientId") $runAlPipelineOverrides = @( "DockerPull" diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index 96401257a..c95ceb365 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -107,6 +107,24 @@ try { MaskValue -key "$($secretName).$($keyName)" -value "$($json."$keyName")" } } + if ($json.ContainsKey('GitHubAppClientId') -and $json.ContainsKey('PrivateKey')) { + # Authenticating as a GitHub App + try { + $jwt = GenerateJwtForTokenRequest -gitHubAppClientId $json.GitHubAppClientId -privateKey $json.PrivateKey + $headers = @{ + "Accept" = "application/vnd.github+json" + "Authorization" = "Bearer $jwt" + "X-GitHub-Api-Version" = "2022-11-28" + } + $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$ENV:GITHUB_API_URL/repos/$ENV:GITHUB_REPOSITORY/installation" + $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url + $secretValue = $tokenResponse.token + MaskValue -key "$secretName PAT" -value $secretValue + } + else { + throw "Unable to authenticate as GitHub App with Client ID $($json.GitHubAppClientId). Error was $($_.Exception.Message)" + } + } if ($json.ContainsKey('clientID') -and !($json.ContainsKey('clientSecret') -or $json.ContainsKey('refreshToken'))) { try { Write-Host "Query federated token" diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index 4f7b48044..ed3435773 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -207,3 +207,29 @@ function GetSecret { } return $value } + +function GenerateJwtForTokenRequest { + Param( + [string] $gitHubAppClientId, + [string] $privateKey + ) + + $header = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + alg = "RS256" + typ = "JWT" + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + $payload = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + iat = [System.DateTimeOffset]::UtcNow.AddSeconds(-10).ToUnixTimeSeconds() + exp = [System.DateTimeOffset]::UtcNow.AddMinutes($expirationMinutes).ToUnixTimeSeconds() + iss = $gitHubAppClientId + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + $signature = pwsh -command { + $rsa = [System.Security.Cryptography.RSA]::Create() + $privateKey = "$($args[1])" + $rsa.ImportFromPem($privateKey) + $signature = [Convert]::ToBase64String($rsa.SignData([System.Text.Encoding]::UTF8.GetBytes($args[0]), [System.Security.Cryptography.HashAlgorithmName]::SHA256, [System.Security.Cryptography.RSASignaturePadding]::Pkcs1)).TrimEnd('=').Replace('+', '-').Replace('/', '_') + Write-OutPut $signature + } -args "$header.$payload", $privateKey + return "$header.$payload.$signature" +} \ No newline at end of file From 9463f40111784fa5bc7471491ca2675b87be632c Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 17 Nov 2024 06:41:53 +0100 Subject: [PATCH 02/34] catch --- Actions/ReadSecrets/ReadSecrets.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index c95ceb365..81d4c8a53 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -121,7 +121,7 @@ try { $secretValue = $tokenResponse.token MaskValue -key "$secretName PAT" -value $secretValue } - else { + catch { throw "Unable to authenticate as GitHub App with Client ID $($json.GitHubAppClientId). Error was $($_.Exception.Message)" } } From dd0cbea3d085fb8fef56a7f6a0c1dadd41be2beb Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 17 Nov 2024 06:46:17 +0100 Subject: [PATCH 03/34] 10 --- Actions/ReadSecrets/ReadSecretsHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index ed3435773..42989ecc8 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -221,7 +221,7 @@ function GenerateJwtForTokenRequest { $payload = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ iat = [System.DateTimeOffset]::UtcNow.AddSeconds(-10).ToUnixTimeSeconds() - exp = [System.DateTimeOffset]::UtcNow.AddMinutes($expirationMinutes).ToUnixTimeSeconds() + exp = [System.DateTimeOffset]::UtcNow.AddMinutes(10).ToUnixTimeSeconds() iss = $gitHubAppClientId }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); $signature = pwsh -command { From 7c7253de79d0d3202f2e5da3f6a0fba768b3c519 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 17 Nov 2024 06:50:15 +0100 Subject: [PATCH 04/34] dumps --- Actions/ReadSecrets/ReadSecrets.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index 81d4c8a53..e3e86759d 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -102,23 +102,31 @@ try { } if ($json.Keys.Count) { foreach($keyName in $json.Keys) { + Write-Host $keyName if ((IsPropertySecret -propertyName $keyName) -and ($json."$keyName" -isnot [boolean])) { # Mask individual values if property is secret MaskValue -key "$($secretName).$($keyName)" -value "$($json."$keyName")" } } + Write-Host "go" if ($json.ContainsKey('GitHubAppClientId') -and $json.ContainsKey('PrivateKey')) { # Authenticating as a GitHub App try { + Write-Host "1" $jwt = GenerateJwtForTokenRequest -gitHubAppClientId $json.GitHubAppClientId -privateKey $json.PrivateKey + Write-Host "2" $headers = @{ "Accept" = "application/vnd.github+json" "Authorization" = "Bearer $jwt" "X-GitHub-Api-Version" = "2022-11-28" } + Write-Host "3" $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$ENV:GITHUB_API_URL/repos/$ENV:GITHUB_REPOSITORY/installation" + Write-Host "4" $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url + Write-Host "5" $secretValue = $tokenResponse.token + Write-Host "6" MaskValue -key "$secretName PAT" -value $secretValue } catch { From 32aab8d428ec00cf3c8f44a0755443b4e13e422a Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 17 Nov 2024 06:55:54 +0100 Subject: [PATCH 05/34] mask multi --- Actions/ReadSecrets/ReadSecrets.ps1 | 7 ------- Actions/ReadSecrets/ReadSecretsHelper.psm1 | 8 ++++++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index e3e86759d..9ef1ebad5 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -108,25 +108,18 @@ try { MaskValue -key "$($secretName).$($keyName)" -value "$($json."$keyName")" } } - Write-Host "go" if ($json.ContainsKey('GitHubAppClientId') -and $json.ContainsKey('PrivateKey')) { # Authenticating as a GitHub App try { - Write-Host "1" $jwt = GenerateJwtForTokenRequest -gitHubAppClientId $json.GitHubAppClientId -privateKey $json.PrivateKey - Write-Host "2" $headers = @{ "Accept" = "application/vnd.github+json" "Authorization" = "Bearer $jwt" "X-GitHub-Api-Version" = "2022-11-28" } - Write-Host "3" $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$ENV:GITHUB_API_URL/repos/$ENV:GITHUB_REPOSITORY/installation" - Write-Host "4" $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url - Write-Host "5" $secretValue = $tokenResponse.token - Write-Host "6" MaskValue -key "$secretName PAT" -value $secretValue } catch { diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index 42989ecc8..1b7b35d2a 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -33,7 +33,9 @@ function MaskValue { ) Write-Host "Masking value for $key" - Write-Host "::add-mask::$value" + $value.Split("`n") | ForEach-Object { + Write-Host "::add-mask::$_" + } $val2 = "" $value.ToCharArray() | ForEach-Object { @@ -47,7 +49,9 @@ function MaskValue { } if ($val2 -ne $value) { - Write-Host "::add-mask::$val2" + $val2.Split("`n") | ForEach-Object { + Write-Host "::add-mask::$_" + } } Write-Host "::add-mask::$([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($value)))" } From 0c191ab1be9ce20c537120b562112f0e653d9288 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 17 Nov 2024 06:59:45 +0100 Subject: [PATCH 06/34] add message --- Actions/ReadSecrets/ReadSecrets.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index 9ef1ebad5..106e9cf9a 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -102,7 +102,6 @@ try { } if ($json.Keys.Count) { foreach($keyName in $json.Keys) { - Write-Host $keyName if ((IsPropertySecret -propertyName $keyName) -and ($json."$keyName" -isnot [boolean])) { # Mask individual values if property is secret MaskValue -key "$($secretName).$($keyName)" -value "$($json."$keyName")" @@ -111,6 +110,7 @@ try { if ($json.ContainsKey('GitHubAppClientId') -and $json.ContainsKey('PrivateKey')) { # Authenticating as a GitHub App try { + Write-Host "Using GitHub App with ClientId $($json.GitHubAppClientId) for authentication" $jwt = GenerateJwtForTokenRequest -gitHubAppClientId $json.GitHubAppClientId -privateKey $json.PrivateKey $headers = @{ "Accept" = "application/vnd.github+json" From df1c6e6be28559a76075409bdbfe14169eec0e4e Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 17 Nov 2024 07:10:46 +0100 Subject: [PATCH 07/34] precommit --- Actions/ReadSecrets/ReadSecretsHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index 1b7b35d2a..8c6474b4b 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -236,4 +236,4 @@ function GenerateJwtForTokenRequest { Write-OutPut $signature } -args "$header.$payload", $privateKey return "$header.$payload.$signature" -} \ No newline at end of file +} From ff16e55f7021602dfa0f870cd09422b908c6cb2c Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 06:48:49 +0100 Subject: [PATCH 08/34] token exchange --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 5 +- .../DetermineDeploymentEnvironments.ps1 | 6 +- .../DetermineProjectsToBuild.psm1 | 5 +- Actions/Github-Helper.psm1 | 82 ++++++++++++++++--- Actions/ReadSecrets/ReadSecrets.ps1 | 19 ----- Actions/ReadSecrets/ReadSecretsHelper.psm1 | 26 ------ e2eTests/e2eTestHelper.psm1 | 10 +-- .../FederatedCredentials/runtest.ps1 | 7 +- 8 files changed, 82 insertions(+), 78 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 3a9454008..0421986e3 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -32,10 +32,7 @@ if ($update -eq 'Y') { } # Use Authenticated API request to avoid the 60 API calls per hour limit -$headers = @{ - "Accept" = "application/vnd.github.baptiste-preview+json" - "Authorization" = "Bearer $token" -} +$headers = GetHeaders -token $token if (-not $templateUrl.Contains('@')) { $templateUrl += "@main" diff --git a/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 b/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 index 6d7870a68..f699120cf 100644 --- a/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 +++ b/Actions/DetermineDeploymentEnvironments/DetermineDeploymentEnvironments.ps1 @@ -7,7 +7,7 @@ ) function IsGitHubPagesAvailable() { - $headers = GetHeader -token $env:GITHUB_TOKEN + $headers = GetHeaders -token $env:GITHUB_TOKEN $url = "$($ENV:GITHUB_API_URL)/repos/$($ENV:GITHUB_REPOSITORY)/pages" try { Write-Host "Requesting GitHub Pages settings from GitHub" @@ -20,7 +20,7 @@ function IsGitHubPagesAvailable() { } function GetGitHubEnvironments() { - $headers = GetHeader -token $env:GITHUB_TOKEN + $headers = GetHeaders -token $env:GITHUB_TOKEN $url = "$($ENV:GITHUB_API_URL)/repos/$($ENV:GITHUB_REPOSITORY)/environments" try { Write-Host "Requesting environments from GitHub" @@ -36,7 +36,7 @@ function GetGitHubEnvironments() { function Get-BranchesFromPolicy($ghEnvironment) { if ($ghEnvironment) { # Environment is defined in GitHub - check protection rules - $headers = GetHeader -token $env:GITHUB_TOKEN + $headers = GetHeaders -token $env:GITHUB_TOKEN $branchPolicy = ($ghEnvironment.protection_rules | Where-Object { $_.type -eq "branch_policy" }) if ($branchPolicy) { if ($ghEnvironment.deployment_branch_policy.protected_branches) { diff --git a/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 b/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 index 42f12ecb8..08be4fc6e 100644 --- a/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 +++ b/Actions/DetermineProjectsToBuild/DetermineProjectsToBuild.psm1 @@ -25,10 +25,7 @@ function Get-ModifiedFiles { $url = "$($env:GITHUB_API_URL)/repos/$($env:GITHUB_REPOSITORY)/compare/$($ghEvent.pull_request.base.sha)...$($ghEvent.pull_request.head.sha)" - $headers = @{ - "Authorization" = "token $token" - "Accept" = "application/vnd.github.baptiste-preview+json" - } + $headers = GetHeaders -token $token $response = (InvokeWebRequest -Headers $headers -Uri $url).Content | ConvertFrom-Json diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index c4439d7f4..758e76931 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -501,7 +501,7 @@ function GetReleases { ) Write-Host "Analyzing releases $api_url/repos/$repository/releases" - $releases = (InvokeWebRequest -Headers (GetHeader -token $token) -Uri "$api_url/repos/$repository/releases").Content | ConvertFrom-Json + $releases = (InvokeWebRequest -Headers (GetHeaders -token $token) -Uri "$api_url/repos/$repository/releases").Content | ConvertFrom-Json if ($releases.Count -gt 1) { # Sort by SemVer tag try { @@ -556,7 +556,39 @@ function GetLatestRelease { $latestRelease } -function GetHeader { +function GetRealToken { + Param( + [string] $token + ) + + if (!($token.StartsWith("{"))) { + # not a json token + return $token + } + else { + try { + $json = $token | ConvertFrom-Json + $gitHubAppClientId = $json.GitHubAppClientId + $privateKey = $json.PrivateKey + Write-Host "Using GitHub App with ClientId $gitHubAppClientId for authentication" + $jwt = GenerateJwtForTokenRequest -gitHubAppClientId $gitHubAppClientId -privateKey $privateKey + $headers = @{ + "Accept" = "application/vnd.github+json" + "Authorization" = "Bearer $jwt" + "X-GitHub-Api-Version" = "2022-11-28" + } + $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$ENV:GITHUB_API_URL/repos/$ENV:GITHUB_REPOSITORY/installation" + $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url + return $tokenResponse.token + } + catch { + # Not a json token + return $token + } + } +} + +function GetHeaders { param ( [string] $token, [string] $accept = "application/vnd.github+json", @@ -567,9 +599,9 @@ function GetHeader { "X-GitHub-Api-Version" = $apiVersion } if (![string]::IsNullOrEmpty($token)) { - $headers["Authorization"] = "token $token" + $realToken = GetRealToken -token $token + $headers["Authorization"] = "token $realToken" } - return $headers } @@ -616,7 +648,7 @@ function GetReleaseNotes { $postParams["target_commitish"] = $target_commitish } - InvokeWebRequest -Headers (GetHeader -token $token) -Method POST -Body ($postParams | ConvertTo-Json) -Uri "$api_url/repos/$repository/releases/generate-notes" + InvokeWebRequest -Headers (GetHeaders -token $token) -Method POST -Body ($postParams | ConvertTo-Json) -Uri "$api_url/repos/$repository/releases/generate-notes" } function DownloadRelease { @@ -636,7 +668,7 @@ function DownloadRelease { if ([string]::IsNullOrEmpty($token)) { $token = invoke-gh -silent -returnValue auth token } - $headers = GetHeader -token $token -accept "application/octet-stream" + $headers = GetHeaders -token $token -accept "application/octet-stream" foreach($project in $projects.Split(',')) { # GitHub replaces series of special characters with a single dot when uploading release assets $project = [Uri]::EscapeDataString($project.Replace('\','_').Replace('/','_').Replace(' ','.')).Replace('%2A','*').Replace('%3F','?').Replace('%','') @@ -674,7 +706,7 @@ function CheckRateLimit { [string] $token = '' ) - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $rate = (InvokeWebRequest -Headers $headers -Uri "https://api.github.com/rate_limit").Content | ConvertFrom-Json $rate | ConvertTo-Json -Depth 99 | Out-Host $rate = $rate.rate @@ -771,7 +803,7 @@ function CheckBuildJobsInWorkflowRun { [string] $workflowRunId ) - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $per_page = 100 $page = 1 @@ -822,7 +854,7 @@ function FindLatestSuccessfulCICDRun { [string] $token ) - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $lastSuccessfulCICDRun = 0 $per_page = 100 $page = 1 @@ -899,7 +931,7 @@ function GetArtifactsFromWorkflowRun { Write-Host "Getting artifacts for workflow run $workflowRun, mask $mask, projects $projects and version $version" - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $foundArtifacts = @() $per_page = 100 @@ -975,7 +1007,7 @@ function GetArtifacts { ) $refname = $branch.Replace('/','_') - $headers = GetHeader -token $token + $headers = GetHeaders -token $token if ($version -eq 'latest') { $version = '*' } # For latest version, use the artifacts from the last successful CICD run @@ -1082,7 +1114,7 @@ function DownloadArtifact { if ([string]::IsNullOrEmpty($token)) { $token = invoke-gh -silent -returnValue auth token } - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $foldername = Join-Path $path $artifact.Name $filename = "$foldername.zip" InvokeWebRequest -Headers $headers -Uri $artifact.archive_download_url -OutFile $filename @@ -1098,3 +1130,29 @@ function DownloadArtifact { return $filename } } + +function GenerateJwtForTokenRequest { + Param( + [string] $gitHubAppClientId, + [string] $privateKey + ) + + $header = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + alg = "RS256" + typ = "JWT" + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + $payload = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + iat = [System.DateTimeOffset]::UtcNow.AddSeconds(-10).ToUnixTimeSeconds() + exp = [System.DateTimeOffset]::UtcNow.AddMinutes(10).ToUnixTimeSeconds() + iss = $gitHubAppClientId + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + $signature = pwsh -command { + $rsa = [System.Security.Cryptography.RSA]::Create() + $privateKey = "$($args[1])" + $rsa.ImportFromPem($privateKey) + $signature = [Convert]::ToBase64String($rsa.SignData([System.Text.Encoding]::UTF8.GetBytes($args[0]), [System.Security.Cryptography.HashAlgorithmName]::SHA256, [System.Security.Cryptography.RSASignaturePadding]::Pkcs1)).TrimEnd('=').Replace('+', '-').Replace('/', '_') + Write-OutPut $signature + } -args "$header.$payload", $privateKey + return "$header.$payload.$signature" +} diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index 106e9cf9a..96401257a 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -107,25 +107,6 @@ try { MaskValue -key "$($secretName).$($keyName)" -value "$($json."$keyName")" } } - if ($json.ContainsKey('GitHubAppClientId') -and $json.ContainsKey('PrivateKey')) { - # Authenticating as a GitHub App - try { - Write-Host "Using GitHub App with ClientId $($json.GitHubAppClientId) for authentication" - $jwt = GenerateJwtForTokenRequest -gitHubAppClientId $json.GitHubAppClientId -privateKey $json.PrivateKey - $headers = @{ - "Accept" = "application/vnd.github+json" - "Authorization" = "Bearer $jwt" - "X-GitHub-Api-Version" = "2022-11-28" - } - $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$ENV:GITHUB_API_URL/repos/$ENV:GITHUB_REPOSITORY/installation" - $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url - $secretValue = $tokenResponse.token - MaskValue -key "$secretName PAT" -value $secretValue - } - catch { - throw "Unable to authenticate as GitHub App with Client ID $($json.GitHubAppClientId). Error was $($_.Exception.Message)" - } - } if ($json.ContainsKey('clientID') -and !($json.ContainsKey('clientSecret') -or $json.ContainsKey('refreshToken'))) { try { Write-Host "Query federated token" diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index 8c6474b4b..978a11b77 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -211,29 +211,3 @@ function GetSecret { } return $value } - -function GenerateJwtForTokenRequest { - Param( - [string] $gitHubAppClientId, - [string] $privateKey - ) - - $header = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ - alg = "RS256" - typ = "JWT" - }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); - - $payload = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ - iat = [System.DateTimeOffset]::UtcNow.AddSeconds(-10).ToUnixTimeSeconds() - exp = [System.DateTimeOffset]::UtcNow.AddMinutes(10).ToUnixTimeSeconds() - iss = $gitHubAppClientId - }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); - $signature = pwsh -command { - $rsa = [System.Security.Cryptography.RSA]::Create() - $privateKey = "$($args[1])" - $rsa.ImportFromPem($privateKey) - $signature = [Convert]::ToBase64String($rsa.SignData([System.Text.Encoding]::UTF8.GetBytes($args[0]), [System.Security.Cryptography.HashAlgorithmName]::SHA256, [System.Security.Cryptography.RSASignaturePadding]::Pkcs1)).TrimEnd('=').Replace('+', '-').Replace('/', '_') - Write-OutPut $signature - } -args "$header.$payload", $privateKey - return "$header.$payload.$signature" -} diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 3b78df681..c4ece1684 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -63,7 +63,7 @@ function Add-PropertiesToJsonFile { ) if ($wait -and $commit) { - $headers = GetHeader -token $token + $headers = GetHeaders -token $token Write-Host "Get Previous runs" $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id @@ -145,7 +145,7 @@ function RunWorkflow { Write-Host ($parameters | ConvertTo-Json) } - $headers = GetHeader -token $token + $headers = GetHeaders -token $token WaitForRateLimit -headers $headers -displayStatus Write-Host "Get Workflows" @@ -210,7 +210,7 @@ function DownloadWorkflowLog { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $url = "https://api.github.com/repos/$repository/actions/runs/$runid" $run = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url).Content | ConvertFrom-Json) $log = InvokeWebRequest -Method Get -Headers $headers -Uri $run.logs_url @@ -266,7 +266,7 @@ function WaitWorkflow { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $status = "" do { if ($delay) { @@ -525,7 +525,7 @@ function MergePRandPull { } Write-Host "Get Previous runs" - $headers = GetHeader -token $token + $headers = GetHeaders -token $token $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id if ($previousrunids) { diff --git a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 index 0a0d2a627..bdd5c3a03 100644 --- a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 +++ b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 @@ -58,11 +58,8 @@ $template = "https://github.com/$appSourceTemplate" $repository = 'microsoft/bcsamples-bingmaps.appsource' SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository -$headers = @{ - "Authorization" = "token $token" - "X-GitHub-Api-Version" = "2022-11-28" - "Accept" = "application/vnd.github+json" -} +$headers = GetHeaders $token + $existingBranch = gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/$repository/branches/$branch 2> $null | ConvertFrom-Json if ($existingBranch.PSObject.Properties.Name -eq 'Name' -and $existingBranch.Name -eq $branch) { Write-Host "Removing existing branch $branch" From add7df981c1a7062d8ca97d43c4e43eace392f40 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 06:58:33 +0100 Subject: [PATCH 09/34] getreal --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 0421986e3..e47f611f9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -27,7 +27,7 @@ if ($update -eq 'Y') { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } else { - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $token = GetRealToken -token ([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token))) } } From 38b7860e8fc5d0d9564eda199144cb7e88055ba7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 07:03:48 +0100 Subject: [PATCH 10/34] tests --- e2eTests/e2eTestHelper.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index c4ece1684..2ac7f9682 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -21,7 +21,7 @@ function SetTokenAndRepository { ) $script:githubOwner = $githubOwner - $script:token = $token + $script:token = GetRealToken -token $token $script:defaultRepository = $repository if ($github) { @@ -32,9 +32,9 @@ function SetTokenAndRepository { $ENV:GITHUB_TOKEN = '' } Write-Host "Authenticating with GitHub using token" - $token | invoke-gh auth login --with-token + $script:token | invoke-gh auth login --with-token if ($github) { - $ENV:GITHUB_TOKEN = $token + $ENV:GITHUB_TOKEN = $script:token } } From af60cca99ac93b1bc2f9945b2cf21351ddb5c587 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 07:09:38 +0100 Subject: [PATCH 11/34] getreal --- Actions/AL-Go-Helper.ps1 | 2 +- Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index acea5e37c..fa3a33377 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -1320,7 +1320,7 @@ function CloneIntoNewFolder { # Environment variables for hub commands $env:GITHUB_USER = $actor - $env:GITHUB_TOKEN = $token + $env:GITHUB_TOKEN = GetRealToken -token $token # Configure git invoke-git config --global user.email "$actor@users.noreply.github.com" diff --git a/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 b/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 index 27775ceee..2f14aa256 100644 --- a/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 +++ b/Actions/PullPowerPlatformChanges/GitCommitChanges.ps1 @@ -24,7 +24,7 @@ Set-Location -Path $location # Environment variables for hub commands $env:GITHUB_USER = $actor -$env:GITHUB_TOKEN = $token +$env:GITHUB_TOKEN = GetRealToken -token $token # Commit from the new folder Write-Host "Committing changes from the new folder $Location\$PowerPlatformSolutionName to branch $gitHubBranch" From 6f09716477efae02240c024b8c1d1c8f3f38dfb4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 07:15:15 +0100 Subject: [PATCH 12/34] x --- Actions/Github-Helper.psm1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 758e76931..351a1e3da 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -577,8 +577,11 @@ function GetRealToken { "Authorization" = "Bearer $jwt" "X-GitHub-Api-Version" = "2022-11-28" } + Write-Host "Get App Info" $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$ENV:GITHUB_API_URL/repos/$ENV:GITHUB_REPOSITORY/installation" + Write-Host "Get Token Response" $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url + Write-Host "return token" return $tokenResponse.token } catch { From e3e49684166aff31305b4050df5dc4d01b76a044 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 07:24:00 +0100 Subject: [PATCH 13/34] use real --- Actions/AL-Go-Helper.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index fa3a33377..4707f55bd 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -1315,13 +1315,14 @@ function CloneIntoNewFolder { $baseFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) New-Item $baseFolder -ItemType Directory | Out-Null Set-Location $baseFolder - $serverUri = [Uri]::new($env:GITHUB_SERVER_URL) - $serverUrl = "$($serverUri.Scheme)://$($actor):$($token)@$($serverUri.Host)/$($env:GITHUB_REPOSITORY)" # Environment variables for hub commands $env:GITHUB_USER = $actor $env:GITHUB_TOKEN = GetRealToken -token $token + $serverUri = [Uri]::new($env:GITHUB_SERVER_URL) + $serverUrl = "$($serverUri.Scheme)://$($env:GITHUB_USER):$($env:GITHUB_TOKEN)@$($serverUri.Host)/$($env:GITHUB_REPOSITORY)" + # Configure git invoke-git config --global user.email "$actor@users.noreply.github.com" invoke-git config --global user.name "$actor" From 0f9ae18a3ca1c35d15d08b4887054a96c24b0e14 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 09:08:54 +0100 Subject: [PATCH 14/34] use real --- Internal/Collect.ps1 | 187 ------------------------------------------- Internal/Deploy.ps1 | 2 +- 2 files changed, 1 insertion(+), 188 deletions(-) delete mode 100644 Internal/Collect.ps1 diff --git a/Internal/Collect.ps1 b/Internal/Collect.ps1 deleted file mode 100644 index add9c467e..000000000 --- a/Internal/Collect.ps1 +++ /dev/null @@ -1,187 +0,0 @@ -Param( - [string] $configName = "", - [string] $githubOwner, - [string] $token, - [string] $algoBranch, - [switch] $directCommit -) - -Import-Module (Join-Path $PSScriptRoot "..\Actions\Github-Helper.psm1" -Resolve) -DisableNameChecking - -$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - -Write-Host "::WARNING::The collect mechanism is deprecated. Use the new direct AL-Go development mechanism instead." - -$oldPath = Get-Location -try { - invoke-git config --global user.email "$githubOwner@users.noreply.github.com" - invoke-git config --global user.name "$githubOwner" - invoke-git config --global hub.protocol https - invoke-git config --global core.autocrlf false - $ENV:GITHUB_TOKEN = '' - - Write-Host "Authenticating with GitHub using token" - $token | invoke-gh auth login --with-token - $ENV:GITHUB_TOKEN = $token - - $originalOwnerAndRepo = @{ - "actionsRepo" = "microsoft/AL-Go-Actions" - "perTenantExtensionRepo" = "microsoft/AL-Go-PTE" - "appSourceAppRepo" = "microsoft/AL-Go-AppSource" - } - $originalBranch = "main" - - Set-Location $PSScriptRoot - $baseRepoPath = invoke-git -returnValue rev-parse --show-toplevel - Write-Host "Base repo path: $baseRepoPath" - $user = invoke-gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" user -silent -returnValue | ConvertFrom-Json - Write-Host "GitHub user: $($user.login)" - - if ($configName -eq "") { $configName = $user.login } - if ([System.IO.Path]::GetExtension($configName) -eq "") { $configName += ".json" } - $config = Get-Content $configName -Encoding UTF8 | ConvertFrom-Json - - Write-Host "Using config file: $configName" - $config | ConvertTo-Json | Out-Host - - Set-Location $baseRepoPath - - if ($algoBranch) { - invoke-git checkout $algoBranch - } - else { - $algoBranch = invoke-git -returnValue branch --show-current - Write-Host "Source branch: $algoBranch" - } - $status = invoke-git -returnValue status --porcelain=v1 | Where-Object { ($_) -and ($_.SubString(3) -notlike "Internal/*") } - if ($status) { - throw "Destination repo is not clean, cannot collect changes into dirty repo" - } - - $srcUrl = invoke-git -returnValue config --get remote.origin.url - if ($srcUrl.EndsWith('.git')) { $srcUrl = $srcUrl.Substring(0, $srcUrl.Length - 4) } - $uri = [Uri]::new($srcUrl) - $srcOwnerAndRepo = $uri.LocalPath.Trim('/') - Write-Host "Source Owner+Repo: $srcOwnerAndRepo" - - if (($config.PSObject.Properties.Name -eq "baseFolder") -and ($config.baseFolder)) { - $baseFolder = Join-Path $config.baseFolder $config.localFolder - } - else { - $baseFolder = Join-Path ([Environment]::GetFolderPath("MyDocuments")) $config.localFolder - } - - if (!(Test-Path $baseFolder)) { - New-Item $baseFolder -ItemType Directory | Out-Null - } - Set-Location $baseFolder - - $config.actionsRepo, $config.perTenantExtensionRepo, $config.appSourceAppRepo | ForEach-Object { - if (Test-Path $_) { - Set-Location $_ - $expectedUrl = "https://github.com/$($config.githubOwner)/$_.git" - $actualUrl = invoke-git -returnValue config --get remote.origin.url - if ($expectedUrl -ne $actualUrl) { - throw "unexpected git repo - was $actualUrl, expected $expectedUrl" - } - Set-Location $baseFolder - } - } - - $actionsRepoPath = Join-Path $baseFolder $config.actionsRepo - $appSourceAppRepoPath = Join-Path $baseFolder $config.appSourceAppRepo - $perTenantExtensionRepoPath = Join-Path $baseFolder $config.perTenantExtensionRepo - - Write-Host "This script will collect the changes in $($config.branch) from three repositories:" - Write-Host "- https://github.com/$($config.githubOwner)/$($config.actionsRepo) (folder $actionsRepoPath)" - Write-Host "- https://github.com/$($config.githubOwner)/$($config.perTenantExtensionRepo) (folder $perTenantExtensionRepoPath)" - Write-Host "- https://github.com/$($config.githubOwner)/$($config.appSourceAppRepo) (folder $appSourceAppRepoPath)" - Write-Host - Write-Host "To the $algoBranch branch from $srcOwnerAndRepo (folder $baseRepoPath)" - Write-Host - - $config.actionsRepo, $config.perTenantExtensionRepo, $config.appSourceAppRepo | ForEach-Object { - if (Test-Path $_) { - Set-Location $_ - invoke-git pull - } - else { - $serverUrl = "https://github.com/$($config.githubOwner)/$_.git" - invoke-git clone --quiet $serverUrl - Set-Location $_ - } - invoke-git checkout $config.branch - Set-Location $baseFolder - } - - $repos = @( - @{ "repo" = $config.actionsRepo; "srcPath" = Join-Path $baseRepoPath "Actions"; "dstPath" = $actionsRepoPath; "branch" = $config.branch } - @{ "repo" = $config.perTenantExtensionRepo; "srcPath" = Join-Path $baseRepoPath "Templates\Per Tenant Extension"; "dstPath" = $perTenantExtensionRepoPath; "branch" = $config.branch } - @{ "repo" = $config.appSourceAppRepo; "srcPath" = Join-Path $baseRepoPath "Templates\AppSource App"; "dstPath" = $appSourceAppRepoPath; "branch" = $config.branch } - ) - - $baseRepoBranch = '' - if (!$directCommit) { - $baseRepoBranch = "collect-from-$($config.branch)/$alGoBranch/$((Get-Date).ToUniversalTime().ToString(`"yyMMddHHmmss`"))" # e.g. collect-from-nopr/main/210101120000 - Set-Location $baseRepoPath - invoke-git checkout -b $baseRepoBranch - } - - $repos | ForEach-Object { - Set-Location $baseFolder - $repo = $_.repo - $srcPath = $_.srcPath - $dstPath = $_.dstPath - - Write-Host "Removing $srcPath content" - Get-ChildItem -Path $srcPath -Force | Where-Object { !($_.PSIsContainer -and $_.Name -eq ".git") } | ForEach-Object { - $name = $_.FullName - Write-Host "Remove $name" - if ($_.PSIsContainer) { - Remove-Item $name -Force -Recurse - } - else { - Remove-Item $name -Force - } - } - - Write-Host -ForegroundColor Yellow "Collecting from $repo" - Get-ChildItem -Path $dstPath -Recurse -File -Force | Where-Object { $_.name -notlike '*.copy.md' } | ForEach-Object { - $dstFile = $_.FullName - $srcFile = $srcPath + $dstFile.Substring($dstPath.Length) - $srcFilePath = [System.IO.Path]::GetDirectoryName($srcFile) - if (!(Test-Path $srcFilePath)) { - New-Item $srcFilePath -ItemType Directory | Out-Null - } - Write-Host "$dstFile -> $srcFile" - $lines = ([string](Get-ContentLF -path $dstFile)).Split("`n") - "actionsRepo", "perTenantExtensionRepo", "appSourceAppRepo" | ForEach-Object { - $regex = "^(.*)$($config.githubOwner)\/$($config."$_")(.*)$($config.branch)(.*)$" - $replace = "`${1}$($originalOwnerAndRepo."$_")`${2}$originalBranch`${3}" - $lines = $lines | ForEach-Object { $_ -replace $regex, $replace } - } - if ($_.Name -eq "AL-Go-Helper.ps1") { - $lines = $lines | ForEach-Object { $_ -replace '^(\s*)\$defaultBcContainerHelperVersion(\s*)=(\s*)"(.*)"(.*)$', "`${1}`$defaultBcContainerHelperVersion`${2}=`${3}""""`${5}" } - } - [System.IO.File]::WriteAllText($srcFile, "$($lines -join "`n")`n") - } - } - Set-Location $baseRepoPath - - $serverUrl = "https://$($user.login):$token@github.com/$($srcOwnerAndRepo).git" - - $commitMessage = "Collect changes from $($config.githubOwner)/*@$($config.branch)" - invoke-git add * - invoke-git commit --allow-empty -m "'$commitMessage'" - if ($baseRepoBranch) { - invoke-git push -u $serverUrl $baseRepoBranch - invoke-gh pr create --fill --head $baseRepoBranch --repo $srcOwnerAndRepo --base $ENV:GITHUB_REF_NAME - invoke-git checkout $algoBranch - } - else { - invoke-git push $serverUrl - } -} -finally { - set-location $oldPath -} diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 62eba2435..a2b1215a1 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -37,7 +37,6 @@ function PushChanges } $branchName = "deploy/$BaseBranch/$((Get-Date).ToUniversalTime().ToString(`"yyMMddHHmmss`"))" - invoke-git checkout -b $branchName origin/$BaseBranch invoke-git commit --allow-empty -m $CommitMessage invoke-git push origin $branchName @@ -45,6 +44,7 @@ function PushChanges } } +$token = GetRealToken -token $token $oldPath = Get-Location try { From d0157f63e711ee720ca36832cf7011259871e32b Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 09:43:28 +0100 Subject: [PATCH 15/34] no user --- Internal/Deploy.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index a2b1215a1..421cfefb6 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -73,8 +73,8 @@ try { Set-Location $baseRepoPath # Whoami - $user = invoke-gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" user -silent -returnValue | ConvertFrom-Json - Write-Host "GitHub user: $($user.login)" + #$user = invoke-gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" user -silent -returnValue | ConvertFrom-Json + #Write-Host "GitHub user: $($user.login)" # Dump configuration Write-Host "Configuration:" @@ -162,7 +162,7 @@ try { Write-Host -ForegroundColor Yellow "Deploying to $repo" try { - $serverUrl = "https://$($user.login):$token@github.com/$($config.githubOwner)/$repo.git" + $serverUrl = "https://$($config.githubOwner):$token@github.com/$($config.githubOwner)/$repo.git" if (Test-Path $repo) { Remove-Item $repo -Recurse -Force } From f358ab8a095bcce2d085d7953e83dae4d6b1fd18 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 10:17:48 +0100 Subject: [PATCH 16/34] use githubOwner --- Actions/Github-Helper.psm1 | 6 ++++-- Internal/Deploy.ps1 | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 351a1e3da..953930486 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -558,7 +558,9 @@ function GetLatestRelease { function GetRealToken { Param( - [string] $token + [string] $token, + [string] $api_url = $ENV:GITHUB_API_URL, + [string] $repository = $ENV:GITHUB_REPOSITORY ) if (!($token.StartsWith("{"))) { @@ -578,7 +580,7 @@ function GetRealToken { "X-GitHub-Api-Version" = "2022-11-28" } Write-Host "Get App Info" - $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$ENV:GITHUB_API_URL/repos/$ENV:GITHUB_REPOSITORY/installation" + $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$api_url/repos/$repository/installation" Write-Host "Get Token Response" $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url Write-Host "return token" diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 421cfefb6..c7547a5b9 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -44,7 +44,7 @@ function PushChanges } } -$token = GetRealToken -token $token +$token = GetRealToken -token $token -repository "$($config.githubOwner)/.github" $oldPath = Get-Location try { From 936c912076d5e8d9752ea4a7841d195b56089b35 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 10:24:19 +0100 Subject: [PATCH 17/34] dump --- Actions/Github-Helper.psm1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 953930486..05f01ea3a 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -579,9 +579,9 @@ function GetRealToken { "Authorization" = "Bearer $jwt" "X-GitHub-Api-Version" = "2022-11-28" } - Write-Host "Get App Info" + Write-Host "Get App Info $api_url/repos/$repository/installation" $appinfo = Invoke-RestMethod -Method GET -UseBasicParsing -Headers $headers -Uri "$api_url/repos/$repository/installation" - Write-Host "Get Token Response" + Write-Host "Get Token Response $($appInfo.access_tokens_url)" $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url Write-Host "return token" return $tokenResponse.token From 945ca71b6a78ecb92d29cca56cffd5e95e8fd7b6 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 10:29:48 +0100 Subject: [PATCH 18/34] use repo --- e2eTests/e2eTestHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 2ac7f9682..5f02aa13e 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -21,7 +21,7 @@ function SetTokenAndRepository { ) $script:githubOwner = $githubOwner - $script:token = GetRealToken -token $token + $script:token = GetRealToken -token $token -repository "$($config.githubOwner)/.github" $script:defaultRepository = $repository if ($github) { From dae2cdcb3eeab459a5aa57f3855d00df27bf91fa Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 10:39:00 +0100 Subject: [PATCH 19/34] no config --- e2eTests/e2eTestHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 5f02aa13e..d8c66609f 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -21,7 +21,7 @@ function SetTokenAndRepository { ) $script:githubOwner = $githubOwner - $script:token = GetRealToken -token $token -repository "$($config.githubOwner)/.github" + $script:token = GetRealToken -token $token -repository "$githubOwner/.github" $script:defaultRepository = $repository if ($github) { From cd8b221eaa4ba89cb6e6e9c8b6ef38f8adca6151 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 10:56:04 +0100 Subject: [PATCH 20/34] use repo --- e2eTests/e2eTestHelper.psm1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index d8c66609f..d01d874f9 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -63,7 +63,7 @@ function Add-PropertiesToJsonFile { ) if ($wait -and $commit) { - $headers = GetHeaders -token $token + $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" Write-Host "Get Previous runs" $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id @@ -145,7 +145,7 @@ function RunWorkflow { Write-Host ($parameters | ConvertTo-Json) } - $headers = GetHeaders -token $token + $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" WaitForRateLimit -headers $headers -displayStatus Write-Host "Get Workflows" @@ -210,7 +210,7 @@ function DownloadWorkflowLog { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeaders -token $token + $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" $url = "https://api.github.com/repos/$repository/actions/runs/$runid" $run = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url).Content | ConvertFrom-Json) $log = InvokeWebRequest -Method Get -Headers $headers -Uri $run.logs_url @@ -266,7 +266,7 @@ function WaitWorkflow { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeaders -token $token + $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" $status = "" do { if ($delay) { @@ -525,7 +525,7 @@ function MergePRandPull { } Write-Host "Get Previous runs" - $headers = GetHeaders -token $token + $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id if ($previousrunids) { From 1524490e1d48f07e6edaf283386e221901e5aaf3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 11:09:06 +0100 Subject: [PATCH 21/34] fix token --- Actions/Github-Helper.psm1 | 19 ------------------- e2eTests/e2eTestHelper.psm1 | 2 +- .../FederatedCredentials/runtest.ps1 | 2 +- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 05f01ea3a..0485754f8 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -706,25 +706,6 @@ function DownloadRelease { } } -function CheckRateLimit { - Param( - [string] $token = '' - ) - - $headers = GetHeaders -token $token - $rate = (InvokeWebRequest -Headers $headers -Uri "https://api.github.com/rate_limit").Content | ConvertFrom-Json - $rate | ConvertTo-Json -Depth 99 | Out-Host - $rate = $rate.rate - $percent = [int]($rate.remaining*100/$rate.limit) - Write-Host "$($rate.remaining) API calls remaining out of $($rate.limit) ($percent%)" - if ($percent -lt 10) { - $resetTimeStamp = ([datetime] '1970-01-01Z').AddSeconds($rate.reset) - $waitTime = $resetTimeStamp.Subtract([datetime]::Now) - Write-Host "Less than 10% API calls left, waiting for $($waitTime.TotalSeconds) seconds for limits to reset." - Start-Sleep -seconds ($waitTime.TotalSeconds+1) - } -} - # Get Content of UTF8 encoded file as a string with LF line endings # No empty line at the end of the file function Get-ContentLF { diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index d01d874f9..2dcb07118 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -301,7 +301,7 @@ function SetRepositorySecret { $repository = $defaultRepository } Write-Host -ForegroundColor Yellow "`nSet Secret $name in $repository" - invoke-gh secret set $name -b $value --repo $repository + invoke-gh secret set $name -b $value --repo $repository -silent } function CreateNewAppInFolder { diff --git a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 index bdd5c3a03..279ea1abb 100644 --- a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 +++ b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 @@ -58,7 +58,7 @@ $template = "https://github.com/$appSourceTemplate" $repository = 'microsoft/bcsamples-bingmaps.appsource' SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository -$headers = GetHeaders $token +$headers = GetHeaders $token -repository "$githubOwner/.github" $existingBranch = gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/$repository/branches/$branch 2> $null | ConvertFrom-Json if ($existingBranch.PSObject.Properties.Name -eq 'Name' -and $existingBranch.Name -eq $branch) { From 28232d694875d60d75bc5bfd9a81b77d0e3afcf4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 12:34:03 +0100 Subject: [PATCH 22/34] mask value --- e2eTests/e2eTestHelper.psm1 | 1 + 1 file changed, 1 insertion(+) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 2dcb07118..3fe70f8f2 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -20,6 +20,7 @@ function SetTokenAndRepository { [switch] $github ) + MaskValue -key "token" -value $token $script:githubOwner = $githubOwner $script:token = GetRealToken -token $token -repository "$githubOwner/.github" $script:defaultRepository = $repository From 1402a6ffb0cec6bcc473c9fd640b2e99cab921f7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 13:00:29 +0100 Subject: [PATCH 23/34] move --- Actions/Github-Helper.psm1 | 32 ++++++++++++++++++++++ Actions/ReadSecrets/ReadSecretsHelper.psm1 | 31 --------------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 0485754f8..f372fa261 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -1,3 +1,35 @@ +$script:escchars = @(' ','!','\"','#','$','%','\u0026','\u0027','(',')','*','+',',','-','.','/','0','1','2','3','4','5','6','7','8','9',':',';','\u003c','=','\u003e','?','@','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','[','\\',']','^','_',[char]96,'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','{','|','}','~') + +function MaskValue { + Param( + [string] $key, + [string] $value + ) + + Write-Host "Masking value for $key" + $value.Split("`n") | ForEach-Object { + Write-Host "::add-mask::$_" + } + + $val2 = "" + $value.ToCharArray() | ForEach-Object { + $chint = [int]$_ + if ($chint -lt 32 -or $chint -gt 126 ) { + $val2 += $_ + } + else { + $val2 += $script:escchars[$chint-32] + } + } + + if ($val2 -ne $value) { + $val2.Split("`n") | ForEach-Object { + Write-Host "::add-mask::$_" + } + } + Write-Host "::add-mask::$([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($value)))" +} + function GetExtendedErrorMessage { [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSAvoidUsingEmptyCatchBlock", "", Justification="We want to ignore errors")] Param( diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index 978a11b77..f6b0533da 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -4,7 +4,6 @@ Param( ) $script:gitHubSecrets = $_gitHubSecrets | ConvertFrom-Json -$script:escchars = @(' ','!','\"','#','$','%','\u0026','\u0027','(',')','*','+',',','-','.','/','0','1','2','3','4','5','6','7','8','9',':',';','\u003c','=','\u003e','?','@','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','[','\\',']','^','_',[char]96,'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','{','|','}','~') . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) @@ -26,36 +25,6 @@ function GetAzureCredentials { return $null } -function MaskValue { - Param( - [string] $key, - [string] $value - ) - - Write-Host "Masking value for $key" - $value.Split("`n") | ForEach-Object { - Write-Host "::add-mask::$_" - } - - $val2 = "" - $value.ToCharArray() | ForEach-Object { - $chint = [int]$_ - if ($chint -lt 32 -or $chint -gt 126 ) { - $val2 += $_ - } - else { - $val2 += $script:escchars[$chint-32] - } - } - - if ($val2 -ne $value) { - $val2.Split("`n") | ForEach-Object { - Write-Host "::add-mask::$_" - } - } - Write-Host "::add-mask::$([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($value)))" -} - function GetGithubSecret { param ( [string] $secretName, From 9491ba7bfb7441c5413d9a2269b84968552468b9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 15:05:39 +0100 Subject: [PATCH 24/34] use gh --- .../CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 9 ++++++++- e2eTests/e2eTestHelper.psm1 | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 766d7ee8b..ed9a223cd 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -27,7 +27,14 @@ function DownloadTemplateRepository { if ($downloadLatest) { # Get latest commit SHA from the template repository - $templateSha.Value = GetLatestTemplateSha -headers $headers -apiUrl $apiUrl -templateUrl $templateUrl + try { + $templateSha.Value = GetLatestTemplateSha -headers $headers -apiUrl $apiUrl -templateUrl $templateUrl + } + catch { + # Try anonymous + $headers.Remove('Authorization') + $templateSha.Value = GetLatestTemplateSha -headers $headers -apiUrl $apiUrl -templateUrl $templateUrl + } Write-Host "Latest SHA for $($templateUrl): $($templateSha.Value)" } $archiveUrl = "$apiUrl/zipball/$($templateSha.Value)" diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 3fe70f8f2..d6e638df4 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -302,7 +302,11 @@ function SetRepositorySecret { $repository = $defaultRepository } Write-Host -ForegroundColor Yellow "`nSet Secret $name in $repository" - invoke-gh secret set $name -b $value --repo $repository -silent + gh secret set $name -b $value --repo $repository -silent + if ($LASTEXITCODE -ne 0) { + $host.SetShouldExit(0) + throw "Error setting secret $name in $repository" + } } function CreateNewAppInFolder { From 9c30d28e66db604aa9dad0315a087598ac9eb7f5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 18 Nov 2024 15:10:21 +0100 Subject: [PATCH 25/34] remove silent --- e2eTests/e2eTestHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index d6e638df4..9db85b93a 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -302,7 +302,7 @@ function SetRepositorySecret { $repository = $defaultRepository } Write-Host -ForegroundColor Yellow "`nSet Secret $name in $repository" - gh secret set $name -b $value --repo $repository -silent + gh secret set $name -b $value --repo $repository if ($LASTEXITCODE -ne 0) { $host.SetShouldExit(0) throw "Error setting secret $name in $repository" From 8b10b201052626d0e1bcda3b20d2282b188cdf9b Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 04:51:40 +0100 Subject: [PATCH 26/34] remove line breaks --- e2eTests/e2eTestHelper.psm1 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 9db85b93a..eb245c8d1 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -302,11 +302,8 @@ function SetRepositorySecret { $repository = $defaultRepository } Write-Host -ForegroundColor Yellow "`nSet Secret $name in $repository" - gh secret set $name -b $value --repo $repository - if ($LASTEXITCODE -ne 0) { - $host.SetShouldExit(0) - throw "Error setting secret $name in $repository" - } + $value = $value.Replace("`r", '').Replace("`n", '') + invoke-gh secret set $name -b $value --repo $repository --silent } function CreateNewAppInFolder { From e9ee5e863f1805ffb4850aa0a3d5830d68980997 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 05:22:38 +0100 Subject: [PATCH 27/34] fix e2e --- .../CheckForUpdates.HelperFunctions.ps1 | 22 +++++++++---------- e2eTests/e2eTestHelper.psm1 | 5 ++++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ed9a223cd..583afee2f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -27,14 +27,7 @@ function DownloadTemplateRepository { if ($downloadLatest) { # Get latest commit SHA from the template repository - try { - $templateSha.Value = GetLatestTemplateSha -headers $headers -apiUrl $apiUrl -templateUrl $templateUrl - } - catch { - # Try anonymous - $headers.Remove('Authorization') - $templateSha.Value = GetLatestTemplateSha -headers $headers -apiUrl $apiUrl -templateUrl $templateUrl - } + $templateSha.Value = GetLatestTemplateSha -headers $headers -apiUrl $apiUrl -templateUrl $templateUrl Write-Host "Latest SHA for $($templateUrl): $($templateSha.Value)" } $archiveUrl = "$apiUrl/zipball/$($templateSha.Value)" @@ -58,15 +51,22 @@ function GetLatestTemplateSha { try { $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry - $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch } } catch { if ($_.Exception.Message -like "*401*") { - throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))" - } else { + try { + $headers.Remove('Authorization') + $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry + } + catch { + throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))" + } + } + else { throw $_.Exception.Message } } + $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch } if (!$branchInfo) { throw "$templateUrl doesn't exist" } diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index eb245c8d1..b416a92fe 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -267,9 +267,12 @@ function WaitWorkflow { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" + $count = 0 $status = "" do { + if ($count % 45 -eq 0) { + $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" + } if ($delay) { Start-Sleep -Seconds 60 } From 112ebc3ecba7d019f66f6ed5496e1df1c0c8f724 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 05:33:20 +0100 Subject: [PATCH 28/34] silent --- e2eTests/e2eTestHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index b416a92fe..40f405a17 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -306,7 +306,7 @@ function SetRepositorySecret { } Write-Host -ForegroundColor Yellow "`nSet Secret $name in $repository" $value = $value.Replace("`r", '').Replace("`n", '') - invoke-gh secret set $name -b $value --repo $repository --silent + invoke-gh secret set $name -b $value --repo $repository -silent } function CreateNewAppInFolder { From 85014aa72bac4b0f84566c8caf0a4a2ced678914 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 05:47:49 +0100 Subject: [PATCH 29/34] use gfh --- e2eTests/e2eTestHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 40f405a17..e1b2f7b60 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -306,7 +306,7 @@ function SetRepositorySecret { } Write-Host -ForegroundColor Yellow "`nSet Secret $name in $repository" $value = $value.Replace("`r", '').Replace("`n", '') - invoke-gh secret set $name -b $value --repo $repository -silent + gh secret set $name -b $value --repo $repository } function CreateNewAppInFolder { From cea8eda09ecd873f8eda90bf68b0dd935ad9dca3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 07:55:05 +0100 Subject: [PATCH 30/34] jit --- e2eTests/e2eTestHelper.psm1 | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index e1b2f7b60..2cee6640a 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -22,8 +22,9 @@ function SetTokenAndRepository { MaskValue -key "token" -value $token $script:githubOwner = $githubOwner - $script:token = GetRealToken -token $token -repository "$githubOwner/.github" + $script:token = $token $script:defaultRepository = $repository + $realToken = GetRealToken -token $script:token -repository "$githubOwner/.github" if ($github) { invoke-git config --global user.email "$githubOwner@users.noreply.github.com" @@ -33,9 +34,9 @@ function SetTokenAndRepository { $ENV:GITHUB_TOKEN = '' } Write-Host "Authenticating with GitHub using token" - $script:token | invoke-gh auth login --with-token + $realToken | invoke-gh auth login --with-token if ($github) { - $ENV:GITHUB_TOKEN = $script:token + $ENV:GITHUB_TOKEN = $realToken } } @@ -64,7 +65,7 @@ function Add-PropertiesToJsonFile { ) if ($wait -and $commit) { - $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" Write-Host "Get Previous runs" $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id @@ -124,7 +125,7 @@ function Remove-PropertiesFromJsonFile { } function DisplayTokenAndRepository { - Write-Host "Token: $token" + Write-Host "Token: $($script:token)" Write-Host "Repo: $defaultRepository" } @@ -146,7 +147,7 @@ function RunWorkflow { Write-Host ($parameters | ConvertTo-Json) } - $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" WaitForRateLimit -headers $headers -displayStatus Write-Host "Get Workflows" @@ -211,7 +212,7 @@ function DownloadWorkflowLog { if (!$repository) { $repository = $defaultRepository } - $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" $url = "https://api.github.com/repos/$repository/actions/runs/$runid" $run = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url).Content | ConvertFrom-Json) $log = InvokeWebRequest -Method Get -Headers $headers -Uri $run.logs_url @@ -271,7 +272,7 @@ function WaitWorkflow { $status = "" do { if ($count % 45 -eq 0) { - $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" } if ($delay) { Start-Sleep -Seconds 60 @@ -490,8 +491,8 @@ function CreateAlGoRepository { invoke-git add * invoke-git commit --allow-empty -m 'init' invoke-git branch -M $branch - if ($githubOwner -and $token) { - invoke-git remote set-url origin "https://$($githubOwner):$token@github.com/$repository.git" + if ($githubOwner -and $script:token) { + invoke-git remote set-url origin "https://$($githubOwner):$($script:token)@github.com/$repository.git" } invoke-git push --set-upstream origin $branch if (!$github) { @@ -530,7 +531,7 @@ function MergePRandPull { } Write-Host "Get Previous runs" - $headers = GetHeaders -token $token -repository "$($script:githubOwner)/.github" + $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" $url = "https://api.github.com/repos/$repository/actions/runs" $previousrunids = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url -retry).Content | ConvertFrom-Json).workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id if ($previousrunids) { From 9ba3af2bd0c30a71ef8d308f92761ca3c77ab176 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 09:37:38 +0100 Subject: [PATCH 31/34] use real token --- e2eTests/e2eTestHelper.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 2cee6640a..3d5140010 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -492,7 +492,8 @@ function CreateAlGoRepository { invoke-git commit --allow-empty -m 'init' invoke-git branch -M $branch if ($githubOwner -and $script:token) { - invoke-git remote set-url origin "https://$($githubOwner):$($script:token)@github.com/$repository.git" + $realToken = GetRealToken -token $script:token -repository "$githubOwner/.github" + invoke-git remote set-url origin "https://$($githubOwner):$($realtoken)@github.com/$repository.git" } invoke-git push --set-upstream origin $branch if (!$github) { From cdd459c29778093bf47966533faa1be103bc312a Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 10:00:05 +0100 Subject: [PATCH 32/34] get token for repo --- Actions/Github-Helper.psm1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index f372fa261..4e1ebc98f 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -629,14 +629,16 @@ function GetHeaders { param ( [string] $token, [string] $accept = "application/vnd.github+json", - [string] $apiVersion = "2022-11-28" + [string] $apiVersion = "2022-11-28", + [string] $api_url = $ENV:GITHUB_API_URL, + [string] $repository = $ENV:GITHUB_REPOSITORY ) $headers = @{ "Accept" = $accept "X-GitHub-Api-Version" = $apiVersion } if (![string]::IsNullOrEmpty($token)) { - $realToken = GetRealToken -token $token + $realToken = GetRealToken -token $token -api_url $api_url -repository $repository $headers["Authorization"] = "token $realToken" } return $headers From d7d440bd70aa57a8917178eab4f1bdcae90946bd Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 10:27:18 +0100 Subject: [PATCH 33/34] inc count --- e2eTests/e2eTestHelper.psm1 | 1 + 1 file changed, 1 insertion(+) diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 3d5140010..f09ca0f7b 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -273,6 +273,7 @@ function WaitWorkflow { do { if ($count % 45 -eq 0) { $headers = GetHeaders -token $script:token -repository "$($script:githubOwner)/.github" + $count++ } if ($delay) { Start-Sleep -Seconds 60 From cd03ea6090f69b2ad96606f05f5d6f72992e57fa Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Nov 2024 17:41:34 +0100 Subject: [PATCH 34/34] token cache --- Actions/Github-Helper.psm1 | 17 +++++++++++++++++ .../ReferenceDocumentation/runtest.ps1 | 3 +++ 2 files changed, 20 insertions(+) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 4e1ebc98f..8e499a974 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -1,4 +1,10 @@ $script:escchars = @(' ','!','\"','#','$','%','\u0026','\u0027','(',')','*','+',',','-','.','/','0','1','2','3','4','5','6','7','8','9',':',';','\u003c','=','\u003e','?','@','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','[','\\',']','^','_',[char]96,'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','{','|','}','~') +$script:realTokenCache = @{ + "token" = '' + "repository" = '' + "realToken" = '' + "expires" = [datetime]::Now +} function MaskValue { Param( @@ -599,6 +605,11 @@ function GetRealToken { # not a json token return $token } + elseif ($script:realTokenCache.token -eq $token -and $script:realTokenCache.repository -eq $repository -and $script:realTokenCache.expires -gt [datetime]::Now.AddMinutes(10)) { + # Same token request and cached token won't expire in 10 minutes + Write-Host "return cached token" + return $script:realTokenCache.realToken + } else { try { $json = $token | ConvertFrom-Json @@ -616,6 +627,12 @@ function GetRealToken { Write-Host "Get Token Response $($appInfo.access_tokens_url)" $tokenResponse = Invoke-RestMethod -Method POST -UseBasicParsing -Headers $headers -Uri $appInfo.access_tokens_url Write-Host "return token" + $script:realTokenCache = @{ + "token" = $token + "repository" = $repository + "realToken" = $tokenResponse.token + "expires" = [datetime]::Now.AddSeconds($tokenResponse.expires_in) + } return $tokenResponse.token } catch { diff --git a/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 b/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 index 8aa200e14..3c86b7332 100644 --- a/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 +++ b/e2eTests/scenarios/ReferenceDocumentation/runtest.ps1 @@ -81,6 +81,7 @@ WaitWorkflow -repository $repository -runid $run.id Test-ArtifactsFromRun -runid $run.id -folder 'artifacts' -expectedArtifacts @{"Apps"=1;"TestApps"=0;"Dependencies"=0;"github-pages"=1} -repoVersion '1.0' -appVersion '1.0' # Set GitHub Pages in repository to GitHub Actions +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository gh api --method POST /repos/$repository/pages -f build_type=workflow | Out-Null # Add setting to deploy to GitHub Pages @@ -92,6 +93,7 @@ CommitAndPush -commitMessage 'DeployToGitHubPages' RunDeployReferenceDocumentation -repository $repository -wait | Out-Null # Get Pages URL and read the content +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository $pagesInfo = gh api /repos/$repository/pages | ConvertFrom-Json $html = (Invoke-WebRequest -Uri $pagesInfo.html_url -UseBasicParsing).Content $html | Should -belike "*Documentation for $repository*" @@ -108,6 +110,7 @@ CommitAndPush -commitMessage 'Continuous Deployment of ALDoc' WaitAllWorkflows -repository $repository -noError # Get Pages URL and read the content +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository $pagesInfo = gh api /repos/$repository/pages | ConvertFrom-Json $html = (Invoke-WebRequest -Uri $pagesInfo.html_url -UseBasicParsing).Content $html | Should -belike "*Documentazione per $repository*"