diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index de22e99a8..aa561db1e 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -14,7 +14,6 @@ namespace OctoshiftCLI.IntegrationTests; [Collection("Integration Tests")] public sealed class BbsToGithub : IDisposable { - private const string SSH_KEY_FILE = "ssh_key.pem"; private const string AWS_REGION = "us-east-1"; @@ -33,6 +32,8 @@ public sealed class BbsToGithub : IDisposable private readonly DateTime _startTime; private readonly string _azureStorageConnectionString; + public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage } + public BbsToGithub(ITestOutputHelper output) { _startTime = DateTime.Now; @@ -67,10 +68,11 @@ public BbsToGithub(ITestOutputHelper output) } [Theory] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, true)] - [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, true)] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, false)] - public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload) + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)] + [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)] + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)] + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)] + public async Task Basic(string bbsServer, bool useSshForArchiveDownload, ArchiveUploadOption uploadOption) { var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}"; var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}"; @@ -110,17 +112,21 @@ await retryPolicy.Retry(async () => } var archiveUploadOptions = ""; - if (useAzureForArchiveUpload) + if (uploadOption == ArchiveUploadOption.AzureStorage) { _tokens.Add("AZURE_STORAGE_CONNECTION_STRING", _azureStorageConnectionString); } - else + else if (uploadOption == ArchiveUploadOption.AwsS3) { _tokens.Add("AWS_ACCESS_KEY_ID", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID")); _tokens.Add("AWS_SECRET_ACCESS_KEY", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY")); var awsBucketName = Environment.GetEnvironmentVariable("AWS_BUCKET_NAME"); archiveUploadOptions = $" --aws-bucket-name {awsBucketName} --aws-region {AWS_REGION}"; } + else if (uploadOption == ArchiveUploadOption.GithubStorage) + { + archiveUploadOptions = " --use-github-storage true"; + } await _targetHelper.RunBbsCliMigration( $"generate-script --github-org {githubTargetOrg} --bbs-server-url {bbsServer} --bbs-project {bbsProjectKey}{archiveDownloadOptions}{archiveUploadOptions}", _tokens); diff --git a/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs b/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs index 727ebc232..f96bdd20b 100644 --- a/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs @@ -63,7 +63,10 @@ public GhesToGithub(ITestOutputHelper output) _targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient, _blobServiceClient); } - public async Task Basic() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Basic(bool useGithubStorage) { var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}"; var githubTargetOrg = $"octoshift-e2e-ghes-{TestHelper.GetOsName()}"; @@ -83,8 +86,14 @@ await retryPolicy.Retry(async () => await _sourceHelper.CreateGithubRepo(githubSourceOrg, repo2); }); - await _targetHelper.RunGeiCliMigration( - $"generate-script --github-source-org {githubSourceOrg} --github-target-org {githubTargetOrg} --ghes-api-url {GHES_API_URL} --download-migration-logs", _tokens); + // Build the command with conditional option + var command = $"generate-script --github-source-org {githubSourceOrg} --github-target-org {githubTargetOrg} --ghes-api-url {GHES_API_URL} --download-migration-logs"; + if (useGithubStorage) + { + command += " --use-github-storage true"; + } + + await _targetHelper.RunGeiCliMigration(command, _tokens); _targetHelper.AssertNoErrorInLogs(_startTime); @@ -96,6 +105,7 @@ await _targetHelper.RunGeiCliMigration( _targetHelper.AssertMigrationLogFileExists(githubTargetOrg, repo1); _targetHelper.AssertMigrationLogFileExists(githubTargetOrg, repo2); } + public void Dispose() { _sourceGithubHttpClient?.Dispose(); diff --git a/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs index ffc34d024..2a5a718c5 100644 --- a/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -1102,6 +1102,63 @@ exit 1 _script.Should().NotContain(expected); } + [Fact] + public async Task Sequential_Github_Single_Repo_With_UseGithubStorage() + { + // Arrange + _mockGithubApi + .Setup(m => m.GetRepos(SOURCE_ORG)) + .ReturnsAsync(new[] { (REPO, "private") }); + + var expected = $"Exec {{ gh gei migrate-repo --github-source-org \"{SOURCE_ORG}\" --source-repo \"{REPO}\" --github-target-org \"{TARGET_ORG}\" --target-repo \"{REPO}\" --target-repo-visibility private --use-github-storage true }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GithubSourceOrg = SOURCE_ORG, + GithubTargetOrg = TARGET_ORG, + Output = new FileInfo("unit-test-output"), + Sequential = true, + UseGithubStorage = true + }; + await _handler.Handle(args); + + _script = TrimNonExecutableLines(_script); + + // Assert + _script.Should().Be(expected); + } + + [Fact] + public async Task Parallel_Github_Single_Repo_With_UseGithubStorage() + { + // Arrange + _mockGithubApi + .Setup(m => m.GetRepos(SOURCE_ORG)) + .ReturnsAsync(new[] { (REPO, "private") }); + + var expected = new StringBuilder(); + expected.AppendLine($"$MigrationID = ExecAndGetMigrationID {{ gh gei migrate-repo --github-source-org \"{SOURCE_ORG}\" --source-repo \"{REPO}\" --github-target-org \"{TARGET_ORG}\" --target-repo \"{REPO}\" --queue-only --target-repo-visibility private --use-github-storage true }}"); + expected.AppendLine($"$RepoMigrations[\"{REPO}\"] = $MigrationID"); + expected.Append($"if ($RepoMigrations[\"{REPO}\"]) {{ gh gei wait-for-migration --migration-id $RepoMigrations[\"{REPO}\"] }}"); + + // Act + var args = new GenerateScriptCommandArgs + { + GithubSourceOrg = SOURCE_ORG, + GithubTargetOrg = TARGET_ORG, + Output = new FileInfo("unit-test-output"), + UseGithubStorage = true + }; + await _handler.Handle(args); + + _script = TrimNonExecutableLines(_script, 19, 7); + + // Assert + _script.Should().Be(expected.ToString()); + } + + [Fact] public async Task Validates_Env_Vars_Blob_Storage_Not_Validated_When_GHES_3_8() { diff --git a/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandTests.cs index a24ff8de7..c7bc43c30 100644 --- a/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandTests.cs +++ b/src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -39,7 +39,7 @@ public void Should_Have_Options() var command = new GenerateScriptCommand(); command.Should().NotBeNull(); command.Name.Should().Be("generate-script"); - command.Options.Count.Should().Be(15); + command.Options.Count.Should().Be(16); TestHelpers.VerifyCommandOption(command.Options, "github-source-org", true); TestHelpers.VerifyCommandOption(command.Options, "github-target-org", true); @@ -56,6 +56,7 @@ public void Should_Have_Options() TestHelpers.VerifyCommandOption(command.Options, "aws-region", false); TestHelpers.VerifyCommandOption(command.Options, "keep-archive", false); TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "use-github-storage", false, true); } [Fact] diff --git a/src/gei/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gei/Commands/GenerateScript/GenerateScriptCommand.cs index 2e20eae87..922f2183d 100644 --- a/src/gei/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/gei/Commands/GenerateScript/GenerateScriptCommand.cs @@ -36,6 +36,7 @@ public GenerateScriptCommand() : base( AddOption(GithubSourcePat); AddOption(Verbose); AddOption(KeepArchive); + AddOption(UseGithubStorage); } public Option GithubSourceOrg { get; } = new("--github-source-org") { @@ -99,6 +100,11 @@ public GenerateScriptCommand() : base( { Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" }; + public Option UseGithubStorage { get; } = new("--use-github-storage") + { + IsHidden = true, + Description = "Enables multipart uploads to a GitHub owned storage for use during migration", + }; public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp) { diff --git a/src/gei/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/gei/Commands/GenerateScript/GenerateScriptCommandArgs.cs index d1f64956a..09c1ce759 100644 --- a/src/gei/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/gei/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -23,6 +23,7 @@ public class GenerateScriptCommandArgs : CommandArgs public string GithubSourcePat { get; set; } public bool KeepArchive { get; set; } public string TargetApiUrl { get; set; } + public bool UseGithubStorage { get; set; } public override void Validate(OctoLogger log) { diff --git a/src/gei/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gei/Commands/GenerateScript/GenerateScriptCommandHandler.cs index 39f8b7aee..a4b3c8d53 100644 --- a/src/gei/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gei/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -44,7 +44,7 @@ public async Task Handle(GenerateScriptCommandArgs args) _log.LogInformation("Generating Script..."); - var script = await GenerateScript(args.GithubSourceOrg, args.GithubTargetOrg, args.GhesApiUrl, args.AwsBucketName, args.AwsRegion, args.NoSslVerify, args.Sequential, args.SkipReleases, args.LockSourceRepo, args.DownloadMigrationLogs, args.KeepArchive, args.TargetApiUrl); + var script = await GenerateScript(args.GithubSourceOrg, args.GithubTargetOrg, args.GhesApiUrl, args.AwsBucketName, args.AwsRegion, args.NoSslVerify, args.Sequential, args.SkipReleases, args.LockSourceRepo, args.DownloadMigrationLogs, args.KeepArchive, args.TargetApiUrl, args.UseGithubStorage); if (script.HasValue() && args.Output.HasValue()) { @@ -52,7 +52,7 @@ public async Task Handle(GenerateScriptCommandArgs args) } } - private async Task GenerateScript(string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool sequential, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl) + private async Task GenerateScript(string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool sequential, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl, bool useGithubStorage) { var repos = await GetGithubRepos(_sourceGithubApi, githubSourceOrg); if (!repos.Any()) @@ -62,8 +62,8 @@ private async Task GenerateScript(string githubSourceOrg, string githubT } return sequential - ? await GenerateSequentialGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl) - : await GenerateParallelGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl); + ? await GenerateSequentialGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl, useGithubStorage) + : await GenerateParallelGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl, useGithubStorage); } private async Task> GetGithubRepos(GithubApi github, string githubOrg) @@ -83,7 +83,7 @@ private async Task GenerateScript(string githubSourceOrg, string githubT return repos; } - private async Task GenerateSequentialGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl) + private async Task GenerateSequentialGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl, bool useGithubStorage) { var content = new StringBuilder(); @@ -110,7 +110,7 @@ private async Task GenerateSequentialGithubScript(IEnumerable<(string Na foreach (var (name, visibility) in repos) { - content.AppendLine(Exec(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, true, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl))); + content.AppendLine(Exec(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, true, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl, useGithubStorage))); if (downloadMigrationLogs) { @@ -121,7 +121,7 @@ private async Task GenerateSequentialGithubScript(IEnumerable<(string Na return content.ToString(); } - private async Task GenerateParallelGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl) + private async Task GenerateParallelGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl, bool useGithubStorage) { var content = new StringBuilder(); @@ -158,7 +158,7 @@ private async Task GenerateParallelGithubScript(IEnumerable<(string Name // Queuing migrations foreach (var (name, visibility) in repos) { - content.AppendLine($"$MigrationID = {ExecAndGetMigrationId(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, false, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl))}"); + content.AppendLine($"$MigrationID = {ExecAndGetMigrationId(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, false, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl, useGithubStorage))}"); content.AppendLine($"$RepoMigrations[\"{name}\"] = $MigrationID"); content.AppendLine(); } @@ -199,11 +199,12 @@ exit 1 return content.ToString(); } - private string MigrateGithubRepoScript(string githubSourceOrg, string githubTargetOrg, string repo, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool wait, bool skipReleases, bool lockSourceRepo, bool keepArchive, string repoVisibility, string targetApiUrl) + private string MigrateGithubRepoScript(string githubSourceOrg, string githubTargetOrg, string repo, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool wait, bool skipReleases, bool lockSourceRepo, bool keepArchive, string repoVisibility, string targetApiUrl, bool useGithubStorage) { var ghesRepoOptions = ghesApiUrl.HasValue() ? GetGhesRepoOptions(ghesApiUrl, awsBucketName, awsRegion, noSslVerify, keepArchive) : null; + var useGithubStorageOption = useGithubStorage ? " --use-github-storage true" : string.Empty; - return $"gh gei migrate-repo{(targetApiUrl.HasValue() ? $" --target-api-url \"{targetApiUrl}\"" : string.Empty)} --github-source-org \"{githubSourceOrg}\" --source-repo \"{repo}\" --github-target-org \"{githubTargetOrg}\" --target-repo \"{repo}\"{(!string.IsNullOrEmpty(ghesRepoOptions) ? $" {ghesRepoOptions}" : string.Empty)}{(_log.Verbose ? " --verbose" : string.Empty)}{(wait ? string.Empty : " --queue-only")}{(skipReleases ? " --skip-releases" : string.Empty)}{(lockSourceRepo ? " --lock-source-repo" : string.Empty)} --target-repo-visibility {repoVisibility}"; + return $"gh gei migrate-repo{(targetApiUrl.HasValue() ? $" --target-api-url \"{targetApiUrl}\"" : string.Empty)} --github-source-org \"{githubSourceOrg}\" --source-repo \"{repo}\" --github-target-org \"{githubTargetOrg}\" --target-repo \"{repo}\"{(!string.IsNullOrEmpty(ghesRepoOptions) ? $" {ghesRepoOptions}" : string.Empty)}{(_log.Verbose ? " --verbose" : string.Empty)}{(wait ? string.Empty : " --queue-only")}{(skipReleases ? " --skip-releases" : string.Empty)}{(lockSourceRepo ? " --lock-source-repo" : string.Empty)} --target-repo-visibility {repoVisibility}{useGithubStorageOption}"; } private string GetGhesRepoOptions(string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool keepArchive) @@ -279,4 +280,4 @@ exit 1 } else { Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" }"; -} +} \ No newline at end of file