Add --github-owned-storage Generate script capabilities + int tests

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -14,7 +14,6 @@ namespace OctoshiftCLI.IntegrationTests;
 [Collection("Integration Tests")]
 public sealed class BbsToGithub : IDisposable
 {
-
     private const string SSH_KEY_FILE = "ssh_key.pem";
     private const string AWS_REGION = "us-east-1";
 
@@ -33,6 +32,8 @@ public sealed class BbsToGithub : IDisposable
     private readonly DateTime _startTime;
     private readonly string _azureStorageConnectionString;
 
+    public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage }
+
     public BbsToGithub(ITestOutputHelper output)
     {
         _startTime = DateTime.Now;
@@ -67,10 +68,11 @@ public BbsToGithub(ITestOutputHelper output)
     }
 
     [Theory]
-    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, true)]
-    [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, true)]
-    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, false)]
-    public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload)
+    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)]
+    [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)]
+    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)]
+    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)]
+    public async Task Basic(string bbsServer, bool useSshForArchiveDownload, ArchiveUploadOption uploadOption)
     {
         var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}";
         var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}";
@@ -110,17 +112,21 @@ await retryPolicy.Retry(async () =>
         }
 
         var archiveUploadOptions = "";
-        if (useAzureForArchiveUpload)
+        if (uploadOption == ArchiveUploadOption.AzureStorage)
         {
             _tokens.Add("AZURE_STORAGE_CONNECTION_STRING", _azureStorageConnectionString);
         }
-        else
+        else if (uploadOption == ArchiveUploadOption.AwsS3)
         {
             _tokens.Add("AWS_ACCESS_KEY_ID", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID"));
             _tokens.Add("AWS_SECRET_ACCESS_KEY", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY"));
             var awsBucketName = Environment.GetEnvironmentVariable("AWS_BUCKET_NAME");
             archiveUploadOptions = $" --aws-bucket-name {awsBucketName} --aws-region {AWS_REGION}";
         }
+        else if (uploadOption == ArchiveUploadOption.GithubStorage)
+        {
+            archiveUploadOptions = " --use-github-storage true";
+        }
 
         await _targetHelper.RunBbsCliMigration(
             $"generate-script --github-org {githubTargetOrg} --bbs-server-url {bbsServer} --bbs-project {bbsProjectKey}{archiveDownloadOptions}{archiveUploadOptions}", _tokens);

src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs

@@ -63,7 +63,10 @@ public GhesToGithub(ITestOutputHelper output)
         _targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient, _blobServiceClient);
     }
 
-    public async Task Basic()
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task Basic(bool useGithubStorage)
     {
         var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}";
         var githubTargetOrg = $"octoshift-e2e-ghes-{TestHelper.GetOsName()}";
@@ -83,8 +86,14 @@ await retryPolicy.Retry(async () =>
             await _sourceHelper.CreateGithubRepo(githubSourceOrg, repo2);
         });
 
-        await _targetHelper.RunGeiCliMigration(
-            $"generate-script --github-source-org {githubSourceOrg} --github-target-org {githubTargetOrg} --ghes-api-url {GHES_API_URL} --download-migration-logs", _tokens);
+        // Build the command with conditional option
+        var command = $"generate-script --github-source-org {githubSourceOrg} --github-target-org {githubTargetOrg} --ghes-api-url {GHES_API_URL} --download-migration-logs";
+        if (useGithubStorage)
+        {
+            command += " --use-github-storage true";
+        }
+
+        await _targetHelper.RunGeiCliMigration(command, _tokens);
 
         _targetHelper.AssertNoErrorInLogs(_startTime);
 
@@ -96,6 +105,7 @@ await _targetHelper.RunGeiCliMigration(
         _targetHelper.AssertMigrationLogFileExists(githubTargetOrg, repo1);
         _targetHelper.AssertMigrationLogFileExists(githubTargetOrg, repo2);
     }
+
     public void Dispose()
     {
         _sourceGithubHttpClient?.Dispose();

src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs

@@ -1102,6 +1102,63 @@ exit 1
             _script.Should().NotContain(expected);
         }
 
+        [Fact]
+        public async Task Sequential_Github_Single_Repo_With_UseGithubStorage()
+        {
+            // Arrange
+            _mockGithubApi
+                .Setup(m => m.GetRepos(SOURCE_ORG))
+                .ReturnsAsync(new[] { (REPO, "private") });
+
+            var expected = $"Exec {{ gh gei migrate-repo --github-source-org \"{SOURCE_ORG}\" --source-repo \"{REPO}\" --github-target-org \"{TARGET_ORG}\" --target-repo \"{REPO}\" --target-repo-visibility private --use-github-storage true }}";
+
+            // Act
+            var args = new GenerateScriptCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                GithubTargetOrg = TARGET_ORG,
+                Output = new FileInfo("unit-test-output"),
+                Sequential = true,
+                UseGithubStorage = true
+            };
+            await _handler.Handle(args);
+
+            _script = TrimNonExecutableLines(_script);
+
+            // Assert
+            _script.Should().Be(expected);
+        }
+
+        [Fact]
+        public async Task Parallel_Github_Single_Repo_With_UseGithubStorage()
+        {
+            // Arrange
+            _mockGithubApi
+                .Setup(m => m.GetRepos(SOURCE_ORG))
+                .ReturnsAsync(new[] { (REPO, "private") });
+
+            var expected = new StringBuilder();
+            expected.AppendLine($"$MigrationID = ExecAndGetMigrationID {{ gh gei migrate-repo --github-source-org \"{SOURCE_ORG}\" --source-repo \"{REPO}\" --github-target-org \"{TARGET_ORG}\" --target-repo \"{REPO}\" --queue-only --target-repo-visibility private --use-github-storage true }}");
+            expected.AppendLine($"$RepoMigrations[\"{REPO}\"] = $MigrationID");
+            expected.Append($"if ($RepoMigrations[\"{REPO}\"]) {{ gh gei wait-for-migration --migration-id $RepoMigrations[\"{REPO}\"] }}");
+
+            // Act
+            var args = new GenerateScriptCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                GithubTargetOrg = TARGET_ORG,
+                Output = new FileInfo("unit-test-output"),
+                UseGithubStorage = true
+            };
+            await _handler.Handle(args);
+
+            _script = TrimNonExecutableLines(_script, 19, 7);
+
+            // Assert
+            _script.Should().Be(expected.ToString());
+        }
+
+
         [Fact]
         public async Task Validates_Env_Vars_Blob_Storage_Not_Validated_When_GHES_3_8()
         {

src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandTests.cs

@@ -39,7 +39,7 @@ public void Should_Have_Options()
             var command = new GenerateScriptCommand();
             command.Should().NotBeNull();
             command.Name.Should().Be("generate-script");
-            command.Options.Count.Should().Be(15);
+            command.Options.Count.Should().Be(16);
 
             TestHelpers.VerifyCommandOption(command.Options, "github-source-org", true);
             TestHelpers.VerifyCommandOption(command.Options, "github-target-org", true);
@@ -56,6 +56,7 @@ public void Should_Have_Options()
             TestHelpers.VerifyCommandOption(command.Options, "aws-region", false);
             TestHelpers.VerifyCommandOption(command.Options, "keep-archive", false);
             TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false);
+            TestHelpers.VerifyCommandOption(command.Options, "use-github-storage", false, true);
         }
 
         [Fact]

src/gei/Commands/GenerateScript/GenerateScriptCommand.cs

@@ -36,6 +36,7 @@ public GenerateScriptCommand() : base(
             AddOption(GithubSourcePat);
             AddOption(Verbose);
             AddOption(KeepArchive);
+            AddOption(UseGithubStorage);
         }
         public Option<string> GithubSourceOrg { get; } = new("--github-source-org")
         {
@@ -99,6 +100,11 @@ public GenerateScriptCommand() : base(
         {
             Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com"
         };
+        public Option<bool> UseGithubStorage { get; } = new("--use-github-storage")
+        {
+            IsHidden = true,
+            Description = "Enables multipart uploads to a GitHub owned storage for use during migration",
+        };
 
         public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp)
         {

src/gei/Commands/GenerateScript/GenerateScriptCommandArgs.cs

@@ -23,6 +23,7 @@ public class GenerateScriptCommandArgs : CommandArgs
         public string GithubSourcePat { get; set; }
         public bool KeepArchive { get; set; }
         public string TargetApiUrl { get; set; }
+        public bool UseGithubStorage { get; set; }
 
         public override void Validate(OctoLogger log)
         {

src/gei/Commands/GenerateScript/GenerateScriptCommandHandler.cs

@@ -44,15 +44,15 @@ public async Task Handle(GenerateScriptCommandArgs args)
 
         _log.LogInformation("Generating Script...");
 
-        var script = await GenerateScript(args.GithubSourceOrg, args.GithubTargetOrg, args.GhesApiUrl, args.AwsBucketName, args.AwsRegion, args.NoSslVerify, args.Sequential, args.SkipReleases, args.LockSourceRepo, args.DownloadMigrationLogs, args.KeepArchive, args.TargetApiUrl);
+        var script = await GenerateScript(args.GithubSourceOrg, args.GithubTargetOrg, args.GhesApiUrl, args.AwsBucketName, args.AwsRegion, args.NoSslVerify, args.Sequential, args.SkipReleases, args.LockSourceRepo, args.DownloadMigrationLogs, args.KeepArchive, args.TargetApiUrl, args.UseGithubStorage);
 
         if (script.HasValue() && args.Output.HasValue())
         {
             await WriteToFile(args.Output.FullName, script);
         }
     }
 
-    private async Task<string> GenerateScript(string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool sequential, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl)
+    private async Task<string> GenerateScript(string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool sequential, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl, bool useGithubStorage)
     {
         var repos = await GetGithubRepos(_sourceGithubApi, githubSourceOrg);
         if (!repos.Any())
@@ -62,8 +62,8 @@ private async Task<string> GenerateScript(string githubSourceOrg, string githubT
         }
 
         return sequential
-            ? await GenerateSequentialGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl)
-            : await GenerateParallelGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl);
+            ? await GenerateSequentialGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl, useGithubStorage)
+            : await GenerateParallelGithubScript(repos, githubSourceOrg, githubTargetOrg, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, skipReleases, lockSourceRepo, downloadMigrationLogs, keepArchive, targetApiUrl, useGithubStorage);
     }
 
     private async Task<IEnumerable<(string Name, string Visibility)>> GetGithubRepos(GithubApi github, string githubOrg)
@@ -83,7 +83,7 @@ private async Task<string> GenerateScript(string githubSourceOrg, string githubT
         return repos;
     }
 
-    private async Task<string> GenerateSequentialGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl)
+    private async Task<string> GenerateSequentialGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl, bool useGithubStorage)
     {
         var content = new StringBuilder();
 
@@ -110,7 +110,7 @@ private async Task<string> GenerateSequentialGithubScript(IEnumerable<(string Na
 
         foreach (var (name, visibility) in repos)
         {
-            content.AppendLine(Exec(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, true, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl)));
+            content.AppendLine(Exec(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, true, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl, useGithubStorage)));
 
             if (downloadMigrationLogs)
             {
@@ -121,7 +121,7 @@ private async Task<string> GenerateSequentialGithubScript(IEnumerable<(string Na
         return content.ToString();
     }
 
-    private async Task<string> GenerateParallelGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl)
+    private async Task<string> GenerateParallelGithubScript(IEnumerable<(string Name, string Visibility)> repos, string githubSourceOrg, string githubTargetOrg, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool skipReleases, bool lockSourceRepo, bool downloadMigrationLogs, bool keepArchive, string targetApiUrl, bool useGithubStorage)
     {
         var content = new StringBuilder();
 
@@ -158,7 +158,7 @@ private async Task<string> GenerateParallelGithubScript(IEnumerable<(string Name
         // Queuing migrations
         foreach (var (name, visibility) in repos)
         {
-            content.AppendLine($"$MigrationID = {ExecAndGetMigrationId(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, false, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl))}");
+            content.AppendLine($"$MigrationID = {ExecAndGetMigrationId(MigrateGithubRepoScript(githubSourceOrg, githubTargetOrg, name, ghesApiUrl, awsBucketName, awsRegion, noSslVerify, false, skipReleases, lockSourceRepo, keepArchive, visibility, targetApiUrl, useGithubStorage))}");
             content.AppendLine($"$RepoMigrations[\"{name}\"] = $MigrationID");
             content.AppendLine();
         }
@@ -199,11 +199,12 @@ exit 1
         return content.ToString();
     }
 
-    private string MigrateGithubRepoScript(string githubSourceOrg, string githubTargetOrg, string repo, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool wait, bool skipReleases, bool lockSourceRepo, bool keepArchive, string repoVisibility, string targetApiUrl)
+    private string MigrateGithubRepoScript(string githubSourceOrg, string githubTargetOrg, string repo, string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool wait, bool skipReleases, bool lockSourceRepo, bool keepArchive, string repoVisibility, string targetApiUrl, bool useGithubStorage)
     {
         var ghesRepoOptions = ghesApiUrl.HasValue() ? GetGhesRepoOptions(ghesApiUrl, awsBucketName, awsRegion, noSslVerify, keepArchive) : null;
+        var useGithubStorageOption = useGithubStorage ? " --use-github-storage true" : string.Empty;
 
-        return $"gh gei migrate-repo{(targetApiUrl.HasValue() ? $" --target-api-url \"{targetApiUrl}\"" : string.Empty)} --github-source-org \"{githubSourceOrg}\" --source-repo \"{repo}\" --github-target-org \"{githubTargetOrg}\" --target-repo \"{repo}\"{(!string.IsNullOrEmpty(ghesRepoOptions) ? $" {ghesRepoOptions}" : string.Empty)}{(_log.Verbose ? " --verbose" : string.Empty)}{(wait ? string.Empty : " --queue-only")}{(skipReleases ? " --skip-releases" : string.Empty)}{(lockSourceRepo ? " --lock-source-repo" : string.Empty)} --target-repo-visibility {repoVisibility}";
+        return $"gh gei migrate-repo{(targetApiUrl.HasValue() ? $" --target-api-url \"{targetApiUrl}\"" : string.Empty)} --github-source-org \"{githubSourceOrg}\" --source-repo \"{repo}\" --github-target-org \"{githubTargetOrg}\" --target-repo \"{repo}\"{(!string.IsNullOrEmpty(ghesRepoOptions) ? $" {ghesRepoOptions}" : string.Empty)}{(_log.Verbose ? " --verbose" : string.Empty)}{(wait ? string.Empty : " --queue-only")}{(skipReleases ? " --skip-releases" : string.Empty)}{(lockSourceRepo ? " --lock-source-repo" : string.Empty)} --target-repo-visibility {repoVisibility}{useGithubStorageOption}";
     }
 
     private string GetGhesRepoOptions(string ghesApiUrl, string awsBucketName, string awsRegion, bool noSslVerify, bool keepArchive)
@@ -279,4 +280,4 @@ exit 1
 } else {
     Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3.""
 }";
-}
+}