diff --git a/arc-validate-package-registry.sln b/arc-validate-package-registry.sln index d792f53..61fd9a1 100644 --- a/arc-validate-package-registry.sln +++ b/arc-validate-package-registry.sln @@ -48,6 +48,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{C4DB scripts\update-index.fsx = scripts\update-index.fsx EndProjectSection EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "AVPRCI", "src\AVPRCI\AVPRCI.fsproj", "{807D2C1D-4EA9-4E01-BD36-45A7B7BA0434}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -82,6 +84,10 @@ Global {72CBB4AF-511D-4A07-B41F-49A5F6F254F5}.Debug|Any CPU.Build.0 = Debug|Any CPU {72CBB4AF-511D-4A07-B41F-49A5F6F254F5}.Release|Any CPU.ActiveCfg = Release|Any CPU {72CBB4AF-511D-4A07-B41F-49A5F6F254F5}.Release|Any CPU.Build.0 = Release|Any CPU + {807D2C1D-4EA9-4E01-BD36-45A7B7BA0434}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {807D2C1D-4EA9-4E01-BD36-45A7B7BA0434}.Debug|Any CPU.Build.0 = Debug|Any CPU + {807D2C1D-4EA9-4E01-BD36-45A7B7BA0434}.Release|Any CPU.ActiveCfg = Release|Any CPU + {807D2C1D-4EA9-4E01-BD36-45A7B7BA0434}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -94,6 +100,7 @@ Global {5096B1A4-D041-4B04-8C90-47050CDBB7BF} = {8061A76B-1B85-4649-A667-6E7979ABA1AE} {72CBB4AF-511D-4A07-B41F-49A5F6F254F5} = {8061A76B-1B85-4649-A667-6E7979ABA1AE} {C4DBD5F6-4A14-44F4-9E5A-AC6B72AAEB81} = {24F7CF58-94B9-4FB4-8A59-FD329B3C431D} + {807D2C1D-4EA9-4E01-BD36-45A7B7BA0434} = {24F7CF58-94B9-4FB4-8A59-FD329B3C431D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D95036E6-C4D5-4E93-8474-BA6F74965635} diff --git a/src/AVPRCI/API.fs b/src/AVPRCI/API.fs new file mode 100644 index 0000000..76d48b8 --- /dev/null +++ b/src/AVPRCI/API.fs @@ -0,0 +1,220 @@ +module API + +open System +open System.IO +open AVPRIndex +open System.IO +open System.Text.Json + +open Argu +open CLIArgs +open Domain + +open AVPRIndex +open AVPRClient + +type GenIndexAPI = + static member generatePreviewIndex (verbose: bool) (repo_root: string) (args: ParseResults) = + + let out_path = + args.TryGetResult(GeneratePreviewIndexArgs.Output_Folder) + |> Option.defaultValue "." + |> fun o -> Path.Combine(o, "avpr-preview-index.json") + + JsonSerializer.Serialize( + value = AVPRRepo.getStagedPackages(repo_root), + options = JsonSerializerOptions(WriteIndented = true) + ) + |> fun json -> File.WriteAllText(out_path, json) + + if verbose then + printfn "Generated preview index at: %s" out_path + printfn "" + + 0 + +type CheckAPI = + static member prePublishChecks (verbose: bool) (repo_root: string) = + + if verbose then + printfn "pulling current preview index..." + printfn "" + + let current_preview_index = + AVPRRepo.getPreviewIndex() + + if verbose then + printfn $"{current_preview_index.Length} packages in the preview index" + printfn "" + printfn "collecting staged packages..." + printfn "" + + let all_packages_in_staging_area = + AVPRRepo.getStagedPackages(repo_root) + + if verbose then + printfn $"{all_packages_in_staging_area.Length} packages in the staging area" + printfn "" + printfn "performing pre-publish checks..." + printfn "" + + // all packages in staging area that are not listed in the current preview index + let staging_diff = + all_packages_in_staging_area + // only keep packages that either differ in content from their entry, or are not there at all + // this filter step keeps packages that incorrectly update published packages, which will be filtered in the next step. + |> Array.filter (fun pending -> + not <| Array.exists (fun indexed -> ValidationPackageIndex.contentEquals pending indexed) current_preview_index + ) + |> Array.filter (fun pending -> + let is_indexed = Array.exists (fun indexed -> ValidationPackageIndex.identityEquals pending indexed) current_preview_index + let is_indexed_and_published = Array.exists (fun indexed -> ValidationPackageIndex.identityEquals pending indexed && indexed.Metadata.Publish) current_preview_index + if is_indexed_and_published then + // package is on the preview index and already set to publish + // we want to fail here, as those packages should stay immutable + failwithf $"package {pending.Metadata.Name} with version {ValidationPackageIndex.getSemanticVersionString pending} is already indexed and set to publish. This is not allowed. Publish a new version instead." + else + if is_indexed then + if verbose then printfn $"package {pending.Metadata.Name} with version {ValidationPackageIndex.getSemanticVersionString pending} is already indexed and will be updated with this release." + true + else + if verbose then printfn $"package {pending.Metadata.Name} with version {ValidationPackageIndex.getSemanticVersionString pending} is new and will be added with this release." + true + ) + + if verbose then + printfn $"{staging_diff.Length} packages are pending for publication" + printfn "" + printfn "writing results to env..." + printfn "" + + // https://stackoverflow.com/questions/70123328/how-to-set-environment-variables-in-github-actions-using-python + let GITHUB_ENV = + let env = Environment.GetEnvironmentVariable("GITHUB_ENV") + if String.IsNullOrEmpty(env) then + printfn "GITHUB_ENV not found in environment, writing to /pre-publish-check.txt" + let p = Path.Combine(repo_root, "check.txt") + let f = new FileInfo(p) + p + else env + + + if staging_diff.Length > 0 then + File.AppendAllLines(GITHUB_ENV, ["UPDATE_PREVIEW_INDEX=true"]) + else + File.AppendAllLines(GITHUB_ENV, ["UPDATE_PREVIEW_INDEX=false"]) + + printfn $"""GITHUB_ENV={File.ReadAllText(GITHUB_ENV)}""" + 0 + +type PublishAPI = + static member publishPendingPackages (verbose: bool) (repo_root: string) (args: ParseResults) = + + let isDryRun = args.TryGetResult(PublishArgs.Dry_Run).IsSome + + if isDryRun then + printfn "Dry run mode enabled. No changes will be pushed to the package database." + printfn "" + + let apiKey = args.GetResult(PublishArgs.API_Key) + + let client = + let httpClient = new System.Net.Http.HttpClient() + httpClient.DefaultRequestHeaders.Add("X-API-KEY", apiKey) + new AVPRClient.Client(httpClient) + + let published_packages = + client.GetAllPackagesAsync() + |> Async.AwaitTask + |> Async.RunSynchronously + |> Array.ofSeq + + //! Paths are relative to the root of the project, since the script is executed from the repo root in CI + let all_indexed_packages = + AVPRRepo.getStagedPackages(repo_root) + + let published_indexed_packages = + all_indexed_packages + |> Array.filter (fun i -> i.Metadata.Publish) + |> Array.filter (fun i -> + Array.exists (fun (p: AVPRClient.ValidationPackage) -> + p.Name = i.Metadata.Name + && p.MajorVersion = i.Metadata.MajorVersion + && p.MinorVersion = i.Metadata.MinorVersion + && p.PatchVersion = i.Metadata.PatchVersion + ) published_packages + ) + + let pending_indexed_packages = + all_indexed_packages + |> Array.filter (fun i -> i.Metadata.Publish) + |> Array.filter (fun i -> + not <| Array.exists (fun (p: AVPRClient.ValidationPackage) -> + p.Name = i.Metadata.Name + && p.MajorVersion = i.Metadata.MajorVersion + && p.MinorVersion = i.Metadata.MinorVersion + && p.PatchVersion = i.Metadata.PatchVersion + ) published_packages + ) + + if verbose then + printfn "" + printfn $"Comparing database and repo content hashes..." + printfn "" + + published_indexed_packages + |> Array.iter (fun i -> + try + i.toPackageContentHash(true) + |> client.VerifyPackageContentAsync + |> Async.AwaitTask + |> Async.RunSynchronously + with e -> + if isDryRun then + printfn $"[E]: {e.Message}" + printfn $"[{i.Metadata.Name}@{i.Metadata.MajorVersion}.{i.Metadata.MinorVersion}.{i.Metadata.PatchVersion}]: Package content hash does not match the published package" + printfn $" Make sure that the package file has not been modified after publication! ({i.RepoPath})" + else + failwith $"[{i.RepoPath}]: Package content hash does not match the published package" + ) + + // Publish the pending packages, and add the content hash to the database + + + if isDryRun then + printfn "" + printfn $"!! the following packages and content hashes will be submitted to the production DB: !!" + printfn "" + + pending_indexed_packages + |> Array.iter (fun i -> printfn $"[{i.Metadata.Name}@{i.Metadata.MajorVersion}.{i.Metadata.MinorVersion}.{i.Metadata.PatchVersion}]") + + if verbose then + printfn "" + printfn "Details:" + printfn "" + pending_indexed_packages + |> Array.iter (fun i -> + let p = i.toValidationPackage() + AVPRClient.ValidationPackage.printJson p + ) + else + printfn "" + printfn $"!! publishing pending packages and content hashes to the production DB: !!" + printfn "" + pending_indexed_packages + |> Array.iter (fun i -> + let p = i.toValidationPackage() + try + printfn $"[{i.Metadata.Name}@{i.Metadata.MajorVersion}.{i.Metadata.MinorVersion}.{i.Metadata.PatchVersion}]: Publishing package..." + p + |> client.CreatePackageAsync + |> Async.AwaitTask + |> Async.RunSynchronously + |> ignore + with e -> + failwith $"CreatePackage: [{i.RepoPath}]: failed with {System.Environment.NewLine}{e.Message}{System.Environment.NewLine}Package info:{System.Environment.NewLine}{AVPRClient.ValidationPackage.toJson p}" + ) + printfn "done." + + 0 diff --git a/src/AVPRCI/AVPRCI.fsproj b/src/AVPRCI/AVPRCI.fsproj new file mode 100644 index 0000000..1f5db93 --- /dev/null +++ b/src/AVPRCI/AVPRCI.fsproj @@ -0,0 +1,26 @@ + + + + Exe + net8.0 + + + + + + + + + + + + + + + + + + + + + diff --git a/src/AVPRCI/CLIArgs.fs b/src/AVPRCI/CLIArgs.fs new file mode 100644 index 0000000..79b3283 --- /dev/null +++ b/src/AVPRCI/CLIArgs.fs @@ -0,0 +1,49 @@ +module CLIArgs + +open Argu + +type GeneratePreviewIndexArgs = + | [] Output_Folder of string + + interface IArgParserTemplate with + member s.Usage = + match s with + | Output_Folder _ -> """Optional | Default = "." | Output folder where "avpr-preview-index.json" will be created.""" + +type PublishArgs = + | [] Dry_Run + | [] API_Key of string + + interface IArgParserTemplate with + member s.Usage = + match s with + | Dry_Run -> """Optional | Default = false | Dry run mode enabled. This will only print a preview of the changes that would be pushed to the package database.""" + | API_Key _ -> """Required | API key for the package database.""" + + +[] +type EntryCommand= + // Parameters + | [] Verbose + | [] Repo_Root_Path of string + + //Commands + | [] Gen_index of ParseResults + | [] Check + | [] Publish of ParseResults + + + interface IArgParserTemplate with + member s.Usage = + match s with + | Verbose -> "Optional | Default = false | Use verbose error messages (with full error stack)." + | Repo_Root_Path _ -> "Optional | Default = '.' | Path to the root of the repository." + | Gen_index _ -> "Subcommand for generating a preview index release of the staged packages." + | Check -> "Subcommand for performing pre-publish checks on the staged packages." + | Publish _ -> "Subcommand for publishing the staged packages to the package database." + + static member createParser() = + + let errorHandler = ProcessExiter(colorizer = function ErrorCode.HelpText -> None | _ -> Some System.ConsoleColor.Red) + + ArgumentParser.Create(programName = "avpr-ci", errorHandler = errorHandler) diff --git a/src/AVPRCI/CommandHandling.fs b/src/AVPRCI/CommandHandling.fs new file mode 100644 index 0000000..f55a321 --- /dev/null +++ b/src/AVPRCI/CommandHandling.fs @@ -0,0 +1,30 @@ +module CommandHandling + +open CLIArgs +open API +open Argu + +let handleEntryCommand (verbose:bool) (repo_root: string) command = + match command with + | EntryCommand.Gen_index subcommand -> + if verbose then + printfn "" + printfn "Command: gen-index" + printfn "" + API.GenIndexAPI.generatePreviewIndex verbose (repo_root) (subcommand) + + | EntryCommand.Check -> + if verbose then + printfn "" + printfn "Command: check" + printfn "" + API.CheckAPI.prePublishChecks verbose (repo_root) + + | EntryCommand.Publish subcommand -> + if verbose then + printfn "" + printfn "Command: publish" + printfn "" + API.PublishAPI.publishPendingPackages verbose (repo_root) (subcommand) + + | _ -> failwith $"unrecognized command '{command}" \ No newline at end of file diff --git a/src/AVPRCI/Domain.fs b/src/AVPRCI/Domain.fs new file mode 100644 index 0000000..bf6237e --- /dev/null +++ b/src/AVPRCI/Domain.fs @@ -0,0 +1,16 @@ +module Domain + +open System.Text.Json + +let jsonSerializerOptions = JsonSerializerOptions(WriteIndented = true) + +type AVPRClient.ValidationPackage with + + static member toJson (p: AVPRClient.ValidationPackage) = + JsonSerializer.Serialize(p, jsonSerializerOptions) + + static member printJson (p: AVPRClient.ValidationPackage) = + let json = AVPRClient.ValidationPackage.toJson p + printfn "" + printfn $"Package info:{System.Environment.NewLine}{json}" + printfn "" \ No newline at end of file diff --git a/src/AVPRCI/Program.fs b/src/AVPRCI/Program.fs new file mode 100644 index 0000000..af66ccb --- /dev/null +++ b/src/AVPRCI/Program.fs @@ -0,0 +1,36 @@ +open Argu +open System.IO + +open API +open CLIArgs +open CommandHandling + +[] +let main argv = + + let parser = EntryCommand.createParser() + + try + let args = parser.ParseCommandLine() + + let verbose = args.TryGetResult(EntryCommand.Verbose) |> Option.isSome + + let repo_root_path = args.TryGetResult(EntryCommand.Repo_Root_Path) |> Option.defaultValue "." + + handleEntryCommand verbose repo_root_path (args.GetSubCommand()) + |> int + + with + | :? ArguParseException as ex -> + match ex.ErrorCode with + | ErrorCode.HelpText -> + printfn "%s" (parser.PrintUsage()) + 0 // printing usage is not an error + + | _ -> + printfn "%A" ex.Message + 1 + + | ex -> + printfn "%A" ex.Message + 1