Large Stream Json Library (#387)

* Initial Library push * Handle Extra parameters * Most tests working * Cleanup * Progress * Progress * Cleanup metadata * Cleanup metadata * Handle larger objects * Good progress * Progress * LargeSboms passing * Fix metadata * Validation fixes * Cleanup * Cleanup * Cleanup * Cleanup * Cleanup * Accessibility * Code coverage * Merge cleanup * PR Feedback * PR feedback * Rename JsonAsynchronousNodeKit * Move JsonKit into existing packages * Missing project * Throw for explicit unhandled field
microsoft · Nov 7, 2023 · 80d0b27 · 80d0b27
1 parent d542111
commit 80d0b27
Show file tree

Hide file tree

Showing 49 changed files with 1,389 additions and 2,737 deletions.
diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs
@@ -11,6 +11,8 @@
 using Microsoft.Sbom.Api.Manifest.FileHashes;
 using Microsoft.Sbom.Common.Config;
 using Microsoft.Sbom.Entities;
+using Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities;
+using Microsoft.Sbom.Utils;
 using Serilog;
 
 namespace Microsoft.Sbom.Api.Workflows.Helpers;
@@ -56,7 +58,7 @@ public FilesValidator(
         this.spdxFileFilterer = spdxFileFilterer ?? throw new ArgumentNullException(nameof(spdxFileFilterer));
     }
 
-    public async Task<(int, List<FileValidationResult>)> Validate(ISbomParser sbomParser)
+    public async Task<(int, List<FileValidationResult>)> Validate(IEnumerable<SPDXFile> files)
     {
         var errors = new List<ChannelReader<FileValidationResult>>();
         var results = new List<ChannelReader<FileValidationResult>>();
@@ -66,7 +68,7 @@ public FilesValidator(
         results.AddRange(onDiskFileResults);
         errors.AddRange(onDiskFileErrors);
 
-        var (inSbomFileResults, inSbomFileErrors) = GetInsideSbomFiles(sbomParser);
+        var (inSbomFileResults, inSbomFileErrors) = GetInsideSbomFiles(files);
         results.AddRange(inSbomFileResults);
         errors.AddRange(inSbomFileErrors);
 
@@ -146,13 +148,13 @@ public FilesValidator(
         return (filesWithHashes, errors);
     }
 
-    private (List<ChannelReader<FileValidationResult>>, List<ChannelReader<FileValidationResult>>) GetInsideSbomFiles(ISbomParser sbomParser)
+    private (List<ChannelReader<FileValidationResult>>, List<ChannelReader<FileValidationResult>>) GetInsideSbomFiles(IEnumerable<SPDXFile> files)
     {
         var errors = new List<ChannelReader<FileValidationResult>>();
         var filesWithHashes = new List<ChannelReader<FileValidationResult>>();
 
         // Enumerate files from SBOM
-        var (sbomFiles, sbomFileErrors) = enumeratorChannel.Enumerate(sbomParser.GetFiles);
+        var (sbomFiles, sbomFileErrors) = enumeratorChannel.Enumerate(() => files.Select(f => f.ToSbomFile()));
         errors.Add(sbomFileErrors);
 
         log.Debug($"Splitting the workflow into {configuration.Parallelism.Value} threads.");

diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs
@@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
 using System.Linq;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -20,6 +19,8 @@
 using Microsoft.Sbom.Common;
 using Microsoft.Sbom.Common.Config;
 using Microsoft.Sbom.Extensions;
+using Microsoft.Sbom.JsonAsynchronousNodeKit;
+using Microsoft.Sbom.Parser;
 using PowerArgs;
 using Serilog;
 using Constants = Microsoft.Sbom.Api.Utils.Constants;
@@ -95,35 +96,35 @@ public async Task<bool> RunAsync()
                 var successfullyValidatedFiles = 0;
                 List<FileValidationResult> fileValidationFailures = null;
 
-                while (sbomParser.Next() != Contracts.Enums.ParserState.FINISHED)
+                ParserStateResult? result = null;
+                do
                 {
-                    switch (sbomParser.CurrentState)
+                    result = sbomParser.Next();
+                    if (result is not null)
                     {
-                        case Contracts.Enums.ParserState.FILES:
-                            (successfullyValidatedFiles, fileValidationFailures) = await filesValidator.Validate(sbomParser);
-                            break;
-                        case Contracts.Enums.ParserState.PACKAGES:
-                            var packages = sbomParser.GetPackages().ToList();
-                            totalNumberOfPackages = packages.Count();
-                            break;
-                        case Contracts.Enums.ParserState.RELATIONSHIPS:
-                            sbomParser.GetRelationships().ToList();
-                            break;
-                        case Contracts.Enums.ParserState.REFERENCES:
-                            sbomParser.GetReferences().ToList();
-                            break;
-                        case Contracts.Enums.ParserState.NONE:
-                            break;
-                        case Contracts.Enums.ParserState.METADATA:
-                            _ = sbomParser.GetMetadata();
-                            break;
-                        case Contracts.Enums.ParserState.INTERNAL_SKIP:
-                            break;
-                        case Contracts.Enums.ParserState.FINISHED:
-                            break;
-                        default: break;
+                        switch (result)
+                        {
+                            case FilesResult filesResult:
+                                (successfullyValidatedFiles, fileValidationFailures) = await filesValidator.Validate(filesResult.Files);
+                                break;
+                            case PackagesResult packagesResult:
+                                var packages = packagesResult.Packages.ToList();
+                                totalNumberOfPackages = packages.Count();
+                                break;
+                            case RelationshipsResult relationshipsResult:
+                                relationshipsResult.Relationships.ToList();
+                                break;
+                            case ExternalDocumentReferencesResult externalRefResult:
+                                externalRefResult.References.ToList();
+                                break;
+                            default:
+                                break;
+                        }
                     }
                 }
+                while (result is not null);
+
+                _ = sbomParser.GetMetadata();
 
                 if (configuration.FailIfNoPackages?.Value == true && totalNumberOfPackages <= 1)
                 {

diff --git a/src/Microsoft.Sbom.Contracts/Contracts/Enums/AlgorithmName.cs b/src/Microsoft.Sbom.Contracts/Contracts/Enums/AlgorithmName.cs
@@ -63,6 +63,36 @@ public override int GetHashCode()
         return !(left == right);
     }
 
+    public static AlgorithmName FromString(string name)
+    {
+        if (string.IsNullOrEmpty(name))
+        {
+            throw new ArgumentNullException(nameof(name));
+        }
+
+        if (string.Equals(name, SHA1.Name, StringComparison.OrdinalIgnoreCase))
+        {
+            return SHA1;
+        }
+
+        if (string.Equals(name, SHA256.Name, StringComparison.OrdinalIgnoreCase))
+        {
+            return SHA256;
+        }
+
+        if (string.Equals(name, SHA512.Name, StringComparison.OrdinalIgnoreCase))
+        {
+            return SHA512;
+        }
+
+        if (string.Equals(name, MD5.Name, StringComparison.OrdinalIgnoreCase))
+        {
+            return MD5;
+        }
+
+        throw new ArgumentException($"Unknown hash algorithm '{name}'.", nameof(name));
+    }
+
     /// <summary>
     /// Gets equivalent to <see cref="HashAlgorithmName.SHA1"/>.
     /// </summary>

diff --git a/src/Microsoft.Sbom.Contracts/Contracts/Enums/ParserState.cs b/src/Microsoft.Sbom.Contracts/Contracts/Enums/ParserState.cs
diff --git a/src/Microsoft.Sbom.Extensions/ISbomParser.cs b/src/Microsoft.Sbom.Extensions/ISbomParser.cs
@@ -1,10 +1,11 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System.Collections.Generic;
+#nullable enable
+
 using Microsoft.Sbom.Contracts;
-using Microsoft.Sbom.Contracts.Enums;
 using Microsoft.Sbom.Extensions.Entities;
+using Microsoft.Sbom.JsonAsynchronousNodeKit;
 
 namespace Microsoft.Sbom;
 
@@ -18,39 +19,7 @@ public interface ISbomParser
     /// </summary>
     /// <param name="stream"></param>
     /// <returns></returns>
-    ParserState Next();
-
-    /// <summary>
-    /// Returns a list of <see cref="SbomFile"/> objects defined in the
-    /// current SBOM.
-    /// </summary>
-    /// <param name="stream"></param>
-    /// <returns></returns>
-    IEnumerable<SbomFile> GetFiles();
-
-    /// <summary>
-    /// Returns a list of <see cref="SbomPackage"/> objects defined in the
-    /// current SBOM.
-    /// </summary>
-    /// <param name="stream"></param>
-    /// <returns></returns>
-    IEnumerable<SbomPackage> GetPackages();
-
-    /// <summary>
-    /// Returns a list of <see cref="SBOMRelationship"/> objects defined in the
-    /// current SBOM.
-    /// </summary>
-    /// <param name="stream"></param>
-    /// <returns></returns>
-    IEnumerable<SBOMRelationship> GetRelationships();
-
-    /// <summary>
-    /// Returns a list of <see cref="SBOMReference"/> objects defined in the
-    /// current SBOM.
-    /// </summary>
-    /// <param name="stream"></param>
-    /// <returns></returns>
-    IEnumerable<SBOMReference> GetReferences();
+    ParserStateResult? Next();
 
     /// <summary>
     /// Returns a <see cref="SBOMMetadata"/> object using the metadata defined in the
@@ -67,9 +36,4 @@ public interface ISbomParser
     /// <returns>An version sorted array in ascending order of
     /// <see cref="ManifestInfo">manifests</see> this library can parse.</returns>
     ManifestInfo[] RegisterManifest();
-
-    /// <summary>
-    /// Get the current state of the parser.
-    /// </summary>
-    public ParserState CurrentState { get; }
 }
diff --git a/src/Microsoft.Sbom.Extensions/ParserStateResult.cs b/src/Microsoft.Sbom.Extensions/ParserStateResult.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Sbom.JsonAsynchronousNodeKit;
+
+#nullable enable
+
+public record ParserStateResult(
+    string FieldName,
+    object? Result,
+    bool ExplicitField,
+    bool YieldReturn);
diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Constants.cs b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Constants.cs
@@ -6,7 +6,7 @@
 
 namespace Microsoft.Sbom.Parsers.Spdx22SbomParser;
 
-internal class Constants
+internal static class Constants
 {
     internal const string SPDXName = "SPDX";
     internal const string SPDXVersion = "2.2";
@@ -31,6 +31,8 @@ internal class Constants
 
     #endregion
 
+    internal const int ReadBufferSize = 4096;
+
     #region Value format strings
 
     internal const string SPDXDocumentNameFormatString = "{0} {1}";
@@ -58,29 +60,7 @@ internal class Constants
 
     internal static ManifestInfo Spdx22ManifestInfo = new ManifestInfo
     {
-        Name = Constants.SPDXName,
-        Version = Constants.SPDXVersion
-    };
-
-    internal const int ReadBufferSize = 4096;
-
-    /// <summary>
-    /// Converts a <see cref="System.Text.Json.JsonTokenType"/> enum to the actual string
-    /// representation of the token.
-    /// </summary>
-    internal static readonly string[] JsonTokenStrings = new string[]
-    {
-        string.Empty, // None
-        "{", // StartObject
-        "}", // EndObject
-        "[", // StartArray
-        "]", // EndArray
-        "PropertyName", // PropertyName
-        "Comment", // Comment
-        "String", // String
-        "Number", // Number
-        "True", // True
-        "False", // False
-        "Null", // Null
+        Name = SPDXName,
+        Version = SPDXVersion
     };
 }
diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Entities/ExternalReference.cs b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Entities/ExternalReference.cs
@@ -16,11 +16,12 @@ public class ExternalReference
     /// <summary>
     /// Gets or sets the category for the external reference.
     /// </summary>
+    [JsonRequired]
     [JsonPropertyName("referenceCategory")]
     public string ReferenceCategory { get; set; }
 
     /// <summary>
-    /// Gets or sets type of the external reference. These are definined in an appendix in the SPDX specification.
+    /// Gets or sets type of the external reference. These are defined in an appendix in the SPDX specification.
     /// https://spdx.github.io/spdx-spec/appendix-VI-external-repository-identifiers/.
     /// </summary>
     [JsonPropertyName("referenceType")]
@@ -30,6 +31,7 @@ public class ExternalReference
     /// Gets or sets a unique string without any spaces that specifies a location where the package specific information
     /// can be located. The locator constraints are defined by the <see cref="Type"/>.
     /// </summary>
+    [JsonRequired]
     [JsonPropertyName("referenceLocator")]
     public string Locator { get; set; }
 }
diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Entities/PackageVerificationCode.cs b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Entities/PackageVerificationCode.cs
@@ -15,6 +15,7 @@ public class PackageVerificationCode
     /// <summary>
     /// Gets or sets the actual package verification code as a hex encoded value.
     /// </summary>
+    [JsonRequired]
     [JsonPropertyName("packageVerificationCodeValue")]
     public string PackageVerificationCodeValue { get; set; }