Added RunTwice Test and Fixed #153

digitalcoyote · digitalcoyote · commit 37b5444bb499 · 2024-02-11T22:15:54.000-06:00
diff --git a/Src/NuGetDefense.Lib/Scanner.cs b/Src/NuGetDefense.Lib/Scanner.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -55,16 +56,19 @@ public class Scanner
     public int Scan(ScanOptions options)
     {
         int exitCode;
+        NumberOfVulnerabilities = 0;
+        _projects = [];
         try
         {
             LoadSettings(options);
             ConfigureLogging();
             ScanVulnerabilities(options);
             exitCode = _settings.WarnOnly ? 0 : NumberOfVulnerabilities;
         }
-        catch (Exception)
+        catch (Exception e)
         {
             exitCode = -1;
+            Debug.WriteLine(e);
         }
 
         return exitCode;
@@ -187,11 +191,20 @@ private void ScanVulnerabilities(ScanOptions options)
             var nonSensitivePackageIDs = nonSensitivePackages.SelectMany(p => p.Value).ToArray();
             if (_settings.OssIndex.Enabled)
             {
-                const string OssIndexSourceID = "OSSIndex";
-                var uncachedPkgs = options.Cache.GetUncachedPackages(nonSensitivePackageIDs, TimeSpan.FromDays(1), OssIndexSourceID, out var cachedPackages);
-
+                const string ossIndexSourceId = "OSSIndex";
+                var uncachedPkgs = options.Cache.GetUncachedPackages(nonSensitivePackageIDs, TimeSpan.FromDays(1), ossIndexSourceId, out var cachedPackages);
+                var modUncached = uncachedPkgs.Count % 128;
+                if (modUncached > 0)
+                {
+                    for (var i = cachedPackages.Length - 1; i >= cachedPackages.Length - modUncached; i--)
+                    {
+                        uncachedPkgs.Add(cachedPackages[i]);
+                    }
+                    
+                    cachedPackages = cachedPackages[..^modUncached];
+                }
                 // Round out the calls to have a full set of packages each to refresh oldest cached packages
-                if (uncachedPkgs.Count > 0) uncachedPkgs.AddRange(cachedPackages.Take(128 - uncachedPkgs.Count % 128));
+                if (uncachedPkgs.Count > 0) uncachedPkgs.AddRange(cachedPackages.Take(uncachedPkgs.Count % 128));
 
                 Log.Logger.Verbose("Checking with OSSIndex for Vulnerabilities");
                 vulnDict =
@@ -201,10 +214,10 @@ private void ScanVulnerabilities(ScanOptions options)
                     // If we failed to update the OSS Index data don't clear out old cached data as that will
                     // increase the number of requests next time, increasing the liklihood of further
                     // TooManyRequeusts responses.
-                    options.Cache.UpdateCache(vulnDict, uncachedPkgs, OssIndexSourceID);
+                    options.Cache.UpdateCache(vulnDict, uncachedPkgs, ossIndexSourceId);
 
                 // Skipping the packages we refreshed
-                options.Cache.GetPackagesCachedVulnerabilitiesForSource(cachedPackages.Skip(128 - uncachedPkgs.Count % 128), OssIndexSourceID, ref vulnDict);
+                options.Cache.GetPackagesCachedVulnerabilitiesForSource(cachedPackages.Skip(uncachedPkgs.Count % 128), ossIndexSourceId, ref vulnDict);
             }
 
             if (_settings.GitHubAdvisoryDatabase.Enabled)
@@ -218,20 +231,29 @@ private void ScanVulnerabilities(ScanOptions options)
                 }
                 else
                 {
-                    const string GitHubAdvisoryDatabaseSourceId = "GitHubSecurityAdvisoryDatabase";
-                    var uncachedPkgs = options.Cache.GetUncachedPackages(nonSensitivePackageIDs, TimeSpan.FromDays(1), GitHubAdvisoryDatabaseSourceId, out var cachedPackages);
-
+                    const string gitHubAdvisoryDatabaseSourceId = "GitHubSecurityAdvisoryDatabase";
+                    var uncachedPkgs = options.Cache.GetUncachedPackages(nonSensitivePackageIDs, TimeSpan.FromDays(1), gitHubAdvisoryDatabaseSourceId, out var cachedPackages);
+                    var modUncached = uncachedPkgs.Count % 128;
+                    if (modUncached > 0)
+                    {
+                        for (var i = cachedPackages.Length - 1; i >= cachedPackages.Length - modUncached; i--)
+                        {
+                            uncachedPkgs.Add(cachedPackages[i]);
+                        }
+                    
+                        cachedPackages = cachedPackages[..^modUncached];
+                    }
                     Log.Logger.Verbose("Checking the GitHub Security Advisory Database for Vulnerabilities");
                     var ghsaVulnDict =
                         new GitHubAdvisoryDatabase.Scanner(_nuGetFile, _settings.GitHubAdvisoryDatabase.ApiToken, _settings.GitHubAdvisoryDatabase.BreakIfCannotRun)
                             .GetVulnerabilitiesForPackages(uncachedPkgs.ToArray());
-                    options.Cache.UpdateCache(ghsaVulnDict, uncachedPkgs, GitHubAdvisoryDatabaseSourceId);
+                    options.Cache.UpdateCache(ghsaVulnDict, uncachedPkgs, gitHubAdvisoryDatabaseSourceId);
 
                     if (vulnDict == null)
                         vulnDict = ghsaVulnDict;
                     else
                         MergeVulnDict(ref vulnDict, ref ghsaVulnDict);
-                    options.Cache.GetPackagesCachedVulnerabilitiesForSource(cachedPackages, GitHubAdvisoryDatabaseSourceId, ref vulnDict);
+                    options.Cache.GetPackagesCachedVulnerabilitiesForSource(cachedPackages, gitHubAdvisoryDatabaseSourceId, ref vulnDict);
                 }
             }
 
diff --git a/Src/NuGetDefense.Lib/VulnerabilityReporter.cs b/Src/NuGetDefense.Lib/VulnerabilityReporter.cs
@@ -35,14 +35,14 @@ public void BuildVulnerabilityReport(
             VulnerabilitiesCount = vulnerabilityDictionary.Sum(x => x.Value.Count),
             Packages = distinctPackages.OrderBy(x => x.Id).ThenBy(x => x.Version)
                 .Where(p => p.LineNumber != null
-                            && vulnerabilityDictionary.ContainsKey(p.PackageUrl.ToLower())
-                            && vulnerabilityDictionary[p.PackageUrl.ToLower()].Any()
+                            && vulnerabilityDictionary.ContainsKey(p.PackageUrl)
+                            && vulnerabilityDictionary[p.PackageUrl].Any()
                 )
                 .Select(p => new VulnerableNuGetPackage
                 {
                     Id = p.Id,
                     Version = p.Version,
-                    Vulnerabilities = vulnerabilityDictionary[p.PackageUrl.ToLower()].Select(
+                    Vulnerabilities = vulnerabilityDictionary[p.PackageUrl].Select(
                         v => new ReportedVulnerability
                         {
                             Description = v.Value.Description,
@@ -64,9 +64,9 @@ public void BuildVulnerabilityTextReport(Dictionary<string, Dictionary<string, V
         var logBuilder = new StringBuilder(VulnerabilityTextReport);
         var nuGetPackages = pkgs as NuGetPackage[] ?? pkgs.ToArray();
         logBuilder.AppendLine($"{vulnerabilityDictionary.Sum(ve => ve.Value.Count)} vulnerabilities found in {nuGetPackages.Count()} packages for {nuGetFile}.");
-        foreach (var pkg in nuGetPackages.Where(p => vulnerabilityDictionary.ContainsKey(p.PackageUrl.ToLower())))
+        foreach (var pkg in nuGetPackages.Where(p => vulnerabilityDictionary.ContainsKey(p.PackageUrl)))
         {
-            var vulnerabilities = vulnerabilityDictionary[pkg.PackageUrl.ToLower()];
+            var vulnerabilities = vulnerabilityDictionary[pkg.PackageUrl];
 
             logBuilder.AppendLine("*************************************");
             // TODO: Dependencies will need to be listed by package url when this is used.
diff --git a/Src/NuGetDefense/NuGetDefense.csproj b/Src/NuGetDefense/NuGetDefense.csproj
@@ -64,7 +64,6 @@
 
     <ItemGroup>
         <None Remove="Tests\**"/>
-        <None Include="TestFiles\test.csproj"/>
     </ItemGroup>
 
     <ItemGroup>
diff --git a/Src/NuGetDefenseTests/GlobalUsings.cs b/Src/NuGetDefenseTests/GlobalUsings.cs
@@ -0,0 +1,7 @@
+// Global using directives
+
+global using NuGetDefense;
+global using NuGetDefense.Core;
+global using Xunit;
+global using Xunit.Abstractions;
+global using static NuGetDefense.UtilityMethods;
diff --git a/Src/NuGetDefenseTests/IssueAttribute.cs b/Src/NuGetDefenseTests/IssueAttribute.cs
@@ -1,8 +1,4 @@
 ﻿using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace NuGetDefenseTests
 {
diff --git a/Src/NuGetDefenseTests/NuGetDefenseTests.csproj b/Src/NuGetDefenseTests/NuGetDefenseTests.csproj
@@ -32,4 +32,9 @@
         <ProjectReference Include="..\NuGetDefense\NuGetDefense.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+        <Content Include="TestFiles\*.*">
+            <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </Content>
+    </ItemGroup>
 </Project>
diff --git a/Src/NuGetDefenseTests/NuGetDefenseTests.csproj.DotSettings b/Src/NuGetDefenseTests/NuGetDefenseTests.csproj.DotSettings
@@ -0,0 +1,2 @@
+﻿<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=testfiles/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
diff --git a/Src/NuGetDefenseTests/SqliteCacheTests.cs b/Src/NuGetDefenseTests/SqliteCacheTests.cs
@@ -3,9 +3,6 @@
 using System.IO;
 using System.Linq;
 using Microsoft.Data.Sqlite;
-using NuGetDefense;
-using NuGetDefense.Core;
-using Xunit;
 
 namespace NuGetDefenseTests;
 
diff --git a/Src/NuGetDefenseTests/TestFiles/NuGetDefense.json b/Src/NuGetDefenseTests/TestFiles/NuGetDefense.json
diff --git a/Src/NuGetDefenseTests/TestFiles/packages.config b/Src/NuGetDefenseTests/TestFiles/packages.config
diff --git a/Src/NuGetDefenseTests/TestFiles/test.csproj b/Src/NuGetDefenseTests/TestFiles/test.csproj
diff --git a/Src/NuGetDefenseTests/VulnerabilityReportsTest.cs b/Src/NuGetDefenseTests/VulnerabilityReportsTest.cs
@@ -1,13 +1,17 @@
 using System.Collections.Generic;
-using NuGetDefense;
-using NuGetDefense.Core;
-using Xunit;
-using static NuGetDefense.UtilityMethods;
+using System.IO;
 
 namespace NuGetDefenseTests;
 
 public class VulnerabilityReportsTest
 {
+    private readonly ITestOutputHelper _testOutputHelper;
+
+    public VulnerabilityReportsTest(ITestOutputHelper testOutputHelper)
+    {
+        _testOutputHelper = testOutputHelper;
+    }
+
     [Fact]
     public void ReportVulnerabilityWithNullReferences()
     {
@@ -64,4 +68,24 @@ public void IgnoreVulnerabilitiesForPackage()
         Assert.True(pkgs[0].Version == "1.0.1");
         Assert.True(pkgs[0].LineNumber == 1);
     }
+
+    [Fact]
+    public void RunTwice()
+    {
+        var options = new ScanOptions()
+        {
+            ProjectFile = new FileInfo("./TestFiles/test.csproj"),
+            CheckTransitiveDependencies = true,
+            WarnOnly = false,    
+        };
+        var scanner = new Scanner();
+        var retVal = scanner.Scan(options);
+        Assert.Equal(4, retVal);
+        _testOutputHelper.WriteLine($"depcheck returned {retVal}");
+        
+        retVal = scanner.Scan(options);        
+        Assert.Equal(4, retVal);
+    }
+    
+    //{"pkg:nuget/jQuery@3.3.1":{"CVE-2019-11358":{},"CVE-2020-11023":{},"CVE-2020-23064":{}},"pkg:nuget/jQuery.Validation@1.17.0":{"CVE-2021-43306":{}}}
 }

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,4 @@`
`1`	`1`	`using System;`
`2`		`-using System.Collections.Generic;`
`3`		`-using System.Linq;`
`4`		`-using System.Text;`
`5`		`-using System.Threading.Tasks;`
`6`	`2`
`7`	`3`	`namespace NuGetDefenseTests`
`8`	`4`	`{`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">`
	`2`	`+ <s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=testfiles/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>`