diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/BenchmarkScore.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/BenchmarkScore.java index 427b36ba..9db9f63d 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/BenchmarkScore.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/BenchmarkScore.java @@ -147,7 +147,7 @@ public void execute() { /** * This is the original main() method used to invoke the scorecard generator. e.g., mvn validate - * -Pscorecard -Dexec.args="-cf ../julietjs/config/julietscoringconfig.yaml" + * -Pscorecard -Dexec.args="-cf ../TESTSUITENAME/config/testsuitescoringconfig.yaml" * * @param args - The command line arguments. */ @@ -576,7 +576,7 @@ private static void printExtraCWE( TestSuiteResults expectedResults, TestSuiteResults actualResults) { Set expectedCWE = new HashSet(); for (String testcase : expectedResults.keySet()) { - List list = expectedResults.get(testcase); + List list = expectedResults.getTestCaseResults(testcase); for (TestCaseResult t : list) { expectedCWE.add(t.getCWE()); } @@ -584,7 +584,7 @@ private static void printExtraCWE( Set actualCWE = new HashSet(); for (String testcase : actualResults.keySet()) { - List list = actualResults.get(testcase); + List list = actualResults.getTestCaseResults(testcase); if (list != null) { for (TestCaseResult t : list) { actualCWE.add(t.getCWE()); @@ -683,7 +683,7 @@ private static Map calculateScores(TestSuiteResults Map map = new TreeMap(); for (String testcase : actualResults.keySet()) { - TestCaseResult tcr = actualResults.get(testcase).get(0); // only one + TestCaseResult tcr = actualResults.getTestCaseResults(testcase).get(0); // only one String cat = Categories.getById(tcr.getCategory()).getName(); TP_FN_TN_FP_Counts c = map.get(cat); @@ -764,9 +764,10 @@ private static TestSuiteResults analyze( boolean pass = false; for (String testcase : expected.keySet()) { - TestCaseResult exp = expected.get(testcase).get(0); // always only one! + TestCaseResult exp = expected.getTestCaseResults(testcase).get(0); // always only one! List act = - rawToolResults.get(testcase); // could be lots of results for this test + rawToolResults.getTestCaseResults( + testcase); // could be lots of results for this test pass = compare(exp, act, rawToolResults.getToolName()); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/TestCaseResult.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/TestCaseResult.java index b4685416..f85b211d 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/TestCaseResult.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/TestCaseResult.java @@ -19,6 +19,8 @@ import org.owasp.benchmarkutils.helpers.Categories; import org.owasp.benchmarkutils.helpers.Category; +import org.owasp.benchmarkutils.score.parsers.Reader; +import org.owasp.benchmarkutils.score.service.ExpectedResultsProvider; import org.owasp.benchmarkutils.tools.AbstractTestCaseRequest; /* This class represents a single test case result. It documents the expected result (real), @@ -28,7 +30,9 @@ public class TestCaseResult { private String testCaseName = ""; // The name of the test case (E.g., BenchmarkTest00001) - private int number = 0; + // testID is the unique ID for this test case. For Benchmark Style, its a number (e.g., 1), for + // nonBenchmark Style its the name of the entire test case (e.g., FooBar_TryThis02). + private String testID = "0"; private boolean truePositive = false; // Is this test case a true or false positive? private boolean result = false; // Did a tool properly detect this as a true or false positive? private int CWE = 0; @@ -58,10 +62,8 @@ public TestCaseResult() { */ public TestCaseResult(AbstractTestCaseRequest request) { this.testCaseName = request.getName(); - this.number = request.getNumber(); this.truePositive = request.isVulnerability(); this.CWE = request.getCategory().getCWE(); - // this.category = request.getCategory().getName(); this.category = Categories.getByCWE(this.CWE).getName(); // fill in optional attributes since we have this data available @@ -82,27 +84,106 @@ public void setTestCaseName(String name) { * The name of the test case. E.g., BenchmarkTest00001 */ public String getTestCaseName() { - return testCaseName; + return this.testCaseName; } public int getConfidence() { - return confidence; + return this.confidence; } public void setConfidence(int confidence) { this.confidence = confidence; } - public int getNumber() { - return number; + public String getTestID() { + return this.testID; } - public void setNumber(int number) { - this.number = number; + /** + * Sets the unique identifier for this test case. For Benchmark style scoring, its the test ID + * number, converted to a String. + * + *

The use of this method should be converted to use setActualResultTestID() for tool + * parsers. + * + * @param id The unique test case number for this Benchmark style test case. + */ + @Deprecated + public void setTestID(int id) { + this.testID = String.valueOf(id); + } + + /** + * Sets the unique identifier for this test case. For Benchmark style, it parses out the test + * case number and uses that as the test ID. For non-Benchmark style scoring, it used the name + * of the tool result file to try to find the matching expected result test case name. In this + * case, if the reported filename from the tool starts with the name of a test case, then the + * test case name, is used as the testID. That way if there are multi-file test cases that all + * start with the same name, they will all match up against the expected result name with that + * name. + * + * @param id The test case file name, without path information. + */ + public void setActualResultTestID(String testCaseFileName) { + if (ExpectedResultsProvider.isBenchmarkStyleScoring()) { + // Sets the test ID to the test case # or -1 if not a match + this.testID = + String.valueOf(Reader.getBenchmarkStyleTestCaseNumber(testCaseFileName.trim())); + } else { + if (testCaseFileName.contains("/") || testCaseFileName.contains("\\")) { + new IllegalArgumentException( + "FATAL ERROR: testCaseFileName value: " + + testCaseFileName + + " passed to setActualResultTestID() can't have any path information") + .printStackTrace(); + System.exit(-1); + } + // For actual results, we look for a matching test case name, and set that as the testID + String matchingID = + ExpectedResultsProvider.getExpectedResults() + .getMatchingTestCaseName(testCaseFileName); + // TODO: Maybe null is OK, and we should simply set the test ID to -1, like we do for + // Benchmark + if (matchingID == null) { + new IllegalArgumentException( + "FATAL ERROR: testCaseFileName value: " + + testCaseFileName + + " passed to setActualResultTestID() doesn't match any expected results test case name.") + .printStackTrace(); + System.exit(-1); + } + this.testID = matchingID; + } + } + + /** + * Sets the unique identifier for this test case. For non-Benchmark style scoring, its the name + * of the test case. For Benchmark style, it parses out the test case number and uses that as + * the test ID. + * + * @param id The test case file name, without path information. + */ + public void setExpectedResultTestID(String testCaseFileName) { + if (ExpectedResultsProvider.isBenchmarkStyleScoring()) { + // Sets the test ID to the test case # or -1 if not a match + this.testID = + String.valueOf(Reader.getBenchmarkStyleTestCaseNumber(testCaseFileName.trim())); + } else { + if (testCaseFileName.contains("/") || testCaseFileName.contains("\\")) { + new IllegalArgumentException( + "FATAL ERROR: testCaseFileName value: " + + testCaseFileName + + " passed to setExpectedResultTestID() can't have any path information") + .printStackTrace(); + System.exit(-1); + } + // For expected results, we don't change the test case file name + this.testID = testCaseFileName.trim(); + } } public boolean isTruePositive() { - return truePositive; + return this.truePositive; } public void setTruePositive(boolean truePositive) { @@ -110,7 +191,7 @@ public void setTruePositive(boolean truePositive) { } public boolean isPassed() { - return result; + return this.result; } public void setPassed(boolean result) { @@ -118,7 +199,7 @@ public void setPassed(boolean result) { } public int getCWE() { - return CWE; + return this.CWE; } public void setCWE(int cwe) { @@ -127,7 +208,25 @@ public void setCWE(int cwe) { if (category != null) { this.category = category.getId(); } else { - this.category = this.UNMAPPED_CATEGORY; + this.category = TestCaseResult.UNMAPPED_CATEGORY; + } + } + + /** + * This method is used to abstract away how filenames are matched against a test case, that way + * it can be enhanced to support different test suite formats, and the logic on how to do this + * isn't implemented in every individual parser. This method expects that + * ExpectedResultsProvider.getExpectedResults().isTestCaseFile(filename) was called first to + * verify this file is a test case in the test suite being scored. It sets the CWE number and + * the test case name in this TestCaseResult if successful. It halts with an error if the + * supplied filename is not a valid test case. + * + * @param cwe The CWE # reported by this tool. + * @param filename The filename that might be a test case. + */ + public void setCWEAndTestCaseID(int cwe, String filename) { + if (ExpectedResultsProvider.getExpectedResults().isTestCaseFile(filename)) { + // TODO } } @@ -137,7 +236,7 @@ public void setCWE(int cwe) { * @return The descriptive name of this CWE, per categories.xml */ public String getCategory() { - return category; + return this.category; } /* @@ -155,7 +254,7 @@ public void setCategory(String category) { } */ public String getEvidence() { - return evidence; + return this.evidence; } public void setEvidence(String evidence) { @@ -188,8 +287,8 @@ public void setSink(String sink) { @Override public String toString() { - return "Testcase #: " - + getNumber() + return "Testcase ID: " + + getTestID() + ", Category: " + getCategory() + ", isVulnerable: " diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/TestSuiteResults.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/TestSuiteResults.java index 82b02551..815de871 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/TestSuiteResults.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/TestSuiteResults.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import org.owasp.benchmarkutils.score.service.ExpectedResultsProvider; /** * TestSuiteResults contains the expected results for each test case in a test suite, if its @@ -48,7 +49,6 @@ public static enum ToolType { // The name and version of the test suite these test results are for private String testSuiteName = "notSet"; private String testSuiteVersion = "notSet"; - private boolean standardBenchmarkStyleScoring = true; private String toolName = "Unknown Tool"; private String toolVersion = null; @@ -95,11 +95,11 @@ public String getTestSuiteVersion() { } public ToolType getToolType() { - return toolType; + return this.toolType; } public boolean isCommercial() { - return isCommercial; + return this.isCommercial; } /** @@ -109,42 +109,51 @@ public boolean isCommercial() { */ public void put(TestCaseResult tcr) { - int testCaseNum = tcr.getNumber(); - String testCaseKey; - - // If we are using test case numbers, we add each result to that specific test case number - if (this.standardBenchmarkStyleScoring - && (testCaseNum != TestCaseResult.NOT_USING_TESTCASE_NUMBERS)) { - // This warning message is added just in case. It can be caused by a buggy parser or - // invalid results file. - if ((testCaseNum <= 0 || testCaseNum > 10000)) { + String testCaseKey = tcr.getTestID(); + + // If we are using test case numbers, just check to make sure the test case number is within + // a reasonable valid range, and if not, provide a warning. + if (ExpectedResultsProvider.isBenchmarkStyleScoring()) { + try { + int testCaseNum = Integer.parseInt(testCaseKey); + // This warning message is added just in case. It can be caused by a buggy parser or + // invalid results file. + if ((testCaseNum <= 0 || testCaseNum > 10000)) { + System.out.println( + "WARNING: Did you really intend to add a test case result for test case: " + + tcr.getTestCaseName() + + " with TestID: " + + testCaseKey + + " and testCaseNum: " + + testCaseNum); + new Exception().printStackTrace(); + } + } catch (NumberFormatException e) { System.out.println( - "WARNING: Did you really intend to add a test case result for test case: " - + testCaseNum); + "FATAL INTERNAL ERROR: testCaseKey: '" + + testCaseKey + + "' for test case: " + + tcr.getTestCaseName() + + " is supposed to be an integer."); + System.exit(-1); } - - testCaseKey = String.valueOf(testCaseNum); - } else { - // otherwise use test case names as the key, and we add each result by test case name - testCaseKey = tcr.getTestCaseName(); - this.standardBenchmarkStyleScoring = false; } // There is a list of results for each test case - List results = testCaseResults.get(testCaseKey); + List results = this.testCaseResults.get(testCaseKey); if (results == null) { // If there are no results yet for this test case, create a List. // Add this entry for this test case to the set of results results = new ArrayList(); - testCaseResults.put(testCaseKey, results); + this.testCaseResults.put(testCaseKey, results); } // Add this specific result to this test case's results results.add(tcr); } - public List get(String tn) { - return testCaseResults.get(tn); + public List getTestCaseResults(String tn) { + return this.testCaseResults.get(tn); } /** @@ -153,7 +162,7 @@ public List get(String tn) { * @return The Set of Keys. */ public Set keySet() { - return testCaseResults.keySet(); + return this.testCaseResults.keySet(); } /** @@ -173,7 +182,7 @@ public String getToolName() { * @return Name of the tool. */ public String getToolNameAndVersion() { - if (!anonymous + if (!this.anonymous && this.toolVersion != null && !"".equals(this.toolVersion) && !(BenchmarkScore.config.anonymousMode && this.isCommercial)) { @@ -188,7 +197,44 @@ public String getToolNameAndVersion() { * @return Version of the tool if determined. Null otherwise. */ public String getToolVersion() { - return toolVersion; + return this.toolVersion; + } + + /** + * Determines whether the provided filename (without path info) is a test case file in this test + * suite or not + * + * @param testCaseFilename The name of the test case file to match against + * @return True if a test case file, false otherwise. + */ + public boolean isTestCaseFile(String testCaseFilename) { + return (getMatchingTestCaseName(testCaseFilename) != null); + } + + /** + * Determines whether the provided filename (without path info) is a test case file in this test + * suite or not. If so, it returns the name of the test cast name this file belongs to. + * + * @param testCaseFilename The name of the test case file to match against + * @return The corresponding test case name, or null if not a match + */ + public String getMatchingTestCaseName(String testCaseFilename) { + if (ExpectedResultsProvider.isBenchmarkStyleScoring()) { + if (testCaseFilename.startsWith(BenchmarkScore.TESTCASENAME)) return testCaseFilename; + } else { + // If filename exactly matches test case file name, return the filename. + // This is done as a quick short circuit, as it is MUCH faster than searching + // through ALL test cases to look for a match + if (this.testCaseResults.get(testCaseFilename) != null) return testCaseFilename; + // Or filename starts with ANY expected results test case name, return the matching test + // case name. + Set testcases = this.testCaseResults.keySet(); + for (String testcasename : testcases) { + if (testCaseFilename.startsWith(testcasename)) return testcasename; + } + // if no match, fall through to return null + } + return null; } /** diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AcunetixReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AcunetixReader.java index 12f2065d..2a286de9 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AcunetixReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AcunetixReader.java @@ -144,8 +144,8 @@ private TestCaseResult parseAcunetixVulnerability(Node vuln) throws Exception { if (testfile.contains("?")) { testfile = testfile.substring(0, testfile.indexOf("?")); } - if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + if (isTestCaseFile(testfile)) { + tcr.setActualResultTestID(testfile); Node classification = getNamedChild("classification", vuln); Node vulnId = getNamedChild("cwe", classification); @@ -203,7 +203,7 @@ private TestCaseResult parseAcunetixReportItem(Node flaw) throws Exception { } if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanDynamicReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanDynamicReader.java index 71369ae0..3f0f2519 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanDynamicReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanDynamicReader.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestCaseResult; import org.owasp.benchmarkutils.score.TestSuiteResults; @@ -160,8 +159,8 @@ private TestCaseResult parseAppScanDynamicVulnerability( String testcase = getNamedChild("Url", issue).getTextContent(); testcase = testcase.substring(testcase.lastIndexOf('/') + 1); testcase = testcase.split("\\.")[0]; - if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase)); + if (isTestCaseFile(testcase)) { + tcr.setActualResultTestID(testcase); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanSourceReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanSourceReader.java index 65efb7f4..5954d7d4 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanSourceReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/AppScanSourceReader.java @@ -93,30 +93,36 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { } if (filename.endsWith(".java") && filename.startsWith(BenchmarkScore.TESTCASENAME)) { filename = filename.substring(0, filename.length() - 5); - int testno = testNumber(filename); - tn = testno; + tn = getBenchmarkStyleTestCaseNumber(filename); } - Set findings = assess.get(fileid); - for (int findingid : findings) { - TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(tn); - int vid = Integer.parseInt(finding.get(findingid)); - String confString = conf.get(findingid); - int confidence = Integer.parseInt(confString); - - String vtype = vulns.get(vid); - - tcr.setCWE(cweLookup(vtype)); - tcr.setEvidence(vtype); - tcr.setConfidence(confidence); - - // Exclude Confidence 3 - apparently these are "scan coverage" - // We tried excluding Confidence 2 as well - as these are "suspect", but AppScan's - // score actually went down because it excludes ALL of the weak randomness findings. - // Confidence 1 - are "definitive" findings - if (confidence < 3) { - tr.put(tcr); + if (isTestCaseFile(filename)) { + Set findings = assess.get(fileid); + for (int findingid : findings) { + TestCaseResult tcr = new TestCaseResult(); + tcr.setActualResultTestID(filename); + int vid = Integer.parseInt(finding.get(findingid)); + String confString = conf.get(findingid); + int confidence = Integer.parseInt(confString); + + String vtype = vulns.get(vid); + + int cwe = cweLookup(vtype); + // Exclude unmapped findings + if (cwe == CweNumber.DONTCARE || cwe == CweNumber.UNKNOWN) continue; + tcr.setCWE(cwe); + tcr.setEvidence(vtype); + tcr.setConfidence(confidence); + + // Exclude Confidence 3 - apparently these are "scan coverage" + // We tried excluding Confidence 2 as well - as these are "suspect", but + // AppScan's + // score actually went down because it excludes ALL of the weak randomness + // findings. + // Confidence 1 - are "definitive" findings + if (confidence < 3) { + tr.put(tcr); + } } } } @@ -137,16 +143,22 @@ private String parseTime(String message) { private int cweLookup(String vtype) { switch (vtype) { - // case "Vulnerability.AppDOS" : return 00; - // case "Vulnerability.Authentication.Entity" : return 00; + case "Vulnerability.AppDOS": + case "Vulnerability.Authentication.Entity": + case "Vulnerability.ErrorHandling.RevealDetails.Message": + case "Vulnerability.ErrorHandling.RevealDetails.StackTrace": + case "Vulnerability.Malicious.DynamicCode": + case "Vulnerability.Malicious.DynamicCode.Execution": + case "Vulnerability.Quality.TestCode": + case "Vulnerability.Quality.Unsupported": + return CweNumber.DONTCARE; + case "Vulnerability.Cryptography.InsecureAlgorithm": return CweNumber.STATIC_CRYPTO_INIT; case "Vulnerability.Cryptography.PoorEntropy": return CweNumber.WEAK_RANDOM; case "Vulnerability.Cryptography.????WeakHash": return CweNumber.WEAK_HASH_ALGO; // They don't have a weak hashing rule - // case "Vulnerability.ErrorHandling.RevealDetails.Message" : return 00; - // case "Vulnerability.ErrorHandling.RevealDetails.StackTrace" : return 00; case "Vulnerability.Injection.HttpResponseSplitting": return CweNumber.HTTP_RESPONSE_SPLITTING; case "Vulnerability.Injection.LDAP": @@ -157,20 +169,19 @@ private int cweLookup(String vtype) { return CweNumber.SQL_INJECTION; case "Vulnerability.Injection.XPath": return CweNumber.XPATH_INJECTION; - // case "Vulnerability.Malicious.DynamicCode" : return 00; - // case "Vulnerability.Malicious.DynamicCode.Execution" : return 00; case "Vulnerability.PathTraversal": return CweNumber.PATH_TRAVERSAL; - // case "Vulnerability.Quality.TestCode" : return 00; - // case "Vulnerability.Quality.Unsupported" : return 00; case "Vulnerability.SessionManagement.Cookies": return CweNumber.INSECURE_COOKIE; case "Vulnerability.Validation.EncodingRequired": return CweNumber.XSS; case "Vulnerability.Validation.Required": return CweNumber.TRUST_BOUNDARY_VIOLATION; + + default: + System.out.println("Unknown vuln type for AppScanSource: " + vtype); } - return 0; + return CweNumber.UNKNOWN; } /** diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ArachniReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ArachniReader.java index 50bf6d6a..f217cdda 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ArachniReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ArachniReader.java @@ -164,7 +164,7 @@ private TestCaseResult parseArachniIssue(Node flaw) throws URISyntaxException { testfile = testfile.substring(testfile.lastIndexOf('/') + 1); if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BearerReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BearerReader.java index ca7688a0..6ba0fd52 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BearerReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BearerReader.java @@ -88,7 +88,7 @@ private TestCaseResult parseBearerFindings(JSONObject result) { tcr.setCWE(cwe); tcr.setEvidence(evidence); tcr.setConfidence(0); - tcr.setNumber(testNumber(className)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(className)); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpJsonReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpJsonReader.java index 31d51c46..4b7eef1b 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpJsonReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpJsonReader.java @@ -97,7 +97,7 @@ private TestCaseResult parseBurpJSONFinding(JSONObject finding) { filename.split("\\.")[ 0]; // If there is any extension on the filename, remove it if (filename.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); int rule = issue.getInt("type_index"); int cwe = BurpReader.cweLookup(new Integer(rule).toString()); tcr.setCWE(cwe); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpReader.java index dc408815..1ad4609f 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/BurpReader.java @@ -92,7 +92,7 @@ private TestCaseResult parseBurpVulnerability(Node issue) { testcase = testcase.substring(testcase.lastIndexOf('/') + 1); testcase = testcase.split("\\.")[0]; if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReader.java index 14056fce..9c190f1d 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReader.java @@ -90,7 +90,7 @@ private TestCaseResult parseCASTAIPIssue(Node flaw) throws Exception { filename = filename.replaceAll("\\\\", "/"); filename = filename.substring(filename.lastIndexOf('/') + 1); if (filename.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxESReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxESReader.java index b6a54233..00f9850b 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxESReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxESReader.java @@ -143,7 +143,7 @@ private TestCaseResult parseCheckmarxFindings( String resultFileName = nodes.getJSONObject(0).getString("FileName").replace("\\", "/"); String testcaseName = resultFileName.substring(resultFileName.lastIndexOf('/') + 1); if (testcaseName.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcaseName)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcaseName)); return tcr; } else { resultFileName = @@ -152,7 +152,7 @@ private TestCaseResult parseCheckmarxFindings( .replace("\\", "/"); testcaseName = resultFileName.substring(resultFileName.lastIndexOf('/') + 1); if (testcaseName.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcaseName)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcaseName)); return tcr; } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReader.java index e1096d41..3ef39f5c 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReader.java @@ -45,7 +45,6 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { for (CSVRecord record : records) { String checkerKey = record.get("Vulnerability Type"); String url = record.get("URL"); - // System.out.println("URL = "+url); //For debugging YE TestCaseResult tcr = new TestCaseResult(); tcr.setCWE(cweLookup(checkerKey)); @@ -58,18 +57,14 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { Matcher testCaseMatcher = testCasePattern.matcher(url); if (testCaseMatcher.find()) { String testCase = testCaseMatcher.group(0); - // System.out.println("testCase = "+testCase+" Test Num = - // "+testCase.substring(testCase.length()-Utils.TESTCASE_DIGITS, - // testCase.length())); // For debugging YE tcr.setTestCaseName(testCase); - // BenchmarkTest00000 - BenchmarkTest99999 - tcr.setNumber(testNumber(testCase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testCase)); if (tcr.getCWE() != 0) { tr.put(tcr); } } } - tr.setTime("100"); + return tr; } @@ -168,7 +163,8 @@ private int cweLookup(String checkerKey) { default: System.out.println( - "WARNING: Unmapped Vulnerability category detected: " + checkerKey); + "WARNING: Unmapped Vulnerability category for CheckmarxIAST detected: " + + checkerKey); } return 0; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReader.java index 3a5a3502..2d9d0379 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReader.java @@ -151,9 +151,9 @@ private TestCaseResult parseCheckmarxVulnerability(Node query, Node result) { tcr.setEvidence(getAttributeValue("name", query)); /* Some results do not appear in the previous version of this parser because it only look for the attribute "FileName" - * Checkmarx have some results where the input does not start in a BenchmarkTest file so it was necessary to make some changes - * We must consider a good result if the result node "FileName" startsWith BenchmarkTest file or if the last PathNode ends in a "FileName" that startsWith BenchmarkTest file. - * An example is the SeparateClassRequest.java file that have some inputs that we catch but are not been considered as a valid result (FN) + * Checkmarx have some results where the input does not start in a BenchmarkTest file so it was necessary to make some changes + * We must consider a good result if the result node "FileName" startsWith BenchmarkTest file or if the last PathNode ends in a "FileName" that startsWith BenchmarkTest file. + * An example is the SeparateClassRequest.java file that have some inputs that we catch but are not been considered as a valid result (FN) */ // Get the Path element inside Result @@ -176,7 +176,7 @@ private TestCaseResult parseCheckmarxVulnerability(Node query, Node result) { testcase = testcase.substring(testcase.lastIndexOf('\\') + 1); } if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); return tcr; } else { // If not, then the last PastNode must end in a FileName that startsWith BenchmarkTest @@ -193,7 +193,7 @@ private TestCaseResult parseCheckmarxVulnerability(Node query, Node result) { testcase2 = testcase2.substring(testcase2.lastIndexOf('\\') + 1); } if (testcase2.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase2)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase2)); return tcr; } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ContrastAssessReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ContrastAssessReader.java index d1129b73..0cc1124f 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ContrastAssessReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ContrastAssessReader.java @@ -107,7 +107,7 @@ private void parseContrastNodeFinding(TestSuiteResults tr, String line) throws E // Node findings look like: // debug: 2021-05-12T21:00:46.118Z 12631 contrast:rules:sinks - crypto-bad-mac: - // /julietjs/sqli-00/JulietJSTest00001 + // /TESTSUITENAME/sqli-00/TestSuiteTest00001 // However, there are similar lines like this we have to avoid: // debug: 2021-05-12T21:00:30.487Z 12631 contrast:rules:sinks - loading provider for // hardcoded-password @@ -131,7 +131,7 @@ private void parseContrastNodeFinding(TestSuiteResults tr, String line) throws E tcr.setCWE(cweLookup(elements[0])); if (tcr.getCWE() != 0 && elements[1].contains(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(elements[1])); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(elements[1])); tr.put(tcr); } } @@ -156,7 +156,7 @@ private void parseContrastJavaFinding(TestSuiteResults tr, String json) throws E // web services, they can look like: // "uri":"/benchmark/rest/xxe-00/BenchmarkTest03915/send" // At this point testNumber could contain '00215', or '03915/send' - tcr.setNumber(testNumber(uri)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(uri)); // System.out.println( tcr.getNumber() + "\t" + tcr.getCWE() + "\t" + // tcr.getCategory() ); tr.put(tcr); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CoverityReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CoverityReader.java index d678705f..fbe6a35d 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CoverityReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CoverityReader.java @@ -72,7 +72,7 @@ private TestCaseResult parseCoverityFinding(JSONObject finding, int version) { filename = filename.replaceAll("\\\\", "/"); filename = filename.substring(filename.lastIndexOf('/') + 1); if (filename.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); JSONObject props = finding.getJSONObject("checkerProperties"); String cweNumber = props.getString("cweCategory"); if (cweNumber == null || cweNumber.equals("none")) { @@ -98,7 +98,7 @@ private TestCaseResult parseCoverityFinding(JSONObject finding, int version) { filename = filename.replaceAll("\\\\", "/"); filename = filename.substring(filename.lastIndexOf('/') + 1); if (filename.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); if (finding.isNull("cweNumber")) { return null; } @@ -137,7 +137,7 @@ private TestCaseResult parseCoverityFindingV2(JSONObject finding) { filename = filename.substring(filename.lastIndexOf('/') + 1); if (filename.startsWith(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); // // *** Warning: serious foefeling and cutting of corners ahead. *** // diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CrashtestReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CrashtestReader.java index b7eaaecf..b015f25d 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CrashtestReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/CrashtestReader.java @@ -152,7 +152,7 @@ private TestCaseResult parseCrashtestIssue(Node testcase) throws URISyntaxExcept testfile = testfile.substring(testfile.lastIndexOf('/') + 1); if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/DatadogReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/DatadogReader.java index 3fcca534..515b5f7b 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/DatadogReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/DatadogReader.java @@ -124,7 +124,7 @@ private void process(final TestSuiteResults tr, String testNumber, final List Parse error: " + line); } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FaastReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FaastReader.java index b5ddfc09..cf6a89d8 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FaastReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FaastReader.java @@ -76,7 +76,7 @@ private TestCaseResult parseFaastFinding(JSONObject finding) { } if (url.contains(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber); + tcr.setTestID(testNumber); tcr.setCWE(cwe); tcr.setEvidence(category); return tcr; @@ -93,6 +93,6 @@ private String getCategory(String url) { } private int getTestCase(String url) { - return testNumber(url); + return getBenchmarkStyleTestCaseNumber(url); } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FindbugsReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FindbugsReader.java index b961921a..439026e7 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FindbugsReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FindbugsReader.java @@ -20,7 +20,7 @@ import java.io.StringReader; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import org.owasp.benchmarkutils.score.BenchmarkScore; +import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestCaseResult; import org.owasp.benchmarkutils.score.TestSuiteResults; @@ -95,14 +95,18 @@ private TestCaseResult parseFindBugsBug(Node n) { Node cl = getNamedNode("Class", n.getChildNodes()); String classname = cl.getAttributes().getNamedItem("classname").getNodeValue(); classname = classname.substring(classname.lastIndexOf('.') + 1); - if (classname.startsWith(BenchmarkScore.TESTCASENAME)) { + // TODO: Globally replace all instances like this across all parsers. + // if (classname.startsWith(BenchmarkScore.TESTCASENAME)) { + if (isTestCaseFile(classname)) { + // System.out.println("Class: " + classname + " IS a test case file."); TestCaseResult tcr = new TestCaseResult(); try { - tcr.setNumber(testNumber(classname)); + // TODO: Globally replace all instances like this across all parsers. + // tcr.setTestID(getBenchmarkStyleTestCaseNumber(classname)); + tcr.setActualResultTestID(classname); Node cwenode = attrs.getNamedItem("cweid"); Node catnode = attrs.getNamedItem("abbrev"); tcr.setCWE(figureCWE(tcr, cwenode, catnode)); - String type = attrs.getNamedItem("type").getNodeValue(); tcr.setEvidence(type); @@ -131,7 +135,7 @@ private int figureCWE(TestCaseResult tcr, Node cwenode, Node catnode) { tcr.setEvidence("FB:" + cwe + "::" + cat); if (cwe != null) { - // FIX path traversal CWEs + // Map 2x path traversal CWEs to the standard CWE if (cwe.equals("23") || cwe.equals("36")) { cwe = "22"; } @@ -149,109 +153,105 @@ else if (cwe.equals("326")) { switch (cat) { // Cookies case "SECIC": - return 614; // insecure cookie use + return CweNumber.INSECURE_COOKIE; case "SECCU": - return 00; // servlet cookie + return CweNumber.DONTCARE; // Cookie Usage- not a vuln case "SECHOC": - return 00; // HTTP Only not set on cookie - Information Leak / Disclosure - // (CWE-200)?? + return CweNumber.COOKIE_WITHOUT_HTTPONLY; // Injections case "SECSQLIHIB": - return 564; // Hibernate Injection, child of SQL Injection + return CweNumber.HIBERNATE_INJECTION; // Hibernate Injection, child of SQL Injection + case "SECSQLIJDO": - return 89; case "SECSQLIJPA": - return 89; case "SECSQLISPRJDBC": - return 89; case "SECSQLIJDBC": - return 89; + return CweNumber.SQL_INJECTION; // LDAP injection case "SECLDAPI": - return 90; // LDAP injection + return CweNumber.LDAP_INJECTION; + + // Potential XML injection + case "SECXML": + return 91; // XML injection // XPath injection case "SECXPI": - return 643; // XPATH injection + return CweNumber.XPATH_INJECTION; // Command injection case "SECCI": - return 78; // command injection + return CweNumber.COMMAND_INJECTION; // Weak random case "SECPR": - return 330; // weak random + return CweNumber.WEAK_RANDOM; // Weak encryption - case "SECDU": - return 327; // weak encryption DES - case "CIPINT": - return 327; // weak encryption - cipher with no integrity - case "PADORA": - return 327; // padding oracle -- FIXME: probably wrong + case "SECDU": // weak encryption DES + case "CIPINT": // weak encryption - cipher with no integrity + case "PADORA": // padding oracle -- FIXME: probably wrong + return CweNumber.WEAK_CRYPTO_ALGO; + case "STAIV": return 329; // static initialization vector for crypto // Weak hash case "SECWMD": - return 328; // weak hash + return CweNumber.WEAK_HASH_ALGO; // Path traversal case "SECPTO": - return 22; // path traversal case "SECPTI": - return 22; // path traversal + return CweNumber.PATH_TRAVERSAL; // XSS case "SECXRW": - return 79; // XSS case "SECXSS1": - return 79; // XSS case "SECXSS2": - return 79; // XSS + return CweNumber.XSS; // XXE case "SECXXEDOC": - return 611; // XXE case "SECXXEREAD": - return 611; // XXE case "SECXXESAX": - return 611; // XXE + return CweNumber.XXE; // Input sources - case "SECSP": - return 00; // servlet parameter - not a vuln - case "SECSH": - return 00; // servlet header - not a vuln - case "SECSHR": - return 00; // Use of Request Header -- spoofable - case "SECSSQ": - return 00; // servlet query - not a vuln + case "SECSCT": // Servlet Content Type - not a vuln + case "SECSP": // Servlet Parameter - not a vuln + case "SECSH": // Servlet Header - not a vuln + case "SECSHR": // Use of Request Header -- spoofable + case "SECSSQ": // Servlet Query - not a vuln + return CweNumber.DONTCARE; // Technology detection - case "SECSC": - return 00; // found Spring endpoint - not a vuln - case "SECJRS": - return 00; // JAX-RS Endpoint + case "SECSC": // found Spring endpoint - not a vuln + case "SECJRS": // JAX-RS Endpoint + return CweNumber.DONTCARE; // Configuration - case "SECOPFP": - return 00; // Overly Permissive File Permissions + case "SECOPFP": // Overly Permissive File Permissions + return 732; // CWE-732: Incorrect Permission Assignment for Critical Resource // Other + case "SECFSM": + return 134; // Format String Manipulation case "SECHPP": - return 235; // HTTP Parameter Polution + return 235; // HTTP Parameter Pollution (HPP) case "SECUNI": - return 00; // Improper Unicode + return CweNumber.DONTCARE; // Improper Unicode case "SECWF": - return 00; // Weak Filename Utils - i.e., not filtering out Null bytes in file names + return CweNumber + .DONTCARE; // Weak Filename Utils - i.e., not filtering out Null bytes in + // file names default: System.out.println("Unknown vuln category for FindBugs: " + cat); } - return 0; + return CweNumber.UNKNOWN; } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReader.java index 5403f736..276cad27 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReader.java @@ -62,7 +62,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { if (testCaseName.startsWith(BenchmarkScore.TESTCASENAME)) { testCaseResult.setCWE(categoryToExpectedCwe(category)); testCaseResult.setEvidence(category); - testCaseResult.setNumber( + testCaseResult.setTestID( Integer.parseInt( testCaseName.substring( testCaseName.length() - BenchmarkScore.TESTIDLENGTH, diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FortifyReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FortifyReader.java index 4446125e..2c6127cd 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FortifyReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FortifyReader.java @@ -176,9 +176,16 @@ private TestCaseResult parseFortifyVulnerability(Node vuln) { // The first block looks for class names for Java findings. String tc = getAttributeValue("enclosingClass", function); - if (tc != null && tc.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(tc)); - return tcr; + if (tc != null) { + // Strip off inner class name from the test case file name if present + int dollar = tc.indexOf('$'); + if (dollar != -1) { + tc = tc.substring(0, dollar); + } + if (isTestCaseFile(tc)) { + tcr.setActualResultTestID(tc); + return tcr; + } } else { /* if tc is null (from attribute enclosingClass), then this might be a NodeJS finding that looks like this: @@ -186,17 +193,18 @@ private TestCaseResult parseFortifyVulnerability(Node vuln) { - + */ if (tc == null) { + // TODO: Test with other test suite and fix use of deprecated API as appropriate. Node functionDecl = getNamedNode("FunctionDeclarationSourceLocation", context.getChildNodes()); if (functionDecl != null) { String path = getAttributeValue("path", functionDecl); if (path != null) { int i = path.indexOf(BenchmarkScore.TESTCASENAME); - if (i > 0) { + if (i >= 0) { tc = path.substring(i); tc = tc.substring( @@ -207,7 +215,7 @@ private TestCaseResult parseFortifyVulnerability(Node vuln) { if (dollar != -1) { tc = tc.substring(0, dollar); } - tcr.setNumber(Integer.parseInt(tc)); + tcr.setTestID(Integer.parseInt(tc)); return tcr; } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FusionLiteInsightReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FusionLiteInsightReader.java index b2754227..90f29ce0 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FusionLiteInsightReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/FusionLiteInsightReader.java @@ -82,7 +82,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { TestCaseResult tcr = new TestCaseResult(); tcr.setCWE(findingCWE); tcr.setEvidence(findingName); - tcr.setNumber(testNumber); + tcr.setTestID(testNumber); tr.put(tcr); } } @@ -104,7 +104,7 @@ private static int extractTestNumber(String uri) throws URISyntaxException { testfile = testfile.substring(testfile.lastIndexOf('/') + 1); if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - int testno = testNumber(testfile); + int testno = getBenchmarkStyleTestCaseNumber(testfile); return testno; } return -1; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReader.java index 4e57376c..da3bc7d7 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReader.java @@ -136,7 +136,7 @@ private void parseFindings(TestSuiteResults tr, String json) throws Exception { String uri = request.getString("uri"); if (uri.contains(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(uri)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(uri)); if (tcr.getCWE() != 0) { // System.out.println( tcr.getNumber() + "\t" + tcr.getCWE() + "\t" + // tcr.getCategory() ); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReader.java index 0a0ec2c1..6573fde9 100755 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReader.java @@ -82,7 +82,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { // Parse out test number from: BenchmarkTest02603:99 try { if (filename.contains(BenchmarkScore.TESTCASENAME)) { - tn = testNumber(filename); + tn = getBenchmarkStyleTestCaseNumber(filename); if (tn < 0) { throw new Exception("Failed to get test number from file: " + filename); } @@ -97,7 +97,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { reportWarning("TestCase Number is bad for file: " + filename); } else { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(tn); + tcr.setTestID(tn); tcr.setCWE(vtype); tcr.setEvidence(issueType); tr.put(tcr); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReader.java index 68537fe1..3fa1b3e7 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReader.java @@ -146,7 +146,7 @@ private TestCaseResult TestCaseLookup(String issueType, String url) { if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { // Add the vuln found in a test case to the results for this tool TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); tcr.setCWE(vtype); tcr.setEvidence(issueType); return tcr; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HorusecReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HorusecReader.java index 0a89b8ac..b51fc7f2 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HorusecReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/HorusecReader.java @@ -80,7 +80,7 @@ private TestCaseResult parseTestCaseResult(JSONObject finding) { if (filename.contains(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); tcr.setCWE(figureCwe(vuln)); return tcr; } @@ -155,7 +155,7 @@ private String guessCwe(String details) { case "Message Digest": return "328"; case "Cookie without the HttpOnly flag": - return "614"; + return "1004"; case "Base64 Encode": return "649"; default: diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/InsiderReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/InsiderReader.java index 32fd757a..0cf122ac 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/InsiderReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/InsiderReader.java @@ -78,7 +78,7 @@ private TestCaseResult parseTestCaseResult(JSONObject finding) { if (filename.contains(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); int cwe = cweNumber(finding); tcr.setCWE(cwe); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/JuliaReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/JuliaReader.java index 996360e3..cea29a6e 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/JuliaReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/JuliaReader.java @@ -84,7 +84,7 @@ private TestCaseResult parseJuliaBug(Node n) { if (childName.equals("source")) { String where = child.getTextContent().replace('/', '.'); // "org.owasp.benchmark.testcode.BenchmarkTest00042.java" - tcr.setNumber(testNumber(where)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(where)); } else if (childName.equals("CWEid")) tcr.setCWE(Integer.parseInt(child.getTextContent())); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KiuwanReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KiuwanReader.java index d397dbb5..a147da51 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KiuwanReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KiuwanReader.java @@ -89,7 +89,7 @@ private TestCaseResult parseKiuwanFinding(JSONObject finding) { String filename = dataFlow.getJSONObject(propagationPathLength).getString("file"); filename = filename.substring(filename.lastIndexOf('/')); if (filename.contains(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); int cwe = -1; try { diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReader.java index d2d03c91..e9efe849 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReader.java @@ -20,7 +20,6 @@ import java.io.StringReader; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVRecord; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestCaseResult; @@ -60,14 +59,15 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { String filename = record.get("File"); // e.g., BenchmarkTest00001 TestCaseResult tcr = new TestCaseResult(); - tcr.setCWE(cweLookup(category)); - tcr.setEvidence(category); - if (filename.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(filename)); - } - - if (tcr.getCWE() != 0) { - tr.put(tcr); + if (isTestCaseFile(filename)) { + tcr.setActualResultTestID(filename); + tcr.setCWE(cweLookup(category)); + tcr.setEvidence(category); + + int cwe = tcr.getCWE(); + if (cwe != CweNumber.DONTCARE && cwe != CweNumber.UNKNOWN) { + tr.put(tcr); + } } } @@ -135,7 +135,7 @@ private int cweLookup(String checkerKey) { default: System.out.println( "WARNING: Unmapped Klocwork Vuln category detected: " + checkerKey); - return 0; + return CweNumber.UNKNOWN; } } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/MendReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/MendReader.java index 0a35935f..1b81b852 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/MendReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/MendReader.java @@ -54,7 +54,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { tcr.setCWE(result.type.cwe.asNumber()); tcr.setEvidence(result.type.name); - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); tr.put(tcr); } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NJSScanReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NJSScanReader.java index 21913968..cf32bb3b 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NJSScanReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NJSScanReader.java @@ -200,7 +200,7 @@ private TestCaseResult produceTestCaseResult(JSONObject file, int cwe_identifier // filename = filename.replaceAll("[^0-9]", ""); // Note: This code should be checked with test cases - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); } catch (JSONException jsonE) { System.out.println("Issue with file JSON : " + jsonE.toString()); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReader.java index dbf3cc15..71bb102e 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReader.java @@ -95,7 +95,7 @@ private TestCaseResult parseNetsparkerIssue(Node flaw) { } if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } return null; @@ -107,7 +107,7 @@ private int cweLookup(String cweNum) { } int cwe = Integer.parseInt(cweNum); switch (cwe) { - case 80: + case 80: // TODO - Is this correct? Shouldn't this be mapped to XSS? return CweNumber.INSECURE_COOKIE; // insecure cookie use // case "insecure-cookie" : return 614; // insecure cookie use // case "sql-injection" : return 89; // sql injection diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NoisyCricketReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NoisyCricketReader.java index 4dd95446..974aabbc 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NoisyCricketReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/NoisyCricketReader.java @@ -62,7 +62,7 @@ private void parseNoisyCricketIssue(Node item, TestSuiteResults tr) { String[] cwes = cwelist.split(", "); for (String cwe : cwes) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); tcr.setCWE(Integer.parseInt(cwe)); tr.put(tcr); } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/PMDReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/PMDReader.java index 927dab34..cb19f8e4 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/PMDReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/PMDReader.java @@ -17,12 +17,12 @@ */ package org.owasp.benchmarkutils.score.parsers; +import java.io.File; import java.io.FileInputStream; import java.util.ArrayList; import java.util.List; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestCaseResult; @@ -60,10 +60,10 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { NodeList rootList = root.getChildNodes(); tr.setToolVersion(version); - List fileList = getNamedNodes("file", rootList); + List fileNodesList = getNamedNodes("file", rootList); - for (Node file : fileList) { - List tcrs = parsePMDItem(file); + for (Node fileNode : fileNodesList) { + List tcrs = parsePMDItem(fileNode); for (TestCaseResult tcr : tcrs) { tr.put(tcr); } @@ -77,43 +77,196 @@ private List parsePMDItem(Node fileNode) { String filename = fileNode.getAttributes().getNamedItem("name").getNodeValue(); List violationNodes = getNamedChildren("violation", fileNode); - for (Node violationNode : violationNodes) { - String violation = violationNode.getAttributes().getNamedItem("rule").getNodeValue(); + String testclass = filename.substring(filename.lastIndexOf(File.separator) + 1); + // System.out.println( + // "DRW: " + violationNodes.size() + " potential violations for file: " + testclass); + if (isTestCaseFile(testclass)) { + // System.out.println("DRW: " + testclass + " is test case file."); + for (Node violationNode : violationNodes) { - String testclass = filename.substring(filename.lastIndexOf("/") + 1); - if (testclass.startsWith(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - - tcr.setNumber(testNumber(testclass)); - tcr.setCWE(figureCWE(violation)); - + tcr.setActualResultTestID(testclass); + String violation = + violationNode.getAttributes().getNamedItem("rule").getNodeValue(); + // System.out.println("DRW: looking up CWE for rule: " + violation); + tcr.setCWE(figureCWE(violation, testclass)); tcr.setEvidence(violation); results.add(tcr); } } - return results; } - private int figureCWE(String rule) { + private int figureCWE(String rule, String testclass) { switch (rule) { + case "AddEmptyString": + case "AtLeastOneConstructor": + case "AvoidBranchingStatementAsLastInLoop": + case "AvoidDeeplyNestedIfStmts": + case "AvoidDuplicateLiterals": + case "AvoidFileStream": + case "AvoidInstantiatingObjectsInLoops": + case "AvoidLiteralsInIfCondition": + case "AvoidReassigningParameters": + case "AvoidStringBufferField": + case "AvoidUsingHardCodedIP": + case "AvoidUsingNativeCode": case "AvoidUsingOctalValues": + case "ClassNamingConventions": + case "ClassWithOnlyPrivateConstructorsShouldBeFinal": + case "CloneMethodMustBePublic": case "CollapsibleIfStatements": + case "CognitiveComplexity": + case "CommentDefaultAccessModifier": // What is this? + case "ConfusingTernary": + case "ControlStatementBraces": + case "CyclomaticComplexity": case "EmptyCatchBlock": + case "EmptyControlStatement": case "EmptyFinallyBlock": - case "EmptyIfStmt": case "EmptyStatementNotInLoop": case "EmptySwitchStatements": + case "ExceptionAsFlowControl": + case "FieldDeclarationsShouldBeAtStartOfClass": + case "FieldNamingConventions": + case "GuardLogStatement": + case "IdenticalCatchBranches": + case "ImmutableField": // One of the static/final but not Immutable CWEs? + case "LawOfDemeter": // Principal of Least Knowledge + case "LinguisticNaming": + case "LocalVariableCouldBeFinal": + case "LocalVariableNamingConventions": + case "LongVariable": + case "LooseCoupling": + case "MethodArgumentCouldBeFinal": + case "MethodNamingConventions": + case "MissingOverride": + case "MissingSerialVersionUID": + case "NcssCount": // What is this? + case "NonStaticInitializer": + case "NPathComplexity": + case "OnlyOneReturn": + case "OverrideBothEqualsAndHashcode": + case "PackageCase": + case "RedundantFieldInitializer": + case "ReplaceVectorWithList": + case "ShortClassName": + case "ShortVariable": + case "SwitchDensity": + case "SystemPrintln": + case "TestClassWithoutTestCases": + case "TooFewBranchesForASwitchStatement": + case "TooManyMethods": + case "UnnecessaryAnnotationValueElement": + case "UnnecessaryBoxing": + case "UnusedAssignment": + case "UnnecessaryCast": case "UnnecessaryConversionTemporary": case "UnnecessaryFullyQualifiedName": + case "UnnecessaryImport": + case "UnnecessaryLocalBeforeReturn": case "UnnecessaryModifier": case "UnnecessaryReturn": - case "UnusedFormalParameter": + case "UnnecessarySemicolon": case "UnusedImports": - case "UnusedLocalVariable": + case "UnusedPrivateField": case "UnusedPrivateMethod": + case "UseDiamondOperator": + case "UseArrayListInsteadOfVector": + case "UselessOperationOnImmutable": case "UselessParentheses": + case "UselessStringValueOf": + case "UseLocaleWithCaseConversions": + case "UseShortArrayInitializer": + case "UseTryWithResources": // CWE 772? + case "UseUnderscoresInNumericLiterals": + case "UseVarargs": + return CweNumber.DONTCARE; + + /*/ Some of these might map to CWEs + case "DoNotThrowExceptionInFinally": + case "LiteralsFirstInComparisons": // CWE for NullPointer? + case "PrematureDeclaration": // ??? + case "UseIndexOfChar": + case "UseProperClassLoader": + return CweNumber.DONTCARE;*/ + + // Are these the CWE for Expression Always True or False? + case "EmptyIfStmt": + case "UnconditionalIfStatement": return CweNumber.DONTCARE; + + case "AvoidPrintStackTrace": + return 209; + + case "AvoidThrowingRawExceptionTypes": + return 248; + + case "HardCodedCryptoKey": + return 321; + + case "DoNotTerminateVM": + return 382; + case "DoNotUseThreads": + return 383; + case "AvoidCatchingNPE": + return 395; + + case "AvoidCatchingGenericException": + case "AvoidCatchingThrowable": + return 396; + + case "IdempotentOperations": + return 398; + + case "CloseResource": + return 400; + + case "BrokenNullCheck": + case "NullAssignment": + return 476; + + case "SwitchStmtsShouldHaveDefault": + return 478; + + case "OneDeclarationPerLine": + return 483; + case "ImplicitSwitchFallThrough": + return 484; + + case "UnusedFormalParameter": + case "UnusedLocalVariable": + return 563; + + case "FinalizeDoesNotCallSuperFinalize": + return 568; + + case "DontCallThreadRun": + return 572; + + case "ProperCloneImplementation": + return 580; + + case "ReturnFromFinallyBlock": + return 584; + + case "AvoidCallingFinalize": + return 586; + + case "CompareObjectsWithEquals": + case "UseEqualsToCompareStrings": + return 597; + + case "MutableStaticState": + return 607; // Or 582? + + // Should any of these be 609?? + // case "AvoidSynchronizedAtMethodLevel": + // case "DoNotUseThreads": + // case "NonThreadSafeSingleton": + case "DoubleCheckedLocking": + return 609; + // Don't think PMD reports any of these: case "??1": return CweNumber.INSECURE_COOKIE; @@ -142,7 +295,11 @@ private int figureCWE(String rule) { return CweNumber.DONTCARE; default: - System.out.println("WARNING: Unknown PMD vuln category: " + rule); + System.out.println( + "WARNING: Unknown PMD vuln category: " + + rule + + " for test case: " + + testclass); } return CweNumber.UNKNOWN; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ParasoftReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ParasoftReader.java index 4014c9cb..057216ff 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ParasoftReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ParasoftReader.java @@ -111,7 +111,7 @@ private TestCaseResult parseStdViol(Node flaw) { String testcase = getAttributeValue("locFile", flaw); testcase = testcase.substring(testcase.lastIndexOf('/')); if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); return tcr; } } @@ -144,7 +144,7 @@ private TestCaseResult parseFlowViol(Node flaw) { String testcase = getAttributeValue("locFile", flaw); testcase = testcase.substring(testcase.lastIndexOf('/') + 1); if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/QualysWASReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/QualysWASReader.java index 2e56d33b..4fdaae8a 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/QualysWASReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/QualysWASReader.java @@ -130,7 +130,7 @@ private TestCaseResult parseQualysVulnerability(Node issue) { testcase = testcase.split("\\?")[0]; if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Rapid7Reader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Rapid7Reader.java index d9e52b78..da283ad1 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Rapid7Reader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Rapid7Reader.java @@ -54,7 +54,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { tcr.setEvidence(vulnerability.vulnType + "::" + vulnerability.attackType); tcr.setCWE(cweLookup(vulnerability.cwe, vulnerability.attackType)); - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); tr.put(tcr); } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Reader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Reader.java index 421330b6..0eaa49e4 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Reader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/Reader.java @@ -34,6 +34,7 @@ import org.owasp.benchmarkutils.score.parsers.sarif.PrecautionReader; import org.owasp.benchmarkutils.score.parsers.sarif.SemgrepSarifReader; import org.owasp.benchmarkutils.score.parsers.sarif.SnykReader; +import org.owasp.benchmarkutils.score.service.ExpectedResultsProvider; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -46,8 +47,7 @@ public abstract class Reader { // TODO: Figure out how to dynamically add all readers here without listing them // out manually // NOTE: There is a unit test that at least automatically verifies that any - // reader with a unit - // test is in this list + // reader with a unit test is in this list public static List allReaders() { return Arrays.asList( new AcunetixReader(), @@ -128,10 +128,16 @@ public static Node getNamedNode(String name, NodeList list) { return null; } - // Returns the node inside this nodelist whose name matches 'name', that also - // has an attribute - // called 'key' whose value matches 'keyvalue' + /** + * Returns the node inside this nodelist whose name matches 'name', that also has an attribute + * called 'key' whose value matches 'keyvalue' + * + * @param name The name of the node to get + * @param keyvalue The name of the key attribute to match against + * @param list The list of Nodes to search through return The matching Node or null, of not a + * match. + */ public static Node getNamedNode(String name, String keyValue, NodeList list) { if ((name == null) || (keyValue == null) || (list == null)) return null; for (int i = 0; i < list.getLength(); i++) { @@ -217,24 +223,46 @@ private static int findFirstNonNumeric(String path) { return -1; } - public static long occurrences(String path, char c) { + /** + * Determined whether this file is part of a test case. + * + * @param testCaseFileName The filename reported by the tool + * @return True if file is part of a test case, false otherwise + */ + public static boolean isTestCaseFile(String testCaseFileName) { + return ExpectedResultsProvider.getExpectedResults().isTestCaseFile(testCaseFileName); + } + + private static long occurrences(String path, char c) { return path.chars().filter(ch -> ch == c).count(); } - public static int testNumber(String path) { - return testNumber(path, BenchmarkScore.TESTCASENAME); + /** + * Parses out the test case number from a Benchmark style test case name. + * + * @param path The test case filename, with or without path. @Return the test case number or -1, + * if it can't parse it out. + */ + public static int getBenchmarkStyleTestCaseNumber(String path) { + return getBenchmarkStyleTestCaseNumber(path, BenchmarkScore.TESTCASENAME); } - /** Get rid of everything except the test name. */ - public static int testNumber(String path, String testCaseName) { + /** + * Parses out the test case number from a Benchmark style test case name. + * + * @param path The test case filename, with or without path. + * @param testCaseName The test case name pattern to parse against @Return the test case number + * or -1, if it can't parse it out. + */ + public static int getBenchmarkStyleTestCaseNumber(String path, String testCaseName) { try { // No BenchmarkTest if (!path.contains(testCaseName)) { return -1; } + int numberStart = path.indexOf(testCaseName) + testCaseName.length() + 1; path = path.substring(numberStart); - // System.out.println("After length: " + path); path = path.replaceAll("\\?.*", ""); path = path.replaceAll(",.*", ""); @@ -245,7 +273,6 @@ public static int testNumber(String path, String testCaseName) { path = removeColon(path); } path = path.replaceAll("[^0-9.]", ""); - // System.out.println("After replace: " + path); if (path.contains(".") && occurrences(path, '.') > 1) { int start = path.indexOf(".") + 1; int end = path.length(); @@ -256,12 +283,10 @@ public static int testNumber(String path, String testCaseName) { if (path.contains(".")) { path = removeFileEnding(path); } - // System.out.println("Before dot cleaning " + path); // Remove remaining dots path = path.replace(".", ""); - // System.out.println("Final: " + path); - // In the case of $innerclass + // In the case of an $innerclass int dollar = path.indexOf("$"); if (dollar != -1) { path = path.substring(0, dollar); @@ -269,7 +294,6 @@ public static int testNumber(String path, String testCaseName) { return Integer.parseInt(path); } catch (Exception e) { - return -1; } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ReshiftReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ReshiftReader.java index ac755623..374255b4 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ReshiftReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ReshiftReader.java @@ -115,7 +115,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { String category = record.get("Category"); tcr.setCWE(cweLookup(category)); tcr.setEvidence(category); - tcr.setNumber(testNumber(url)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(url)); if (tcr.getCWE() != 0) { tr.put(tcr); } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ScnrReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ScnrReader.java index 5d4fa534..41bd2dd8 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ScnrReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ScnrReader.java @@ -79,12 +79,13 @@ private TestCaseResult parseJsonResult(JSONObject issue) { return null; } - int testNumber = testNumber(issue.getJSONObject("vector").getString("url")); + int testNumber = + getBenchmarkStyleTestCaseNumber(issue.getJSONObject("vector").getString("url")); TestCaseResult tcr = new TestCaseResult(); tcr.setCWE(issue.getInt("cwe")); - tcr.setNumber(testNumber); + tcr.setTestID(testNumber); return tcr; } @@ -137,12 +138,13 @@ private TestCaseResult parseXmlResult(Node issue) { } int testNumber = - testNumber(getNamedChild("url", getNamedChild("vector", issue)).getTextContent()); + getBenchmarkStyleTestCaseNumber( + getNamedChild("url", getNamedChild("vector", issue)).getTextContent()); TestCaseResult tcr = new TestCaseResult(); tcr.setCWE(parseInt(textContentOf(issue, "cwe"))); - tcr.setNumber(testNumber); + tcr.setTestID(testNumber); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SeekerReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SeekerReader.java index b690dd6f..7c23e99f 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SeekerReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SeekerReader.java @@ -46,7 +46,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { TestCaseResult tcr = new TestCaseResult(); tcr.setCWE(cweLookup(checkerKey)); tcr.setEvidence(checkerKey); - tcr.setNumber(testNumber(url)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(url)); if (tcr.getCWE() != 0) { tr.put(tcr); } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SemgrepReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SemgrepReader.java index f9464f72..f1e96aee 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SemgrepReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SemgrepReader.java @@ -287,7 +287,7 @@ private TestCaseResult parseSemgrepFindings(JSONObject result) { tcr.setCWE(cwe); tcr.setEvidence(category + "::" + evidence); tcr.setConfidence(0); - tcr.setNumber(testNumber(className)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(className)); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftReader.java index 38cac7fd..b0965f24 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftReader.java @@ -55,7 +55,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { String category = split[1]; TestCaseResult testCaseResult = new TestCaseResult(); - testCaseResult.setNumber(Integer.parseInt(split[0])); + testCaseResult.setTestID(Integer.parseInt(split[0])); testCaseResult.setCWE(categoryToCWE(category)); testCaseResult.setEvidence(category + "::" + line); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReader.java index 941b5fb9..e6e6b595 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReader.java @@ -80,7 +80,7 @@ private TestCaseResult parseTestCaseResult(JSONObject finding) { if (filename.contains(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); tcr.setCWE(cweNumber(finding)); return tcr; @@ -119,8 +119,6 @@ private int cweNumber(JSONObject finding) { return 22; case "COMMAND_INJECTION": return 78; - case "HTTP_RESPONSE_SPLITTING": - return 113; case "XSS_SERVLET": case "HRS_REQUEST_PARAMETER_TO_COOKIE": case "XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER": @@ -132,6 +130,8 @@ private int cweNumber(JSONObject finding) { return 89; case "LDAP_INJECTION": return 90; + case "HTTP_RESPONSE_SPLITTING": + return 113; case "PADDING_ORACLE": return 209; case "DES_USAGE": @@ -146,12 +146,12 @@ private int cweNumber(JSONObject finding) { return 330; case "TRUST_BOUNDARY_VIOLATION": return 501; - case "HTTPONLY_COOKIE": - return 1004; case "INSECURE_COOKIE": return 614; case "XPATH_INJECTION": return 643; + case "HTTPONLY_COOKIE": + return 1004; default: System.out.println( diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SnappyTickReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SnappyTickReader.java index 1fb7f951..81f26f0c 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SnappyTickReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SnappyTickReader.java @@ -90,7 +90,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { TestCaseResult tcr = new TestCaseResult(); tcr.setCWE(findingCWE); tcr.setEvidence(findingName); - tcr.setNumber(extractTestNumber(findingName)); + tcr.setTestID(extractTestNumber(findingName)); tr.put(tcr); } } @@ -102,7 +102,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { private int extractTestNumber(String testfile) { if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - int testno = testNumber(testfile); + int testno = getBenchmarkStyleTestCaseNumber(testfile); return testno; } return -1; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReader.java index 6220be4d..b458eeb4 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReader.java @@ -82,7 +82,7 @@ private void parseResults(TestSuiteResults tr, JSONObject obj, boolean isHotspot ? parseSonarQubeHotSpotIssue(arr.getJSONObject(i)) : parseSonarQubeQualityIssue(arr.getJSONObject(i))); if (tcr != null) { - if (tcr.getNumber() == 0) { + if (tcr.getTestID().equals("0")) { System.out.println( "SQ Error: JSON object parsed with isHotspot key: '" + key @@ -128,7 +128,7 @@ private TestCaseResult parseSonarQubeQualityIssue(JSONObject finding) { filename = filename.substring(filename.lastIndexOf('/')); if (filename.contains(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); String rule = finding.getString("rule"); String squid = rule.substring(rule.indexOf(":") + 1); if (squid == null || squid.equals("none")) { @@ -181,7 +181,7 @@ private TestCaseResult parseSonarQubeHotSpotIssue(JSONObject finding) { filename = filename.substring(filename.lastIndexOf('/')); if (filename.contains(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); String secCat = finding.getString("securityCategory"); if (secCat == null || secCat.equals("none")) { return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReader.java index a4ab259c..a0c42f5b 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReader.java @@ -112,7 +112,7 @@ private TestCaseResult parseSonarIssue(Node flaw) { String testfile = getNamedChild("component", flaw).getTextContent().trim(); testfile = testfile.substring(testfile.lastIndexOf('/') + 1); if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } return null; @@ -130,7 +130,7 @@ private TestCaseResult parseSonarPluginIssue(Node flaw) { String testfile = getNamedChild("component", flaw).getTextContent().trim(); testfile = testfile.substring(testfile.lastIndexOf('/') + 1); if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SourceMeterReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SourceMeterReader.java index 1a4a66d0..e60e875d 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SourceMeterReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/SourceMeterReader.java @@ -81,9 +81,9 @@ private TestCaseResult parseSourceMeterItem(String vuln, String file) throws Exc tcr.setCWE(cweLookup(vuln)); tcr.setEvidence(vuln + "::" + file); - int testno = testNumber(file); + int testno = getBenchmarkStyleTestCaseNumber(file); if (testno > 0) { - tcr.setNumber(testno); + tcr.setTestID(testno); return tcr; } else { return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReader.java index 020967b4..137fcef5 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReader.java @@ -62,9 +62,9 @@ private TestCaseResult toTestCaseResult( tcResult.setCWE( figureCwe(vulnerabilityType.name, vulnerability.function, vulnerability.filename)); - int testcasenum = testNumber(vulnerability.filename); + int testcasenum = getBenchmarkStyleTestCaseNumber(vulnerability.filename); if (testcasenum > 0) { - tcResult.setNumber(testcasenum); + tcResult.setTestID(testcasenum); tcResult.setEvidence(vulnerabilityType.name + "::" + lineNumber(vulnerability)); return tcResult; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VeracodeReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VeracodeReader.java index 52418124..173cfad1 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VeracodeReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VeracodeReader.java @@ -116,7 +116,7 @@ private TestCaseResult parseVeracodeVulnerability(Node flaw) { String testcase = getAttributeValue("sourcefile", flaw); if (testcase.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testcase)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testcase)); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReader.java index e9ad2346..e5f4ce18 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReader.java @@ -93,7 +93,7 @@ private TestCaseResult parseVisualCodeGrepperIssue(Node n) { classname = (classname.substring(classname.lastIndexOf('\\') + 1)).split("\\.")[0]; if (classname.startsWith(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(classname)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(classname)); Node catnode = getNamedNode("Title", n.getChildNodes()); tcr.setCWE(figureCWE(tcr, catnode)); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/W3AFReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/W3AFReader.java index d5642162..7bc02f9d 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/W3AFReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/W3AFReader.java @@ -93,7 +93,7 @@ private TestCaseResult parseW3AFIssue(Node flaw) { } if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReader.java index 2ccac737..3a10c697 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReader.java @@ -132,7 +132,7 @@ private static TestCaseResult parseTestCaseResult(JSONObject finding, Integer cw if (filename.contains(BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(filename)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(filename)); tcr.setCWE(cwe); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiReader.java index 7c08e22a..2b5b35df 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WapitiReader.java @@ -92,7 +92,7 @@ public TestSuiteResults parse(ResultFile resultFile) throws Exception { getAttributeValue("name", vuln) + "::" + getNamedChild("curl_command", entry).getTextContent()); - tcr.setNumber(testNumber(path)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(path)); tr.put(tcr); } } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WebInspectReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WebInspectReader.java index 25957ad3..65c1d7ed 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WebInspectReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/WebInspectReader.java @@ -98,7 +98,7 @@ private TestCaseResult parseWebInspectIssue(Node flaw) throws Exception { } if (testfile.startsWith(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testfile)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testfile)); return tcr; } return null; diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReader.java index 97c6e3a0..d2bae5e5 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReader.java @@ -100,7 +100,7 @@ private void handleOldReportFormat(ResultFile resultFile, TestSuiteResults tr) { String testName = extractTestName(finding.getString("url")); if (testName.contains(BenchmarkScore.TESTCASENAME)) { - tcr.setNumber(testNumber(testName)); + tcr.setTestID(getBenchmarkStyleTestCaseNumber(testName)); tcr.setCWE(figureCwe(finding)); tr.put(tcr); @@ -131,7 +131,9 @@ private void handleNewReportFormat(ResultFile resultFile, TestSuiteResults tr) { BenchmarkScore.TESTCASENAME)) { TestCaseResult tcr = new TestCaseResult(); - tcr.setNumber(testNumber(testName)); + tcr.setTestID( + getBenchmarkStyleTestCaseNumber( + testName)); tcr.setCWE(cwe); tr.put(tcr); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapReader.java index 38943199..801d4f08 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/ZapReader.java @@ -145,7 +145,8 @@ private void parseAndAddZapIssues(Node flaw, TestSuiteResults tr) throws URISynt private void addIssue( Node alertData, TestSuiteResults tr, int cwe, String category, int confidence) { - int testNumber = testNumber(getNamedChild("uri", alertData).getTextContent()); + int testNumber = + getBenchmarkStyleTestCaseNumber(getNamedChild("uri", alertData).getTextContent()); if (testNumber > 0) { tr.put(createTestCaseResult(cwe, category, confidence, testNumber)); } @@ -159,7 +160,7 @@ private TestCaseResult createTestCaseResult( } tcr.setEvidence(category); tcr.setConfidence(confidence); - tcr.setNumber(testNumber); + tcr.setTestID(testNumber); return tcr; } diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/sarif/SarifReader.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/sarif/SarifReader.java index d9498ef7..70b3a388 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/sarif/SarifReader.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/parsers/sarif/SarifReader.java @@ -30,7 +30,6 @@ import java.util.regex.Pattern; import org.json.JSONArray; import org.json.JSONObject; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestCaseResult; import org.owasp.benchmarkutils.score.TestSuiteResults; @@ -238,7 +237,7 @@ private TestCaseResult testCaseResultFor(JSONObject result, Map String className = extractFilename(resultUri(result)); - if (!className.startsWith(BenchmarkScore.TESTCASENAME)) { + if (!isTestCaseFile(className)) { return null; } @@ -253,8 +252,9 @@ private TestCaseResult testCaseResultFor(JSONObject result, Map tcr.setCWE(cwe); tcr.setEvidence(evidence); - tcr.setConfidence(0); - tcr.setNumber(testNumber(className)); + // tcr.setConfidence(0); + // tcr.setTestID(getBenchmarkStyleTestCaseNumber(className)); + tcr.setActualResultTestID(className); return tcr; } @@ -268,7 +268,7 @@ private static String resultUri(JSONObject result) { } /** - * Allows extending classes to map/change detected CWE numbers to match Benchmark expected + * Allows extending classes to map/change detected CWE numbers to match test suite expected * numbers (if required) */ public int mapCwe(int cwe) { diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterHome.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterHome.java index 81953d0d..195529ca 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterHome.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterHome.java @@ -148,8 +148,8 @@ private JFreeChart display(String title, Set tools) { } /** - * Add the letter, from the key on the right, next to the plot point on the chart for for each - * tool to the supplied xyplot. + * Add the letter, from the key on the right, next to the plot point on the chart for each tool + * to the supplied xyplot. * * @param tools - THe set of tool results. * @param xyplot - The chart to make the Data labels on. diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterPlot.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterPlot.java index 7e01f94d..5fb6ed3f 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterPlot.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterPlot.java @@ -167,8 +167,8 @@ static String sort(String value) { } /** - * Add the letter, from the key on the right, next to the plot point on the chart for for each - * tool to the supplied xyplot. + * Add the letter, from the key on the right, next to the plot point on the chart for each tool + * to the supplied xyplot. * * @param map - A HashMap of plot points and the label to plot next to them. * @param xyplot - The chart to make the Data labels on. diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterTools.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterTools.java index 6c90b476..c4b45bf7 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterTools.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterTools.java @@ -20,7 +20,6 @@ import java.awt.Color; import java.awt.geom.Point2D; import java.security.SecureRandom; -import java.text.DecimalFormat; import java.util.HashMap; import java.util.List; import org.jfree.chart.ChartFactory; @@ -90,7 +89,11 @@ private JFreeChart display(String title, int height, ToolResults toolResults) { XYPlot xyplot = this.chart.getXYPlot(); makeDataLabels(toolResults, xyplot); - makeLegend(toolResults, 103, 93, dataset, xyplot); + double yCoordinate = 93; // default + // If there are more than 30 rows to plot, move the legend up to the very top of the legend + // box so more rows can be displayed + if (toolResults.getCategories().size() > 30) yCoordinate = 108; + makeLegend(toolResults, 103, yCoordinate, dataset, xyplot); // TODO: Make this into a method, or add it to makeDataLabels for (XYDataItem item : (List) series.getItems()) { @@ -112,8 +115,8 @@ private JFreeChart display(String title, int height, ToolResults toolResults) { } /** - * Add the letter, from the key on the right, next to the plot point on the chart for for each - * tool to the supplied xyplot. + * Add the letter, from the key on the right, next to the plot point on the chart for each tool + * to the supplied xyplot. * * @param tools - THe set of tool results. * @param xyplot - The chart to make the Data labels on. @@ -131,17 +134,21 @@ private HashMap makePointList(ToolResults toolResults) { int size = 0; // make a list of all points. Add in a tiny random to prevent exact // duplicate coordinates in map + int wrapNumber = 1; for (CategoryResults r : toolResults.getCategoryResults()) { size++; double x = r.falsePositiveRate * 100 + sr.nextDouble() * .000001; // this puts the label just below the point double y = r.truePositiveRate * 100 + sr.nextDouble() * .000001 - 1; Point2D p = new Point2D.Double(x, y); - String label = "" + ch; + String label = (wrapNumber == 1 ? "" + ch : ch + String.valueOf(wrapNumber)); map.put(p, label); // Weak hack if there are more than 26 tools scored. This will only get us to 52. if (ch == 'Z') ch = 'a'; - else ch++; + else if (ch == 'z') { + ch = 'A'; // Wrap to use A again + wrapNumber++; + } else ch++; } // add average point if (size > 1) { @@ -156,21 +163,33 @@ private HashMap makePointList(ToolResults toolResults) { return map; } + /** + * Add the vulnerability results per category to this tool's scorecard. + * + * @param or The set of results for this tool. + * @param x The X coordinate to start writing the legend. + * @param y The Y coordinate to start writing the legend. + * @param dataset The dataset to plot + * @param xyplot The XYPlot to use to plot this. + */ private void makeLegend( ToolResults or, double x, double y, XYSeriesCollection dataset, XYPlot xyplot) { char ch = ScatterHome.INITIAL_LABEL; int i = 0; int toolCount = 0; - double totalScore = 0; double totalTPR = 0; double totalFPR = 0; - final DecimalFormat DF = new DecimalFormat("#0.0"); + int wrapNumber = 1; for (CategoryResults r : or.getCategoryResults()) { toolCount++; - // Special hack to make it line up better if the letter is an 'I' or 'i' - String label = (ch == 'I' || ch == 'i' ? ch + ": " : ch + ": "); - // Another hack to make it line up better if the letter is a 'J' or 'j' - label = (ch == 'J' || ch == 'j' ? ch + ": " : label); + // Special hack to make it line up better for these skinny letters + // By default, don't add a number unless there are over 52 entries + String extraNumeric = (wrapNumber == 1 ? "" : String.valueOf(wrapNumber)); + String label = ((ch == 'I') ? ch + extraNumeric + ": " : ch + extraNumeric + ": "); + label = + ((ch == 'f' || ch == 'i' || ch == 'J' || ch == 'j') + ? ch + extraNumeric + ": " + : label); addEntryToKey( xyplot, @@ -183,14 +202,15 @@ private void makeLegend( r.truePositiveRate, r.falsePositiveRate); - double score = 100 * (r.truePositiveRate - r.falsePositiveRate); - totalScore += score; // Already multiplied by 100 totalTPR += r.truePositiveRate; // From 0-1, additive totalFPR += r.falsePositiveRate; // From 0-1, additive; i++; // Weak hack if there are more than 26 tools scored. This will only get us to 52. if (ch == 'Z') ch = 'a'; - else ch++; + else if (ch == 'z') { + ch = 'A'; // Wrap to use A again + wrapNumber++; + } else ch++; } if (toolCount > 1) { diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterVulns.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterVulns.java index cd511a03..b8d7db9c 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterVulns.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/report/ScatterVulns.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.IOException; import java.security.SecureRandom; -import java.text.DecimalFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -209,8 +208,8 @@ private JFreeChart display(String title, int height, String category, Set } /** - * Add the letter, from the key on the right, next to the plot point on the chart for for each - * tool to the supplied xyplot. + * Add the letter, from the key on the right, next to the plot point on the chart for each tool + * to the supplied xyplot. * * @param tools - THe set of tool results. * @param xyplot - The chart to make the Data labels on. @@ -392,7 +391,6 @@ private void makeLegend( double commercialTotalPrecision = 0; double commercialTotalTPR = 0; double commercialTotalFPR = 0; - final DecimalFormat DF = new DecimalFormat("#0.0"); for (Tool tool : toolResults) { ToolResults or = tool.getOverallResults(); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProvider.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProvider.java index c82cf995..1137ea9e 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProvider.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProvider.java @@ -19,9 +19,9 @@ import static java.lang.Boolean.parseBoolean; import static java.lang.Integer.parseInt; -import static org.owasp.benchmarkutils.score.parsers.Reader.testNumber; import java.io.IOException; +import java.util.List; import java.util.Optional; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -29,6 +29,7 @@ import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestCaseResult; import org.owasp.benchmarkutils.score.TestSuiteResults; +import org.owasp.benchmarkutils.score.domain.TestSuiteName; public class ExpectedResultsProvider { @@ -43,19 +44,28 @@ public class ExpectedResultsProvider { private static final String DATA_FLOW = " vuln src"; private static final String SINK = " vuln df"; + private static boolean standardBenchmarkStyleScoring; + private static TestSuiteResults expectedResults; + public static TestSuiteResults parse(ResultFile resultFile) throws IOException { TestSuiteResults tr = new TestSuiteResults("Expected", true, null); try (final CSVParser parser = resultFile.csvRecords()) { - setResultsMetadata(parser, tr); - final String TESTCASENAME = tr.getTestSuiteName() + BenchmarkScore.TEST; + List allExpectedResults = parser.getRecords(); + + CSVRecord firstRecord = allExpectedResults.get(0); + // setExpectedResultsMetadata() has side affects of setting + // BenchmarkScore.TESTSUITENAME, BenchmarkScore.TESTCASENAME, and + // ExpectedResultsProvider.standardBenchmarkStyleScoring + setExpectedResultsMetadata(parser, firstRecord, tr); - for (CSVRecord record : parser) { + // Parse all the expected results + for (CSVRecord record : allExpectedResults) { TestCaseResult tcr = new TestCaseResult(); - tcr.setTestCaseName(record.get(TEST_NAME).trim()); - // tcr.setCategory(record.get(CATEGORY).trim()); // Autoset by setCWE() below. + String testCaseFileName = record.get(TEST_NAME).trim(); + tcr.setTestCaseName(testCaseFileName); tcr.setTruePositive(parseBoolean(record.get(REAL_VULNERABILITY).trim())); int cwe = parseInt(record.get(CWE).trim()); tcr.setCWE(cwe); @@ -68,11 +78,10 @@ public static TestSuiteResults parse(ResultFile resultFile) throws IOException { + ". Add missing data to categories.xml to address."); System.exit(-1); } - if (record.get(TEST_NAME) - .trim() - .startsWith(tr.getTestSuiteName() + BenchmarkScore.TEST)) - tcr.setNumber(testNumber(record.get(TEST_NAME).trim(), TESTCASENAME)); - else tcr.setNumber(TestCaseResult.NOT_USING_TESTCASE_NUMBERS); + // This method sets the expected result testID based on the scoring style, + // previously determined + // DRW TODO: Combine with setTestCaseName in future? + tcr.setExpectedResultTestID(testCaseFileName); if (isExtendedResultsFile(parser)) { tcr.setSource(record.get(SOURCE).trim()); @@ -84,17 +93,44 @@ public static TestSuiteResults parse(ResultFile resultFile) throws IOException { } } + // Set static variable so any class can access the expected results + expectedResults = tr; + return tr; } - private static void setResultsMetadata(CSVParser parser, TestSuiteResults tr) - throws IOException { + /** + * The expected results for this scoring run. + * + * @return The expected results object. + */ + public static TestSuiteResults getExpectedResults() { + if (expectedResults == null) { + System.out.println( + "FATAL INTERNAL ERROR: Expected Results for this scoring run not initialized yet."); + System.exit(-1); + } + return expectedResults; + } + + /** + * Whether standard Benchmark style test case file names are being scored per the expected + * results file. + * + * @return True if so, false otherwise + */ + public static boolean isBenchmarkStyleScoring() { + return standardBenchmarkStyleScoring; + } + + private static void setExpectedResultsMetadata( + CSVParser parser, CSVRecord firstRecord, TestSuiteResults tr) throws IOException { Optional maybeVersionHeader = parser.getHeaderMap().keySet().stream().filter(h -> h.contains(PREFIX)).findFirst(); if (maybeVersionHeader.isEmpty()) { String versionNumError = - "Couldn't find " + PREFIX + " on first line of expected results file"; + "ERROR: Couldn't find " + PREFIX + " on first line of expected results file"; System.out.println(versionNumError); throw new IOException(versionNumError); } @@ -103,12 +139,32 @@ private static void setResultsMetadata(CSVParser parser, TestSuiteResults tr) int start = versionHeader.indexOf(PREFIX); - final String testSuiteName = versionHeader.substring(0, start).trim(); + String testSuiteName = versionHeader.substring(0, start).trim(); + + // These values must be set here before parsing any expected/actual results + // Maybe these two statics should be moved to ExpectedResultsProvider?? + BenchmarkScore.TESTSUITENAME = new TestSuiteName(testSuiteName); + BenchmarkScore.TESTCASENAME = testSuiteName + BenchmarkScore.TEST; start += PREFIX.length(); tr.setTestSuiteName(testSuiteName); + tr.setTestSuiteVersion(versionHeader.substring(start)); + + // Now check the 1st row of results to determine the scoring style + if (firstRecord + .get(TEST_NAME) + .trim() + .startsWith(tr.getTestSuiteName() + BenchmarkScore.TEST)) { + ExpectedResultsProvider.standardBenchmarkStyleScoring = true; + System.out.println( + "INFO: Using Default Benchmark style scoring based on contents of supplied expected results file."); + } else { + ExpectedResultsProvider.standardBenchmarkStyleScoring = false; + System.out.println( + "INFO: Using non-Benchmark style scoring based on contents of supplied expected results file."); + } } private static boolean isExtendedResultsFile(CSVParser parser) { diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ResultsFileCreator.java b/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ResultsFileCreator.java index afbd6715..cd18f1e0 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ResultsFileCreator.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/score/service/ResultsFileCreator.java @@ -74,7 +74,8 @@ public String createFor(TestSuiteResults actual) { private boolean isFullDetails(TestSuiteResults actual) { Iterator iterator = actual.keySet().iterator(); - return iterator.hasNext() && (actual.get(iterator.next()).get(0).getSource() != null); + return iterator.hasNext() + && (actual.getTestCaseResults(iterator.next()).get(0).getSource() != null); } private void writeHeader(PrintStream ps, boolean fullDetails, String testSuiteVersion) { @@ -95,7 +96,7 @@ private void writeHeader(PrintStream ps, boolean fullDetails, String testSuiteVe private void appendRow( PrintStream ps, TestSuiteResults actual, String testcaseID, boolean fullDetails) { - TestCaseResult actualResult = actual.get(testcaseID).get(0); + TestCaseResult actualResult = actual.getTestCaseResults(testcaseID).get(0); boolean isReal = actualResult.isTruePositive(); boolean passed = actualResult.isPassed(); diff --git a/plugin/src/main/java/org/owasp/benchmarkutils/tools/CalculateToolCodeBlocksSupport.java b/plugin/src/main/java/org/owasp/benchmarkutils/tools/CalculateToolCodeBlocksSupport.java index e6d3b1bd..e30d9bb4 100644 --- a/plugin/src/main/java/org/owasp/benchmarkutils/tools/CalculateToolCodeBlocksSupport.java +++ b/plugin/src/main/java/org/owasp/benchmarkutils/tools/CalculateToolCodeBlocksSupport.java @@ -373,7 +373,8 @@ protected void run() { // 3a. Loop through all the results in theToolResults to calculate the initial // statistics across all of them. for (String tc : theToolResults.keySet()) { - TestCaseResult theResult = theToolResults.get(tc).get(0); // Always only one. + TestCaseResult theResult = + theToolResults.getTestCaseResults(tc).get(0); // Always only one. boolean passed = theResult.isPassed(); CodeBlockSupportResults source = sourceCodeBlocksResults.get(theResult.getSource()); CodeBlockSupportResults dataflow = @@ -433,7 +434,8 @@ protected void run() { // Loop through them all again and calculate the number of TPs for each source that // pass/fail, ignoring any failures that are caused by unsupported SINKs. for (String tc : theToolResults.keySet()) { - TestCaseResult theResult = theToolResults.get(tc).get(0); // Always only one. + TestCaseResult theResult = + theToolResults.getTestCaseResults(tc).get(0); // Always only one. boolean passed = theResult.isPassed(); CodeBlockSupportResults source = sourceCodeBlocksResults.get(theResult.getSource()); CodeBlockSupportResults dataflow = @@ -484,7 +486,8 @@ protected void run() { // Positive and the dataflow is 'null'. boolean foundFPorFN = false; for (String tc : theToolResults.keySet()) { - TestCaseResult theResult = theToolResults.get(tc).get(0); // Always only one. + TestCaseResult theResult = + theToolResults.getTestCaseResults(tc).get(0); // Always only one. boolean passed = theResult.isPassed(); CodeBlockSupportResults source = sourceCodeBlocksResults.get(theResult.getSource()); CodeBlockSupportResults dataflow = @@ -591,7 +594,8 @@ protected void run() { // sinks or sources already known to cause FPs int FPCount = 1; for (String tc : theToolResults.keySet()) { - TestCaseResult theResult = theToolResults.get(tc).get(0); // Always only one. + TestCaseResult theResult = + theToolResults.getTestCaseResults(tc).get(0); // Always only one. boolean passed = theResult.isPassed(); CodeBlockSupportResults source = sourceCodeBlocksResults.get(theResult.getSource()); CodeBlockSupportResults dataflow = diff --git a/plugin/src/main/resources/scorecard/content/dashboard.css b/plugin/src/main/resources/scorecard/content/dashboard.css new file mode 100644 index 00000000..dd0b5dfe --- /dev/null +++ b/plugin/src/main/resources/scorecard/content/dashboard.css @@ -0,0 +1,4 @@ +.dropdown-menu { + overflow-y: auto; + max-height: calc(100vh - 50px); +} diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/ConfigurationTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/ConfigurationTest.java index 5a4ad623..f0265856 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/ConfigurationTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/ConfigurationTest.java @@ -32,6 +32,7 @@ import java.io.PrintStream; import java.util.HashMap; import java.util.Map; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.yaml.snakeyaml.DumperOptions; @@ -39,14 +40,14 @@ public class ConfigurationTest { - private Map defaultConfig; - private Yaml yaml; - private final ClassLoader classLoader = Configuration.class.getClassLoader(); - private ByteArrayOutputStream out; + private static Map defaultConfig; + private static Yaml yaml; + private static final ClassLoader classLoader = Configuration.class.getClassLoader(); + private static ByteArrayOutputStream out; private static final String SEP = System.getProperty("line.separator"); - @BeforeEach - public void setUp() { + @BeforeAll + static void overallSetUp() { // Prevent JSON-like output (https://stackoverflow.com/a/62305688) final DumperOptions options = new DumperOptions(); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); @@ -54,7 +55,10 @@ public void setUp() { yaml = new Yaml(options); defaultConfig = yaml.load(classLoader.getResourceAsStream(Configuration.DEFAULT_CONFIG)); + } + @BeforeEach + void setUp() { out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); } @@ -66,7 +70,7 @@ public void canReadConfigFromDefaultFile() { private void assertConfigEquals( Map expectedConfig, Configuration actualConfig) { - assertEquals(expectedConfig.get("expectedresults"), actualConfig.expectedResultsFileName); + assertEquals(expectedConfig.get("expectedresults"), Configuration.expectedResultsFileName); assertEquals(expectedConfig.get("resultsfileordir"), actualConfig.resultsFileOrDirName); assertEquals(expectedConfig.get("focustool"), actualConfig.focus); assertEquals(expectedConfig.get("anonymousmode"), actualConfig.anonymousMode); diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/TestHelper.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/TestHelper.java index fd025d39..0f58213f 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/TestHelper.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/TestHelper.java @@ -17,13 +17,66 @@ */ package org.owasp.benchmarkutils.score; +import static org.junit.jupiter.api.extension.ExtensionContext.Namespace.GLOBAL; + +import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; import java.util.Objects; import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.owasp.benchmarkutils.score.service.ExpectedResultsProvider; + +/** + * This class provides an initialization method to be run before any tests start, as well as some + * static helper methods. + * + *

The initialization sets up an ExpectedResults file before any of the scoring tests are run, as + * that is required to generate a scorecard. + */ +public class TestHelper implements BeforeAllCallback, ExtensionContext.Store.CloseableResource { + + private static boolean started = false; + + @Override + public void beforeAll(ExtensionContext context) { + if (!started) { + started = true; -public class TestHelper { + // Custom initialization is here + + // Initialize Expected Results File to use during testing + String EXPECTED_RESULTS_TESTFILENAME = + "src/test/resources/expectedresults-testfile.csv"; + try { + File EXPECTED_RESULTS_FILE = new File(EXPECTED_RESULTS_TESTFILENAME); + TestSuiteResults tr = + ExpectedResultsProvider.parse(new ResultFile(EXPECTED_RESULTS_FILE)); + } catch (FileNotFoundException e) { + System.out.println( + "FATAL ERROR: Can't find expected results test file: " + + EXPECTED_RESULTS_TESTFILENAME); + System.exit(-1); + } catch (IOException e) { + System.out.println( + "FATL ERROR: Reading contents of expected results test file: " + + EXPECTED_RESULTS_TESTFILENAME); + e.printStackTrace(); + System.exit(-1); + } + + // Don't know if registering this callback hook is needed, but doing it anyway. + context.getRoot().getStore(GLOBAL).put("BenchmarkTestSetup", this); + } + } + + @Override + public void close() { + // Any after all tests logic goes here. None currently. + } public static ResultFile resultFileOf(String filename) { try { diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/builder/TestCaseResultBuilder.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/builder/TestCaseResultBuilder.java index 3cf001c0..dacdbd29 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/builder/TestCaseResultBuilder.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/builder/TestCaseResultBuilder.java @@ -24,7 +24,6 @@ public class TestCaseResultBuilder { private String testCaseName = "SomeTest"; private int testNumber = -1; - private String category = "somecategory"; private int cwe = CweNumber.DONTCARE; private boolean truePositive = false; private boolean passed = false; @@ -51,12 +50,6 @@ public TestCaseResultBuilder setTestNumber(int testNumber) { return this; } - /* public TestCaseResultBuilder setCategory(String category) { - this.category = category; - - return this; - } - */ public TestCaseResultBuilder setCwe(int cwe) { this.cwe = cwe; @@ -97,8 +90,7 @@ public TestCaseResult build() { TestCaseResult testCaseResult = new TestCaseResult(); testCaseResult.setTestCaseName(testCaseName); - testCaseResult.setNumber(testNumber); - // testCaseResult.setCategory(category); + testCaseResult.setTestID(testNumber); testCaseResult.setCWE(cwe); testCaseResult.setSource(source); testCaseResult.setDataFlow(dataFlow); diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/AcunetixReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/AcunetixReaderTest.java index b0f22b2a..7073af1a 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/AcunetixReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/AcunetixReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,19 +29,18 @@ public class AcunetixReaderTest extends ReaderTestBase { - private ResultFile resultFile_360, resultFile_WVS; + private static ResultFile resultFile_360, resultFile_WVS; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile_360 = TestHelper.resultFileOf("testfiles/Benchmark_Acunetix-v1.4.1.xml"); resultFile_WVS = TestHelper.resultFileOf("testfiles/Benchmark_Acunetix-v15.3.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyAcunetixReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile_360, AcunetixReader.class); - assertOnlyMatcherClassIs(this.resultFile_WVS, AcunetixReader.class); + assertOnlyMatcherClassIs(resultFile_360, AcunetixReader.class); + assertOnlyMatcherClassIs(resultFile_WVS, AcunetixReader.class); } @Test @@ -57,8 +55,8 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); // For Acunetix WVS reader = new AcunetixReader(); @@ -70,7 +68,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.LDAP_INJECTION, result.get("44").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2629").get(0).getCWE()); + assertEquals(CweNumber.LDAP_INJECTION, result.getTestCaseResults("44").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2629").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ArachniReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ArachniReaderTest.java index 67c51588..f092bf27 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ArachniReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ArachniReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class ArachniReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Arachni-v2.0dev.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyArachniReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, ArachniReader.class); + assertOnlyMatcherClassIs(resultFile, ArachniReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BearerReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BearerReaderTest.java index a09e1787..8e8bbf50 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BearerReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BearerReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class BearerReaderTest extends ReaderTestBase { - private ResultFile resultFileV1_30; + private static ResultFile resultFileV1_30; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFileV1_30 = TestHelper.resultFileOf("testfiles/Benchmark_Bearer-v1.30.0.jsonv2"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyBearerReaderReportsCanReadAsTrueForV1_30() { - assertOnlyMatcherClassIs(this.resultFileV1_30, BearerReader.class); + assertOnlyMatcherClassIs(resultFileV1_30, BearerReader.class); } @Test @@ -55,8 +53,8 @@ void readerHandlesGivenResultFileInV1_30() throws Exception { assertEquals(3, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("7").get(0).getCWE()); - assertEquals(CweNumber.WEAK_HASH_ALGO, result.get("5").get(0).getCWE()); - assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.get("35").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("7").get(0).getCWE()); + assertEquals(CweNumber.WEAK_HASH_ALGO, result.getTestCaseResults("5").get(0).getCWE()); + assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.getTestCaseResults("35").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BurpReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BurpReaderTest.java index 8fe2427f..2b60d7f5 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BurpReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/BurpReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class BurpReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_BurpPro-v2020.2.1.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyBurpReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, BurpReader.class); + assertOnlyMatcherClassIs(resultFile, BurpReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReaderTest.java index 2ad021ab..f468cb4e 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CASTAIPReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class CASTAIPReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_CAST_AIP-v8.2.3.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyCASTAIPReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, CASTAIPReader.class); + assertOnlyMatcherClassIs(resultFile, CASTAIPReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReaderTest.java index 1427f788..d30f3baf 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxIASTReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class CheckmarxIASTReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_CxIAST.csv"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyCheckmarxIASTReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, CheckmarxIASTReader.class); + assertOnlyMatcherClassIs(resultFile, CheckmarxIASTReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReaderTest.java index 8d456b56..d50f7c80 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CheckmarxReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class CheckmarxReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Checkmarx-v8.2.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyCheckmarxReaderTestReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, CheckmarxReader.class); + assertOnlyMatcherClassIs(resultFile, CheckmarxReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CoverityReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CoverityReaderTest.java index 6ff05913..3275a8c7 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CoverityReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/CoverityReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class CoverityReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Coverity-v3.0.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyCoverityReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, CoverityReader.class); + assertOnlyMatcherClassIs(resultFile, CoverityReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/DatadogReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/DatadogReaderTest.java index 9c2b9f45..ed762d20 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/DatadogReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/DatadogReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class DatadogReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_1.2-Datadog.log"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyDatadogReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, DatadogReader.class); + assertOnlyMatcherClassIs(resultFile, DatadogReader.class); } @Test @@ -55,9 +53,11 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(4, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1609").get(0).getCWE()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("2").get(0).getCWE()); - assertEquals(CweNumber.WEAK_HASH_ALGO, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.TRUST_BOUNDARY_VIOLATION, result.get("4").get(0).getCWE()); + assertEquals( + CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1609").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("2").get(0).getCWE()); + assertEquals(CweNumber.WEAK_HASH_ALGO, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals( + CweNumber.TRUST_BOUNDARY_VIOLATION, result.getTestCaseResults("4").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FindbugsReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FindbugsReaderTest.java index 4bdc42a5..7ace5882 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FindbugsReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FindbugsReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,25 +29,24 @@ public class FindbugsReaderTest extends ReaderTestBase { - private ResultFile findSecBugsResultFile; - private ResultFile spotBugsResultFile; + private static ResultFile findSecBugsResultFile; + private static ResultFile spotBugsResultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { findSecBugsResultFile = TestHelper.resultFileOf("testfiles/Benchmark_findsecbugs-v1.11.0-105.xml"); spotBugsResultFile = TestHelper.resultFileOf("testfiles/Benchmark_spotbugs-v4.1.4-104.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyFindbugsReaderReportsCanReadAsTrueForFindSecBugsFile() { - assertOnlyMatcherClassIs(this.findSecBugsResultFile, FindbugsReader.class); + assertOnlyMatcherClassIs(findSecBugsResultFile, FindbugsReader.class); } @Test public void onlyFindbugsReaderReportsCanReadAsTrueForSpotBugsFile() { - assertOnlyMatcherClassIs(this.spotBugsResultFile, FindbugsReader.class); + assertOnlyMatcherClassIs(spotBugsResultFile, FindbugsReader.class); } @Test @@ -62,8 +60,8 @@ void readerHandlesGivenFindSecBugsResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } @Test @@ -77,7 +75,7 @@ void readerHandlesGivenSpotBugsResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReaderTest.java index 777b6135..6fedadd7 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FluidAttacksReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class FluidAttacksReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Fluid-Attacks-v20210416.csv"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyFluidReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, FluidAttacksReader.class); + assertOnlyMatcherClassIs(resultFile, FluidAttacksReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FortifyReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FortifyReaderTest.java index 5a29c471..d4fba527 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FortifyReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/FortifyReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class FortifyReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Fortify20.20_2020Q1-1234.fpr"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyFortifyReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, FortifyReader.class); + assertOnlyMatcherClassIs(resultFile, FortifyReader.class); } @Test @@ -55,7 +53,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReaderTest.java index 62146e05..19e2ba04 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanIASTReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class HCLAppScanIASTReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_HCL-IAST.hcl"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyHCLReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, HCLAppScanIASTReader.class); + assertOnlyMatcherClassIs(resultFile, HCLAppScanIASTReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReaderTest.java index e2b96108..7cdf9cdc 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanSourceReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class HCLAppScanSourceReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_HCLAppScanSource-2.4.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyHCLAppScanSourceReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, HCLAppScanSourceReader.class); + assertOnlyMatcherClassIs(resultFile, HCLAppScanSourceReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReaderTest.java index cb517146..56a13123 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HCLAppScanStandardReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,18 +29,17 @@ public class HCLAppScanStandardReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_HCLAppScanStandardReader-v10.0.6.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyHCLAppScanStandardReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, HCLAppScanStandardReader.class); + assertOnlyMatcherClassIs(resultFile, HCLAppScanStandardReader.class); } @Test @@ -56,9 +54,9 @@ void readerHandlesGivenV10ResultFile() throws Exception { assertEquals(4, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("300").get(0).getCWE()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("348").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("300").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("348").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HorusecReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HorusecReaderTest.java index b34afd33..6e084a1b 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HorusecReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/HorusecReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class HorusecReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_horusec-v2.5.0.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyHorusecReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, HorusecReader.class); + assertOnlyMatcherClassIs(resultFile, HorusecReader.class); } @Test @@ -56,7 +54,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/InsiderReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/InsiderReaderTest.java index eaf64262..029c6ddf 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/InsiderReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/InsiderReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class InsiderReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_insider-v3.0.0.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyInsiderReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, InsiderReader.class); + assertOnlyMatcherClassIs(resultFile, InsiderReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/JuliaReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/JuliaReaderTest.java index 6f99f296..280d5d73 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/JuliaReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/JuliaReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class JuliaReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_JuliaSoft-v2.3.2.1.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyJuliaReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, JuliaReader.class); + assertOnlyMatcherClassIs(resultFile, JuliaReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KiuwanReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KiuwanReaderTest.java index b59e8d27..cfefc51c 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KiuwanReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KiuwanReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class KiuwanReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Kiuwan-20191012.threadfix"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyKiuwanReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, KiuwanReader.class); + assertOnlyMatcherClassIs(resultFile, KiuwanReader.class); } @Test @@ -56,7 +54,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReaderTest.java index b2f67753..3cd08ede 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/KlocworkCSVReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class KlocworkCSVReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Klocwork.csv"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyKlocworkCSVReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, KlocworkCSVReader.class); + assertOnlyMatcherClassIs(resultFile, KlocworkCSVReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/MendReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/MendReaderTest.java index 42664529..93b970b3 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/MendReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/MendReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class MendReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Mend.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyMendReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, MendReader.class); + assertOnlyMatcherClassIs(resultFile, MendReader.class); } @Test @@ -55,7 +53,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReaderTest.java index 477df4ec..c304e9f2 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/NetsparkerReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class NetsparkerReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Netsparker.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyNetsparkerReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, NetsparkerReader.class); + assertOnlyMatcherClassIs(resultFile, NetsparkerReader.class); } @Test @@ -54,7 +52,8 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.COOKIE_WITHOUT_HTTPONLY, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals( + CweNumber.COOKIE_WITHOUT_HTTPONLY, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ParasoftReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ParasoftReaderTest.java index 24b5c1b6..a73dac9d 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ParasoftReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ParasoftReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class ParasoftReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_ParasoftJTest-v10.2.3.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyParasoftReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, ParasoftReader.class); + assertOnlyMatcherClassIs(resultFile, ParasoftReader.class); } @Test @@ -55,7 +53,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/Rapid7ReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/Rapid7ReaderTest.java index 44fda6a2..dfab45d0 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/Rapid7ReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/Rapid7ReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class Rapid7ReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_AppSpider-v7.2.119-1234.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyZapReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, Rapid7Reader.class); + assertOnlyMatcherClassIs(resultFile, Rapid7Reader.class); } @Test @@ -56,7 +54,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ReaderTest.java index fa2ae2ea..de310932 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ReaderTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -10,6 +11,11 @@ public class ReaderTest { + @AfterAll + static void restoreAfter() { + BenchmarkScore.TESTCASENAME = "BenchmarkTest"; + } + @ParameterizedTest(name = "{index} {0}") @ValueSource( strings = { @@ -46,19 +52,19 @@ public class ReaderTest { }) public void readsTestNumberFromPath(String path) { BenchmarkScore.TESTCASENAME = "BenchmarkTest"; - assertEquals(42, Reader.testNumber(path)); + assertEquals(42, Reader.getBenchmarkStyleTestCaseNumber(path)); } @Test public void returnsInvalidNumberForNonMatchingPrefix() { BenchmarkScore.TESTCASENAME = "SomethingElse"; - assertEquals(-1, Reader.testNumber("/somepath/BenchmarkTest00042")); + assertEquals(-1, Reader.getBenchmarkStyleTestCaseNumber("/somepath/BenchmarkTest00042")); } @Test public void returnsInvalidNumberForPathWithoutTestfile() { BenchmarkScore.TESTCASENAME = "BenchmarkTest"; - assertEquals(-1, Reader.testNumber("/somepath/someotherfile")); + assertEquals(-1, Reader.getBenchmarkStyleTestCaseNumber("/somepath/someotherfile")); } @ParameterizedTest(name = "{index} {0}") diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ScnrReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ScnrReaderTest.java index c509632f..36fb12bd 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ScnrReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ScnrReaderTest.java @@ -3,9 +3,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -13,14 +12,13 @@ public class ScnrReaderTest extends ReaderTestBase { - private ResultFile jsonResultFile; - private ResultFile xmlResultFile; + private static ResultFile jsonResultFile; + private static ResultFile xmlResultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { jsonResultFile = TestHelper.resultFileOf("testfiles/Benchmark_SCNR.json"); xmlResultFile = TestHelper.resultFileOf("testfiles/Benchmark_SCNR.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test @@ -42,8 +40,8 @@ void readerHandlesGivenJsonResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } private static void assertToolData(TestSuiteResults result) { @@ -63,7 +61,7 @@ void readerHandlesGivenXmlResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SeekerReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SeekerReaderTest.java index ef4a6b14..1a5edb3d 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SeekerReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SeekerReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class SeekerReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Seeker.csv"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlySeekerReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, SeekerReader.class); + assertOnlyMatcherClassIs(resultFile, SeekerReader.class); } @Test @@ -54,7 +52,8 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.TRUST_BOUNDARY_VIOLATION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals( + CweNumber.TRUST_BOUNDARY_VIOLATION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SemgrepReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SemgrepReaderTest.java index b832813f..7853d863 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SemgrepReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SemgrepReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,26 +29,25 @@ public class SemgrepReaderTest extends ReaderTestBase { - private ResultFile resultFileV65; - private ResultFile resultFileV121; + private static ResultFile resultFileV65; + private static ResultFile resultFileV121; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFileV65 = TestHelper.resultFileOf("testfiles/Benchmark_semgrep-v0.65.0.json"); resultFileV121 = TestHelper.resultFileWithoutLineBreaksOf( "testfiles/Benchmark_semgrep-v0.121.0.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlySemgrepReaderReportsCanReadAsTrueForV65() { - assertOnlyMatcherClassIs(this.resultFileV65, SemgrepReader.class); + assertOnlyMatcherClassIs(resultFileV65, SemgrepReader.class); } @Test public void onlySemgrepReaderReportsCanReadAsTrueForV121() { - assertOnlyMatcherClassIs(this.resultFileV121, SemgrepReader.class); + assertOnlyMatcherClassIs(resultFileV121, SemgrepReader.class); } @Test @@ -63,8 +61,8 @@ void readerHandlesGivenResultFileInV65() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("2").get(0).getCWE()); } @Test @@ -78,7 +76,8 @@ void readerHandlesGivenResultFileInV121() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("3").get(0).getCWE()); - assertEquals(CweNumber.COOKIE_WITHOUT_HTTPONLY, result.get("4").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("3").get(0).getCWE()); + assertEquals( + CweNumber.COOKIE_WITHOUT_HTTPONLY, result.getTestCaseResults("4").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReaderTest.java index 679b997c..af4d1a91 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ShiftLeftScanReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class ShiftLeftScanReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_shiftleftscan-v2.0.3.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyShiftLeftScanReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, ShiftLeftScanReader.class); + assertOnlyMatcherClassIs(resultFile, ShiftLeftScanReader.class); } @Test @@ -54,7 +52,8 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COOKIE_WITHOUT_HTTPONLY, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("2").get(0).getCWE()); + assertEquals( + CweNumber.COOKIE_WITHOUT_HTTPONLY, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReaderTest.java index 8aa8de35..c2b7ca49 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeJsonReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class SonarQubeJsonReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_sonarqube-v9.1.0.47736.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlySonarQubeJsonReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, SonarQubeJsonReader.class); + assertOnlyMatcherClassIs(resultFile, SonarQubeJsonReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.WEAK_CRYPTO_ALGO, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReaderTest.java index a8d36fd8..afb6e655 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/SonarQubeReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,13 +29,12 @@ public class SonarQubeReaderTest extends ReaderTestBase { - private ResultFile pluginResultFile; + private static ResultFile pluginResultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { pluginResultFile = TestHelper.resultFileOf("testfiles/Benchmark_sonar-Java-Plugin-v3.14-1234.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test @@ -56,7 +54,7 @@ void readerHandlesGivenPluginResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.WEAK_RANDOM, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.WEAK_RANDOM, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReaderTest.java index 065e83d4..3ae97c44 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ThunderScanReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class ThunderScanReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_ThunderScan.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyZapReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, ThunderScanReader.class); + assertOnlyMatcherClassIs(resultFile, ThunderScanReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XPATH_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XPATH_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VeracodeReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VeracodeReaderTest.java index 16a21c30..f210d1f5 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VeracodeReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VeracodeReaderTest.java @@ -3,23 +3,25 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.*; +import org.owasp.benchmarkutils.score.CweNumber; +import org.owasp.benchmarkutils.score.ResultFile; +import org.owasp.benchmarkutils.score.TestHelper; +import org.owasp.benchmarkutils.score.TestSuiteResults; class VeracodeReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Veracode.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test void onlyVeracodeReportCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, VeracodeReader.class); + assertOnlyMatcherClassIs(resultFile, VeracodeReader.class); } @Test @@ -33,7 +35,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(3, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("7").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("8").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("7").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("8").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReaderTest.java index 436f4be6..4a407bc0 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/VisualCodeGrepperReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class VisualCodeGrepperReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_visualcodegrepper-v2.2.0.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyFindbugsReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, VisualCodeGrepperReader.class); + assertOnlyMatcherClassIs(resultFile, VisualCodeGrepperReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/W3AFReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/W3AFReaderTest.java index df533b82..b141cae4 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/W3AFReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/W3AFReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class W3AFReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_w3af-v1.7.6.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyW3AFReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, W3AFReader.class); + assertOnlyMatcherClassIs(resultFile, W3AFReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReaderTest.java index 14112594..27ae56b3 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiJsonReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class WapitiJsonReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Wapiti-v3.0.5.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyWapitiJsonReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, WapitiJsonReader.class); + assertOnlyMatcherClassIs(resultFile, WapitiJsonReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiReaderTest.java index eedbf951..e8efc8ab 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/WapitiReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class WapitiReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Wapiti-v3.0.3.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyWapitiReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, WapitiReader.class); + assertOnlyMatcherClassIs(resultFile, WapitiReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.SQL_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReaderTest.java index 056e4bf8..19f9c292 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapJsonReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,25 +29,24 @@ public class ZapJsonReaderTest extends ReaderTestBase { - private ResultFile resultFileOldFormat; - private ResultFile resultFileNewFormat; + private static ResultFile resultFileOldFormat; + private static ResultFile resultFileNewFormat; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFileOldFormat = TestHelper.resultFileOf("testfiles/Benchmark_ZAP-v2.10.0-oldfmt.json"); resultFileNewFormat = TestHelper.resultFileOf("testfiles/Benchmark_ZAP-v2.11.1.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyZapJsonReaderReportsCanReadAsTrueForOldFormat() { - assertOnlyMatcherClassIs(this.resultFileOldFormat, ZapJsonReader.class); + assertOnlyMatcherClassIs(resultFileOldFormat, ZapJsonReader.class); } @Test public void onlyZapJsonReaderReportsCanReadAsTrueForNewFormat() { - assertOnlyMatcherClassIs(this.resultFileNewFormat, ZapJsonReader.class); + assertOnlyMatcherClassIs(resultFileNewFormat, ZapJsonReader.class); } @Test @@ -62,8 +60,8 @@ void readerHandlesGivenOldFormatResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } @Test @@ -78,7 +76,8 @@ void readerHandlesGivenNewFormatResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.CSRF, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.COOKIE_WITHOUT_HTTPONLY, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.CSRF, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals( + CweNumber.COOKIE_WITHOUT_HTTPONLY, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapReaderTest.java index b630ee56..8ca6a29b 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/ZapReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class ZapReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_ZAP_WEEKLY.xml"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyZapReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, ZapReader.class); + assertOnlyMatcherClassIs(resultFile, ZapReader.class); } @Test @@ -54,7 +52,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/CodeQLReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/CodeQLReaderTest.java index 62753f8a..c1071ab4 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/CodeQLReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/CodeQLReaderTest.java @@ -19,9 +19,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -30,17 +29,16 @@ public class CodeQLReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_CodeQL-v2.13.sarif"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyCodeQLReaderTestReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, CodeQLReader.class); + assertOnlyMatcherClassIs(resultFile, CodeQLReader.class); } @Test @@ -55,8 +53,8 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } @Test @@ -72,7 +70,7 @@ void readerHandlesAlternativeResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.XSS, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.SQL_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.SQL_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/ContrastScanReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/ContrastScanReaderTest.java index 3fca82f3..786a05be 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/ContrastScanReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/ContrastScanReaderTest.java @@ -21,9 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -32,17 +31,16 @@ public class ContrastScanReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Contrast_3.9.0.sarif.json"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyContrastJsonReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, ContrastScanReader.class); + assertOnlyMatcherClassIs(resultFile, ContrastScanReader.class); } @Test @@ -58,7 +56,7 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/DatadogSastReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/DatadogSastReaderTest.java index bcbdacf7..8ccfcb93 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/DatadogSastReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/DatadogSastReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -31,17 +30,16 @@ public class DatadogSastReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_DatadogSast.sarif"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void canReadFile() { - assertOnlyMatcherClassIs(this.resultFile, DatadogSastReader.class); + assertOnlyMatcherClassIs(resultFile, DatadogSastReader.class); } @Test @@ -56,6 +54,6 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(1, result.getTotalResults()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("10").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("10").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/PrecautionReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/PrecautionReaderTest.java index 2b21897a..54b3008d 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/PrecautionReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/PrecautionReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -31,17 +30,16 @@ class PrecautionReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_Precaution.sarif"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlyPrecautionReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, PrecautionReader.class); + assertOnlyMatcherClassIs(resultFile, PrecautionReader.class); } @Test @@ -55,6 +53,6 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals("0.5.0", result.getToolVersion()); assertEquals(1, result.getTotalResults()); - assertEquals(CweNumber.WEAK_HASH_ALGO, result.get("73").get(0).getCWE()); + assertEquals(CweNumber.WEAK_HASH_ALGO, result.getTestCaseResults("73").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SemgrepSarifReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SemgrepSarifReaderTest.java index b522e055..8d1d8cbe 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SemgrepSarifReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SemgrepSarifReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -31,19 +30,18 @@ class SemgrepSarifReaderTest extends ReaderTestBase { - private ResultFile resultFileOSS, resultFilePRO; + private static ResultFile resultFileOSS, resultFilePRO; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFileOSS = TestHelper.resultFileOf("testfiles/Benchmark_semgrep-oss-v1.67.0.sarif"); resultFilePRO = TestHelper.resultFileOf("testfiles/Benchmark_semgrep-pro-v1.68.1.sarif"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test public void onlySemgrepSarifReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFileOSS, SemgrepSarifReader.class); - assertOnlyMatcherClassIs(this.resultFilePRO, SemgrepSarifReader.class); + assertOnlyMatcherClassIs(resultFileOSS, SemgrepSarifReader.class); + assertOnlyMatcherClassIs(resultFilePRO, SemgrepSarifReader.class); } @Test @@ -58,8 +56,9 @@ void readerHandlesSemgrepOSSResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COOKIE_WITHOUT_HTTPONLY, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals( + CweNumber.COOKIE_WITHOUT_HTTPONLY, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } @Test @@ -74,7 +73,8 @@ void readerHandlesSemgrepPROResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.COOKIE_WITHOUT_HTTPONLY, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XSS, result.get("2").get(0).getCWE()); + assertEquals( + CweNumber.COOKIE_WITHOUT_HTTPONLY, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XSS, result.getTestCaseResults("2").get(0).getCWE()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SnykReaderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SnykReaderTest.java index 105812ca..51c25106 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SnykReaderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/parsers/sarif/SnykReaderTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; import org.owasp.benchmarkutils.score.TestHelper; @@ -31,17 +30,16 @@ class SnykReaderTest extends ReaderTestBase { - private ResultFile resultFile; + private static ResultFile resultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { resultFile = TestHelper.resultFileOf("testfiles/Benchmark_SnykCodeCli.sarif"); - BenchmarkScore.TESTCASENAME = "BenchmarkTest"; } @Test void onlySnykReaderReportsCanReadAsTrue() { - assertOnlyMatcherClassIs(this.resultFile, SnykReader.class); + assertOnlyMatcherClassIs(resultFile, SnykReader.class); } @Test @@ -56,8 +54,8 @@ void readerHandlesGivenResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.INSECURE_COOKIE, result.get("1").get(0).getCWE()); - assertEquals(CweNumber.XPATH_INJECTION, result.get("2").get(0).getCWE()); + assertEquals(CweNumber.INSECURE_COOKIE, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals(CweNumber.XPATH_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); } @Test diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/MenuUpdaterTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/MenuUpdaterTest.java index 96ab91eb..24564d5c 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/MenuUpdaterTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/MenuUpdaterTest.java @@ -30,7 +30,7 @@ import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CategoryResults; @@ -45,10 +45,10 @@ class MenuUpdaterTest { - private File tmpDir; + private static File tmpDir; - @BeforeEach - void setUp() throws IOException { + @BeforeAll + static void setUp() throws IOException { tmpDir = Files.createTempDirectory("Benchmark.MenuUpdaterTest").toFile(); } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/OverallStatsTableTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/OverallStatsTableTest.java index 2d15b8ac..e964fc92 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/OverallStatsTableTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/OverallStatsTableTest.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.Configuration; @@ -38,8 +38,8 @@ class OverallStatsTableTest { - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { BenchmarkScore.config = Configuration.fromDefaultConfig(); } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/ToolScorecardTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/ToolScorecardTest.java index 356f9013..d6c53091 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/ToolScorecardTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/ToolScorecardTest.java @@ -44,7 +44,7 @@ class ToolScorecardTest { - private File tmpDir; + private static File tmpDir; @BeforeEach void setUp() throws IOException { diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/VulnerabilityStatsTableTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/VulnerabilityStatsTableTest.java index 7d412362..5e9a6ac5 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/VulnerabilityStatsTableTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/report/html/VulnerabilityStatsTableTest.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CategoryResults; @@ -39,8 +39,8 @@ class VulnerabilityStatsTableTest { - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { BenchmarkScore.config = Configuration.fromDefaultConfig(); } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProviderTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProviderTest.java index c4c4fbb1..955b581a 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProviderTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProviderTest.java @@ -20,7 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.owasp.benchmarkutils.score.CweNumber; import org.owasp.benchmarkutils.score.ResultFile; @@ -29,11 +29,11 @@ class ExpectedResultsProviderTest { - private ResultFile simpleResultFile; - private ResultFile extendedResultFile; + private static ResultFile simpleResultFile; + private static ResultFile extendedResultFile; - @BeforeEach - void setUp() { + @BeforeAll + static void setUp() { simpleResultFile = TestHelper.resultFileOf("expectedresults-1.2-simple.csv"); extendedResultFile = TestHelper.resultFileOf("expectedresults-1.2-extended.csv"); } @@ -47,11 +47,11 @@ void providerHandlesGivenSimpleResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("1").get(0).getCWE()); - assertNull(result.get("1").get(0).getSource()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("1").get(0).getCWE()); + assertNull(result.getTestCaseResults("1").get(0).getSource()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("2").get(0).getCWE()); - assertNull(result.get("2").get(0).getSource()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); + assertNull(result.getTestCaseResults("2").get(0).getSource()); } @Test @@ -63,10 +63,10 @@ void providerHandlesGivenExtendedResultFile() throws Exception { assertEquals(2, result.getTotalResults()); - assertEquals(CweNumber.PATH_TRAVERSAL, result.get("1").get(0).getCWE()); - assertEquals("RequestGetCookies.code", result.get("1").get(0).getSource()); + assertEquals(CweNumber.PATH_TRAVERSAL, result.getTestCaseResults("1").get(0).getCWE()); + assertEquals("RequestGetCookies.code", result.getTestCaseResults("1").get(0).getSource()); - assertEquals(CweNumber.COMMAND_INJECTION, result.get("2").get(0).getCWE()); - assertEquals("RequestGetHeader.code", result.get("2").get(0).getSource()); + assertEquals(CweNumber.COMMAND_INJECTION, result.getTestCaseResults("2").get(0).getCWE()); + assertEquals("RequestGetHeader.code", result.getTestCaseResults("2").get(0).getSource()); } } diff --git a/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ResultsFileCreatorTest.java b/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ResultsFileCreatorTest.java index f1596ee0..92f3a5ab 100644 --- a/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ResultsFileCreatorTest.java +++ b/plugin/src/test/java/org/owasp/benchmarkutils/score/service/ResultsFileCreatorTest.java @@ -26,7 +26,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.owasp.benchmarkutils.score.BenchmarkScore; import org.owasp.benchmarkutils.score.CweNumber; @@ -38,12 +38,12 @@ class ResultsFileCreatorTest { - private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - private File tmpDir; + private static File tmpDir; - @BeforeEach - void setUp() throws IOException { + @BeforeAll + static void setUp() throws IOException { tmpDir = Files.createTempDirectory("Benchmark.ResultsFileCreatorTest").toFile(); } @@ -62,7 +62,7 @@ void createsSimpleResultsFile() throws IOException { TestCaseResultBuilder.builder() .setTestCaseName("BenchmarkTest00001") .setTestNumber(1) - // .setCategory("pathtraver") + // .setCategory("pathtraver") .setCwe(CweNumber.PATH_TRAVERSAL) .setTruePositive(true) .setPassed(true) @@ -71,7 +71,7 @@ void createsSimpleResultsFile() throws IOException { TestCaseResultBuilder.builder() .setTestCaseName("BenchmarkTest00002") .setTestNumber(2) - // .setCategory("trustbound") + // .setCategory("trustbound") .setCwe(CweNumber.TRUST_BOUNDARY_VIOLATION) .setTruePositive(false) .setPassed(false) @@ -80,7 +80,7 @@ void createsSimpleResultsFile() throws IOException { TestCaseResultBuilder.builder() .setTestCaseName("BenchmarkTest00003") .setTestNumber(3) - // .setCategory("sqli") + // .setCategory("sqli") .setCwe(CweNumber.SQL_INJECTION) .setTruePositive(false) .setPassed(true) @@ -89,7 +89,7 @@ void createsSimpleResultsFile() throws IOException { TestCaseResultBuilder.builder() .setTestCaseName("BenchmarkTest00004") .setTestNumber(4) - // .setCategory("cmdi") + // .setCategory("cmdi") .setCwe(CweNumber.COMMAND_INJECTION) .setTruePositive(true) .setPassed(false) @@ -138,7 +138,7 @@ void createsFullDetailsResultFile() throws IOException { TestCaseResultBuilder.builder() .setTestCaseName("BenchmarkTest00001") .setTestNumber(1) - // .setCategory("pathtraver") + // .setCategory("pathtraver") .setCwe(CweNumber.PATH_TRAVERSAL) .setTruePositive(true) .setPassed(true) @@ -150,7 +150,7 @@ void createsFullDetailsResultFile() throws IOException { TestCaseResultBuilder.builder() .setTestCaseName("BenchmarkTest00002") .setTestNumber(2) - // .setCategory("trustbound") + // .setCategory("trustbound") .setCwe(CweNumber.TRUST_BOUNDARY_VIOLATION) .setTruePositive(false) .setPassed(false) @@ -174,7 +174,7 @@ void createsFullDetailsResultFile() throws IOException { TestCaseResultBuilder.builder() .setTestCaseName("BenchmarkTest00004") .setTestNumber(4) - // .setCategory("cmdi") + // .setCategory("cmdi") .setCwe(CweNumber.COMMAND_INJECTION) .setTruePositive(true) .setPassed(false) diff --git a/plugin/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/plugin/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 00000000..5b912d22 --- /dev/null +++ b/plugin/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1 @@ +org.owasp.benchmarkutils.score.TestHelper diff --git a/plugin/src/test/resources/expectedresults-testfile.csv b/plugin/src/test/resources/expectedresults-testfile.csv new file mode 100644 index 00000000..05b1e08d --- /dev/null +++ b/plugin/src/test/resources/expectedresults-testfile.csv @@ -0,0 +1,101 @@ +# test name, category, real vulnerability, cwe, Benchmark version: 1.2, 2016-06-1 +BenchmarkTest00001,pathtraver,true,22 +BenchmarkTest00002,pathtraver,true,22 +BenchmarkTest00003,hash,true,328 +BenchmarkTest00004,trustbound,true,501 +BenchmarkTest00005,crypto,true,327 +BenchmarkTest00006,cmdi,true,78 +BenchmarkTest00007,cmdi,true,78 +BenchmarkTest00008,sqli,true,89 +BenchmarkTest00009,hash,false,328 +BenchmarkTest00010,weakrand,false,330 +BenchmarkTest00011,pathtraver,true,22 +BenchmarkTest00012,ldapi,true,90 +BenchmarkTest00013,xss,true,79 +BenchmarkTest00014,xss,true,79 +BenchmarkTest00015,cmdi,true,78 +BenchmarkTest00016,securecookie,false,614 +BenchmarkTest00017,cmdi,true,78 +BenchmarkTest00018,sqli,true,89 +BenchmarkTest00019,crypto,true,327 +BenchmarkTest00020,crypto,true,327 +BenchmarkTest00021,ldapi,true,90 +BenchmarkTest00022,hash,false,328 +BenchmarkTest00023,weakrand,true,330 +BenchmarkTest00024,sqli,true,89 +BenchmarkTest00025,sqli,true,89 +BenchmarkTest00026,sqli,true,89 +BenchmarkTest00027,sqli,true,89 +BenchmarkTest00028,pathtraver,true,22 +BenchmarkTest00029,hash,true,328 +BenchmarkTest00030,xss,true,79 +BenchmarkTest00031,trustbound,true,501 +BenchmarkTest00032,sqli,true,89 +BenchmarkTest00033,sqli,true,89 +BenchmarkTest00034,sqli,true,89 +BenchmarkTest00035,crypto,true,327 +BenchmarkTest00036,xss,true,79 +BenchmarkTest00037,sqli,true,89 +BenchmarkTest00038,sqli,true,89 +BenchmarkTest00039,sqli,true,89 +BenchmarkTest00040,pathtraver,true,22 +BenchmarkTest00041,xss,true,79 +BenchmarkTest00042,weakrand,false,330 +BenchmarkTest00043,sqli,true,89 +BenchmarkTest00044,ldapi,true,90 +BenchmarkTest00045,pathtraver,true,22 +BenchmarkTest00046,hash,true,328 +BenchmarkTest00047,xss,true,79 +BenchmarkTest00048,xss,true,79 +BenchmarkTest00049,xss,true,79 +BenchmarkTest00050,crypto,true,327 +BenchmarkTest00051,cmdi,false,78 +BenchmarkTest00052,sqli,false,89 +BenchmarkTest00053,crypto,true,327 +BenchmarkTest00054,crypto,false,327 +BenchmarkTest00055,crypto,true,327 +BenchmarkTest00056,crypto,true,327 +BenchmarkTest00057,crypto,true,327 +BenchmarkTest00058,crypto,false,327 +BenchmarkTest00059,crypto,false,327 +BenchmarkTest00060,pathtraver,true,22 +BenchmarkTest00061,pathtraver,true,22 +BenchmarkTest00062,pathtraver,true,22 +BenchmarkTest00063,pathtraver,false,22 +BenchmarkTest00064,pathtraver,false,22 +BenchmarkTest00065,pathtraver,true,22 +BenchmarkTest00066,weakrand,true,330 +BenchmarkTest00067,weakrand,true,330 +BenchmarkTest00068,weakrand,true,330 +BenchmarkTest00069,hash,false,328 +BenchmarkTest00070,hash,true,328 +BenchmarkTest00071,hash,true,328 +BenchmarkTest00072,hash,false,328 +BenchmarkTest00073,hash,true,328 +BenchmarkTest00074,hash,true,328 +BenchmarkTest00075,hash,false,328 +BenchmarkTest00076,hash,false,328 +BenchmarkTest00077,cmdi,true,78 +BenchmarkTest00078,weakrand,true,330 +BenchmarkTest00079,weakrand,true,330 +BenchmarkTest00080,weakrand,true,330 +BenchmarkTest00081,weakrand,true,330 +BenchmarkTest00082,weakrand,true,330 +BenchmarkTest00083,weakrand,true,330 +BenchmarkTest00084,weakrand,true,330 +BenchmarkTest00085,weakrand,true,330 +BenchmarkTest00086,weakrand,true,330 +BenchmarkTest00087,securecookie,true,614 +BenchmarkTest00088,securecookie,false,614 +BenchmarkTest00089,securecookie,false,614 +BenchmarkTest00090,cmdi,false,78 +BenchmarkTest00091,cmdi,true,78 +BenchmarkTest00092,cmdi,true,78 +BenchmarkTest00093,cmdi,false,78 +BenchmarkTest00094,weakrand,false,330 +BenchmarkTest00095,weakrand,false,330 +BenchmarkTest00096,weakrand,false,330 +BenchmarkTest00097,trustbound,false,501 +BenchmarkTest00098,trustbound,true,501 +BenchmarkTest00099,trustbound,false,501 +BenchmarkTest00100,sqli,true,89 diff --git a/plugin/src/test/resources/junit-platform.properties b/plugin/src/test/resources/junit-platform.properties new file mode 100644 index 00000000..49ab5718 --- /dev/null +++ b/plugin/src/test/resources/junit-platform.properties @@ -0,0 +1,2 @@ +junit.jupiter.extensions.autodetection.enabled=true +