workaround for gtest XML files not containing specialcahrs in test names

csoltenborn · Feb 3, 2019 · bfa71f8 · bfa71f8
1 parent e9e57e8
commit bfa71f8
Show file tree

Hide file tree

Showing 10 changed files with 408 additions and 55 deletions.
diff --git a/GoogleTestAdapter/Core.Tests/TestResults/StandardOutputTestResultParserTests.cs b/GoogleTestAdapter/Core.Tests/TestResults/StandardOutputTestResultParserTests.cs
@@ -119,6 +119,34 @@ [ RUN      ] Test.Fail
 [  FAILED  ] Test.Fail (0 ms)
 [----------] 3 tests from Test (3 ms total)
 
+[----------] Global test environment tear-down
+[==========] 3 tests from 1 test suite ran. (6 ms total)
+[  PASSED  ] 1 test.
+[  SKIPPED ] 1 test, listed below:
+[  SKIPPED ] Test.Skip
+[  FAILED  ] 1 test, listed below:
+[  FAILED  ] Test.Fail
+
+ 1 FAILED TEST
+".Split('\n');
+
+        /// <summary>
+        /// <see cref="https://github.com/csoltenborn/GoogleTestAdapter/issues/260"/>
+        /// </summary>
+        private string[] ConsoleOutputWithSkippedTestAsLastTest { get; } = @"[==========] Running 3 tests from 1 test suite.
+[----------] Global test environment set-up.
+[----------] 3 tests from Test
+[ RUN      ] Test.Succeed
+[       OK ] Test.Succeed (0 ms)
+[ RUN      ] Test.Fail
+C:\...\test.cpp(14): error: Value of: false
+  Actual: false
+Expected: true
+[  FAILED  ] Test.Fail (0 ms)
+[ RUN      ] Test.Skip
+[  SKIPPED ] Test.Skip (1 ms)
+[----------] 3 tests from Test (3 ms total)
+
 [----------] Global test environment tear-down
 [==========] 3 tests from 1 test suite ran. (6 ms total)
 [  PASSED  ] 1 test.
@@ -167,7 +195,7 @@ public override void SetUp()
         [TestCategory(Unit)]
         public void GetTestResults_CompleteOutput_ParsedCorrectly()
         {
-            List<TestResult> results = ComputeTestResults(Complete);
+            var results = ComputeTestResults(Complete);
 
             results.Should().HaveCount(3);
 
@@ -193,7 +221,7 @@ public void GetTestResults_CompleteOutput_ParsedCorrectly()
         [TestCategory(Unit)]
         public void GetTestResults_OutputWithImmediateCrash_CorrectResultHasCrashText()
         {
-            List<TestResult> results = ComputeTestResults(CrashesImmediately);
+            var results = ComputeTestResults(CrashesImmediately);
 
             results.Should().HaveCount(2);
 
@@ -214,7 +242,7 @@ public void GetTestResults_OutputWithImmediateCrash_CorrectResultHasCrashText()
         [TestCategory(Unit)]
         public void GetTestResults_OutputWithCrashAfterErrorMessage_CorrectResultHasCrashText()
         {
-            List<TestResult> results = ComputeTestResults(CrashesAfterErrorMsg);
+            var results = ComputeTestResults(CrashesAfterErrorMsg);
 
             results.Should().HaveCount(3);
 
@@ -240,7 +268,7 @@ public void GetTestResults_OutputWithCrashAfterErrorMessage_CorrectResultHasCras
         [TestCategory(Unit)]
         public void GetTestResults_OutputWithInvalidDurationUnit_DefaultDurationIsUsedAndWarningIsProduced()
         {
-            List<TestResult> results = ComputeTestResults(WrongDurationUnit);
+            var results = ComputeTestResults(WrongDurationUnit);
 
             results.Should().ContainSingle();
             results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
@@ -259,7 +287,7 @@ public void GetTestResults_OutputWithThousandsSeparatorInDuration_ParsedCorrectl
             Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo("en-US");
             try
             {
-                List<TestResult> results = ComputeTestResults(ThousandsSeparatorInDuration);
+                var results = ComputeTestResults(ThousandsSeparatorInDuration);
 
                 results.Should().ContainSingle();
                 results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
@@ -275,7 +303,7 @@ public void GetTestResults_OutputWithThousandsSeparatorInDuration_ParsedCorrectl
         [TestCategory(Unit)]
         public void GetTestResults_OutputWithConsoleOutput_ConsoleOutputIsIgnored()
         {
-            List<TestResult> results = ComputeTestResults(PassingTestProducesConsoleOutput);
+            var results = ComputeTestResults(PassingTestProducesConsoleOutput);
 
             results.Should().ContainSingle();
             results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
@@ -332,6 +360,34 @@ public void GetTestResults_OutputWithSkippedTest_AllResultsAreFound()
             XmlTestResultParserTests.AssertTestResultIsFailure(result);
         }
 
+        [TestMethod]
+        [TestCategory(Unit)]
+        public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
+        {
+            var cases = new List<TestCase>
+            {
+                TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
+                TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
+                TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
+            };
+
+            var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithSkippedTestAsLastTest, TestEnvironment.Logger).GetTestResults();
+
+            results.Should().HaveCount(3);
+
+            var result = results[0];
+            result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
+            XmlTestResultParserTests.AssertTestResultIsPassed(result);
+
+            result = results[1];
+            result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
+            XmlTestResultParserTests.AssertTestResultIsFailure(result);
+
+            result = results[2];
+            result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
+            XmlTestResultParserTests.AssertTestResultIsSkipped(result);
+        }
+
         [TestMethod]
         [TestCategory(Unit)]
         public void OutputHandling_OutputManyLinesWithNewlines_IsParsedCorrectly()
@@ -413,7 +469,7 @@ private IList<TestResult> GetTestResultsFromCompleteOutputFile()
         }
 
 
-        private List<TestResult> ComputeTestResults(List<string> consoleOutput)
+        private IList<TestResult> ComputeTestResults(List<string> consoleOutput)
         {
             var cases = new List<TestCase>
             {

diff --git a/GoogleTestAdapter/Core.Tests/TestResults/StreamingStandardOutputTestResultParserTests.cs b/GoogleTestAdapter/Core.Tests/TestResults/StreamingStandardOutputTestResultParserTests.cs
@@ -130,6 +130,34 @@ [ RUN      ] Test.Fail
 [  FAILED  ] Test.Fail (0 ms)
 [----------] 3 tests from Test (3 ms total)
 
+[----------] Global test environment tear-down
+[==========] 3 tests from 1 test suite ran. (6 ms total)
+[  PASSED  ] 1 test.
+[  SKIPPED ] 1 test, listed below:
+[  SKIPPED ] Test.Skip
+[  FAILED  ] 1 test, listed below:
+[  FAILED  ] Test.Fail
+
+ 1 FAILED TEST
+".Split('\n');
+
+        /// <summary>
+        /// <see cref="https://github.com/csoltenborn/GoogleTestAdapter/issues/260"/>
+        /// </summary>
+        private string[] ConsoleOutputWithSkippedTestAsLastTest { get; } = @"[==========] Running 3 tests from 1 test suite.
+[----------] Global test environment set-up.
+[----------] 3 tests from Test
+[ RUN      ] Test.Succeed
+[       OK ] Test.Succeed (0 ms)
+[ RUN      ] Test.Fail
+C:\...\test.cpp(14): error: Value of: false
+  Actual: false
+Expected: true
+[  FAILED  ] Test.Fail (0 ms)
+[ RUN      ] Test.Skip
+[  SKIPPED ] Test.Skip (1 ms)
+[----------] 3 tests from Test (3 ms total)
+
 [----------] Global test environment tear-down
 [==========] 3 tests from 1 test suite ran. (6 ms total)
 [  PASSED  ] 1 test.
@@ -383,6 +411,37 @@ public void GetTestResults_OutputWithSkippedTest_AllResultsAreFound()
             XmlTestResultParserTests.AssertTestResultIsFailure(result);
         }
 
+        [TestMethod]
+        [TestCategory(Unit)]
+        public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
+        {
+            var cases = new List<TestCase>
+            {
+                TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
+                TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
+                TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
+            };
+
+            var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
+            ConsoleOutputWithSkippedTestAsLastTest.ToList().ForEach(parser.ReportLine);
+            parser.Flush();
+            var results = parser.TestResults;
+
+            results.Should().HaveCount(3);
+
+            var result = results[0];
+            result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
+            XmlTestResultParserTests.AssertTestResultIsPassed(result);
+
+            result = results[1];
+            result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
+            XmlTestResultParserTests.AssertTestResultIsFailure(result);
+
+            result = results[2];
+            result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
+            XmlTestResultParserTests.AssertTestResultIsSkipped(result);
+        }
+
         [TestMethod]
         [TestCategory(Unit)]
         public void OutputHandling_OutputManyLinesWithNewlines_IsParsedCorrectly()

diff --git a/GoogleTestAdapter/Core.Tests/TestResults/XmlTestResultParserTests.cs b/GoogleTestAdapter/Core.Tests/TestResults/XmlTestResultParserTests.cs
@@ -20,7 +20,7 @@ public void GetTestResults_FileDoesNotExist_WarningAndEmptyResult()
             IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest",
                 "FooSuite.BazTest", "BarSuite.BazTest2");
 
-            var parser = new XmlTestResultParser(testCases, "somefile", TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", "somefile", TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().BeEmpty();
@@ -35,7 +35,7 @@ public void GetTestResults_InvalidFile_WarningAndEmptyResult()
                 "GoogleTestSuiteName1.TestMethod_002");
             MockOptions.Setup(o => o.DebugMode).Returns(true);
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFileBroken, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFileBroken, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().BeEmpty();
@@ -50,7 +50,7 @@ public void GetTestResults_FileWithInvalidStatusAttribute_WarningAndEmptyResult(
                 "GoogleTestSuiteName1.TestMethod_002");
             MockOptions.Setup(o => o.DebugMode).Returns(true);
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFileBroken_InvalidStatusAttibute, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFileBroken_InvalidStatusAttibute, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().ContainSingle();
@@ -63,7 +63,7 @@ public void GetTestResults_Sample1_FindsPassedAndSkipptedResults()
         {
             IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001", "SimpleTest.DISABLED_TestMethodDisabled");
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().HaveCount(2);
@@ -77,7 +77,7 @@ public void GetTestResults_Sample1_UnexpectedTestOutcome_LogsErrorAndThrows()
         {
             IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_007");
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
             parser.Invoking(p => p.GetTestResults()).Should().NotThrow<Exception>();
             MockLogger.Verify(l => l.LogError(It.Is<string>(s => s.Contains("Foo"))), Times.Exactly(1));
         }
@@ -88,7 +88,7 @@ public void GetTestResults_Sample1_FindsPassedParameterizedResult()
         {
             IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7");
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().ContainSingle();
@@ -101,7 +101,7 @@ public void GetTestResults_Sample1_FindsFailureResult()
         {
             IEnumerable<Model.TestCase> testCases = TestDataCreator.ToTestCase("AnimalsTest.testGetEnoughAnimals", TestDataCreator.DummyExecutable, @"x:\prod\company\util\util.cpp").Yield();
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().ContainSingle();
@@ -119,7 +119,7 @@ public void GetTestResults_Sample1_FindsParamterizedFailureResult()
         {
             IEnumerable<Model.TestCase> testCases = TestDataCreator.ToTestCase("ParameterizedTestsTest1/AllEnabledTest.TestInstance/11", TestDataCreator.DummyExecutable, @"someSimpleParameterizedTest.cpp").Yield();
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().ContainSingle();
@@ -134,7 +134,7 @@ public void GetTestResults_Sample2_FindsPassedResult()
         {
             IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("FooTest.DoesXyz");
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFile2, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().ContainSingle();
@@ -148,7 +148,7 @@ public void GetTestResults_Sample2_FindsFailureResult()
             IEnumerable<Model.TestCase> testCases = TestDataCreator.ToTestCase("FooTest.MethodBarDoesAbc", TestDataCreator.DummyExecutable,
                 @"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp").Yield();
 
-            var parser = new XmlTestResultParser(testCases, TestResources.XmlFile2, TestEnvironment.Logger);
+            var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger);
             List<Model.TestResult> results = parser.GetTestResults();
 
             results.Should().ContainSingle();

diff --git a/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs b/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs
@@ -157,7 +157,7 @@ private IEnumerable<TestResult> TryRunTests(string executable, string workingDir
             var remainingTestCases =
                 arguments.TestCases.Except(streamingParser.TestResults.Select(tr => tr.TestCase));
             var testResults = new TestResultCollector(_logger, _threadName)
-                .CollectTestResults(remainingTestCases, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase);
+                .CollectTestResults(remainingTestCases, executable, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase);
             testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();
 
             return testResults;

diff --git a/GoogleTestAdapter/Core/Runners/TestResultCollector.cs b/GoogleTestAdapter/Core/Runners/TestResultCollector.cs
@@ -18,13 +18,13 @@ public TestResultCollector(ILogger logger, string threadName)
             _threadName = threadName;
         }
 
-        public List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, string resultXmlFile, List<string> consoleOutput, TestCase crashedTestCase)
+        public List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, string testExecutable, string resultXmlFile, List<string> consoleOutput, TestCase crashedTestCase)
         {
             var testResults = new List<TestResult>();
             TestCase[] arrTestCasesRun = testCasesRun as TestCase[] ?? testCasesRun.ToArray();
 
             if (testResults.Count < arrTestCasesRun.Length)
-                CollectResultsFromXmlFile(arrTestCasesRun, resultXmlFile, testResults);
+                CollectResultsFromXmlFile(arrTestCasesRun, testExecutable, resultXmlFile, testResults);
 
             var consoleParser = new StandardOutputTestResultParser(arrTestCasesRun, consoleOutput, _logger);
             if (testResults.Count < arrTestCasesRun.Length)
@@ -48,9 +48,9 @@ public List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, s
             return testResults;
         }
 
-        private void CollectResultsFromXmlFile(TestCase[] testCasesRun, string resultXmlFile, List<TestResult> testResults)
+        private void CollectResultsFromXmlFile(TestCase[] testCasesRun, string testExecutable, string resultXmlFile, List<TestResult> testResults)
         {
-            var xmlParser = new XmlTestResultParser(testCasesRun, resultXmlFile, _logger);
+            var xmlParser = new XmlTestResultParser(testCasesRun, testExecutable, resultXmlFile, _logger);
             List<TestResult> xmlResults = xmlParser.GetTestResults();
             int nrOfCollectedTestResults = 0;
             foreach (TestResult testResult in xmlResults.Where(