From bfa71f8d13f1df636df8a966c3ac316c249900c3 Mon Sep 17 00:00:00 2001 From: Christian Soltenborn Date: Sun, 3 Feb 2019 07:21:18 +0100 Subject: [PATCH] workaround for gtest XML files not containing specialcahrs in test names --- .../StandardOutputTestResultParserTests.cs | 70 ++++++- ...mingStandardOutputTestResultParserTests.cs | 59 ++++++ .../TestResults/XmlTestResultParserTests.cs | 20 +- .../Core/Runners/SequentialTestRunner.cs | 2 +- .../Core/Runners/TestResultCollector.cs | 8 +- .../Core/TestResults/XmlTestResultParser.cs | 55 ++++- .../TestAdapter.Tests.csproj | 31 +-- .../TestExecutorSequentialTests.cs | 20 +- ...cutorSequentialTests_FrameworkDebugging.cs | 191 ++++++++++++++++++ .../TestExecutorTestsBase.cs | 7 + 10 files changed, 408 insertions(+), 55 deletions(-) create mode 100644 GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests_FrameworkDebugging.cs diff --git a/GoogleTestAdapter/Core.Tests/TestResults/StandardOutputTestResultParserTests.cs b/GoogleTestAdapter/Core.Tests/TestResults/StandardOutputTestResultParserTests.cs index 550dfeac8..f62145120 100644 --- a/GoogleTestAdapter/Core.Tests/TestResults/StandardOutputTestResultParserTests.cs +++ b/GoogleTestAdapter/Core.Tests/TestResults/StandardOutputTestResultParserTests.cs @@ -119,6 +119,34 @@ [ RUN ] Test.Fail [ FAILED ] Test.Fail (0 ms) [----------] 3 tests from Test (3 ms total) +[----------] Global test environment tear-down +[==========] 3 tests from 1 test suite ran. (6 ms total) +[ PASSED ] 1 test. +[ SKIPPED ] 1 test, listed below: +[ SKIPPED ] Test.Skip +[ FAILED ] 1 test, listed below: +[ FAILED ] Test.Fail + + 1 FAILED TEST +".Split('\n'); + + /// + /// + /// + private string[] ConsoleOutputWithSkippedTestAsLastTest { get; } = @"[==========] Running 3 tests from 1 test suite. +[----------] Global test environment set-up. +[----------] 3 tests from Test +[ RUN ] Test.Succeed +[ OK ] Test.Succeed (0 ms) +[ RUN ] Test.Fail +C:\...\test.cpp(14): error: Value of: false + Actual: false +Expected: true +[ FAILED ] Test.Fail (0 ms) +[ RUN ] Test.Skip +[ SKIPPED ] Test.Skip (1 ms) +[----------] 3 tests from Test (3 ms total) + [----------] Global test environment tear-down [==========] 3 tests from 1 test suite ran. (6 ms total) [ PASSED ] 1 test. @@ -167,7 +195,7 @@ public override void SetUp() [TestCategory(Unit)] public void GetTestResults_CompleteOutput_ParsedCorrectly() { - List results = ComputeTestResults(Complete); + var results = ComputeTestResults(Complete); results.Should().HaveCount(3); @@ -193,7 +221,7 @@ public void GetTestResults_CompleteOutput_ParsedCorrectly() [TestCategory(Unit)] public void GetTestResults_OutputWithImmediateCrash_CorrectResultHasCrashText() { - List results = ComputeTestResults(CrashesImmediately); + var results = ComputeTestResults(CrashesImmediately); results.Should().HaveCount(2); @@ -214,7 +242,7 @@ public void GetTestResults_OutputWithImmediateCrash_CorrectResultHasCrashText() [TestCategory(Unit)] public void GetTestResults_OutputWithCrashAfterErrorMessage_CorrectResultHasCrashText() { - List results = ComputeTestResults(CrashesAfterErrorMsg); + var results = ComputeTestResults(CrashesAfterErrorMsg); results.Should().HaveCount(3); @@ -240,7 +268,7 @@ public void GetTestResults_OutputWithCrashAfterErrorMessage_CorrectResultHasCras [TestCategory(Unit)] public void GetTestResults_OutputWithInvalidDurationUnit_DefaultDurationIsUsedAndWarningIsProduced() { - List results = ComputeTestResults(WrongDurationUnit); + var results = ComputeTestResults(WrongDurationUnit); results.Should().ContainSingle(); results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails"); @@ -259,7 +287,7 @@ public void GetTestResults_OutputWithThousandsSeparatorInDuration_ParsedCorrectl Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo("en-US"); try { - List results = ComputeTestResults(ThousandsSeparatorInDuration); + var results = ComputeTestResults(ThousandsSeparatorInDuration); results.Should().ContainSingle(); results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails"); @@ -275,7 +303,7 @@ public void GetTestResults_OutputWithThousandsSeparatorInDuration_ParsedCorrectl [TestCategory(Unit)] public void GetTestResults_OutputWithConsoleOutput_ConsoleOutputIsIgnored() { - List results = ComputeTestResults(PassingTestProducesConsoleOutput); + var results = ComputeTestResults(PassingTestProducesConsoleOutput); results.Should().ContainSingle(); results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses"); @@ -332,6 +360,34 @@ public void GetTestResults_OutputWithSkippedTest_AllResultsAreFound() XmlTestResultParserTests.AssertTestResultIsFailure(result); } + [TestMethod] + [TestCategory(Unit)] + public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound() + { + var cases = new List + { + TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), + TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), + TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), + }; + + var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithSkippedTestAsLastTest, TestEnvironment.Logger).GetTestResults(); + + results.Should().HaveCount(3); + + var result = results[0]; + result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed"); + XmlTestResultParserTests.AssertTestResultIsPassed(result); + + result = results[1]; + result.TestCase.FullyQualifiedName.Should().Be("Test.Fail"); + XmlTestResultParserTests.AssertTestResultIsFailure(result); + + result = results[2]; + result.TestCase.FullyQualifiedName.Should().Be("Test.Skip"); + XmlTestResultParserTests.AssertTestResultIsSkipped(result); + } + [TestMethod] [TestCategory(Unit)] public void OutputHandling_OutputManyLinesWithNewlines_IsParsedCorrectly() @@ -413,7 +469,7 @@ private IList GetTestResultsFromCompleteOutputFile() } - private List ComputeTestResults(List consoleOutput) + private IList ComputeTestResults(List consoleOutput) { var cases = new List { diff --git a/GoogleTestAdapter/Core.Tests/TestResults/StreamingStandardOutputTestResultParserTests.cs b/GoogleTestAdapter/Core.Tests/TestResults/StreamingStandardOutputTestResultParserTests.cs index b12437f50..b2fa5ac32 100644 --- a/GoogleTestAdapter/Core.Tests/TestResults/StreamingStandardOutputTestResultParserTests.cs +++ b/GoogleTestAdapter/Core.Tests/TestResults/StreamingStandardOutputTestResultParserTests.cs @@ -130,6 +130,34 @@ [ RUN ] Test.Fail [ FAILED ] Test.Fail (0 ms) [----------] 3 tests from Test (3 ms total) +[----------] Global test environment tear-down +[==========] 3 tests from 1 test suite ran. (6 ms total) +[ PASSED ] 1 test. +[ SKIPPED ] 1 test, listed below: +[ SKIPPED ] Test.Skip +[ FAILED ] 1 test, listed below: +[ FAILED ] Test.Fail + + 1 FAILED TEST +".Split('\n'); + + /// + /// + /// + private string[] ConsoleOutputWithSkippedTestAsLastTest { get; } = @"[==========] Running 3 tests from 1 test suite. +[----------] Global test environment set-up. +[----------] 3 tests from Test +[ RUN ] Test.Succeed +[ OK ] Test.Succeed (0 ms) +[ RUN ] Test.Fail +C:\...\test.cpp(14): error: Value of: false + Actual: false +Expected: true +[ FAILED ] Test.Fail (0 ms) +[ RUN ] Test.Skip +[ SKIPPED ] Test.Skip (1 ms) +[----------] 3 tests from Test (3 ms total) + [----------] Global test environment tear-down [==========] 3 tests from 1 test suite ran. (6 ms total) [ PASSED ] 1 test. @@ -383,6 +411,37 @@ public void GetTestResults_OutputWithSkippedTest_AllResultsAreFound() XmlTestResultParserTests.AssertTestResultIsFailure(result); } + [TestMethod] + [TestCategory(Unit)] + public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound() + { + var cases = new List + { + TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), + TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), + TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), + }; + + var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object); + ConsoleOutputWithSkippedTestAsLastTest.ToList().ForEach(parser.ReportLine); + parser.Flush(); + var results = parser.TestResults; + + results.Should().HaveCount(3); + + var result = results[0]; + result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed"); + XmlTestResultParserTests.AssertTestResultIsPassed(result); + + result = results[1]; + result.TestCase.FullyQualifiedName.Should().Be("Test.Fail"); + XmlTestResultParserTests.AssertTestResultIsFailure(result); + + result = results[2]; + result.TestCase.FullyQualifiedName.Should().Be("Test.Skip"); + XmlTestResultParserTests.AssertTestResultIsSkipped(result); + } + [TestMethod] [TestCategory(Unit)] public void OutputHandling_OutputManyLinesWithNewlines_IsParsedCorrectly() diff --git a/GoogleTestAdapter/Core.Tests/TestResults/XmlTestResultParserTests.cs b/GoogleTestAdapter/Core.Tests/TestResults/XmlTestResultParserTests.cs index 9caecf5f2..1cc9203de 100644 --- a/GoogleTestAdapter/Core.Tests/TestResults/XmlTestResultParserTests.cs +++ b/GoogleTestAdapter/Core.Tests/TestResults/XmlTestResultParserTests.cs @@ -20,7 +20,7 @@ public void GetTestResults_FileDoesNotExist_WarningAndEmptyResult() IEnumerable testCases = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest", "FooSuite.BazTest", "BarSuite.BazTest2"); - var parser = new XmlTestResultParser(testCases, "somefile", TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", "somefile", TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().BeEmpty(); @@ -35,7 +35,7 @@ public void GetTestResults_InvalidFile_WarningAndEmptyResult() "GoogleTestSuiteName1.TestMethod_002"); MockOptions.Setup(o => o.DebugMode).Returns(true); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFileBroken, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFileBroken, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().BeEmpty(); @@ -50,7 +50,7 @@ public void GetTestResults_FileWithInvalidStatusAttribute_WarningAndEmptyResult( "GoogleTestSuiteName1.TestMethod_002"); MockOptions.Setup(o => o.DebugMode).Returns(true); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFileBroken_InvalidStatusAttibute, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFileBroken_InvalidStatusAttibute, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().ContainSingle(); @@ -63,7 +63,7 @@ public void GetTestResults_Sample1_FindsPassedAndSkipptedResults() { IEnumerable testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001", "SimpleTest.DISABLED_TestMethodDisabled"); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().HaveCount(2); @@ -77,7 +77,7 @@ public void GetTestResults_Sample1_UnexpectedTestOutcome_LogsErrorAndThrows() { IEnumerable testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_007"); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger); parser.Invoking(p => p.GetTestResults()).Should().NotThrow(); MockLogger.Verify(l => l.LogError(It.Is(s => s.Contains("Foo"))), Times.Exactly(1)); } @@ -88,7 +88,7 @@ public void GetTestResults_Sample1_FindsPassedParameterizedResult() { IEnumerable testCases = TestDataCreator.CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7"); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().ContainSingle(); @@ -101,7 +101,7 @@ public void GetTestResults_Sample1_FindsFailureResult() { IEnumerable testCases = TestDataCreator.ToTestCase("AnimalsTest.testGetEnoughAnimals", TestDataCreator.DummyExecutable, @"x:\prod\company\util\util.cpp").Yield(); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().ContainSingle(); @@ -119,7 +119,7 @@ public void GetTestResults_Sample1_FindsParamterizedFailureResult() { IEnumerable testCases = TestDataCreator.ToTestCase("ParameterizedTestsTest1/AllEnabledTest.TestInstance/11", TestDataCreator.DummyExecutable, @"someSimpleParameterizedTest.cpp").Yield(); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().ContainSingle(); @@ -134,7 +134,7 @@ public void GetTestResults_Sample2_FindsPassedResult() { IEnumerable testCases = TestDataCreator.CreateDummyTestCases("FooTest.DoesXyz"); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFile2, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().ContainSingle(); @@ -148,7 +148,7 @@ public void GetTestResults_Sample2_FindsFailureResult() IEnumerable testCases = TestDataCreator.ToTestCase("FooTest.MethodBarDoesAbc", TestDataCreator.DummyExecutable, @"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp").Yield(); - var parser = new XmlTestResultParser(testCases, TestResources.XmlFile2, TestEnvironment.Logger); + var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger); List results = parser.GetTestResults(); results.Should().ContainSingle(); diff --git a/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs b/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs index 98a4d840e..fc6a6c64d 100644 --- a/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs +++ b/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs @@ -157,7 +157,7 @@ private IEnumerable TryRunTests(string executable, string workingDir var remainingTestCases = arguments.TestCases.Except(streamingParser.TestResults.Select(tr => tr.TestCase)); var testResults = new TestResultCollector(_logger, _threadName) - .CollectTestResults(remainingTestCases, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase); + .CollectTestResults(remainingTestCases, executable, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase); testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList(); return testResults; diff --git a/GoogleTestAdapter/Core/Runners/TestResultCollector.cs b/GoogleTestAdapter/Core/Runners/TestResultCollector.cs index 3ac00fff5..9f4d82b5e 100644 --- a/GoogleTestAdapter/Core/Runners/TestResultCollector.cs +++ b/GoogleTestAdapter/Core/Runners/TestResultCollector.cs @@ -18,13 +18,13 @@ public TestResultCollector(ILogger logger, string threadName) _threadName = threadName; } - public List CollectTestResults(IEnumerable testCasesRun, string resultXmlFile, List consoleOutput, TestCase crashedTestCase) + public List CollectTestResults(IEnumerable testCasesRun, string testExecutable, string resultXmlFile, List consoleOutput, TestCase crashedTestCase) { var testResults = new List(); TestCase[] arrTestCasesRun = testCasesRun as TestCase[] ?? testCasesRun.ToArray(); if (testResults.Count < arrTestCasesRun.Length) - CollectResultsFromXmlFile(arrTestCasesRun, resultXmlFile, testResults); + CollectResultsFromXmlFile(arrTestCasesRun, testExecutable, resultXmlFile, testResults); var consoleParser = new StandardOutputTestResultParser(arrTestCasesRun, consoleOutput, _logger); if (testResults.Count < arrTestCasesRun.Length) @@ -48,9 +48,9 @@ public List CollectTestResults(IEnumerable testCasesRun, s return testResults; } - private void CollectResultsFromXmlFile(TestCase[] testCasesRun, string resultXmlFile, List testResults) + private void CollectResultsFromXmlFile(TestCase[] testCasesRun, string testExecutable, string resultXmlFile, List testResults) { - var xmlParser = new XmlTestResultParser(testCasesRun, resultXmlFile, _logger); + var xmlParser = new XmlTestResultParser(testCasesRun, testExecutable, resultXmlFile, _logger); List xmlResults = xmlParser.GetTestResults(); int nrOfCollectedTestResults = 0; foreach (TestResult testResult in xmlResults.Where( diff --git a/GoogleTestAdapter/Core/TestResults/XmlTestResultParser.cs b/GoogleTestAdapter/Core/TestResults/XmlTestResultParser.cs index 7fc2ddc8b..d1ef25c8d 100644 --- a/GoogleTestAdapter/Core/TestResults/XmlTestResultParser.cs +++ b/GoogleTestAdapter/Core/TestResults/XmlTestResultParser.cs @@ -6,9 +6,11 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using System.Xml; using GoogleTestAdapter.Common; using GoogleTestAdapter.Model; +using GoogleTestAdapter.Settings; namespace GoogleTestAdapter.TestResults { @@ -18,17 +20,63 @@ public class XmlTestResultParser private static readonly NumberFormatInfo NumberFormatInfo = new CultureInfo("en-US").NumberFormat; + private static readonly Regex SpecialCharsRegex = new Regex("[äöüÄÖÜß]+", RegexOptions.Compiled); + private readonly ILogger _logger; private readonly string _xmlResultFile; + private readonly string _testExecutable; private readonly IDictionary _testCasesMap; + private readonly Lazy> _workaroundMapLazy; - public XmlTestResultParser(IEnumerable testCasesRun, string xmlResultFile, ILogger logger) + public XmlTestResultParser(IEnumerable testCasesRun, string testExecutable, string xmlResultFile, ILogger logger) { _logger = logger; + _testExecutable = testExecutable; _xmlResultFile = xmlResultFile; _testCasesMap = testCasesRun.ToDictionary(tc => tc.FullyQualifiedName, tc => tc); + _workaroundMapLazy = new Lazy>(CreateWorkaroundMap); + } + + // workaround for special chars not ending up in gtest xml output file + private IDictionary CreateWorkaroundMap() + { + var map = new Dictionary(); + var duplicates = new HashSet(); + var duplicateTestCases = new List(); + + foreach (var kvp in _testCasesMap) + { + string workaroundKey = SpecialCharsRegex.Replace(kvp.Key, ""); + if (map.ContainsKey(workaroundKey)) + { + duplicates.Add(workaroundKey); + duplicateTestCases.Add(kvp.Value); + } + else + { + map[workaroundKey] = kvp.Value; + } + } + + foreach (var duplicate in duplicates) + { + duplicateTestCases.Add(map[duplicate]); + map.Remove(duplicate); + } + + if (duplicateTestCases.Any()) + { + string message = + $"Executable {_testExecutable} has test names containing characters which happen to not end up in the XML file produced by Google Test. " + + "This has caused ambiguities while resolving the according test results, which are thus not available. " + + $"Note that this problem does not occur if GTA option '{SettingsWrapper.OptionUseNewTestExecutionFramework}' is enabled." + + $"\nThe following tests are affected: {string.Join(", ", duplicateTestCases.Select(tc => tc.FullyQualifiedName).OrderBy(n => n))}"; + _logger.LogWarning(message); + } + + return map; } @@ -99,8 +147,11 @@ private TestResult ParseTestResult(XmlNode testcaseNode) string qualifiedName = GetQualifiedName(testcaseNode); TestCase testCase; - if (!_testCasesMap.TryGetValue(qualifiedName, out testCase)) + if (!_testCasesMap.TryGetValue(qualifiedName, out testCase) && + !_workaroundMapLazy.Value.TryGetValue(qualifiedName, out testCase)) + { return null; + } var testResult = new TestResult(testCase) { diff --git a/GoogleTestAdapter/TestAdapter.Tests/TestAdapter.Tests.csproj b/GoogleTestAdapter/TestAdapter.Tests/TestAdapter.Tests.csproj index d0f7e4875..e3ec4d04d 100644 --- a/GoogleTestAdapter/TestAdapter.Tests/TestAdapter.Tests.csproj +++ b/GoogleTestAdapter/TestAdapter.Tests/TestAdapter.Tests.csproj @@ -41,23 +41,23 @@ - - - $(NuGetPackages)Microsoft.TestPlatform.ObjectModel.11.0.0\lib\net35\Microsoft.VisualStudio.TestPlatform.ObjectModel.dll - True - - + + + $(NuGetPackages)Microsoft.TestPlatform.ObjectModel.11.0.0\lib\net35\Microsoft.VisualStudio.TestPlatform.ObjectModel.dll + True + + - - - $(NuGetPackages)Microsoft.TestPlatform.ObjectModel.15.0.1\lib\net46\Microsoft.TestPlatform.CoreUtilities.dll - - - $(NuGetPackages)Microsoft.TestPlatform.ObjectModel.15.0.1\lib\net46\Microsoft.VisualStudio.TestPlatform.ObjectModel.dll - True - - + + + $(NuGetPackages)Microsoft.TestPlatform.ObjectModel.15.0.1\lib\net46\Microsoft.TestPlatform.CoreUtilities.dll + + + $(NuGetPackages)Microsoft.TestPlatform.ObjectModel.15.0.1\lib\net46\Microsoft.VisualStudio.TestPlatform.ObjectModel.dll + True + + @@ -99,6 +99,7 @@ + diff --git a/GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests.cs b/GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests.cs index 9b0b08b96..5f803d43e 100644 --- a/GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests.cs +++ b/GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests.cs @@ -46,14 +46,14 @@ protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailed [TestMethod] [TestCategory(Integration)] - public void RunTests_CancelingExecutor_StopsTestExecution() + public virtual void RunTests_CancelingExecutor_StopsTestExecution() { DoRunCancelingTests(false, DurationOfEachLongRunningTestInMs); // (only) 1st test should be executed } [TestMethod] [TestCategory(Integration)] - public void RunTests_CancelingExecutorAndKillProcesses_StopsTestExecutionFaster() + public virtual void RunTests_CancelingExecutorAndKillProcesses_StopsTestExecutionFaster() { DoRunCancelingTests(true, WaitBeforeCancelInMs); // 1st test should be actively canceled } @@ -123,14 +123,14 @@ public override void RunTests_StaticallyLinkedX64Tests_CorrectTestResults() [TestMethod] [TestCategory(Integration)] - public void RunTests_CrashingX64Tests_CorrectTestResults() + public virtual void RunTests_CrashingX64Tests_CorrectTestResults() { RunAndVerifyTests(TestResources.CrashingTests_ReleaseX64, 1, 2, 0, 3); } [TestMethod] [TestCategory(Integration)] - public void RunTests_CrashingX86Tests_CorrectTestResults() + public virtual void RunTests_CrashingX86Tests_CorrectTestResults() { RunAndVerifyTests(TestResources.CrashingTests_ReleaseX86, 1, 2, 0, 3); } @@ -186,16 +186,4 @@ public override void RunTests_WithoutPathExtension_ExecutionFails() #endregion } - - [TestClass] - public class TestExecutorSequentialTests_OldTestExecutionFramework : TestExecutorSequentialTests - { - [TestInitialize] - public override void SetUp() - { - base.SetUp(); - MockOptions.Setup(o => o.UseNewTestExecutionFramework).Returns(false); - } - } - } \ No newline at end of file diff --git a/GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests_FrameworkDebugging.cs b/GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests_FrameworkDebugging.cs new file mode 100644 index 000000000..752492f1c --- /dev/null +++ b/GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests_FrameworkDebugging.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using GoogleTestAdapter.Helpers; +using GoogleTestAdapter.Tests.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace GoogleTestAdapter.TestAdapter +{ + [TestClass] + public class TestExecutorSequentialTests_FrameworkDebugging : TestExecutorSequentialTests + { + [TestInitialize] + public override void SetUp() + { + base.SetUp(); + MockOptions.Setup(o => o.UseNewTestExecutionFramework).Returns(false); + + MockRunContext.Setup(c => c.IsBeingDebugged).Returns(true); + SetUpMockFrameworkHandle(); + } + + protected override void SetUpMockFrameworkHandle() + { + MockFrameworkHandle.Setup( + h => h.LaunchProcessWithDebuggerAttached( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Returns((string filePath, + string workingDirectory, + string arguments, + IDictionary environmentVariables) => + { + var processStartInfo = new ProcessStartInfo(filePath, arguments) + { + WorkingDirectory = workingDirectory, + UseShellExecute = false, + CreateNoWindow = true + }; + foreach (var kvp in environmentVariables) + { + processStartInfo.EnvironmentVariables[kvp.Key] = kvp.Value; + } + + var process = new Process {StartInfo = processStartInfo}; + if (!process.Start()) + throw new Exception("WTF!"); + return process.Id; + }); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_CrashingX64Tests_CorrectTestResults() + { + // test crashes, no info available if debugged via framework + RunAndVerifyTests(TestResources.CrashingTests_ReleaseX64, 0, 0, 0); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_CrashingX86Tests_CorrectTestResults() + { + // test crashes, no info available if debugged via framework + RunAndVerifyTests(TestResources.CrashingTests_ReleaseX86, 0, 0, 0); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_HardCrashingX86Tests_CorrectTestResults() + { + TestExecutor executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options); + executor.RunTests(TestResources.CrashingTests_DebugX86.Yield(), MockRunContext.Object, MockFrameworkHandle.Object); + + // test crashes, no info available if debugged via framework + CheckMockInvocations(0, 0, 0, 0); + } + + #region Method stubs for code coverage + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_ExternallyLinkedX64_CorrectTestResults() + { + base.RunTests_ExternallyLinkedX64_CorrectTestResults(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_ExternallyLinkedX86TestsInDebugMode_CorrectTestResults() + { + base.RunTests_ExternallyLinkedX86TestsInDebugMode_CorrectTestResults(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_ExternallyLinkedX86Tests_CorrectTestResults() + { + base.RunTests_ExternallyLinkedX86Tests_CorrectTestResults(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_StaticallyLinkedX64Tests_CorrectTestResults() + { + base.RunTests_StaticallyLinkedX64Tests_CorrectTestResults(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_StaticallyLinkedX86Tests_CorrectTestResults() + { + base.RunTests_StaticallyLinkedX86Tests_CorrectTestResults(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineArg() + { + base.RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineArg(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_WithNonexistingSetupBatch_LogsError() + { + base.RunTests_WithNonexistingSetupBatch_LogsError(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_WithPathExtension_ExecutionOk() + { + base.RunTests_WithPathExtension_ExecutionOk(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_WithSetupAndTeardownBatchesWhereSetupFails_LogsWarning() + { + base.RunTests_WithSetupAndTeardownBatchesWhereSetupFails_LogsWarning(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_WithSetupAndTeardownBatchesWhereTeardownFails_LogsWarning() + { + base.RunTests_WithSetupAndTeardownBatchesWhereTeardownFails_LogsWarning(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_WithoutBatches_NoLogging() + { + base.RunTests_WithoutBatches_NoLogging(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_WithoutPathExtension_ExecutionFails() + { + base.RunTests_WithoutPathExtension_ExecutionFails(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_WorkingDir_IsSetCorrectly() + { + base.RunTests_WorkingDir_IsSetCorrectly(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_CancelingExecutorAndKillProcesses_StopsTestExecutionFaster() + { + base.RunTests_CancelingExecutorAndKillProcesses_StopsTestExecutionFaster(); + } + + [TestMethod] + [TestCategory(TestMetadata.TestCategories.Integration)] + public override void RunTests_CancelingExecutor_StopsTestExecution() + { + base.RunTests_CancelingExecutor_StopsTestExecution(); + } + + #endregion + } +} \ No newline at end of file diff --git a/GoogleTestAdapter/TestAdapter.Tests/TestExecutorTestsBase.cs b/GoogleTestAdapter/TestAdapter.Tests/TestExecutorTestsBase.cs index f487915cf..12f9f9973 100644 --- a/GoogleTestAdapter/TestAdapter.Tests/TestExecutorTestsBase.cs +++ b/GoogleTestAdapter/TestAdapter.Tests/TestExecutorTestsBase.cs @@ -37,6 +37,10 @@ protected virtual void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedT Times.Exactly(nrOfUnexecutedTests)); } + protected virtual void SetUpMockFrameworkHandle() + { + } + [TestInitialize] public override void SetUp() { @@ -67,6 +71,7 @@ public virtual void RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineAr RunAndVerifySingleTest(testCase, VsTestOutcome.Failed); MockFrameworkHandle.Reset(); + SetUpMockFrameworkHandle(); MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-testdirectory=\"" + SettingsWrapper.TestDirPlaceholder + "\""); RunAndVerifySingleTest(testCase, VsTestOutcome.Passed); @@ -82,7 +87,9 @@ public virtual void RunTests_WorkingDir_IsSetCorrectly() RunAndVerifySingleTest(testCase, VsTestOutcome.Failed); MockFrameworkHandle.Reset(); + SetUpMockFrameworkHandle(); MockOptions.Setup(o => o.WorkingDir).Returns(SettingsWrapper.SolutionDirPlaceholder); + RunAndVerifySingleTest(testCase, VsTestOutcome.Passed); }