From 998f1e0a51166a38e235d20a80f0cf8f945e040d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 13 Nov 2024 06:03:02 +0100 Subject: [PATCH 1/8] Fix DataSourceAttribute not being discovered with TestDataSourceDiscoveryOption.DuringDiscovery --- .../Discovery/AssemblyEnumerator.cs | 4 +- .../DataSourceTests.cs | 111 ++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index c820ee0e9c..5dcae8fb17 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -403,8 +403,10 @@ private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMeth private static bool ProcessTestDataSourceTests(UnitTestElement test, ReflectionTestMethodInfo methodInfo, IEnumerable testDataSources, List tests) { + bool hasAtLeastOneITestDataSource = false; foreach (ITestDataSource dataSource in testDataSources) { + hasAtLeastOneITestDataSource = true; IEnumerable? data; // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests. @@ -476,6 +478,6 @@ private static bool ProcessTestDataSourceTests(UnitTestElement test, ReflectionT tests.AddRange(discoveredTests); } - return true; + return hasAtLeastOneITestDataSource; } } diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs new file mode 100644 index 0000000000..e4ca8aba2b --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestGroup] +public class DataSourceTests : AcceptanceTestBase +{ + private const string SourceCode = """ +#file DataSourceTests.csproj + + + net472 + enable + Exe + preview + true + + + + + + + + PreserveNewest + + + + +#file App.config + + + +
+ + + + + + + + + + + + + + + +#file MyTestClass.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class MyTestClass +{ + public TestContext TestContext { get; set; } + + [DataTestMethod] + [DataSource("TestData")] + public void TestSum() + { + int expected = (int)TestContext.DataRow["expectedSum"]; + int num1 = (int)TestContext.DataRow["num1"]; + int num2 = (int)TestContext.DataRow["num2"]; + Assert.AreEqual(expected, num1 + num2); + } + + [TestMethod] + public void MyTest() + { + } +} + +#file TestData.csv +num1,num2,expectedSum +1,1,2 +5,6,11 +10,30,40 +1,1,1 +"""; + + private readonly AcceptanceFixture _acceptanceFixture; + + public DataSourceTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture) + : base(testExecutionContext) => _acceptanceFixture = acceptanceFixture; + + public async Task TestDataSourceFromAppConfig() + { + using TestAsset generator = await TestAsset.GenerateAssetAsync( + "DataSourceTests", + SourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion), + addPublicFeeds: true); + + await DotnetCli.RunAsync( + $"build {generator.TargetAssetPath} -c Release", + _acceptanceFixture.NuGetGlobalPackagesFolder.Path, + retryCount: 0); + + var testHost = TestHost.LocateFrom(generator.TargetAssetPath, "DataSourceTests", "net472"); + + TestHostResult result = await testHost.ExecuteAsync(); + result.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + result.AssertOutputContainsSummary(failed: 1, passed: 4, skipped: 0); + } +} From b43b5ddc76d60771282f0f3d8dc43f0a91ef6eff Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 13 Nov 2024 06:08:42 +0100 Subject: [PATCH 2/8] Add comment --- .../MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 5dcae8fb17..0910673538 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -403,6 +403,10 @@ private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMeth private static bool ProcessTestDataSourceTests(UnitTestElement test, ReflectionTestMethodInfo methodInfo, IEnumerable testDataSources, List tests) { + // The data source tests that we can process currently are those using attributes that + // implement ITestDataSource (i.e, DataRow and DynamicData attributes). + // However, for DataSourceAttribute, we currently don't have anyway to process it during discovery. + // For that case, we want to return false from this method. Otherwise, the test will be completely skipped which is wrong behavior. bool hasAtLeastOneITestDataSource = false; foreach (ITestDataSource dataSource in testDataSources) { From 93888d189fe57cc8ee116fe899ea383c358ed463 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 13 Nov 2024 06:52:20 +0100 Subject: [PATCH 3/8] Run only on Windows --- .../MSTest.Acceptance.IntegrationTests/DataSourceTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs index e4ca8aba2b..375dedad5f 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs @@ -90,6 +90,12 @@ public DataSourceTests(ITestExecutionContext testExecutionContext, AcceptanceFix public async Task TestDataSourceFromAppConfig() { + if (!OperatingSystem.IsWindows()) + { + // Test is specific to .NET Framework. + return; + } + using TestAsset generator = await TestAsset.GenerateAssetAsync( "DataSourceTests", SourceCode From 044fb5ba4897023474110438e29c9f7a829d59f4 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 13 Nov 2024 07:54:28 +0100 Subject: [PATCH 4/8] Install Access Database Engine in CI --- azure-pipelines.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 529b3077ac..f172e0d049 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -77,6 +77,17 @@ stages: failOnStderr: true showWarnings: true + # This is required for DataSourceTests + # Otherwise, the tests will fail with: + # The unit test adapter failed to connect to the data source or to read the data. For more information on troubleshooting this error, see "Troubleshooting Data-Driven Unit Tests" (http://go.microsoft.com/fwlink/?LinkId=62412) in the MSDN Library. Error details: The 'Microsoft.Ace.OLEDB.12.0' provider is not registered on the local machine. + # at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.TestDataSource.GetData(ITestMethod testMethodInfo, ITestContext testContext) in /_/src/Adapter/MSTestAdapter.PlatformServices/Services/TestDataSource.cs:84 + # The direct download link originates from https://www.microsoft.com/en-us/download/details.aspx?id=54920&msockid=01fa77be234c617f31936293223560aa + - task: PowerShell@2 + displayName: 'Install Access Database Engine' + run : | + Invoke-RestMethod https://download.microsoft.com/download/3/5/C/35C84C36-661A-44E6-9324-8786B8DBE231/accessdatabaseengine_X64.exe -OutFile ./accessdatabaseengine_X64.exe + Start-Process ./accessdatabaseengine_X64.exe -Wait -ArgumentList "/quiet /passive /norestart" + - script: eng\common\CIBuild.cmd -configuration $(_BuildConfig) -prepareMachine From 742fd9dbc60885f59dbf6413e5aa61efd791717b Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 13 Nov 2024 07:59:37 +0100 Subject: [PATCH 5/8] Adjust Access Database Engine installation --- eng/install-access-database-engine.ps1 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 eng/install-access-database-engine.ps1 diff --git a/eng/install-access-database-engine.ps1 b/eng/install-access-database-engine.ps1 new file mode 100644 index 0000000000..3ff7705794 --- /dev/null +++ b/eng/install-access-database-engine.ps1 @@ -0,0 +1,7 @@ +# This is required for DataSourceTests +# Otherwise, the tests will fail with: +# The unit test adapter failed to connect to the data source or to read the data. For more information on troubleshooting this error, see "Troubleshooting Data-Driven Unit Tests" (http://go.microsoft.com/fwlink/?LinkId=62412) in the MSDN Library. Error details: The 'Microsoft.Ace.OLEDB.12.0' provider is not registered on the local machine. +# at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.TestDataSource.GetData(ITestMethod testMethodInfo, ITestContext testContext) in /_/src/Adapter/MSTestAdapter.PlatformServices/Services/TestDataSource.cs:84 +# The direct download link originates from https://www.microsoft.com/en-us/download/details.aspx?id=54920&msockid=01fa77be234c617f31936293223560aa +Invoke-RestMethod https://download.microsoft.com/download/3/5/C/35C84C36-661A-44E6-9324-8786B8DBE231/accessdatabaseengine_X64.exe -OutFile ./accessdatabaseengine_X64.exe +Start-Process ./accessdatabaseengine_X64.exe -Wait -ArgumentList "/quiet /passive /norestart" From 04f901ceb6e7069a65c2734ab49081ba1fd5c07c Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 13 Nov 2024 08:01:42 +0100 Subject: [PATCH 6/8] Fix YML --- azure-pipelines.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f172e0d049..c7327adfd6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -77,16 +77,13 @@ stages: failOnStderr: true showWarnings: true - # This is required for DataSourceTests - # Otherwise, the tests will fail with: - # The unit test adapter failed to connect to the data source or to read the data. For more information on troubleshooting this error, see "Troubleshooting Data-Driven Unit Tests" (http://go.microsoft.com/fwlink/?LinkId=62412) in the MSDN Library. Error details: The 'Microsoft.Ace.OLEDB.12.0' provider is not registered on the local machine. - # at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.TestDataSource.GetData(ITestMethod testMethodInfo, ITestContext testContext) in /_/src/Adapter/MSTestAdapter.PlatformServices/Services/TestDataSource.cs:84 - # The direct download link originates from https://www.microsoft.com/en-us/download/details.aspx?id=54920&msockid=01fa77be234c617f31936293223560aa - task: PowerShell@2 displayName: 'Install Access Database Engine' - run : | - Invoke-RestMethod https://download.microsoft.com/download/3/5/C/35C84C36-661A-44E6-9324-8786B8DBE231/accessdatabaseengine_X64.exe -OutFile ./accessdatabaseengine_X64.exe - Start-Process ./accessdatabaseengine_X64.exe -Wait -ArgumentList "/quiet /passive /norestart" + inputs: + targetType: filePath + filePath: ./eng/install-access-database-engine.ps1 + failOnStderr: true + showWarnings: true - script: eng\common\CIBuild.cmd -configuration $(_BuildConfig) From afc6e2f71adb04f56c2d41043530caaf8f3c372a Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 13 Nov 2024 10:26:12 +0100 Subject: [PATCH 7/8] Address review comments --- .../Discovery/AssemblyEnumerator.cs | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 0910673538..1b51692f5f 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -285,11 +285,6 @@ private static bool DynamicDataAttached(UnitTestElement test, Lazy tests, HashSet fixtureTests) @@ -381,7 +384,7 @@ static UnitTestElement GetFixtureTest(string classFullName, string assemblyLocat } } - private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, List tests) + private static bool TryProcessITestDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, List tests) { // We don't have a special method to filter attributes that are not derived from Attribute, so we take all // attributes and filter them. We don't have to care if there is one, because this method is only entered when @@ -390,7 +393,7 @@ private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMeth try { - return ProcessTestDataSourceTests(test, new(testMethodInfo.MethodInfo, test.DisplayName), testDataSources, tests); + return ProcessITestDataSourceTests(test, new(testMethodInfo.MethodInfo, test.DisplayName), testDataSources, tests); } catch (Exception ex) { @@ -400,17 +403,11 @@ private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMeth } } - private static bool ProcessTestDataSourceTests(UnitTestElement test, ReflectionTestMethodInfo methodInfo, IEnumerable testDataSources, + private static bool ProcessITestDataSourceTests(UnitTestElement test, ReflectionTestMethodInfo methodInfo, IEnumerable testDataSources, List tests) { - // The data source tests that we can process currently are those using attributes that - // implement ITestDataSource (i.e, DataRow and DynamicData attributes). - // However, for DataSourceAttribute, we currently don't have anyway to process it during discovery. - // For that case, we want to return false from this method. Otherwise, the test will be completely skipped which is wrong behavior. - bool hasAtLeastOneITestDataSource = false; foreach (ITestDataSource dataSource in testDataSources) { - hasAtLeastOneITestDataSource = true; IEnumerable? data; // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests. @@ -482,6 +479,6 @@ private static bool ProcessTestDataSourceTests(UnitTestElement test, ReflectionT tests.AddRange(discoveredTests); } - return hasAtLeastOneITestDataSource; + return true; } } From 270d010e4ec0eb2f131828c04205cbba46f1803e Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Wed, 13 Nov 2024 22:17:29 +0100 Subject: [PATCH 8/8] Fix --- .../MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 1b51692f5f..f2015d0c41 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -285,6 +285,8 @@ private static bool DynamicDataAttached(UnitTestElement test, Lazy