Fix DataSourceAttribute not being discovered with TestDataSourceDiscoveryOption.DuringDiscovery

azure-pipelines.yml

@@ -77,6 +77,14 @@ stages:
             failOnStderr: true
             showWarnings: true
 
+        - task: PowerShell@2
+          displayName: 'Install Access Database Engine'
+          inputs:
+            targetType: filePath
+            filePath: ./eng/install-access-database-engine.ps1
+            failOnStderr: true
+            showWarnings: true
+
         - script: eng\common\CIBuild.cmd
             -configuration $(_BuildConfig)
             -prepareMachine

eng/install-access-database-engine.ps1

@@ -0,0 +1,7 @@
+# This is required for DataSourceTests
+# Otherwise, the tests will fail with:
+# The unit test adapter failed to connect to the data source or to read the data. For more information on troubleshooting this error, see "Troubleshooting Data-Driven Unit Tests" (http://go.microsoft.com/fwlink/?LinkId=62412) in the MSDN Library. Error details: The 'Microsoft.Ace.OLEDB.12.0' provider is not registered on the local machine.
+#     at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.TestDataSource.GetData(ITestMethod testMethodInfo, ITestContext testContext) in /_/src/Adapter/MSTestAdapter.PlatformServices/Services/TestDataSource.cs:84
+# The direct download link originates from https://www.microsoft.com/en-us/download/details.aspx?id=54920&msockid=01fa77be234c617f31936293223560aa
+Invoke-RestMethod https://download.microsoft.com/download/3/5/C/35C84C36-661A-44E6-9324-8786B8DBE231/accessdatabaseengine_X64.exe -OutFile ./accessdatabaseengine_X64.exe
+Start-Process ./accessdatabaseengine_X64.exe -Wait -ArgumentList "/quiet /passive /norestart"

src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs

@@ -403,8 +403,14 @@ private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMeth
     private static bool ProcessTestDataSourceTests(UnitTestElement test, ReflectionTestMethodInfo methodInfo, IEnumerable<ITestDataSource> testDataSources,
         List<UnitTestElement> tests)
     {
+        // The data source tests that we can process currently are those using attributes that
+        // implement ITestDataSource (i.e, DataRow and DynamicData attributes).
+        // However, for DataSourceAttribute, we currently don't have anyway to process it during discovery.
+        // For that case, we want to return false from this method. Otherwise, the test will be completely skipped which is wrong behavior.
+        bool hasAtLeastOneITestDataSource = false;
         foreach (ITestDataSource dataSource in testDataSources)
         {
+            hasAtLeastOneITestDataSource = true;
             IEnumerable<object?[]>? data;
 
             // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests.
@@ -476,6 +482,6 @@ private static bool ProcessTestDataSourceTests(UnitTestElement test, ReflectionT
             tests.AddRange(discoveredTests);
         }
 
-        return true;
+        return hasAtLeastOneITestDataSource;
     }
 }

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DataSourceTests.cs

@@ -0,0 +1,117 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Platform.Acceptance.IntegrationTests;
+using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers;
+using Microsoft.Testing.Platform.Helpers;
+
+namespace MSTest.Acceptance.IntegrationTests;
+
+[TestGroup]
+public class DataSourceTests : AcceptanceTestBase
+{
+    private const string SourceCode = """
+#file DataSourceTests.csproj
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <TargetFramework>net472</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <OutputType>Exe</OutputType>
+        <LangVersion>preview</LangVersion>
+        <EnableMSTestRunner>true</EnableMSTestRunner>
+    </PropertyGroup>
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$MicrosoftNETTestSdkVersion$" />
+        <PackageReference Include="MSTest.TestAdapter" Version="$MSTestVersion$" />
+        <PackageReference Include="MSTest.TestFramework" Version="$MSTestVersion$" />
+
+        <None Include="TestData.csv">
+            <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        </None>
+    </ItemGroup>
+</Project>
+
+#file App.config
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <configSections>
+    <section name="microsoft.visualstudio.testtools" type="Microsoft.VisualStudio.TestTools.UnitTesting.TestConfigurationSection, Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions"/>
+  </configSections>
+  <connectionStrings>
+    <add name="ConnString" connectionString="TestData.csv" providerName="Microsoft.VisualStudio.TestTools.DataSource.CSV"/>
+  </connectionStrings>
+  <microsoft.visualstudio.testtools>
+    <dataSources>
+      <add name="TestData" connectionString="ConnString" dataTableName="TestData#csv" dataAccessMethod="Sequential"/>
+    </dataSources>
+  </microsoft.visualstudio.testtools>
+  <startup>
+    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8"/>
+  </startup>
+</configuration>
+
+
+#file MyTestClass.cs
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+[TestClass]
+public class MyTestClass
+{
+    public TestContext TestContext { get; set; }
+
+    [DataTestMethod]
+    [DataSource("TestData")]
+    public void TestSum()
+    {
+        int expected = (int)TestContext.DataRow["expectedSum"];
+        int num1 = (int)TestContext.DataRow["num1"];
+        int num2 = (int)TestContext.DataRow["num2"];
+        Assert.AreEqual(expected, num1 + num2);
+    }
+
+    [TestMethod]
+    public void MyTest()
+    {
+    }
+}
+
+#file TestData.csv
+num1,num2,expectedSum
+1,1,2
+5,6,11
+10,30,40
+1,1,1
+""";
+
+    private readonly AcceptanceFixture _acceptanceFixture;
+
+    public DataSourceTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture)
+        : base(testExecutionContext) => _acceptanceFixture = acceptanceFixture;
+
+    public async Task TestDataSourceFromAppConfig()
+    {
+        if (!OperatingSystem.IsWindows())
+        {
+            // Test is specific to .NET Framework.
+            return;
+        }
+
+        using TestAsset generator = await TestAsset.GenerateAssetAsync(
+            "DataSourceTests",
+            SourceCode
+            .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)
+            .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion),
+            addPublicFeeds: true);
+
+        await DotnetCli.RunAsync(
+            $"build {generator.TargetAssetPath} -c Release",
+            _acceptanceFixture.NuGetGlobalPackagesFolder.Path,
+            retryCount: 0);
+
+        var testHost = TestHost.LocateFrom(generator.TargetAssetPath, "DataSourceTests", "net472");
+
+        TestHostResult result = await testHost.ExecuteAsync();
+        result.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed);
+        result.AssertOutputContainsSummary(failed: 1, passed: 4, skipped: 0);
+    }
+}