Evaluate VS Test Platform as a better replacement for test discovery and execution

package.json

@@ -274,6 +274,9 @@
     "fkill": "^5.3.0",
     "glob": "^7.1.2",
     "ts-node": "^7.0.1",
-    "xmldom": "^0.1.27"
+    "xmldom": "^0.1.27",
+    "typescript-collections": "^1.3.2",
+    "freeport": "^1.0.5",
+    "amd-loader": "^0.0.8"
   }
 }

src/dotnetTestExplorer.ts

@@ -10,6 +10,7 @@ import { TestNode } from "./testNode";
 import { ITestResult, TestResult } from "./testResult";
 import { TestResultsFile } from "./testResultsFile";
 import { Utility } from "./utility";
+import { TestManager } from "./vsTestPlatform/vsCode/vsTest/vsTestManager";
 
 export class DotnetTestExplorer implements TreeDataProvider<TestNode> {
 
@@ -39,6 +40,16 @@ export class DotnetTestExplorer implements TreeDataProvider<TestNode> {
     public refreshTestExplorer(): void {
         this.testCommands.discoverTests();
 
+        // VTP testing code
+        TestManager.initialize(this.context, vscode.workspace.rootPath).then(() => {
+            // this.isTestExplorerInitialized = true;
+            // this._onDidChangeTreeData.fire();
+            const testManagerInstance = TestManager.getInstance();
+            const testService = testManagerInstance.getTestService();
+            Logger.Log("Test Manager initialised", "vsTest");
+            this.testCommands.vsDiscoverTests(testService);
+        });
+
         AppInsightsClient.sendEvent("refreshTestExplorer");
     }
 

src/extension.ts

@@ -17,6 +17,7 @@ import { TestNode } from "./testNode";
 import { TestResultsFile } from "./testResultsFile";
 import { TestStatusCodeLensProvider } from "./testStatusCodeLensProvider";
 import { Utility } from "./utility";
+import { TestManager } from "./vsTestPlatform/vsCode/vsTest/vsTestManager";
 import { Watch } from "./watch";
 
 export function activate(context: vscode.ExtensionContext) {
@@ -56,6 +57,15 @@ export function activate(context: vscode.ExtensionContext) {
 
     testCommands.discoverTests();
 
+    TestManager.initialize(this.context, vscode.workspace.rootPath).then(() => {
+        // this.isTestExplorerInitialized = true;
+        // this._onDidChangeTreeData.fire();
+        const testManagerInstance = TestManager.getInstance();
+        const testService = testManagerInstance.getTestService();
+        Logger.Log("Test Manager initialised", "vsTest");
+        testCommands.vsDiscoverTests(testService);
+    });
+
     const codeLensProvider = new TestStatusCodeLensProvider(testCommands);
     context.subscriptions.push(codeLensProvider);
     context.subscriptions.push(vscode.languages.registerCodeLensProvider(
@@ -83,7 +93,7 @@ export function activate(context: vscode.ExtensionContext) {
     }));
 
     context.subscriptions.push(vscode.commands.registerTextEditorCommand("dotnet-test-explorer.runTestInContext", (editor: vscode.TextEditor) => {
-        findTestInContext.find(editor.document, editor.selection.start).then( (testRunContext) => {
+        findTestInContext.find(editor.document, editor.selection.start).then((testRunContext) => {
             testCommands.runTestByName(testRunContext.testName, testRunContext.isSingleTest);
         });
     }));

src/testCommands.ts

@@ -1,4 +1,5 @@
 import * as path from "path";
+import * as vscode from "vscode";
 import { commands, Event, EventEmitter } from "vscode";
 import { AppInsightsClient } from "./appInsightsClient";
 import { Executor } from "./executor";
@@ -9,6 +10,7 @@ import { TestNode } from "./testNode";
 import { ITestResult, TestResult } from "./testResult";
 import { TestResultsFile } from "./testResultsFile";
 import { Utility } from "./utility";
+import { VSTestServiceIDE } from "./vsTestPlatform/vsCode/vsTest/vsTestServiceIDE";
 
 export interface ITestRunContext {
     testName: string;
@@ -39,7 +41,7 @@ export class TestCommands {
             try {
                 for (const dir of this.testDirectories.getTestDirectories()) {
                     const testsForDir: IDiscoverTestsResult = await discoverTests(dir, Utility.additionalArgumentsOption);
-                    this.testDirectories.addTestsForDirectory(testsForDir.testNames.map( (tn) => ({dir, name: tn})));
+                    this.testDirectories.addTestsForDirectory(testsForDir.testNames.map((tn) => ({ dir, name: tn })));
                     discoveredTests.push(testsForDir);
                 }
 
@@ -93,6 +95,23 @@ export class TestCommands {
         }
     }
 
+    public vsDiscoverTests(testService: VSTestServiceIDE) {
+        const date1 = new Date();
+        Logger.Log(date1.toLocaleString() + " - Start discovery VTP2");
+        testService.discoveryTests(vscode.workspace.rootPath).then((result) => {
+            if (result) {
+                Logger.Log(result.TotalTests.toString() + " tests discovered using VTP");
+                const date2 = new Date();
+                Logger.Log(date1.toLocaleString() + " - End discovery VTP2 in " + (date2.getMilliseconds() - date1.getMilliseconds()).toString() + "ms");
+                Logger.Log("");
+                // this._onDidChangeTreeData.fire();
+            } else {
+                // tslint:disable-next-line:no-console
+                console.info("vsTestPlatform: No tests discovered.");
+            }
+        });
+    }
+
     private runTestCommand(testName: string, isSingleTest: boolean): void {
 
         commands.executeCommand("workbench.view.extension.test", "workbench.view.extension.test");
@@ -107,12 +126,20 @@ export class TestCommands {
         const runSeq = async () => {
 
             try {
+                const date1 = new Date();
+                Logger.Log(date1.toLocaleString() + " - Start execution via cli");
+
                 for (let i = 0; i < testDirectories.length; i++) {
                     testResults.push(await this.runTestCommandForSpecificDirectory(testDirectories[i], testName, isSingleTest, i));
                 }
 
                 const merged = [].concat(...testResults);
-                this.sendNewTestResults({ testName, testResults: merged});
+
+                const date2 = new Date();
+                Logger.Log(date2.toLocaleString() + " - End execution via cli in " + (date2.getMilliseconds() - date1.getMilliseconds()).toString() + "ms");
+                Logger.Log("");
+
+                this.sendNewTestResults({ testName, testResults: merged });
             } catch (err) {
                 Logger.Log(`Error while executing test command: ${err}`);
                 this.discoverTests();
@@ -122,7 +149,7 @@ export class TestCommands {
         runSeq();
     }
 
-    private runBuildCommandForSpecificDirectory(testDirectoryPath: string): Promise<any>  {
+    private runBuildCommandForSpecificDirectory(testDirectoryPath: string): Promise<any> {
         return new Promise((resolve, reject) => {
 
             Logger.Log(`Executing dotnet build in ${testDirectoryPath}`);
@@ -140,7 +167,7 @@ export class TestCommands {
 
         const trxTestName = index + ".trx";
 
-        const textContext = {testName, isSingleTest};
+        const textContext = { testName, isSingleTest };
 
         return new Promise((resolve, reject) => {
             const testResultFile = path.join(Utility.pathForResultFile, "test-explorer", trxTestName);
@@ -159,7 +186,7 @@ export class TestCommands {
             this.onTestRunEmitter.fire(textContext);
 
             this.runBuildCommandForSpecificDirectory(testDirectoryPath)
-                .then( () => {
+                .then(() => {
                     Logger.Log(`Executing ${command} in ${testDirectoryPath}`);
 
                     return Executor.exec(command, (err, stdout: string) => {
@@ -171,12 +198,12 @@ export class TestCommands {
 
                         Logger.Log(stdout);
 
-                        this.resultsFile.parseResults(testResultFile).then( (result) => {
+                        this.resultsFile.parseResults(testResultFile).then((result) => {
                             resolve(result);
                         });
                     }, testDirectoryPath, true);
                 })
-                .catch( (err) => {
+                .catch((err) => {
                     reject(err);
                 });
         });

src/testDiscovery.ts

@@ -47,6 +47,9 @@ function executeDotnetTest(testDirectoryPath: string, dotnetTestOptions: string)
     return new Promise((resolve, reject) => {
         const command = `dotnet test -t -v=q${dotnetTestOptions}`;
 
+        const date1 = new Date();
+        Logger.Log(date1.toLocaleString() + " - Start discovery via cli");
+
         Logger.Log(`Executing ${command} in ${testDirectoryPath}`);
 
         Executor.exec(command, (err: Error, stdout: string, stderr: string) => {
@@ -56,9 +59,12 @@ function executeDotnetTest(testDirectoryPath: string, dotnetTestOptions: string)
                 reject(err);
                 return;
             }
-
+            const date2 = new Date();
+            Logger.Log(date2.toLocaleString() + " - End discovery via cli in " + (date2.getMilliseconds() - date1.getMilliseconds()).toString() + "ms");
+            Logger.Log("");
             resolve(stdout);
         }, testDirectoryPath);
+
     });
 }
 

src/vsTestPlatform/README.md

@@ -0,0 +1,87 @@
+# Evaluation TODO
+
+- [x] Get vstest platform code minimally working
+  - [x] convert spawn to macOS
+  - [x] vstest server spawns
+  - [x] can make version request and get response successfully
+- [x] Add license from source to all files
+- [x] Test discovery works and can see result data in debug mode
+  - [x] Mstest project
+  - [x] Nunit project
+  - [x] Xunit projet
+- [ ] Test execution works and can see result data in debug mode
+  - [ ] Mstest project
+  - [ ] Nunit project
+  - [ ] Xunit project
+- [ ] Test discovery performs better than running `dotnet vstest` and parsing TRX?
+  - with a 100+ tests, Discovery seem to be slightly (20-30ms) quicker for the first run. Refershes are quicker with the cli. Not sure why. VTP is the same-ish as the first run.
+- [ ] Test exection performs better than running with the cli
+- [x] Check if test **discovery** results data schema are the smae across test frameworks for key data?
+  - displayName value is FQN for Xunit but not for Nunit and Mstest. See examples below. Believe this is the same as TRX.
+- [ ] Check if test **execution** results data schema are the smae across test frameworks for key data?
+- [ ] Check what new data do we have available?
+- [ ] Can it handle multiple test projects in a single workspace?
+
+## additional npm dependencies
+
+    "typescript-collections": "^1.3.2",
+    "freeport": "^1.0.5",
+    "amd-loader": "^0.0.8"
+
+## Configuring the test project
+
+This is currently required manual editting of `/vsTestPlatform/vsCode/config.ts`
+
+## Check before shipping TODO
+
+- Works cross platform
+- Unit tests for VS Test Platform
+- Auto discover config for execution host i.e. what is currently hardcoded in config.ts
+- TSLint currently ignores **/src/vsTestPlatform/**/**.*. Undo and make TSLint happy (there'll be a lot). Leave winbase* excluded.
+
+## Test Discovery data responses
+
+The following are key/value from the json
+
+### Nunit
+
+```javascript
+isRunning:false
+plainObject:Object
+fullyQualifiedName:"NunitTests.TestClass1.AnotherPass"
+executorUri:"executor://NUnit3TestExecutor"
+source:"/Users/janaka/code-projects/vscode-extentions/vscode-dotnet-test-explorer/test/nunit/bin/debug/netcoreapp2.0/NunitTests.dll"
+codeFilePath:"/Users/janaka/code-projects/vscode-extentions/vscode-dotnet-test-explorer/test/nunit/TestClass1.cs"
+displayName:"AnotherPass"
+id:"8b95092a-43d6-e6c1-8755-e9886078ed34"
+lineNumber:18
+```
+
+### Xunit
+
+```javascript
+isRunning:false
+plainObject:Object
+fullyQualifiedName:"XunitTests.TestClass1.Pass"
+executorUri:"executor://xunit/VsTestRunner2/netcoreapp"
+source:"/Users/janaka/code-projects/vscode-extentions/vscode-dotnet-test-explorer/test/xunittests/bin/debug/netcoreapp2.0/XunitTests.dll"
+codeFilePath:"/Users/janaka/code-projects/vscode-extentions/vscode-dotnet-test-explorer/test/xunittests/TestClass1.cs"
+displayName:"XunitTests.TestClass1.Pass"
+id:"bd43f0f0-1d15-11da-f753-733a2d22cb4b"
+lineNumber:11
+```
+
+### Mstest
+
+```javascript
+isRunning:false
+plainObject:Object
+fullyQualifiedName:"MsTestTests.TestClass1.Pass"
+executorUri:"executor://MSTestAdapter/v2"
+source:"/Users/janaka/code-projects/vscode-extentions/vscode-dotnet-test-explorer/test/mstest/bin/debug/netcoreapp2.0/MsTestTests.dll"
+codeFilePath:"/Users/janaka/code-projects/vscode-extentions/vscode-dotnet-test-explorer/test/mstest/TestClass1.cs"
+displayName:"Pass"
+id:"5736fbf0-a673-f686-0c30-35bd962d1f20"
+lineNumber:12
+testClassName:"MsTestTests.TestClass1"
+```

src/vsTestPlatform/vsCode/commands/testDiscoveryCommands.ts

@@ -0,0 +1,19 @@
+import * as vscode from "vscode";
+import { VSTestServiceIDE } from "../vsTest/vsTestServiceIDE"
+import { TestManager } from "../vsTest/vsTestManager";
+import { TestModel, Test, TestResult, TestOutcome } from "../../vsTest/vsTestModel"
+
+export function RegisterTestDiscoveryCommands(context: vscode.ExtensionContext) {
+    new TestDiscoveryCommands(context);
+}
+
+export class TestDiscoveryCommands {
+    private testService: VSTestServiceIDE;
+    constructor(private context: vscode.ExtensionContext) {
+        //this.testService = TestManager.getInstance().getTestService();
+
+        //const runCommand = vscode.commands.registerCommand("vstest.execution.run",
+        //    test => this.runTests([test]));
+        //context.subscriptions.push(runCommand);
+    }
+}

src/vsTestPlatform/vsCode/commands/testExecutionCommands.ts

@@ -0,0 +1,32 @@
+import * as vscode from "vscode";
+import { VSTestServiceIDE } from "../vsTest/vsTestServiceIDE"
+import { TestManager } from "../vsTest/vsTestManager";
+import { TestModel, Test, TestResult, TestOutcome } from "../../vsTest/vsTestModel"
+
+export function RegisterTestExecutionCommands(context: vscode.ExtensionContext) {
+    new TestExecutionCommands(context);
+}
+
+export class TestExecutionCommands {
+    private testService: VSTestServiceIDE;
+    constructor(private context: vscode.ExtensionContext) {
+        //this.testService = TestManager.getInstance().getTestService();
+
+
+
+        //const runAllCommand = vscode.commands.registerCommand("vstest.execution.runAll",
+        //    test => this.runTests(this.testService.getModel().getTests()));
+        //context.subscriptions.push(runAllCommand);
+
+        //const debugAllCommand = vscode.commands.registerCommand("vstest.execution.debugAll",
+        //    test => this.runTests(this.testService.getModel().getTests(), true));
+        //context.subscriptions.push(debugAllCommand);
+    }
+
+
+    runTests(test: Array<Test>, debuggingEnabled: boolean = false): void {
+        this.testService.runTests(test, debuggingEnabled);
+    }
+
+
+}

src/vsTestPlatform/vsCode/config.ts

@@ -0,0 +1,31 @@
+import * as vscode from "vscode";
+import { IVSTestConfig } from "../vsTest/vsTestConfig";
+
+export function getCurrentAdapterName() {
+    // return vscode.workspace.getConfiguration("vstest.adapterName");
+    return "dotnet";
+}
+
+export function getConfigurationForAdatper(): IVSTestConfig {
+    // return vscode.workspace.getConfiguration(`vstest.${getCurrentAdapterName()}`) as any;
+    // FIXME: fix hardcoded config. Why do we need these? Should be able to deduce these values. 
+    return {
+        "output": "bin/debug", // isn't this the convention
+        "framework": "netcoreapp2.0", // used to construct path. can come from *.test.csproj
+        // "outputFileName": "NunitTests.dll"
+         "outputFileName": "XunitTests.dll"
+        //"outputFileName": "MsTestTests.dll"
+    }
+}
+
+export function isExtensionEnabled(): boolean {
+    const configuration  = vscode.workspace.getConfiguration(`vstest`);
+    const value = configuration.get("enable");
+    return value == true || !value;
+}
+
+export function isAutoInitializeEnabled(): boolean {
+    const configuration  = vscode.workspace.getConfiguration(`vstest`);
+    const value = configuration.get("autoInitialize");
+    return value == true;
+}

src/vsTestPlatform/vsCode/console/testOutputChannel.ts

@@ -0,0 +1,9 @@
+import * as vscode from "vscode";
+
+export class TestOutputChannel {
+    private outputChannel = vscode.window.createOutputChannel("vsTest");
+
+    public appendData(value : string) {
+        this.outputChannel.appendLine(value);
+    }
+}