feat - allow other extensions to register test runner

src/commands/projectExplorerCommands.ts

@@ -8,7 +8,7 @@ import { loadChildren, runTests, testController } from '../controller/testContro
 import { loadJavaProjects, updateItemForDocument } from '../controller/utils';
 import { IProgressReporter } from '../debugger.api';
 import { progressProvider } from '../extension';
-import { TestLevel } from '../types';
+import { TestLevel } from '../java-test-runner.api';
 
 export async function runTestsFromJavaProjectExplorer(node: any, isDebug: boolean): Promise<void> {
     const testLevel: TestLevel = getTestLevel(node._nodeData);

src/commands/testDependenciesCommands.ts

@@ -9,11 +9,12 @@ import * as fse from 'fs-extra';
 import * as _ from 'lodash';
 import * as os from 'os';
 import { getJavaProjects, getProjectType } from '../controller/utils';
-import { IJavaTestItem, ProjectType, TestKind } from '../types';
+import { IJavaTestItem, ProjectType } from '../types';
 import { createWriteStream, WriteStream } from 'fs';
 import { URL } from 'url';
 import { ClientRequest, IncomingMessage } from 'http';
 import { sendError } from 'vscode-extension-telemetry-wrapper';
+import { TestKind } from '../java-test-runner.api';
 
 export async function enableTests(testKind?: TestKind): Promise<void> {
     const project: IJavaTestItem | undefined = await getTargetProject();

src/controller/testController.ts

@@ -3,22 +3,25 @@
 
 import * as _ from 'lodash';
 import * as path from 'path';
-import { CancellationToken, DebugConfiguration, Disposable, FileCoverage, FileCoverageDetail, FileSystemWatcher, RelativePattern, TestController, TestItem, TestRun, TestRunProfileKind, TestRunRequest, tests, TestTag, Uri, window, workspace, WorkspaceFolder } from 'vscode';
+import { CancellationToken, DebugConfiguration, Disposable, FileCoverage, FileCoverageDetail, FileSystemWatcher, Location, MarkdownString, RelativePattern, TestController, TestItem, TestMessage, TestRun, TestRunProfileKind, TestRunRequest, tests, TestTag, Uri, window, workspace, WorkspaceFolder } from 'vscode';
 import { instrumentOperation, sendError, sendInfo } from 'vscode-extension-telemetry-wrapper';
 import { refreshExplorer } from '../commands/testExplorerCommands';
 import { IProgressReporter } from '../debugger.api';
 import { progressProvider } from '../extension';
 import { testSourceProvider } from '../provider/testSourceProvider';
-import { IExecutionConfig } from '../runConfigs';
 import { BaseRunner } from '../runners/baseRunner/BaseRunner';
 import { JUnitRunner } from '../runners/junitRunner/JunitRunner';
 import { TestNGRunner } from '../runners/testngRunner/TestNGRunner';
-import { IJavaTestItem, IRunTestContext, TestKind, TestLevel } from '../types';
+import { IJavaTestItem } from '../types';
 import { loadRunConfig } from '../utils/configUtils';
 import { resolveLaunchConfigurationForRunner } from '../utils/launchUtils';
 import { dataCache, ITestItemData } from './testItemDataCache';
-import { findDirectTestChildrenForClass, findTestPackagesAndTypes, findTestTypesAndMethods, loadJavaProjects, resolvePath, synchronizeItemsRecursively, updateItemForDocumentWithDebounce } from './utils';
+import { createTestItem, findDirectTestChildrenForClass, findTestPackagesAndTypes, findTestTypesAndMethods, loadJavaProjects, resolvePath, synchronizeItemsRecursively, updateItemForDocumentWithDebounce } from './utils';
 import { JavaTestCoverageProvider } from '../provider/JavaTestCoverageProvider';
+import { testRunnerService } from './testRunnerService';
+import { IRunTestContext, TestRunner, TestFinishEvent, TestItemStatusChangeEvent, TestKind, TestLevel, TestResultState, TestIdParts } from '../java-test-runner.api';
+import { processStackTraceLine } from '../runners/utils';
+import { parsePartsFromTestId } from '../utils/testItemUtils';
 
 export let testController: TestController | undefined;
 export const watchers: Disposable[] = [];
@@ -43,6 +46,10 @@
     startWatchingWorkspace();
 }
 
+export function creatTestProfile(name: string, kind: TestRunProfileKind): void {
+    testController?.createRunProfile(name, kind, runHandler, false, runnableTag);
+}
+
 export const loadChildren: (item: TestItem, token?: CancellationToken) => any = instrumentOperation('java.test.explorer.loadChildren', async (_operationId: string, item: TestItem, token?: CancellationToken) => {
     if (!item) {
         await loadJavaProjects();
@@ -170,7 +177,7 @@
     let coverageProvider: JavaTestCoverageProvider | undefined;
     if (request.profile?.kind === TestRunProfileKind.Coverage) {
         coverageProvider = new JavaTestCoverageProvider();
         request.profile.loadDetailedCoverage = (_testRun: TestRun, fileCoverage: FileCoverage, _token: CancellationToken): Promise<FileCoverageDetail[]> => {
             return Promise.resolve(coverageProvider!.getCoverageDetails(fileCoverage.uri));
         };
     }
@@ -178,21 +185,48 @@
     try {
         await new Promise<void>(async (resolve: () => void): Promise<void> => {
             const token: CancellationToken = option.token ?? run.token;
+            let disposables: Disposable[] = [];
             token.onCancellationRequested(() => {
                 option.progressReporter?.done();
                 run.end();
+                disposables.forEach((d: Disposable) => d.dispose());
                 return resolve();
             });
             enqueueTestMethods(testItems, run);
+            // TODO: first group by project, then merge test methods.
             const queue: TestItem[][] = mergeTestMethods(testItems);
             for (const testsInQueue of queue) {
                 if (testsInQueue.length === 0) {
                     continue;
                 }
                 const testProjectMapping: Map<string, TestItem[]> = mapTestItemsByProject(testsInQueue);
                 for (const [projectName, itemsPerProject] of testProjectMapping.entries()) {
+                    const workspaceFolder: WorkspaceFolder | undefined = workspace.getWorkspaceFolder(itemsPerProject[0].uri!);
+                    if (!workspaceFolder) {
+                        window.showErrorMessage(`Failed to get workspace folder from test item: ${itemsPerProject[0].label}.`);
+                        continue;
+                    }
+                    const testContext: IRunTestContext = {
+                        isDebug: option.isDebug,
+                        kind: TestKind.None,
+                        projectName,
+                        testItems: itemsPerProject,
+                        testRun: run,
+                        workspaceFolder,
+                        profile: request.profile,
+                        testConfig: await loadRunConfig(itemsPerProject, workspaceFolder),
+                    };
+                    const testRunner: TestRunner | undefined = testRunnerService.getRunner(request.profile?.label, request.profile?.kind);
+                    if (testRunner) {
+                        await executeWithTestRunner(option, testRunner, testContext, run, disposables);
+                        disposables.forEach((d: Disposable) => d.dispose());
+                        disposables = [];
+                        continue;
+                    }
                     const testKindMapping: Map<TestKind, TestItem[]> = mapTestItemsByKind(itemsPerProject);
                     for (const [kind, items] of testKindMapping.entries()) {
+                        testContext.kind = kind;
+                        testContext.testItems = items;
                         if (option.progressReporter?.isCancelled()) {
                             option.progressReporter = progressProvider?.createProgressReporter(option.isDebug ? 'Debug Tests' : 'Run Tests');
                         }
@@ -208,33 +242,17 @@
                             return resolve();
                         });
                         option.progressReporter?.report('Resolving launch configuration...');
-                        // TODO: improve the config experience
-                        const workspaceFolder: WorkspaceFolder | undefined = workspace.getWorkspaceFolder(items[0].uri!);
-                        if (!workspaceFolder) {
-                            window.showErrorMessage(`Failed to get workspace folder from test item: ${items[0].label}.`);
-                            continue;
-                        }
-                        const config: IExecutionConfig | undefined = await loadRunConfig(items, workspaceFolder);
-                        if (!config) {
+                        if (!testContext.testConfig) {
                             continue;
                         }
-                        const testContext: IRunTestContext = {
-                            isDebug: option.isDebug,
-                            kind,
-                            projectName,
-                            testItems: items,
-                            testRun: run,
-                            workspaceFolder,
-                            profile: request.profile,
-                        };
                         const runner: BaseRunner | undefined = getRunnerByContext(testContext);
                         if (!runner) {
                             window.showErrorMessage(`Failed to get suitable runner for the test kind: ${testContext.kind}.`);
                             continue;
                         }
                         try {
                             await runner.setup();
-                            const resolvedConfiguration: DebugConfiguration = mergeConfigurations(option.launchConfiguration, config) ?? await resolveLaunchConfigurationForRunner(runner, testContext, config);
+                            const resolvedConfiguration: DebugConfiguration = mergeConfigurations(option.launchConfiguration, testContext.testConfig) ?? await resolveLaunchConfigurationForRunner(runner, testContext, testContext.testConfig);
                             resolvedConfiguration.__progressId = option.progressReporter?.getId();
                             delegatedToDebugger = true;
                             trackTestFrameworkVersion(testContext.kind, resolvedConfiguration.classPaths, resolvedConfiguration.modulePaths);
@@ -258,6 +276,133 @@
     }
 });
 
+async function executeWithTestRunner(option: IRunOption, testRunner: TestRunner, testContext: IRunTestContext, run: TestRun, disposables: Disposable[]) {
+    option.progressReporter?.done();
+    await new Promise<void>(async (resolve: () => void): Promise<void> => {
+        disposables.push(testRunner.onDidChangeTestItemStatus((event: TestItemStatusChangeEvent) => {
+            const parts: TestIdParts = parsePartsFromTestId(event.testId);
+            let parentItem: TestItem;
+            try {
+                parentItem = findTestClass(parts);
+            } catch (e) {
+                sendError(e);
+                window.showErrorMessage(e.message);
+                return resolve();
+            }
+            let currentItem: TestItem | undefined;
+            const invocations: string[] | undefined = parts.invocations;
+            if (invocations?.length) {
+                let i: number = 0;
+                for (; i < invocations.length; i++) {
+                    currentItem = parentItem.children.get(`${parentItem.id}#${invocations[i]}`);
+                    if (!currentItem) {
+                        break;
+                    }
+                    parentItem = currentItem;
+                }
+
+                if (i < invocations.length - 1) {
+                    window.showErrorMessage('Test not found:' + event.testId);
+                    sendError(new Error('Test not found:' + event.testId));
+                    return resolve();
+                }
+
+                if (!currentItem) {
+                    currentItem = createTestItem({
+                        children: [],
+                        uri: parentItem.uri?.toString(),
+                        range: parentItem.range,
+                        jdtHandler: '',
+                        fullName: `${parentItem.id}#${invocations[invocations.length - 1]}`,
+                        label: event.displayName || invocations[invocations.length - 1],
+                        id: `${parentItem.id}#${invocations[invocations.length - 1]}`,
+                        projectName: testContext.projectName,
+                        testKind: TestKind.None,
+                        testLevel: TestLevel.Invocation,
+                    }, parentItem);
+                }
+            } else {
+                currentItem = parentItem;
+            }
+
+            if (event.displayName && getLabelWithoutCodicon(currentItem.label) !== event.displayName) {
+                currentItem.description = event.displayName;
+            }
+            switch (event.state) {
+                case TestResultState.Running:
+                    run.started(currentItem);
+                    break;
+                case TestResultState.Passed:
+                    run.passed(currentItem);
+                    break;
+                case TestResultState.Failed:
+                case TestResultState.Errored:
+                    const testMessages: TestMessage[] = [];
+                    if (event.message) {
+                        const markdownTrace: MarkdownString = new MarkdownString();
+                        markdownTrace.supportHtml = true;
+                        markdownTrace.isTrusted = true;
+                        const testMessage: TestMessage = new TestMessage(markdownTrace);
+                        testMessages.push(testMessage);
+                        const lines: string[] = event.message.split(/\r?\n/);
+                        for (const line of lines) {
+                            const location: Location | undefined = processStackTraceLine(line, markdownTrace, currentItem, testContext.projectName);
+                            if (location) {
+                                testMessage.location = location;
+                            }
+                        }
+                    }
+                    run.failed(currentItem, testMessages);
+                    break;
+                case TestResultState.Skipped:
+                    run.skipped(currentItem);
+                    break;
+                default:
+                    break;
+            }
+        }));
+
+        disposables.push(testRunner.onDidFinishTestRun((_event: TestFinishEvent) => {
+            return resolve();
+        }));
+
+        testRunner.launch(testContext);
+    });
+
+    function findTestClass(parts: TestIdParts): TestItem {
+        const projectItem: TestItem | undefined = testController?.items.get(parts.project);
+        if (!projectItem) {
+            throw new Error('Failed to get the project test item.');
+        }
+
+        if (parts.package === undefined) { // '' means default package
+            throw new Error('package is undefined in the id parts.');
+        }
+
+        const packageItem: TestItem | undefined = projectItem.children.get(`${projectItem.id}@${parts.package}`);
+        if (!packageItem) {
+            throw new Error('Failed to get the package test item.');
+        }
+
+        if (!parts.class) {
+            throw new Error('class is undefined in the id parts.');
+        }
+
+        const classes: string[] = parts.class.split('$'); // handle nested classes
+        let current: TestItem | undefined = packageItem.children.get(`${projectItem.id}@${classes[0]}`);
+        if (!current) {
+            throw new Error('Failed to get the class test item.');
+        }
+        for (let i: number = 1; i < classes.length; i++) {
+            current = current.children.get(`${current.id}$${classes[i]}`);
+            if (!current) {
+                throw new Error('Failed to get the class test item.');
+            }
+        }
+        return current;
+    }
+}
+
 function mergeConfigurations(launchConfiguration: DebugConfiguration | undefined, config: any): DebugConfiguration | undefined {
     if (!launchConfiguration) {
         return undefined;
@@ -586,6 +731,18 @@
     });
 }
 
+function getLabelWithoutCodicon(name: string): string {
+    if (name.includes('#')) {
+        name = name.substring(name.indexOf('#') + 1);
+    }
+
+    const result: RegExpMatchArray | null = name.match(/(?:\$\(.+\) )?(.*)/);
+    if (result?.length === 2) {
+        return result[1];
+    }
+    return name;
+}
+
 interface IRunOption {
     isDebug: boolean;
     progressReporter?: IProgressReporter;

src/controller/testItemDataCache.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 
 import { TestItem } from 'vscode';
-import { TestKind, TestLevel } from '../types';
+import { TestKind, TestLevel } from '../java-test-runner.api';
 
 /**
  * A map cache to save the metadata of the test item.

src/controller/testRunnerService.ts

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+import { TestRunProfileKind } from 'vscode';
+import { creatTestProfile } from './testController';
+import { TestRunner } from '../java-test-runner.api';
+
+// TODO: this should be refactored. The test controller should be extended and hosting the registered runners.
+class TestRunnerService {
+
+    private registeredRunners: Map<string, TestRunner>;
+
+    constructor() {
+        this.registeredRunners = new Map<string, TestRunner>();
+    }
+
+    public registerTestRunner(name: string, kind: TestRunProfileKind, runner: TestRunner) {
+        const key: string = `${name}:${kind}`;
+        if (this.registeredRunners.has(key)) {
+            throw new Error(`Runner ${key} has already been registered.`);
+        }
+        creatTestProfile(name, kind);
+        this.registeredRunners.set(key, runner);
+    }
+
+    public getRunner(name: string | undefined, kind: TestRunProfileKind | undefined): TestRunner | undefined {
+        if (!name || !kind) {
+            return undefined;
+        }
+        const key: string = `${name}:${kind}`;
+        return this.registeredRunners.get(key);
+    }
+}
+
+export const testRunnerService: TestRunnerService = new TestRunnerService();

src/controller/utils.ts

@@ -8,11 +8,12 @@ import { performance } from 'perf_hooks';
 import { CancellationToken, commands, Range, TestItem, Uri, workspace, WorkspaceFolder } from 'vscode';
 import { sendError } from 'vscode-extension-telemetry-wrapper';
 import { JavaTestRunnerDelegateCommands } from '../constants';
-import { IJavaTestItem, ProjectType, TestKind, TestLevel } from '../types';
+import { IJavaTestItem, ProjectType } from '../types';
 import { executeJavaLanguageServerCommand } from '../utils/commandUtils';
 import { getRequestDelay, lruCache, MovingAverage } from './debouncing';
 import { runnableTag, testController } from './testController';
 import { dataCache } from './testItemDataCache';
+import { TestKind, TestLevel } from '../java-test-runner.api';
 
 /**
  * Load the Java projects, which are the root nodes of the test explorer
@@ -129,8 +130,8 @@ function updateTestItem(testItem: TestItem, metaInfo: IJavaTestItem): void {
 
 /**
  * Create test item which will be shown in the test explorer
- * @param metaInfo The data from the server side of the test item
- * @param parent The parent node of the test item (if it has)
+ * @param metaInfo The data from the server side of the test item.
+ * @param parent The parent node of the test item (if it has).
  * @returns The created test item
  */
 export function createTestItem(metaInfo: IJavaTestItem, parent?: TestItem): TestItem {
@@ -139,7 +140,7 @@ export function createTestItem(metaInfo: IJavaTestItem, parent?: TestItem): Test
     }
     const item: TestItem = testController.createTestItem(
         metaInfo.id,
-        `${getCodiconLabel(metaInfo.testLevel)} ${metaInfo.label}`,
+        `${getCodiconLabel(metaInfo.testLevel)} ${metaInfo.label}`.trim(),
         metaInfo.uri ? Uri.parse(metaInfo.uri) : undefined,
     );
     item.range = asRange(metaInfo.range);