Enable JUnit 5 test suite engine in tycho-surefire #2462

Currently, Tycho is not capable of running JUnit 5 test suites, since
the required SuiteTestEngine is not provided while running tests.

This change adds the SuiteTestEngine to tycho-surefire executions using
JUnit Platform 1.8 (the first with suite support) or newer. To this end,
the SuiteTestEngine is always enabled when running JUnit 5 tests. It
also adds a regression test for executing a test suite based on JUnit
Jupiter 5.9.

Fixes #2462

tycho-its/projects/surefire.junit59suite/bundle.test/META-INF/MANIFEST.MF

@@ -0,0 +1,8 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: JUnit5 Suite Test Plug-in
+Bundle-SymbolicName: bundle.test.junit59suite
+Bundle-Version: 1.0.0
+Bundle-RequiredExecutionEnvironment: JavaSE-1.8
+Import-Package: org.junit.jupiter.api;version="5.9.0",
+ org.junit.platform.suite.api;version="1.9.0"

tycho-its/projects/surefire.junit59suite/bundle.test/build.properties

@@ -0,0 +1,4 @@
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+               .

tycho-its/projects/surefire.junit59suite/bundle.test/pom.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.eclipse.tycho.tycho-its.surefire-junit5</groupId>
+  <artifactId>bundle.test.junit59suite</artifactId>
+  <packaging>eclipse-test-plugin</packaging>
+  <version>1.0.0</version>
+
+  <repositories>
+    <repository>
+      <id>eclipse</id>
+      <layout>p2</layout>
+      <url>${target-platform}</url>
+    </repository>
+  </repositories>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.eclipse.tycho</groupId>
+        <artifactId>tycho-maven-plugin</artifactId>
+        <version>${tycho-version}</version>
+        <extensions>true</extensions>
+      </plugin>
+      <plugin>
+        <groupId>org.eclipse.tycho</groupId>
+        <artifactId>tycho-surefire-plugin</artifactId>
+        <version>${tycho-version}</version>
+        <configuration>
+          <includes>
+            <include>**/SuiteWithAllTests.java</include>
+          </includes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

tycho-its/projects/surefire.junit59suite/bundle.test/src/bundle/test/JUnit59Test.java

@@ -0,0 +1,28 @@
+/*******************************************************************************
+ * Copyright (c) 2023 Vector Informatik GmbH and others.
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Vector Informatik GmbH - initial API and implementation
+ *******************************************************************************/
+package bundle.test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+class JUnit59Test {
+
+    @Test
+    @DisplayName("started from test suite")
+    void startedFromSuite() {
+        assertEquals(2, 1 + 1);
+    }
+
+}

tycho-its/projects/surefire.junit59suite/bundle.test/src/bundle/test/SuiteWithAllTests.java

@@ -0,0 +1,22 @@
+/*******************************************************************************
+ * Copyright (c) 2023 Vector Informatik GmbH and others.
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Vector Informatik GmbH - initial API and implementation
+ *******************************************************************************/
+package bundle.test;
+
+import org.junit.platform.suite.api.Suite;
+import org.junit.platform.suite.api.SelectClasses;
+
+@Suite
+@SelectClasses({ JUnit59Test.class })
+public class SuiteWithAllTests {
+
+}

tycho-its/src/test/java/org/eclipse/tycho/test/surefire/JUnit5Test.java

@@ -93,4 +93,17 @@ public void testJUnit59Runner() throws Exception {
 		// make sure test tagged as 'slow' was skipped
 		assertNumberOfSuccessfulTests(projectBasedir, "bundle.test.JUnit59Test", 4);
 	}
+
+	@Test
+	public void testJUnit59Suite() throws Exception {
+		final Verifier verifier = getVerifier("/surefire.junit59suite/bundle.test");
+		verifier.executeGoal("verify");
+		verifier.verifyErrorFreeLog();
+		final String projectBasedir = verifier.getBasedir();
+		assertTestMethodWasSuccessfullyExecuted(projectBasedir, "SuiteWithAllTests", "bundle.test.JUnit59Test",
+				"started from test suite");
+		// make sure tests from suite were executed
+		assertNumberOfSuccessfulTests(projectBasedir, "bundle.test.JUnit59Test", 1);
+	}
+
 }

tycho-its/src/test/java/org/eclipse/tycho/test/util/SurefireUtil.java

@@ -35,11 +35,23 @@ public static File testResultFile(String baseDir, String testSuffix) {
 
 	public static void assertTestMethodWasSuccessfullyExecuted(String baseDir, String className, String methodName,
 			int iterations) throws Exception {
-		File resultFile = getTestResultFile(baseDir, className);
-		Document document = readDocument(resultFile);
+		assertTestMethodWasSuccessfullyExecuted(baseDir, className, className, methodName, iterations);
+	}
+
+	public static void assertTestMethodWasSuccessfullyExecuted(String baseDir, String suiteClassSimpleName,
+			String testClassQualifiedName, String methodName) throws Exception {
+		String testClassSimpleName = testClassQualifiedName.substring(testClassQualifiedName.lastIndexOf(".") + 1);
+		assertTestMethodWasSuccessfullyExecuted(baseDir, testClassQualifiedName,
+				String.join(" ", suiteClassSimpleName, testClassSimpleName), methodName, 1);
+	}
+
+	private static void assertTestMethodWasSuccessfullyExecuted(String baseDir, String qualifiedClassName,
+			String classNameInReport, String methodName, int iterations) throws Exception {
 		// surefire-test-report XML schema:
 		// https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report.xsd
-		String testCaseXPath = String.format("/testsuite/testcase[@classname='%s' and @name='%s']", className,
+		File resultFile = getTestResultFile(baseDir, qualifiedClassName);
+		Document document = readDocument(resultFile);
+		String testCaseXPath = String.format("/testsuite/testcase[@classname='%s' and @name='%s']", classNameInReport,
 				methodName);
 		List<Node> testCaseNodes2 = XMLTool.getMatchingNodes(document, testCaseXPath);
 		assertEquals(resultFile.getAbsolutePath() + " with xpath " + testCaseXPath

tycho-surefire/org.eclipse.tycho.surefire.junit58/bnd.bnd

@@ -1,5 +1,6 @@
 Import-Package: \
 	org.junit.jupiter.api.*;version='[5.8,5.9)',\
+	org.junit.platform.suite.api;version='[1.8,1.9)',\
 	org.opentest4j;version='[1.2,2)',\
 	!org.apache.maven.surefire.*,\
 	!org.apache.maven.plugin.surefire.*,\

tycho-surefire/org.eclipse.tycho.surefire.junit58/pom.xml

@@ -62,6 +62,21 @@
 							<artifactId>junit-jupiter-engine</artifactId>
 							<version>${junit-jupiter-version}</version>
 						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-api</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-commons</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-engine</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
 						<artifactItem>
 							<groupId>org.apache.maven.surefire</groupId>
 							<artifactId>common-java5</artifactId>

tycho-surefire/org.eclipse.tycho.surefire.junit58/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine

@@ -1 +1,2 @@
 org.junit.jupiter.engine.JupiterTestEngine
+org.junit.platform.suite.engine.SuiteTestEngine

tycho-surefire/org.eclipse.tycho.surefire.junit58withvintage/bnd.bnd

@@ -1,5 +1,6 @@
 Import-Package: \
 	org.junit.jupiter.api.*;version='[5.8,5.9)',\
+	org.junit.platform.suite.api;version='[1.8,1.9)',\
 	org.junit.runner.*;resolution:=optional;version='[4.12,5)',\
 	org.junit.runners.*;resolution:=optional;version='[4.12,5)',\
 	org.junit.experimental.categories;resolution:=optional;version='[4.12,5)',\

tycho-surefire/org.eclipse.tycho.surefire.junit58withvintage/pom.xml

@@ -72,6 +72,21 @@
 							<artifactId>junit-vintage-engine</artifactId>
 							<version>${junit-jupiter-version}</version>
 						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-api</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-commons</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-engine</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
 						<artifactItem>
 							<groupId>org.opentest4j</groupId>
 							<artifactId>opentest4j</artifactId>

tycho-surefire/org.eclipse.tycho.surefire.junit58withvintage/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine

@@ -1,4 +1,5 @@
 # Manual merge of META-INF/services/org.junit.platform.engine.TestEngine of both junit-vintage-engine and junit-jupiter-engine
 # otherwise only one of the two files would win when extracting dependencies
 org.junit.jupiter.engine.JupiterTestEngine
-org.junit.vintage.engine.VintageTestEngine
+org.junit.vintage.engine.VintageTestEngine
+org.junit.platform.suite.engine.SuiteTestEngine

tycho-surefire/org.eclipse.tycho.surefire.junit59/bnd.bnd

@@ -1,5 +1,6 @@
 Import-Package: \
 	org.junit.jupiter.api.*;version='[5.9,6)',\
+	org.junit.platform.suite.api;version='[1.9,2)',\
 	org.opentest4j;version='[1.2,2)',\
 	!org.apache.maven.surefire.*,\
 	!org.apache.maven.plugin.surefire.*,\

tycho-surefire/org.eclipse.tycho.surefire.junit59/pom.xml

@@ -62,6 +62,21 @@
 							<artifactId>junit-jupiter-engine</artifactId>
 							<version>${junit-jupiter-version}</version>
 						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-api</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-commons</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-engine</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
 						<artifactItem>
 							<groupId>org.apache.maven.surefire</groupId>
 							<artifactId>common-java5</artifactId>

tycho-surefire/org.eclipse.tycho.surefire.junit59/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine

@@ -1 +1,2 @@
 org.junit.jupiter.engine.JupiterTestEngine
+org.junit.platform.suite.engine.SuiteTestEngine

tycho-surefire/org.eclipse.tycho.surefire.junit59withvintage/bnd.bnd

@@ -1,5 +1,6 @@
 Import-Package: \
 	org.junit.jupiter.api.*;version='[5.9,6)',\
+	org.junit.platform.suite.api;version='[1.9,2)',\
 	org.junit.runner.*;resolution:=optional;version='[4.12,5)',\
 	org.junit.runners.*;resolution:=optional;version='[4.12,5)',\
 	org.junit.experimental.categories;resolution:=optional;version='[4.12,5)',\

tycho-surefire/org.eclipse.tycho.surefire.junit59withvintage/pom.xml

@@ -72,6 +72,21 @@
 							<artifactId>junit-vintage-engine</artifactId>
 							<version>${junit-jupiter-version}</version>
 						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-api</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-commons</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
+						<artifactItem>
+							<groupId>org.junit.platform</groupId>
+							<artifactId>junit-platform-suite-engine</artifactId>
+							<version>${junit-platform-version}</version>
+						</artifactItem>
 						<artifactItem>
 							<groupId>org.opentest4j</groupId>
 							<artifactId>opentest4j</artifactId>

tycho-surefire/org.eclipse.tycho.surefire.junit59withvintage/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine

@@ -1,4 +1,5 @@
 # Manual merge of META-INF/services/org.junit.platform.engine.TestEngine of both junit-vintage-engine and junit-jupiter-engine
 # otherwise only one of the two files would win when extracting dependencies
 org.junit.jupiter.engine.JupiterTestEngine
-org.junit.vintage.engine.VintageTestEngine
+org.junit.vintage.engine.VintageTestEngine
+org.junit.platform.suite.engine.SuiteTestEngine