Provide only compatible JUnit 5 test engines in RemotePluginTestRunner

Currently, it is possible that the RemotePluginTestRunner loads bundles
with "incompatible" test engines. This means, the engine is
version-incompatible with the TestEngine interface. In such a case, the
execution will fail, because the ServiceLoader tries to load this engine
as a provider for the TestEngine service but fails because it cannot
assign the engine to the TestEngine interface.
One such case is the execution of tests with Tycho, which can use a
different JUnit version than the one used in the JDT test bundles.

This change ensures that bundles providing incompatible engines are not
added to the class loader used for executing the test to ensure that
service loading for the engines does not fail.

ui/org.eclipse.pde.junit.runtime/META-INF/MANIFEST.MF

@@ -11,5 +11,6 @@ Require-Bundle: org.eclipse.jdt.junit.runtime;bundle-version="[3.4.0,4.0.0)",
 Export-Package: org.eclipse.pde.internal.junit.runtime;x-internal:=true
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
 Bundle-ActivationPolicy: lazy
-Import-Package: org.eclipse.ui.testing;resolution:=optional
+Import-Package: org.eclipse.ui.testing;resolution:=optional,
+ org.junit.platform.engine
 Automatic-Module-Name: org.eclipse.pde.junit.runtime

ui/org.eclipse.pde.junit.runtime/src/org/eclipse/pde/internal/junit/runtime/RemotePluginTestRunner.java

@@ -14,16 +14,20 @@
  *******************************************************************************/
 package org.eclipse.pde.internal.junit.runtime;
 
+import static java.util.stream.Collectors.toList;
+
 import java.io.IOException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.function.Predicate;
 
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.jdt.internal.junit.runner.RemoteTestRunner;
+import org.junit.platform.engine.TestEngine;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.FrameworkUtil;
@@ -89,7 +93,8 @@ private static ClassLoader createJUnit5PluginClassLoader(String testPluginName)
 			throw new IllegalArgumentException("Bundle \"" + testPluginName + "\" not found. Possible causes include missing dependencies, too restrictive version ranges, or a non-matching required execution environment."); //$NON-NLS-1$ //$NON-NLS-2$
 		}
 		Bundle junit5RuntimeBundle = Platform.getBundle("org.eclipse.jdt.junit5.runtime"); //$NON-NLS-1$
-		List<Bundle> platformEngineBundles = findTestEngineBundles();
+		List<Bundle> platformEngineBundles = new ArrayList<>();
+		platformEngineBundles.addAll(findTestEngineBundles());
 		platformEngineBundles.add(testBundle);
 		if (junit5RuntimeBundle != null) {
 			platformEngineBundles.add(junit5RuntimeBundle);
@@ -99,19 +104,38 @@ private static ClassLoader createJUnit5PluginClassLoader(String testPluginName)
 
 	private static List<Bundle> findTestEngineBundles() {
 		BundleContext bundleContext = FrameworkUtil.getBundle(RemotePluginTestRunner.class).getBundleContext();
-		List<Bundle> engineBundles = new ArrayList<>();
-		for (Bundle bundle : bundleContext.getBundles()) {
-			try {
-				BundleWiring bundleWiring = bundle.adapt(BundleWiring.class);
-				Collection<String> listResources = bundleWiring.listResources("META-INF/services", "org.junit.platform.engine.TestEngine", BundleWiring.LISTRESOURCES_LOCAL); //$NON-NLS-1$//$NON-NLS-2$
-				if (!listResources.isEmpty()) {
-					engineBundles.add(bundle);
-				}
-			} catch (Exception e) {
-				// check the next bundle
+		return Arrays.stream(bundleContext.getBundles()).filter(RemotePluginTestRunner::providesCompatibleTestEngine).collect(toList());
+	}
+
+	/**
+	 * Checks whether the bundle provides test engines.
+	 * Ensures that test engines that can be loaded from the bundle
+	 * are compatible with the TestEngine version in current scope.
+	 * Otherwise, the JUnit platform's call to the ServiceLoader for
+	 * retrieving available engines will fail.
+	 * Incompatibilities can happen, e.g., in Tycho builds, where
+	 * the org.eclipse.tycho.surefire.osgibooter bundle is found
+	 * that may provide a different JUnit platform version than the
+	 * one available via the Eclipse target platform.
+	 */
+	private static boolean providesCompatibleTestEngine(Bundle bundle) {
+		try {
+			BundleWiring bundleWiring = bundle.adapt(BundleWiring.class);
+			Collection<String> engineProviderSpecifications = bundleWiring.listResources("META-INF/services", "org.junit.platform.engine.TestEngine", BundleWiring.LISTRESOURCES_LOCAL); //$NON-NLS-1$//$NON-NLS-2$
+			if (!engineProviderSpecifications.isEmpty()) {
+				Bundle engineBundleInThisClassLoader = FrameworkUtil.getBundle(TestEngine.class);
+				Class<?> testEngineClassInCurrentLoader = getTestEngineClassInBundle(engineBundleInThisClassLoader);
+				Class<?> testEngineInEngineProviderBundle = getTestEngineClassInBundle(bundle);
+				return testEngineInEngineProviderBundle != null && testEngineClassInCurrentLoader.isAssignableFrom(testEngineInEngineProviderBundle);
 			}
+		} catch (Exception e) {
+			// skip this bundle
 		}
-		return engineBundles;
+		return false;
+	}
+
+	private static Class<?> getTestEngineClassInBundle(Bundle bundle) throws ClassNotFoundException {
+		return bundle.loadClass(TestEngine.class.getName());
 	}
 
 	/**