Provide a session test framework for JUnit 5

The existing implementation of session tests (including
SessionTestRunner, SessionTestSuite and WorkspaceSessionTestSuite)
relies on JUnit 3 functionality. Because of the changed way in which
tests are set up in JUnit 5, they cannot be used to run session tests
with JUnit 5. In addition the session test framework is currently overly
complex, as it provides some functionality that is not used (anymore).

This commit introduces a new JUnit 5 extension, the
SessionTestExtension, to be used for running test cases of a class in
separate Eclipse application sessions with JUnit 5. This extension
intercepts test methods calls to start a separate Eclipse session in
which the tests are executed. The results are evaluated in the
intercepted method execution in way they were evaluated for existing
JUnit-3-based session tests. Session that are expected to fail can be
annotated with an added @SessionShouldFail annotation.

Functionality is demonstrated in a manually executed sample session
test.

runtime/tests/org.eclipse.core.tests.harness/META-INF/MANIFEST.MF

@@ -5,22 +5,35 @@ Bundle-SymbolicName: org.eclipse.core.tests.harness;singleton:=true
 Bundle-Version: 3.15.400.qualifier
 Bundle-Vendor: Eclipse.org
 Export-Package: org.eclipse.core.tests.harness;version="2.0",
+ org.eclipse.core.tests.harness.session,
  org.eclipse.core.tests.session;version="2.0"
 Require-Bundle: org.junit,
  org.eclipse.test.performance,
  org.eclipse.core.runtime;bundle-version="[3.29.0,4.0.0)",
  org.eclipse.jdt.junit.runtime,
  org.eclipse.jdt.junit4.runtime,
- org.eclipse.pde.junit.runtime
+ org.eclipse.jdt.junit5.runtime,
+ org.eclipse.pde.junit.runtime,
+ junit-platform-launcher,
+ junit-platform-engine,
+ junit-jupiter-engine,
+ junit-vintage-engine,
+ junit-platform-suite-commons,
+ junit-platform-suite-engine
 Bundle-ActivationPolicy: lazy
 Bundle-RequiredExecutionEnvironment: JavaSE-17
 Import-Package: javax.xml.parsers,
  net.bytebuddy,
  org.apiguardian.api,
  org.assertj.core.api,
  org.junit.jupiter.api,
- org.junit.platform.suite.api,
+ org.junit.jupiter.api.condition,
+ org.junit.jupiter.api.extension,
+ org.junit.jupiter.migrationsupport,
+ org.junit.jupiter.params,
  org.junit.platform.commons,
+ org.junit.platform.runner,
+ org.junit.platform.suite.api,
  org.opentest4j,
  org.w3c.dom,
  org.xml.sax

runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/RemoteTestExecutor.java

@@ -0,0 +1,299 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Vector Informatik GmbH and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.core.tests.harness.session;
+
+import static org.eclipse.core.tests.harness.TestHarnessPlugin.PI_HARNESS;
+import static org.eclipse.core.tests.harness.TestHarnessPlugin.log;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.ServerSocket;
+import java.net.Socket;
+import org.eclipse.core.runtime.Assert;
+import org.eclipse.core.runtime.AssertionFailedException;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.core.tests.session.RemoteAssertionFailedError;
+import org.eclipse.core.tests.session.RemoteTestException;
+import org.eclipse.core.tests.session.Setup;
+import org.eclipse.core.tests.session.TestDescriptor;
+
+class RemoteTestExecutor {
+	private static final String REMOTE_EXECUTION_INDICATION_SYSTEM_PROPERY = "org.eclipse.core.tests.session.isRemoteExecution"; //$NON-NLS-1$
+
+	/**
+	 * {@return whether the current test execution is done remotely}
+	 */
+	static boolean isRemoteExecution() {
+		return System.getProperty(REMOTE_EXECUTION_INDICATION_SYSTEM_PROPERY, "").equals(Boolean.toString(true));
+	}
+
+	private final String pluginId;
+	private final String applicationId;
+	private final Setup setup;
+
+	RemoteTestExecutor(Setup setup, String applicationId, String pluginId) {
+		this.setup = setup;
+		this.applicationId = applicationId;
+		this.pluginId = pluginId;
+	}
+
+	void executeRemotely(String testClass, String testMethod, boolean shouldFail) throws Throwable {
+		Setup localSetup = createSingleTestSetup(testClass, testMethod);
+		localSetup.setSystemProperty(REMOTE_EXECUTION_INDICATION_SYSTEM_PROPERY, Boolean.toString(true));
+
+		TestDescriptor descriptor = new TestDescriptor(testClass, testMethod);
+		ResultCollector collector = new ResultCollector(getTestId(testClass, testMethod));
+		localSetup.setEclipseArgument("port", Integer.toString(collector.getPort()));
+		new Thread(collector, "Test result collector").start();
+		IStatus status = launch(localSetup);
+		collector.shutdown();
+		if (!shouldFail) {
+			if (!status.isOK()) {
+				log(status);
+				throw new CoreException(status);
+			}
+			if (!collector.didTestFinish()) {
+				throw new Exception("session test did not run: " + descriptor);
+			}
+			if (!collector.wasTestSuccessful()) {
+				throw collector.getError();
+			}
+		} else {
+			if (status.isOK() && collector.wasTestSuccessful()) {
+				throw new AssertionFailedException("test should fail but did not: " + descriptor);
+			}
+		}
+	}
+
+	private static String getTestId(String testClass, String testMethod) {
+		return testMethod + "(" + testClass + ")";
+	}
+
+	private Setup createSingleTestSetup(String testClassName, String testMethodName) {
+		Setup localSetup = (Setup) setup.clone();
+		localSetup.setEclipseArgument(Setup.APPLICATION, applicationId);
+		localSetup.setEclipseArgument("loaderpluginname", "org.eclipse.core.tests.harness");
+		localSetup.setEclipseArgument("testloaderclass", "org.eclipse.jdt.internal.junit5.runner.JUnit5TestLoader");
+		localSetup.setEclipseArgument("testpluginname", pluginId);
+		localSetup.setEclipseArgument("test", testClassName + ":" + testMethodName);
+		return localSetup;
+	}
+
+	/**
+	 * Runs the setup. Returns a status object indicating the outcome of the
+	 * operation.
+	 *
+	 * @return a status object indicating the outcome
+	 */
+	private static IStatus launch(Setup setup) {
+		Assert.isNotNull(setup.getEclipseArgument(Setup.APPLICATION), "test application is not defined");
+		Assert.isNotNull(setup.getEclipseArgument("testpluginname"), "test plug-in id not defined");
+		Assert.isTrue(setup.getEclipseArgument("classname") != null ^ setup.getEclipseArgument("test") != null,
+				"either a test suite or a test case must be provided");
+		// to prevent changes in the protocol from breaking us,
+		// force the version we know we can work with
+		setup.setEclipseArgument("version", "4");
+		IStatus outcome = Status.OK_STATUS;
+		try {
+			int returnCode = setup.run();
+			if (returnCode == 23) {
+				// asked to restart; for now just do this once.
+				// Note that 23 is our magic return code indicating that a restart is required.
+				// This can happen for tests that update framework extensions which requires a
+				// restart.
+				returnCode = setup.run();
+			}
+			if (returnCode != 0) {
+				outcome = new Status(IStatus.WARNING, Platform.PI_RUNTIME, returnCode,
+						"Process returned non-zero code: " + returnCode + "\n\tCommand: " + setup, null);
+			}
+		} catch (Exception e) {
+			outcome = new Status(IStatus.ERROR, Platform.PI_RUNTIME, -1, "Error running process\n\tCommand: " + setup,
+					e);
+		}
+		return outcome;
+	}
+
+	/**
+	 * Partly taken from
+	 * {@code org.eclipse.core.tests.session.SessionTestRunner#ResultCollector}.
+	 */
+	private static class ResultCollector implements Runnable {
+		private final String testId;
+		private final ServerSocket serverSocket;
+
+		private volatile boolean shouldRun = true;
+		private volatile boolean executionFinished;
+
+		private static enum STATE {
+			SUCCESS, FAILURE, ERROR;
+		}
+
+		private STATE state = STATE.SUCCESS;
+		private StringBuilder stackBuilder;
+
+		private String stackTrace;
+		private boolean testFinished;
+		private Throwable error;
+
+		ResultCollector(String testId) throws IOException {
+			this.serverSocket = new ServerSocket(0);
+			this.testId = testId;
+		}
+
+		int getPort() {
+			return serverSocket.getLocalPort();
+		}
+
+		boolean didTestFinish() {
+			return testFinished;
+		}
+
+		boolean wasTestSuccessful() {
+			return testFinished && error == null;
+		}
+
+		Throwable getError() {
+			return error;
+		}
+
+		@Override
+		public void run() {
+			// someone asked us to stop before we could do anything
+			if (!shouldRun) {
+				return;
+			}
+			try (Socket s = serverSocket.accept();
+					BufferedReader messageReader = new BufferedReader(
+							new InputStreamReader(s.getInputStream(), "UTF-8"))) {
+				// main loop
+				while (true) {
+					synchronized (this) {
+						processAvailableMessages(messageReader);
+						if (!shouldRun) {
+							return;
+						}
+						this.wait(150);
+					}
+				}
+			} catch (InterruptedException e) {
+				// not expected
+			} catch (IOException e) {
+				if (!shouldRun) {
+					// we have been finished without ever getting any connections
+					// no need to throw exception
+					return;
+				}
+				log(new Status(IStatus.WARNING, PI_HARNESS, IStatus.ERROR, "Error", e));
+			} finally {
+				executionFinished = true;
+				synchronized (this) {
+					notifyAll();
+				}
+			}
+		}
+
+		/*
+		 * Politely asks the collector thread to stop and wait until it is finished.
+		 */
+		void shutdown() {
+			// ask the collector to stop
+			synchronized (this) {
+				if (executionFinished) {
+					return;
+				}
+				shouldRun = false;
+				try {
+					serverSocket.close();
+				} catch (IOException e) {
+					log(new Status(IStatus.ERROR, PI_HARNESS, IStatus.ERROR, "Error", e));
+				}
+				notifyAll();
+			}
+			// wait until the collector is done
+			synchronized (this) {
+				while (!executionFinished) {
+					try {
+						wait(100);
+					} catch (InterruptedException e) {
+						// we don't care
+					}
+				}
+			}
+		}
+
+		private void processAvailableMessages(BufferedReader messageReader) throws IOException {
+			while (messageReader.ready()) {
+				String message = messageReader.readLine();
+				processMessage(message);
+			}
+		}
+
+		private void processMessage(String message) {
+			if (message.startsWith("%TESTS")) {
+				String receivedTestId = parseTestId(message);
+				if (!testId.equals(receivedTestId)) {
+					throw new IllegalStateException("unknown test id: " + receivedTestId + message);
+				}
+				return;
+			} else if (message.startsWith("%TESTE")) {
+				switch (state) {
+				case FAILURE:
+					error = new RemoteAssertionFailedError("", stackTrace);
+					break;
+				case ERROR:
+					error = new RemoteTestException("", stackTrace);
+				}
+				testFinished = true;
+			} else if (message.startsWith("%ERROR")) {
+				state = STATE.ERROR;
+			} else if (message.startsWith("%FAILED")) {
+				state = STATE.FAILURE;
+			} else if (message.startsWith("%TRACES")) {
+				// just create the string buffer that will hold all the frames of the stack
+				// trace
+				stackBuilder = new StringBuilder();
+			} else if (message.startsWith("%TRACEE")) {
+				// stack trace fully read - fill the slot in the result object and reset the
+				// string buffer
+				stackTrace = stackBuilder.toString();
+				stackBuilder = null;
+			} else if (message.startsWith("%")) {
+				// ignore any other messages
+			} else if (stackBuilder != null) {
+				// build the stack trace line by line
+				stackBuilder.append(message);
+				stackBuilder.append(System.lineSeparator());
+			}
+		}
+
+		private String parseTestId(String message) {
+			if (message.isEmpty() || message.charAt(0) != '%') {
+				return null;
+			}
+			int firstComma = message.indexOf(',');
+			if (firstComma == -1) {
+				return null;
+			}
+			int secondComma = message.indexOf(',', firstComma + 1);
+			if (secondComma == -1) {
+				secondComma = message.length();
+			}
+			return message.substring(firstComma + 1, secondComma);
+		}
+
+	}
+
+}

runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionShouldError.java

@@ -0,0 +1,27 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Vector Informatik GmbH and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.core.tests.harness.session;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that a session of a test case is expected to terminate with a
+ * failure or error. To be used in combination with a
+ * {@link SessionTestExtension}.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface SessionShouldError {
+	// Marker annotation
+}