From 555d3b17513003d623b9bfc5312bfc890389ae4f Mon Sep 17 00:00:00 2001 From: Heiko Klare Date: Mon, 8 Jan 2024 21:31:50 +0100 Subject: [PATCH] Provide a session test framework for JUnit 5 The existing implementation of session tests (including SessionTestRunner, SessionTestSuite and WorkspaceSessionTestSuite) relies on JUnit 3 functionality. Because of the changed way in which tests are set up in JUnit 5, they cannot be used to run session tests with JUnit 5. In addition the session test framework is currently overly complex, as it provides some functionality that is not used (anymore). This commit introduces a new JUnit 5 extension, the SessionTestExtension, to be used for running test cases of a class in separate Eclipse application sessions with JUnit 5. This extension intercepts test methods calls to start a separate Eclipse session in which the tests are executed. The results are evaluated in the intercepted method execution in way they were evaluated for existing JUnit-3-based session tests. Session that are expected to fail can be annotated with an added @SessionShouldFail annotation. Functionality is demonstrated in a manually executed sample session test. --- .../META-INF/MANIFEST.MF | 17 +- .../harness/session/RemoteTestExecutor.java | 299 ++++++++++++++++++ .../harness/session/SessionShouldError.java | 27 ++ .../harness/session/SessionTestExtension.java | 110 +++++++ .../session/samples/SampleSessionTests.java | 82 +++++ .../ConfigurationSessionTestSuite.java | 11 + 6 files changed, 544 insertions(+), 2 deletions(-) create mode 100644 runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/RemoteTestExecutor.java create mode 100644 runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionShouldError.java create mode 100644 runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtension.java create mode 100644 runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SampleSessionTests.java diff --git a/runtime/tests/org.eclipse.core.tests.harness/META-INF/MANIFEST.MF b/runtime/tests/org.eclipse.core.tests.harness/META-INF/MANIFEST.MF index 2ca87960b26..77dc9aef985 100644 --- a/runtime/tests/org.eclipse.core.tests.harness/META-INF/MANIFEST.MF +++ b/runtime/tests/org.eclipse.core.tests.harness/META-INF/MANIFEST.MF @@ -5,13 +5,21 @@ Bundle-SymbolicName: org.eclipse.core.tests.harness;singleton:=true Bundle-Version: 3.15.400.qualifier Bundle-Vendor: Eclipse.org Export-Package: org.eclipse.core.tests.harness;version="2.0", + org.eclipse.core.tests.harness.session, org.eclipse.core.tests.session;version="2.0" Require-Bundle: org.junit, org.eclipse.test.performance, org.eclipse.core.runtime;bundle-version="[3.29.0,4.0.0)", org.eclipse.jdt.junit.runtime, org.eclipse.jdt.junit4.runtime, - org.eclipse.pde.junit.runtime + org.eclipse.jdt.junit5.runtime, + org.eclipse.pde.junit.runtime, + junit-platform-launcher, + junit-platform-engine, + junit-jupiter-engine, + junit-vintage-engine, + junit-platform-suite-commons, + junit-platform-suite-engine Bundle-ActivationPolicy: lazy Bundle-RequiredExecutionEnvironment: JavaSE-17 Import-Package: javax.xml.parsers, @@ -19,8 +27,13 @@ Import-Package: javax.xml.parsers, org.apiguardian.api, org.assertj.core.api, org.junit.jupiter.api, - org.junit.platform.suite.api, + org.junit.jupiter.api.condition, + org.junit.jupiter.api.extension, + org.junit.jupiter.migrationsupport, + org.junit.jupiter.params, org.junit.platform.commons, + org.junit.platform.runner, + org.junit.platform.suite.api, org.opentest4j, org.w3c.dom, org.xml.sax diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/RemoteTestExecutor.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/RemoteTestExecutor.java new file mode 100644 index 00000000000..50c1433a4f8 --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/RemoteTestExecutor.java @@ -0,0 +1,299 @@ +/******************************************************************************* + * Copyright (c) 2024 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session; + +import static org.eclipse.core.tests.harness.TestHarnessPlugin.PI_HARNESS; +import static org.eclipse.core.tests.harness.TestHarnessPlugin.log; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.Socket; +import org.eclipse.core.runtime.Assert; +import org.eclipse.core.runtime.AssertionFailedException; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Platform; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.tests.session.RemoteAssertionFailedError; +import org.eclipse.core.tests.session.RemoteTestException; +import org.eclipse.core.tests.session.Setup; +import org.eclipse.core.tests.session.TestDescriptor; + +class RemoteTestExecutor { + private static final String REMOTE_EXECUTION_INDICATION_SYSTEM_PROPERY = "org.eclipse.core.tests.session.isRemoteExecution"; //$NON-NLS-1$ + + /** + * {@return whether the current test execution is done remotely} + */ + public static boolean isRemoteExecution() { + return System.getProperty(REMOTE_EXECUTION_INDICATION_SYSTEM_PROPERY, "").equals(Boolean.toString(true)); + } + + private final String pluginId; + private final String applicationId; + private final Setup setup; + + RemoteTestExecutor(Setup setup, String applicationId, String pluginId) { + this.setup = setup; + this.applicationId = applicationId; + this.pluginId = pluginId; + } + + public void executeRemotely(String testClass, String testMethod, boolean shouldFail) throws Throwable { + Setup localSetup = createSingleTestSetup(testClass, testMethod); + localSetup.setSystemProperty(REMOTE_EXECUTION_INDICATION_SYSTEM_PROPERY, Boolean.toString(true)); + + TestDescriptor descriptor = new TestDescriptor(testClass, testMethod); + ResultCollector collector = new ResultCollector(getTestId(testClass, testMethod)); + localSetup.setEclipseArgument("port", Integer.toString(collector.getPort())); + new Thread(collector, "Test result collector").start(); + IStatus status = launch(localSetup); + collector.shutdown(); + if (!shouldFail) { + if (!status.isOK()) { + log(status); + throw new CoreException(status); + } + if (!collector.didTestFinish()) { + throw new Exception("session test did not run: " + descriptor); + } + if (!collector.wasTestSuccessful()) { + throw collector.getError(); + } + } else { + if (status.isOK() && collector.wasTestSuccessful()) { + throw new AssertionFailedException("test should fail but did not: " + descriptor); + } + } + } + + private static String getTestId(String testClass, String testMethod) { + return testMethod + "(" + testClass + ")"; + } + + private Setup createSingleTestSetup(String testClassName, String testMethodName) { + Setup localSetup = (Setup) setup.clone(); + localSetup.setEclipseArgument(Setup.APPLICATION, applicationId); + localSetup.setEclipseArgument("loaderpluginname", "org.eclipse.core.tests.harness"); + localSetup.setEclipseArgument("testloaderclass", "org.eclipse.jdt.internal.junit5.runner.JUnit5TestLoader"); + localSetup.setEclipseArgument("testpluginname", pluginId); + localSetup.setEclipseArgument("test", testClassName + ":" + testMethodName); + return localSetup; + } + + /** + * Runs the setup. Returns a status object indicating the outcome of the + * operation. + * + * @return a status object indicating the outcome + */ + private static IStatus launch(Setup setup) { + Assert.isNotNull(setup.getEclipseArgument(Setup.APPLICATION), "test application is not defined"); + Assert.isNotNull(setup.getEclipseArgument("testpluginname"), "test plug-in id not defined"); + Assert.isTrue(setup.getEclipseArgument("classname") != null ^ setup.getEclipseArgument("test") != null, + "either a test suite or a test case must be provided"); + // to prevent changes in the protocol from breaking us, + // force the version we know we can work with + setup.setEclipseArgument("version", "4"); + IStatus outcome = Status.OK_STATUS; + try { + int returnCode = setup.run(); + if (returnCode == 23) { + // asked to restart; for now just do this once. + // Note that 23 is our magic return code indicating that a restart is required. + // This can happen for tests that update framework extensions which requires a + // restart. + returnCode = setup.run(); + } + if (returnCode != 0) { + outcome = new Status(IStatus.WARNING, Platform.PI_RUNTIME, returnCode, + "Process returned non-zero code: " + returnCode + "\n\tCommand: " + setup, null); + } + } catch (Exception e) { + outcome = new Status(IStatus.ERROR, Platform.PI_RUNTIME, -1, "Error running process\n\tCommand: " + setup, + e); + } + return outcome; + } + + /** + * Partly taken from + * {@code org.eclipse.core.tests.session.SessionTestRunner#ResultCollector}. + */ + static class ResultCollector implements Runnable { + private final String testId; + private final ServerSocket serverSocket; + + private volatile boolean shouldRun = true; + private volatile boolean executionFinished; + + private static enum STATE { + SUCCESS, FAILURE, ERROR; + } + + private STATE state = STATE.SUCCESS; + private StringBuilder stackBuilder; + + private String stackTrace; + private boolean testFinished; + private Throwable error; + + ResultCollector(String testId) throws IOException { + this.serverSocket = new ServerSocket(0); + this.testId = testId; + } + + int getPort() { + return serverSocket.getLocalPort(); + } + + boolean didTestFinish() { + return testFinished; + } + + boolean wasTestSuccessful() { + return testFinished && error == null; + } + + Throwable getError() { + return error; + } + + @Override + public void run() { + // someone asked us to stop before we could do anything + if (!shouldRun) { + return; + } + try (Socket s = serverSocket.accept(); + BufferedReader messageReader = new BufferedReader( + new InputStreamReader(s.getInputStream(), "UTF-8"))) { + // main loop + while (true) { + synchronized (this) { + processAvailableMessages(messageReader); + if (!shouldRun) { + return; + } + this.wait(150); + } + } + } catch (InterruptedException e) { + // not expected + } catch (IOException e) { + if (!shouldRun) { + // we have been finished without ever getting any connections + // no need to throw exception + return; + } + log(new Status(IStatus.WARNING, PI_HARNESS, IStatus.ERROR, "Error", e)); + } finally { + executionFinished = true; + synchronized (this) { + notifyAll(); + } + } + } + + /* + * Politely asks the collector thread to stop and wait until it is finished. + */ + void shutdown() { + // ask the collector to stop + synchronized (this) { + if (executionFinished) { + return; + } + shouldRun = false; + try { + serverSocket.close(); + } catch (IOException e) { + log(new Status(IStatus.ERROR, PI_HARNESS, IStatus.ERROR, "Error", e)); + } + notifyAll(); + } + // wait until the collector is done + synchronized (this) { + while (!executionFinished) { + try { + wait(100); + } catch (InterruptedException e) { + // we don't care + } + } + } + } + + private void processAvailableMessages(BufferedReader messageReader) throws IOException { + while (messageReader.ready()) { + String message = messageReader.readLine(); + processMessage(message); + } + } + + private void processMessage(String message) { + if (message.startsWith("%TESTS")) { + String receivedTestId = parseTestId(message); + if (!testId.equals(receivedTestId)) { + throw new IllegalStateException("unknown test id: " + receivedTestId + message); + } + return; + } else if (message.startsWith("%TESTE")) { + switch (state) { + case FAILURE: + error = new RemoteAssertionFailedError("", stackTrace); + break; + case ERROR: + error = new RemoteTestException("", stackTrace); + } + testFinished = true; + } else if (message.startsWith("%ERROR")) { + state = STATE.ERROR; + } else if (message.startsWith("%FAILED")) { + state = STATE.FAILURE; + } else if (message.startsWith("%TRACES")) { + // just create the string buffer that will hold all the frames of the stack + // trace + stackBuilder = new StringBuilder(); + } else if (message.startsWith("%TRACEE")) { + // stack trace fully read - fill the slot in the result object and reset the + // string buffer + stackTrace = stackBuilder.toString(); + stackBuilder = null; + } else if (message.startsWith("%")) { + // ignore any other messages + } else if (stackBuilder != null) { + // build the stack trace line by line + stackBuilder.append(message); + stackBuilder.append(System.lineSeparator()); + } + } + + private String parseTestId(String message) { + if (message.isEmpty() || message.charAt(0) != '%') { + return null; + } + int firstComma = message.indexOf(','); + if (firstComma == -1) { + return null; + } + int secondComma = message.indexOf(',', firstComma + 1); + if (secondComma == -1) { + secondComma = message.length(); + } + return message.substring(firstComma + 1, secondComma); + } + + } + +} diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionShouldError.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionShouldError.java new file mode 100644 index 00000000000..5645bdfe590 --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionShouldError.java @@ -0,0 +1,27 @@ +/******************************************************************************* + * Copyright (c) 2024 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that a session of a test case is expected to terminate with a + * failure or error. To be used in combination with a + * {@link SessionTestExtension}. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface SessionShouldError { + // Marker annotation +} diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtension.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtension.java new file mode 100644 index 00000000000..5381cf53433 --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtension.java @@ -0,0 +1,110 @@ +/******************************************************************************* + * Copyright (c) 2024 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session; + +import java.lang.reflect.Method; +import java.util.Objects; +import org.eclipse.core.tests.harness.session.samples.SampleSessionTests; +import org.eclipse.core.tests.session.Setup; +import org.eclipse.core.tests.session.SetupManager; +import org.eclipse.core.tests.session.SetupManager.SetupException; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; + +/** + * A JUnit 5 extension that will execute every test method in a class in its own + * session, i.e., in an own Eclipse application instance. It is instantiate via + * a builder created with {@link SessionTestExtension#forPlugin(String)}. For an + * example, see the {@link SampleSessionTests} class. + *

+ * Example: + * + * 

+ * @RegisterExtension
+ * static SessionTestExtension sessionTestExtension = SessionTestExtension.forPlugin(PI_HARNESS).create();
+ *
+ * + * Note: This does not enforce an execution order of test cases. If a + * specific execution order is required, it has to be ensured by different + * means, such as using a test method order annotation like + * {@link TestMethodOrder}. + * + * @see SessionShouldError + * + */ +public class SessionTestExtension implements InvocationInterceptor { + public static final String CORE_TEST_APPLICATION = "org.eclipse.pde.junit.runtime.coretestapplication"; //$NON-NLS-1$ + + private final RemoteTestExecutor testExecutor; + + private SessionTestExtension(String pluginId, String applicationId) { + try { + Setup setup = SetupManager.getInstance().getDefaultSetup(); + setup.setSystemProperty("org.eclipse.update.reconcile", "false"); + testExecutor = new RemoteTestExecutor(setup, applicationId, pluginId); + } catch (SetupException e) { + throw new IllegalStateException("unable to create setup", e); + } + } + + /** + * Creates a builder for the session test extension. Make sure to finally call + * {@link SessionTestExtensionBuilder#create()} to create a + * {@link SessionTestExtension} out of the builder. + * + * @param pluginId the id of the plug-in in which the test class is placed + * @return the builder to create a session test extension with + */ + public static SessionTestExtensionBuilder forPlugin(String pluginId) { + return new SessionTestExtensionBuilder(pluginId); + } + + public static class SessionTestExtensionBuilder { + private final String storedPluginId; + private String storedApplicationId = CORE_TEST_APPLICATION; + + private SessionTestExtensionBuilder(String pluginId) { + Objects.requireNonNull(pluginId); + this.storedPluginId = pluginId; + } + + public SessionTestExtensionBuilder withApplicationId(String applicationId) { + storedApplicationId = applicationId; + return this; + } + + public SessionTestExtension create() { + return new SessionTestExtension(storedPluginId, storedApplicationId); + } + } + + @Override + public void interceptTestMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) throws Throwable { + /** + * Ensure that we do not recursively make a remote call if we are already in + * remote execution + */ + if (RemoteTestExecutor.isRemoteExecution()) { + invocation.proceed(); + return; + } + String testClass = extensionContext.getTestClass().get().getName(); + String testMethod = extensionContext.getTestMethod().get().getName(); + + boolean shouldFail = extensionContext.getTestMethod().get().getAnnotation(SessionShouldError.class) != null; + invocation.skip(); + testExecutor.executeRemotely(testClass, testMethod, shouldFail); + } + +} diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SampleSessionTests.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SampleSessionTests.java new file mode 100644 index 00000000000..1048315c866 --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SampleSessionTests.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (c) 2024 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session.samples; + +import static org.eclipse.core.tests.harness.TestHarnessPlugin.PI_HARNESS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import org.eclipse.core.tests.harness.session.SessionShouldError; +import org.eclipse.core.tests.harness.session.SessionTestExtension; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Examples demonstrating the behavior of session tests using the JUnit 5 + * platform. When executed, the {@code Successful} tests will succeed, while the + * {@code Failing} ones will fail. + */ +public class SampleSessionTests { + @RegisterExtension + static SessionTestExtension sessionTestExtension = SessionTestExtension.forPlugin(PI_HARNESS).create(); + + @Nested + public class Successful { + @Test + public void asIntended() { + assertEquals(1, 1); + } + + @Test + @SessionShouldError + public void withExpectedFailure() { + fail("fail"); + } + + @Test + @SessionShouldError + public void withExpectedError_viaException() { + throw new RuntimeException("error"); + } + + @Test + @SessionShouldError + public void withExpectedError_viaExit() { + System.exit(1); + } + } + + @Nested + public class Failing { + @Test + @SessionShouldError + public void unexpectedlySuccessful() { + assertEquals(1, 1); + } + + @Test + public void withFailure() { + fail("fail"); + } + + @Test + public void withError_viaException() { + throw new RuntimeException("error"); + } + + @Test + public void withError_viaExit() { + System.exit(1); + } + } + +} diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/session/ConfigurationSessionTestSuite.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/session/ConfigurationSessionTestSuite.java index 1f1ab78844a..3e610c84988 100644 --- a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/session/ConfigurationSessionTestSuite.java +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/session/ConfigurationSessionTestSuite.java @@ -74,6 +74,7 @@ public ConfigurationSessionTestSuite(String pluginId, String name) { super(pluginId, name); } + @SuppressWarnings("deprecation") public void addMinimalBundleSet() { // Just use any class from the bundles we want to add as minimal bundle set @@ -91,6 +92,7 @@ public void addMinimalBundleSet() { addBundle(org.eclipse.jdt.internal.junit.runner.ITestLoader.class); // org.eclipse.jdt.junit.runtime addBundle(org.eclipse.jdt.internal.junit4.runner.JUnit4TestLoader.class); // org.eclipse.jdt.junit4.runtime + addBundle(org.eclipse.jdt.internal.junit5.runner.JUnit5TestLoader.class); // org.eclipse.jdt.junit5.runtime addBundle(org.eclipse.pde.internal.junit.runtime.CoreTestApplication.class); // org.eclipse.pde.junit.runtime addBundle(net.bytebuddy.ByteBuddy.class); // net.bytebuddy for org.assertj.core.api @@ -120,8 +122,17 @@ public void addMinimalBundleSet() { addBundle(org.junit.Test.class); // org.junit addBundle(org.junit.jupiter.api.Test.class); // junit-jupiter-api + addBundle(org.junit.jupiter.engine.JupiterTestEngine.class); // junit-jupiter-engine + addBundle(org.junit.jupiter.migrationsupport.EnableJUnit4MigrationSupport.class); // junit-jupiter-migrationsupport + addBundle(org.junit.jupiter.params.ParameterizedTest.class); // junit-jupiter-params + addBundle(org.junit.vintage.engine.VintageTestEngine.class); // junit-vintage-engine addBundle(org.junit.platform.commons.JUnitException.class); // junit-platform-commons + addBundle(org.junit.platform.engine.TestEngine.class); // junit-platform-engine + addBundle(org.junit.platform.launcher.Launcher.class); // junit-platform-launcher + addBundle(org.junit.platform.runner.JUnitPlatform.class); // junit-platform-runner addBundle(org.junit.platform.suite.api.Suite.class); // junit-platform-suite-api + addBundle(org.junit.platform.suite.commons.SuiteLauncherDiscoveryRequestBuilder.class); // junit-platform-suite-commons + addBundle(org.junit.platform.suite.engine.SuiteTestEngine.class); // junit-platform-suite-engine addBundle(org.apiguardian.api.API.class); // org.apiguardian.api addBundle(org.opentest4j.AssertionFailedError.class); // org.opentest4j }