diff --git a/test-framework/maven/src/main/java/io/quarkus/maven/it/RunAndCheckMojoTestBase.java b/test-framework/maven/src/main/java/io/quarkus/maven/it/RunAndCheckMojoTestBase.java
index 810d934b13852..03f5e1293c2b9 100644
--- a/test-framework/maven/src/main/java/io/quarkus/maven/it/RunAndCheckMojoTestBase.java
+++ b/test-framework/maven/src/main/java/io/quarkus/maven/it/RunAndCheckMojoTestBase.java
@@ -1,9 +1,11 @@
 package io.quarkus.maven.it;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -75,6 +77,8 @@ protected void run(boolean performCompile, LaunchMode mode, String... options)
     protected void run(boolean performCompile, LaunchMode mode, boolean skipAnalytics, String... options)
             throws FileNotFoundException, MavenInvocationException {
         assertThat(testDir).isDirectory();
+        assertThatPortIsFree();
+
         running = new RunningInvoker(testDir, false);
 
         final List<String> args = new ArrayList<>(3 + options.length);
@@ -114,6 +118,17 @@ protected void run(boolean performCompile, LaunchMode mode, boolean skipAnalytic
         running.execute(args, Map.of());
     }
 
+    private void assertThatPortIsFree() {
+        try {
+            // Call get(), which doesn't retry - otherwise, tests will go very slow
+            devModeClient.get();
+            fail("The port " + getPort()
+                    + " appears to be in use before starting the test instance of Quarkus, so any tests will give unpredictable results.");
+        } catch (IOException e) {
+            // All good, we wanted this
+        }
+    }
+
     protected void runAndCheck(String... options) throws FileNotFoundException, MavenInvocationException {
         runAndCheck(true, options);
     }