diff --git a/refaster-runner/src/test/java/tech/picnic/errorprone/refaster/runner/RefasterTest.java b/refaster-runner/src/test/java/tech/picnic/errorprone/refaster/runner/RefasterTest.java
index d73aa924d4a..4c99a90032f 100644
--- a/refaster-runner/src/test/java/tech/picnic/errorprone/refaster/runner/RefasterTest.java
+++ b/refaster-runner/src/test/java/tech/picnic/errorprone/refaster/runner/RefasterTest.java
@@ -234,7 +234,7 @@ private static boolean isBuiltWithErrorProneFork() {
               /* initialize= */ false,
               Thread.currentThread().getContextClassLoader());
     } catch (ClassNotFoundException e) {
-      return false;
+      throw new IllegalStateException("Can't load `ErrorProneOptions` class", e);
     }
     return Arrays.stream(clazz.getDeclaredMethods())
         .filter(m -> Modifier.isPublic(m.getModifiers()))