diff --git a/harp/commandline/start.py b/harp/commandline/start.py
index 6c2f7cca..348d6929 100644
--- a/harp/commandline/start.py
+++ b/harp/commandline/start.py
@@ -11,6 +11,16 @@ def _get_service_name_for_humans(x: str):
     return x
 
 
+def assert_package_is_available(package_name: str):
+    if importlib.util.find_spec(package_name) is None:
+        raise ModuleNotFoundError(f"Package {package_name!r} is not available.")
+
+
+def assert_development_packages_are_available():
+    assert_package_is_available("honcho")
+    assert_package_is_available("watchfiles")
+
+
 @click.command(short_help="Starts the development environment.")
 @click.option("--set", "options", default=(), multiple=True, help="Set proxy configuration options.")
 @click.option("--file", "-f", "files", default=(), multiple=True, help="Load configuration from file.")
@@ -26,7 +36,9 @@ def _get_service_name_for_humans(x: str):
 )
 @click.argument("services", nargs=-1)
 def start(with_docs, with_ui, options, files, disable, services, server_subprocesses):
-    if importlib.util.find_spec("honcho") is None or importlib.util.find_spec("watchfiles") is None:
+    try:
+        assert_development_packages_are_available()
+    except ModuleNotFoundError as exc:
         raise click.UsageError(
             "\n".join(
                 (
@@ -35,7 +47,7 @@ def start(with_docs, with_ui, options, files, disable, services, server_subproce
                     'Try to install the "dev" extra.',
                 )
             )
-        )
+        ) from exc
 
     from harp.commandline.utils.manager import (
         HARP_DOCS_SERVICE,
diff --git a/harp/utils/testing/benchmarking.py b/harp/utils/testing/benchmarking.py
index 3f520449..d84242c0 100644
--- a/harp/utils/testing/benchmarking.py
+++ b/harp/utils/testing/benchmarking.py
@@ -8,6 +8,7 @@
 import pytest
 
 from harp import get_logger
+from harp.commandline.start import assert_development_packages_are_available
 from harp.utils.network import get_available_network_port, wait_for_port
 
 logger = get_logger(__name__)
@@ -25,6 +26,10 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, *,
                 _tmpfile_config.write(config or "")
                 self.config_filename = _tmpfile_config.name
 
+        # XXX we may not need dev environment (yet maybe we want to test it works too). To avoid cryptic error, we
+        # double check here to get an exception if not available.
+        assert_development_packages_are_available()
+
     def run(self):
         args = [
             "harp",
@@ -34,6 +39,7 @@ def run(self):
             *("--disable", "harp_apps.telemetry"),
         ]
         logger.info('Starting subprocess: "%s"', " ".join(args))
+
         self.process = subprocess.Popen(args, shell=False)
 
     def join(self, timeout=None):
@@ -48,9 +54,14 @@ class AbstractProxyBenchmark:
     @pytest.fixture(scope="class")
     def proxy(self, httpbin, database_url):
         port = get_available_network_port()
-        thread = RunHarpProxyInSubprocessThread(
-            config=self.config.substitute(port=port, httpbin=httpbin, database=database_url)
-        )
+
+        try:
+            thread = RunHarpProxyInSubprocessThread(
+                config=self.config.substitute(port=port, httpbin=httpbin, database=database_url)
+            )
+        except Exception as exc:
+            pytest.skip(f"Failed to create subprocess thread: {exc}")
+
         try:
             try:
                 from pytest_cov.embed import cleanup_on_sigterm
@@ -59,7 +70,7 @@ def proxy(self, httpbin, database_url):
             else:
                 cleanup_on_sigterm()
             thread.start()
-            wait_for_port(port, timeout=10.0)
+            wait_for_port(port, timeout=5.0)
             yield f"localhost:{port}"
         finally:
             thread.join()