diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c7b0a1..07d73ed 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -130,14 +130,16 @@ jobs:
           # Pull images to avoid making smoke tests vulnerable to system behavior (docker pull speed)
           docker pull localstack/localstack-pro
           cd dist-bin
-          # start pro
+          # start pro with an API key (legacy)
           LOCALSTACK_API_KEY=${{ secrets.TEST_LOCALSTACK_API_KEY }} ./localstack start -d
           ./localstack wait -t 180
           ./localstack status services --format plain
           ./localstack status services --format plain | grep "xray=available"
           ./localstack stop
-          LOCALSTACK_AUTH_TOKEN=${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }} ./localstack start -d
+          # start pro with an auth token and extensions dev mode (see issue #20)
+          LOCALSTACK_AUTH_TOKEN=${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }} EXTENSION_DEV_MODE=1 ./localstack start -d
           ./localstack wait -t 180
+          ./localstack logs | grep "extension developer mode enabled"
           ./localstack status services --format plain
           ./localstack status services --format plain | grep "xray=available"
           ./localstack stop
diff --git a/Makefile b/Makefile
index 107a249..b2b370a 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,9 @@ dist-bin/localstack build: $(VENV_ACTIVATE) main.py
 		--hidden-import cookiecutter.extensions \
 		--hidden-import localstack_ext.cli.localstack \
 		--hidden-import localstack_ext.plugins \
+		--hidden-import localstack.dev.run.configurators \
 		--hidden-import localstack_ext.extensions.plugins \
+		--hidden-import localstack_ext.extensions.bootstrap \
 		--additional-hooks-dir hooks
 
 dist-dir/localstack: PYINSTALLER_ARGS=--distpath=dist-dir