diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index b5b725e..445b91b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -19,6 +19,26 @@ jobs:
         run: |
           bundle exec rubocop --extra-details --display-style-guide --no-server --parallel
 
+  smoke_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby_version: ["3.0", "3.1", "3.2", "3.3"]
+
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_RUBYGEMS__PKG__GITHUB__COM: gocardless-robot-readonly:${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+          ruby-version: "${{ matrix.ruby-version }}"
+      - name: Start bin/que
+        run: |
+          bin/que ./lib/que.rb --metrics-port=8080 --ci
+
   rspec:
     strategy:
       fail-fast: false
@@ -40,7 +60,7 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
-      
+
     env:
       PGDATABASE: que-test
       PGUSER: ubuntu
@@ -56,4 +76,4 @@ jobs:
           ruby-version: "${{ matrix.ruby-version }}"
       - name: Run specs
         run: |
-          bundle exec rspec
\ No newline at end of file
+          bundle exec rspec
diff --git a/bin/que b/bin/que
index ec9ff68..1d6cc39 100755
--- a/bin/que
+++ b/bin/que
@@ -7,8 +7,14 @@ require "ostruct"
 require "que"
 require "rack"
 require "prometheus/middleware/exporter"
+require "prometheus_gcstat"
 require "webrick"
 
+if Rack.release[0] == "3"
+  # Required if using Rack 3.x
+  require "rackup"
+end
+
 $stdout.sync = true
 
 options = OpenStruct.new
@@ -67,6 +73,10 @@ OptionParser.new do |opts|
     $stdout.puts opts
     exit 0
   end
+
+  opts.on("--ci", "Don't wait for sigterm exit after boot") do
+    options.ci = true
+  end
 end.parse!(ARGV)
 # rubocop:enable Layout/LineLength
 
@@ -150,15 +160,31 @@ if options.metrics_port
       end,
     )
 
-    Rack::Handler::WEBrick.run(
-      app,
-      Host: "0.0.0.0",
-      Port: options.metrics_port,
-      Logger: WEBrick::Log.new("/dev/null"),
-      AccessLog: [],
-    )
+    host = "0.0.0.0"
+    logger = WEBrick::Log.new("/dev/null")
+
+    if Rack.release[0] == "3"
+      Rackup::Handler::WEBrick.run(
+        app,
+        Host: host,
+        Port: options.metrics_port,
+        Logger: logger,
+        AccessLog: [],
+      )
+    else
+      Rack::Handler::WEBrick.run(
+        app,
+        Host: host,
+        Port: options.metrics_port,
+        Logger: logger,
+        AccessLog: [],
+      )
+    end
   end
 end
 
-wait_for_signals("INT", "TERM")
-worker_group.stop(timeout)
+# For a basic CI check we ensure the app boots etc
+unless options.ci
+  wait_for_signals("INT", "TERM")
+  worker_group.stop(timeout)
+end
diff --git a/que.gemspec b/que.gemspec
index 129996a..c784645 100644
--- a/que.gemspec
+++ b/que.gemspec
@@ -26,9 +26,11 @@ Gem::Specification.new do |spec|
   # instead, and in any other clients of `Que`.
   # This is highly non ideal, but unless we properly fork, we have to do this for now.
   spec.add_dependency "prometheus-client"
+  spec.add_dependency "prometheus_gcstat"
 
-  spec.add_dependency "rack", "~> 3.0"
-  spec.add_dependency "webrick", "~> 1.8"
+  spec.add_dependency "rack", ">= 2", "< 4"
+  spec.add_dependency "rackup"
+  spec.add_dependency "webrick"
 
   spec.add_runtime_dependency "activesupport"
   spec.metadata["rubygems_mfa_required"] = "true"