diff --git a/MANIFEST.in b/MANIFEST.in
index c3a072f4f..1a97a32ed 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -7,6 +7,7 @@
 
 include pyproject.toml
 include *.md
+include test/*.sh
 recursive-include benchexec/tablegenerator/react-table/build *.min.js
 recursive-include benchexec/tablegenerator/react-table/build *.min.css
 recursive-include bin *
diff --git a/debian/rules b/debian/rules
index 12b2195bc..a3fc4cb19 100755
--- a/debian/rules
+++ b/debian/rules
@@ -17,6 +17,7 @@ export PYBUILD_NAME=benchexec
 # Our tests need some additional files in {build_dir} to execute succesfully
 export PYBUILD_BEFORE_TEST=cp -r {dir}/bin {dir}/doc {build_dir}/
 export PYBUILD_AFTER_TEST=rm -r {build_dir}/bin {build_dir}/doc
+export PYBUILD_TEST_ARGS=""
 
 %:
 	dh $@ --with python3 --buildsystem=pybuild
diff --git a/release.sh b/release.sh
index a19cb7b09..5906a7e91 100755
--- a/release.sh
+++ b/release.sh
@@ -103,7 +103,10 @@ cd "BenchExec-$VERSION"
 dh_make -p "benchexec_$VERSION" --createorig -f "../$TAR" -i -c apache || true
 
 dpkg-buildpackage --build=source -sa "--sign-key=$DEBKEY"
-podman run --rm -w "$(pwd)" -v "$TEMP_DEB:$TEMP_DEB:rw" ubuntu:20.04 bash -c '
+podman run --security-opt unmask=/sys/fs/cgroup --cgroups=split \
+  --security-opt unmask=/proc/* --security-opt seccomp=unconfined --device /dev/fuse \
+  --rm -w "$(pwd)" -v "$TEMP_DEB:$TEMP_DEB:rw" --rm ubuntu:20.04 \
+  "$TEMP_DEB/BenchExec-$VERSION/test/setup_cgroupsv2_in_container.sh" bash -c '
   apt-get update
   apt-get install -y --no-install-recommends dpkg-dev
   TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y $(dpkg-checkbuilddeps 2>&1 | grep -o "Unmet build dependencies:.*" | cut -d: -f2- | sed "s/([^)]*)//g")