diff --git a/test/certs/generate-certs.sh b/test/certs/generate-certs.sh
index 026c500..ee15748 100755
--- a/test/certs/generate-certs.sh
+++ b/test/certs/generate-certs.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+set -euo pipefail
+
+rm -f -- ca.pem client.key client.pem server.key server.pem
+
 # Generate CA
 openssl req -nodes -new -x509 -days 3650 -extensions v3_ca -keyout ca.key -out ca.pem -config openssl.cnf