diff --git a/bin/tests/system/checkconf/clean.sh b/bin/tests/system/checkconf/clean.sh
index 3b424be321..088c2f4731 100644
--- a/bin/tests/system/checkconf/clean.sh
+++ b/bin/tests/system/checkconf/clean.sh
@@ -22,5 +22,6 @@ rm -f diff.out*
 rm -f good-kasp.conf.in
 rm -f good-server-christmas-tree.conf
 rm -f good.conf good.conf.raw good.conf.out badzero.conf *.out
+rm -rf keys
 rm -f ns*/named.lock
 rm -rf test.keydir
diff --git a/bin/tests/system/checkconf/tests.sh b/bin/tests/system/checkconf/tests.sh
index f21746f0e1..6c18835c16 100644
--- a/bin/tests/system/checkconf/tests.sh
+++ b/bin/tests/system/checkconf/tests.sh
@@ -18,7 +18,7 @@ set -e
 status=0
 n=0
 
-mkdir keys
+mkdir -p keys
 
 n=$((n + 1))
 echo_i "checking that named-checkconf handles a known good config ($n)"
@@ -619,7 +619,5 @@ grep -F "warning: ownername 'foo.*.check-wildcard' contains an non-terminal wild
 if [ $ret != 0 ]; then echo_i "failed"; ret=1; fi
 status=$((status + ret))
 
-rmdir keys
-
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1