diff --git a/tests/vault/env_cleanup.yml b/tests/vault/env_cleanup.yml
index 081a9d96460e24db37d45d8ff535ef4006d7040a..326c6202681db0e0cc0065dd2f7e329e0c00b852 100644
--- a/tests/vault/env_cleanup.yml
+++ b/tests/vault/env_cleanup.yml
@@ -62,3 +62,11 @@
     file:
       path: "{{ ansible_env.HOME }}/in.txt"
       state: absent
+
+  - name: Remove private/public key files.
+    shell:
+      cmd: rm -f private.pem public.pem
+    delegate_to: localhost
+    become: no
+    args:
+      warn: no  # suppres warning for not using the `file` module.
diff --git a/tests/vault/env_setup.yml b/tests/vault/env_setup.yml
index a8437b86053c1bec6e204ac64e60b58451c0b0d2..2b4cca33f2ecf3d5ba4648bcd296fee330a98a51 100644
--- a/tests/vault/env_setup.yml
+++ b/tests/vault/env_setup.yml
@@ -1,4 +1,7 @@
-# Tasks executed to ensure a sane environment to test IPA Vault module.
+---
+  # Tasks executed to ensure a sane environment to test IPA Vault module.
+  - name: Ensure environment is clean.
+    import_tasks: env_cleanup.yml
 
   - name: Create private key file.
     shell:
@@ -12,9 +15,6 @@
     delegate_to: localhost
     become: no
 
-  - name: Ensure environment is clean.
-    import_tasks: env_cleanup.yml
-
   - name: Copy password file to target host.
     copy:
       src: "{{ playbook_dir }}/password.txt"