diff --git a/.github/workflows/qe-tests.yml b/.github/workflows/qe-tests.yml
index 58cfda11768..e50531f347c 100644
--- a/.github/workflows/qe-tests.yml
+++ b/.github/workflows/qe-tests.yml
@@ -106,6 +106,15 @@ jobs:
         - name: Set up PKI deployment by running playbook on "controller"
           run: vagrant provision controller
 
+        - name: Run CA related tests
+          run: |
+                vagrant ssh controller -c \
+                'pytest --ansible-host-pattern master \
+                --ansible-inventory /vagrant/inventory \
+                --ansible-module-path /vagrant/tests/dogtag/pytest-ansible/common-modules/ \
+                /vagrant/tests/dogtag/pytest-ansible/pytest/sanity/test_role_users.py \
+                --junit-xml pki_cert_junit.xml -qsvvv'
+
         - name: Setup tmate session
           if: failure()
           uses: mxschmitt/action-tmate@v2
diff --git a/ci/Vagrantfile b/ci/Vagrantfile
index 79526dc04f2..3756380a24b 100644
--- a/ci/Vagrantfile
+++ b/ci/Vagrantfile
@@ -23,6 +23,8 @@ Vagrant.configure("2") do |config|
       ansible.raw_arguments  = "-M /vagrant/tests/dogtag/pytest-ansible/common-modules/"
     end
 
+    controller.vm.provision "shell", inline: "export ANSIBLE_LIBRARY=/vagrant/tests/dogtag/pytest-ansible/common-modules/"
+    controller.vm.provision "shell", inline: "pip install -e /vagrant/tests/dogtag/pytest-ansible/"
     controller.vm.network :private_network, ip: "CONTROLLER_IP"
   end
 
diff --git a/tests/dogtag/pytest-ansible/requirements.txt b/tests/dogtag/pytest-ansible/requirements.txt
index a658d2267b8..ae4b80a5994 100644
--- a/tests/dogtag/pytest-ansible/requirements.txt
+++ b/tests/dogtag/pytest-ansible/requirements.txt
@@ -6,3 +6,4 @@ pytest-autochecklog==0.2.0
 configparser
 pytest-html
 pytest_marker_bugzilla
+pyOpenSSL