diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index aacd282b..0e57c2d2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -9,9 +9,35 @@ env:
   COBAYA_PACKAGES_PATH: ../packages
 
 jobs:
+  should_run:
+    # only run on pushes that are not also part of PR
+    runs-on: ubuntu-latest
+    outputs:
+      run_tests: ${{ steps.check.outputs.run_tests }}
+    steps:
+      - name: Check if tests should run
+        id: check
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            if (context.eventName === 'pull_request') {
+              core.setOutput('run_tests', 'true');
+            } else if (context.eventName === 'push') {
+              const { data: prs } = await github.rest.pulls.list({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                head: `${context.repo.owner}:${context.ref.replace('refs/heads/', '')}`,
+                state: 'open'
+              });
+              const shouldRun = prs.length === 0;
+              core.setOutput('run_tests', shouldRun ? 'true' : 'false');
+            }  
+
   tests:
+    needs: should_run
     runs-on: ${{ matrix.os }}
-    if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.event.pull_request == null)
+    if: needs.should_run.outputs.run_tests == 'true'
     strategy:
       fail-fast: false
       matrix: