run tests as separate jobs

.github/workflows/integration.yaml

@@ -14,13 +14,26 @@ jobs:
     name: Build rock
     uses: canonical/data-platform-workflows/.github/workflows/build_rock.yaml@v16
 
+  collect-tests:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+    steps:
+      - name: Generate matrix with all tests
+        id: set-matrix
+        run: |
+          echo "matrix=$(find tests/integration/ -name "test_*.py" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+
   integration:
     runs-on: ubuntu-latest
     timeout-minutes: 120
-    needs: build
+    needs:
+      - build
+      - collect-tests
     strategy:
       matrix:
-        env: [integration]
+        file: ${{ fromJson(needs.collect-tests.outputs.matrix) }}
       fail-fast: false
     steps:
       - name: Checkout repository
@@ -50,7 +63,8 @@ jobs:
           pipx install poetry
       - name: Integration Tests
         run: |
-          sg snap_microk8s -c "tox -e ${{ matrix.env }}"
+          echo "Running test ${{ matrix.file }}"
+          sg snap_microk8s -c "tox -e ${{ matrix.file }}"
       - name: Free disk space
         run: |
           echo "Free disk space after int tests"