From 60e6ab503afe95038daeb2531385302e5be301b9 Mon Sep 17 00:00:00 2001
From: Jan Rous <rousik@gmail.com>
Date: Fri, 1 Dec 2023 12:08:19 -0700
Subject: [PATCH] TESTONLY: Run matrix of concurrency options.

To test the speed-up characteristics of different tuning params, and
to determine what fits on the worker.
---
 .github/workflows/pytest.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 401ce2c09a..f3f3cd89db 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -121,6 +121,11 @@ jobs:
     defaults:
       run:
         shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        dagster_workers: [0, 1, 2, 4]
+        xbrl_workers: [1, 2, 3, 4, 8]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -181,8 +186,11 @@ jobs:
         run: |
           coverage run ${{ env.COVERAGE_OPTIONS }} \
             src/pudl/ferc_to_sqlite/cli.py \
-              --workers ${{ env.XBRL_WORKERS }} \
+              --dagster-workers ${{ matrix.dagster_workers }} \
+              --workers ${{ matrix.xbrl_workers }} \
               --clobber ${{ env.ETL_COMMANDLINE_OPTIONS }} ${{ env.ETL_CONFIG }}
+      - name: Fail pipeline early
+        run: exit 1
       - name: Run pudl_etl
         env:
           COVERAGE_FILE: .coverage.pudl_etl