Add performance testing start

.github/workflows/examples.yml

@@ -23,7 +23,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - name: "Checkout repo"
+      - name: Check out repo
         uses: actions/checkout@v2
 
       - name: Setup Python & Poetry Environment
@@ -36,19 +36,15 @@ jobs:
         id: fix-vm-crash
         run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
 
-      - name: "Start test db"
+      - name: Start test db
         run: >
-          poetry run -- itde spawn-test-environment --environment-name test
-          --database-port-forward 8563 --bucketfs-port-forward 2580
-          --docker-db-image-version ${{ matrix.exasol-version }} --db-mem-size 4GB
+          poetry run -- nox -s db:start -- --db-version ${{ matrix.exasol-version }}
 
-      - name: "Wait for Exasol container (UDF extraction)"
+      - name: Wait for Exasol container (UDF extraction)
         run: poetry run -- python examples/_wait_container.py
 
-      - name: "Run examples"
-        run: |
-          poetry run -- nox -s run:examples
+      - name: Run examples
+        run: poetry run -- nox -s run:examples
 
-      - name: "Stop test db"
-        run: |
-          docker kill db_container_test
+      - name: Stop test db
+        run: poetry run -- nox -s db:stop

.github/workflows/merge-gate.yml

@@ -45,14 +45,20 @@ jobs:
     permissions:
       contents: read
 
+  performance-checks:
+    needs: [ run-slow-tests ]
+    uses: ./.github/workflows/performance-checks.yml
+    permissions:
+      contents: read
+
   # This job ensures inputs have been executed successfully.
   approve-merge:
     name: Allow Merge
     runs-on: ubuntu-24.04
     permissions:
       contents: read
     # If you need additional jobs to be part of the merge gate, add them below
-    needs: [ fast-checks, slow-checks, examples, ssl_cert ]
+    needs: [ fast-checks, slow-checks, examples, ssl_cert, performance-checks ]
 
     # Each job requires a step, so we added this dummy step.
     steps:

.github/workflows/performance-checks.yml

@@ -0,0 +1,47 @@
+name: Performance-Checks
+
+on:
+  workflow_call:
+
+jobs:
+  performance-checks:
+    name: Performance-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}), Run-${{ matrix.run-number }}
+    runs-on:  ubuntu-24.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+        exasol-version: ["8.32.0"]
+        run-number: ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"]
+    permissions:
+      contents: read
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v2
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/python-environment@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          extras: "all"
+
+      - name: Start test db
+        run: >
+          poetry run -- nox -s db:start -- --db-version ${{ matrix.exasol-version }}
+
+      - name: Run performance tests
+        run: poetry run -- nox -s performance:test
+
+      - name: Upload Artifacts
+        uses: actions/[email protected]
+        with:
+          name: performance-python${{ matrix.python-version }}-exasol${{ matrix.exasol-version}}-run${{ matrix.run-number }}
+          path: test/performance/.benchmarks/0002_performance.json
+          include-hidden-files: true
+
+      - name: Stop test db
+        run: poetry run -- nox -s db:stop
+
+      - name: Run performance check against saved reference
+        run: poetry run -- nox -s performance:check

.github/workflows/ssl_cert.yml

@@ -23,7 +23,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - name: "Checkout repo"
+      - name: Check out repo
         uses: actions/checkout@v2
 
       - name: Setup Python & Poetry Environment
@@ -32,18 +32,19 @@ jobs:
           python-version: ${{ matrix.python-version }}
           extras: "all"
 
-      - name: "Start test db"
+      - name: Start test db
         run: >
-          poetry run -- itde spawn-test-environment --environment-name ssl
-          --database-port-forward 8563 --bucketfs-port-forward 2580
-          --docker-db-image-version ${{ matrix.exasol-version }} --db-mem-size 4GB
-          --create-certificates
+          poetry run -- nox -s db:start -- --db-version ${{ matrix.exasol-version }}
+          --with-certificate
 
-      - name: "Pull certificate from container"
-        run: docker cp db_container_ssl:/certificates/rootCA.crt .
+      - name: Pull certificate from container
+        run: docker cp db_container_test:/certificates/rootCA.crt .
 
-      - name: "Setup custom host"
+      - name: Set up custom host
         run: echo "127.0.0.1 exasol-test-database" | sudo tee -a /etc/hosts
 
-      - name: "Run test"
+      - name: Run test
         run: poetry run -- python examples/_ssl_test.py
+
+      - name: Stop test db
+        run: poetry run -- nox -s db:stop

doc/changes/unreleased.md

@@ -2,3 +2,4 @@
 
 ## ✨Features
 * #208: Added `ExaConnection.import_from_parquet` which can read parquet file(s)
+* #220: Added performance tests & checks for http_transport export & import methods via nox sessions `performance:test` and `performance:check`

doc/developer_guide.rst

@@ -79,12 +79,47 @@ Integration Tests
 
 Passing additional arguments to pytest works the same as for the unit tests.
 
+Performance Tests
+-----------------
+
+The CI executes and compares the results of the performance tests in `performance-checks.yml`,
+using two nox sessions:
+
+* ``performance:test`` - this uses the ``pytest-benchmark`` to capture the results
+  of the **current CI run** in ``test/performance/.benchmarks/0002_performance.json``
+* ``performance:check`` - this utilizes scipy's implementation of the Wilcoxon
+  signed-rank test to determine if the results in the **current CI run** are statistically
+  similar to the **saved reference results** in ``test/performance/.benchmarks/0001_performance.json``.
+  If a test is not present in both the **current CI run** and **saved reference results**,
+  this is noted as an error.
+
+If the results of executing ``nox -s performance:check`` breaks a build, this requires
+manual checks from a developer:
+
+#. When the CI run was executed, the file generated from executing the performance tests
+is uploaded to the GitHub Action as an artifact, ``performance-python${{ python-version }}-exasol${{ exasol-version}}``.
+The developer should retrieve this file and compare the results to the **saved reference results**.
+#. If the changed results can be explained by significant differences in the CI runner
+or expected changes in the implementation, then the developer would overwrite the **saved reference results**
+with the GitHub Action artifact.
+
+.. note::
+
+    As the results of performance tests can vary by machine & the active environment on said machine,
+    the performance tests are benchmarked against the CI runner, and the reference results are saved
+    in ``test/performance/.benchmarks/0001_performance.json`` of the repository.
+    Naturally, this makes the results dependent upon the consistency & stability of GitHub's runners.
+    While it may not cover all use cases, the results saved by the ``nox -s performance:test`` include
+    the machine information, which may help developers determine if the observed shift in performance results
+    could have resulted from differences in the past and current execution of the performance tests
+    due to apparent macroscopic runner differences.
+
 DB
 --
 If you manually run some tests or want to try something out, you can start and stop the database manually using ``nox -s db:start`` and ``nox -s db:stop``.
 
 Preparing & Triggering a Release
---------------------------------
+++++++++++++++++++++++++++++++++
 
 The `exasol-toolbox` provides nox tasks to semi-automate the release process:
 

noxconfig.py

@@ -14,13 +14,15 @@
 
 
 def start_test_db(
-    session: Session, port: int = DEFAULT_PORT, db_version: str = DEFAULT_DB_VERSION
+    session: Session,
+    port: int = DEFAULT_PORT,
+    db_version: str = DEFAULT_DB_VERSION,
+    with_certificate: bool = True,
 ) -> None:
     # For Docker in a VM setup, refer to the ``doc/user_guide/developer_guide.rst``
-    session.run(
+    command = [
         "itde",
         "spawn-test-environment",
-        "--create-certificates",
         "--environment-name",
         CONTAINER_SUFFIX,
         "--database-port-forward",
@@ -31,8 +33,13 @@ def start_test_db(
         db_version,
         "--db-mem-size",
         "4GB",
-        external=True,
-    )
+    ]
+    if with_certificate:
+        command.append(
+            "--create-certificates",
+        )
+
+    session.run(*command, external=True)
 
 
 def stop_test_db(session: Session) -> None: