Test

.github/workflows/ci.yaml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  get-flavors:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: '3.10'
+          poetry-version: '2.1.2'
+      - id: set-matrix
+        run: |
+          poetry -- run nox -s ci:find-available-flavors -- --github-var matrix
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+  ci:
+    needs: get-flavors
+    strategy:
+      fail-fast: true
+      matrix:
+        flavor: ${{fromJson(needs.get-flavors.outputs.matrix)}}
+    uses: ./.github/workflows/ci_flavor.yaml
+    secrets: inherit
+    with:
+      flavor: ${{ matrix.flavor }}
+
+

.github/workflows/ci_build_slc.yaml

@@ -0,0 +1,71 @@
+name: Build SLC
+
+on:
+  workflow_call:
+    inputs:
+      flavor:
+          description: 'A flavor'
+          required: true
+          type: string
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_PASSWORD:
+        required: true
+
+jobs:
+
+  detect-runner:
+    runs-on: ubuntu-22.04
+    steps:
+     - uses: actions/checkout@v4
+     - name: Setup Python & Poetry Environment
+       uses: exasol/python-toolbox/.github/actions/[email protected]
+       with:
+         python-version: '3.10'
+         poetry-version: '2.1.2'
+     - name: Detect runner
+       id: detect-runner
+       run: |
+         poetry run -- nox -s "ci:get-build-runner-for-flavor" -- --flavor "${{ inputs.flavor }}" --github-var runner
+    outputs:
+     runner: ${{ steps.detect-runner.outputs.runner }}
+  build:
+    needs: detect-runner
+    runs-on:
+      labels: ${{needs.detect-runner.outputs.runner}}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: '3.10'
+          poetry-version: '2.1.2'
+
+      - name: Build SLC ${{ inputs.flavor }}
+        id: build-slc
+        run: |
+          branch_name="${{github.head_ref || github.ref_name}}"
+          poetry run -- nox -s "ci:export-and-scan-vulnerabilities" -- --flavor "${{inputs.flavor}}" --branch-name ${{github.ref_name}} --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD" --commit-sha "${{ github.sha }}" --github-var slc_path
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+      - uses: actions/upload-artifact@v4
+        id: artifact-upload-logs-step
+        with:
+          name: logs_${{inputs.flavor}}
+          path: |
+            .build_output/jobs/**/*
+            .build_output/security_scan/**/*
+            .build_output/metadata/**/*
+          retention-days: 30
+
+      - uses: actions/upload-artifact@v4
+        id: artifact-upload-step
+        with:
+          name: slc_${{inputs.flavor}}
+          path: "${{ steps.build-slc.outputs.slc_path }}"
+          compression-level: 0
+          retention-days: 7

.github/workflows/ci_check_for_build.yaml

@@ -0,0 +1,34 @@
+name: CI
+
+on:
+  workflow_call:
+    inputs:
+      flavor:
+          description: 'A flavor'
+          required: true
+          type: string
+    outputs:
+      continue:
+        description: "Indicates if need to build and test SLC"
+        value: ${{ jobs.check_for_build.outputs.continue }}
+
+jobs:
+  check_for_build:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: "3.10"
+          poetry-version: "2.1.2"
+      - name: Run check
+        id: run-check
+        run: |
+          branch_name="${{github.head_ref || github.ref_name}}"
+          poetry run -- nox -s  ci:check-if-build-need -- --flavor ${{ inputs.flavor }} --branch-name "$branch_name" --github-var continue
+
+    outputs:
+      continue: ${{ steps.run-check.outputs.continue }}

.github/workflows/ci_flavor.yaml

@@ -0,0 +1,43 @@
+name: CI
+
+on:
+  workflow_call:
+    inputs:
+      flavor:
+          description: 'A flavor'
+          required: true
+          type: string
+    secrets:
+      DOCKER_USERNAME:
+          required: true
+      DOCKER_PASSWORD:
+          required: true
+
+jobs:
+  check_for_build:
+    uses: ./.github/workflows/ci_check_for_build.yaml
+    with:
+      flavor: ${{ inputs.flavor }}
+
+  do_nothing:
+    if: ${{ needs.check_for_build.outputs.continue != 'True' }}
+    needs: [check_for_build]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Print status
+        run: Skipping build and test for flavor ${{ inputs.flavor }}
+
+  build:
+    needs: check_for_build
+    uses: ./.github/workflows/ci_build_slc.yaml
+    if: ${{ needs.check_for_build.outputs.continue == 'True' }}
+    with:
+      flavor: ${{ inputs.flavor }}
+    secrets: inherit
+  test:
+    needs: [check_for_build, build]
+    if: ${{ needs.check_for_build.outputs.continue == 'True' }}
+    uses: ./.github/workflows/ci_test_slc.yaml
+    with:
+      flavor: ${{ inputs.flavor }}
+    secrets: inherit

.github/workflows/ci_test_slc.yaml

@@ -0,0 +1,75 @@
+name: Test SLC
+
+on:
+  workflow_call:
+    inputs:
+      flavor:
+          description: 'A flavor'
+          required: true
+          type: string
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_PASSWORD:
+        required: true
+jobs:
+
+  detect-test-runner:
+    runs-on: ubuntu-22.04
+    steps:
+     - uses: actions/checkout@v4
+     - name: Setup Python & Poetry Environment
+       uses: exasol/python-toolbox/.github/actions/[email protected]
+       with:
+         python-version: '3.10'
+         poetry-version: '2.1.2'
+     - name: Detect Test runner
+       id: detect-test-runner
+       run: |
+         poetry run -- nox -s ci:get-test-runner-for-flavor -- --flavor "${{ inputs.flavor }}" --github-var runner
+    outputs:
+     runner: ${{ steps.detect-test-runner.outputs.runner }}
+
+  detect-test-sets:
+    runs-on: ubuntu-22.04
+    steps:
+     - uses: actions/checkout@v4
+     - name: Setup Python & Poetry Environment
+       uses: exasol/python-toolbox/.github/actions/[email protected]
+       with:
+         python-version: '3.10'
+         poetry-version: '2.1.2'
+     - name: Detect Test runner
+       id: detect-test-sets
+       run: |
+         poetry run -- nox -s ci:get-test-set-names-for-flavor -- --flavor "${{ inputs.flavor }}" --github-var test_set_names
+    outputs:
+     test-sets: ${{ steps.detect-test-sets.outputs.test_set_names }}
+
+  run-tests:
+    needs: [detect-test-runner, detect-test-sets]
+    runs-on:
+      labels: ${{needs.detect-test-runner.outputs.runner}}
+    strategy:
+      fail-fast: true
+      matrix:
+        test-set: ${{fromJson(needs.detect-test-sets.outputs.test-sets)}}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: '3.10'
+          poetry-version: '2.1.2'
+      - name: Download SLC
+        id: download-slc
+        uses: actions/download-artifact@v4
+        with:
+          name: slc_${{inputs.flavor}}
+          path: "slc"
+      - name: Run Test-Set ${{ matrix.test-set }}
+        run: |
+          poetry run -- nox -s ci:run-db-tests -- --flavor "${{ inputs.flavor }}" --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD"  --test-set-name ${{ matrix.test-set }} --slc-directory "${{ steps.download-slc.outputs.download-path }}"
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

flavors/template-Exasol-8-python-3.10-cuda-conda/ci.json

@@ -0,0 +1,16 @@
+{
+  "build_runner": "int-linux-x64-4core-ubuntu24.04-1",
+  "test_config": {
+    "test_runner": "int-linux-x64-4core-ubuntu24.04-1",
+    "test_sets": [
+      {
+        "name": "all",
+        "folders": ["python3/all"]
+      },
+      {
+        "name": "pandas",
+        "folders": ["pandas/all", "pandas/pandas2"]
+      }
+    ]
+  }
+}

flavors/template-Exasol-all-python-3.10-conda/ci.json

@@ -0,0 +1,16 @@
+{
+  "build_runner": "ubuntu-22.04",
+  "test_config": {
+    "test_runner": "ubuntu-22.04",
+    "test_sets": [
+      {
+        "name": "all",
+        "folders": ["python3/all"]
+      },
+      {
+        "name": "pandas",
+        "folders": ["pandas/all", "pandas/pandas2"]
+      }
+    ]
+  }
+}

flavors/template-Exasol-all-python-3.10/ci.json

@@ -0,0 +1,20 @@
+{
+  "build_runner": "ubuntu-22.04",
+  "test_config": {
+    "test_runner": "ubuntu-22.04",
+    "test_sets": [
+      {
+        "name": "all",
+        "folders": ["python3/all"]
+      },
+      {
+        "name": "pandas",
+        "folders": ["pandas/all", "pandas/pandas2"]
+      },
+      {
+        "name": "linker_namespace",
+        "folders": ["pandas/all", "pandas/pandas2"]
+      }
+    ]
+  }
+}

flavors/test-Exasol-8-cuda-ml/ci.json

@@ -0,0 +1,20 @@
+{
+  "build_runner": "int-linux-x64-4core-ubuntu24.04-1",
+  "test_config": {
+    "test_runner": "int-linux-x64-4core-ubuntu24.04-1",
+    "test_sets": [
+      {
+        "name": "all",
+        "folders": ["python3/all"]
+      },
+      {
+        "name": "pandas",
+        "folders": ["pandas/all", "pandas/pandas2"]
+      },
+      {
+        "name": "cuda",
+        "folders": ["python3-cuda-flavor"]
+      }
+    ]
+  }
+}

noxfile.py

@@ -1,15 +1,20 @@
+import argparse
+import json
+from argparse import ArgumentParser
 from pathlib import Path
-
+from typing import List
 
 import nox
+from exasol.slc.api import run_db_tests as exaslct_run_db_tests
 
-ROOT = Path(__file__).parent
+from exasol.slc_ci.nox.tasks import *
 
+ROOT = Path(__file__).parent
+FLAVOR_PATH = ROOT / "flavors"
 
 # default actions to be run if nothing is explicitly specified with the -s option
 nox.options.sessions = []
 
-
 def get_oft_jar(session: nox.Session) -> Path:
     oft_version = "4.1.0"
     oft_jar = Path.home() / ".m2" / "repository" / "org" / "itsallcode" / "openfasttrace" / "openfasttrace" / oft_version / f"openfasttrace-{oft_version}.jar"
@@ -54,3 +59,4 @@ def run_oft_udf_client_html(session: nox.Session):
     """
     html_file = session.posargs[0] if session.posargs else "report.html"
     run_oft_for_udf_client(session, "-o", "html", "-f", html_file)
+