refactor: share python setup across test jobs (#1219)

* refactor: remove unused os workflows * feat: add reusable python setup * refactor: DRY for python setup, bump studio --------- Co-authored-by: Sebastian Niehus <[email protected]>
Aleph-Alpha · Jan 21, 2025 · 2b3a1db · 2b3a1db
1 parent d51be8d
commit 2b3a1db
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 216 deletions.
diff --git a/.github/workflows/get_jfrog_access_token.sh → .github/composites/get_jfrog_access_token.sh b/.github/workflows/get_jfrog_access_token.sh → .github/composites/get_jfrog_access_token.sh
diff --git a/...rkflows/get_jfrog_access_token_subject.sh → ...posites/get_jfrog_access_token_subject.sh b/...rkflows/get_jfrog_access_token_subject.sh → ...posites/get_jfrog_access_token_subject.sh
diff --git a/.github/composites/python-setup/action.yml b/.github/composites/python-setup/action.yml
@@ -0,0 +1,30 @@
+name: Checkout and set up python
+description: "Installs python, dependencies and handles venv caching"
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+
+    - name: Install and configure Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
+        virtualenvs-path: .venv
+
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+    - name: Install dependencies
+      shell: bash
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: |
+        poetry config installer.max-workers 10
+        poetry install --no-interaction
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml
@@ -4,49 +4,25 @@ on:
       runner:
         type: string
         default: "ubuntu-latest"
-    secrets:
-      AA_TOKEN:
-        required: true
-      HUGGING_FACE_TOKEN:
-        required: true
+      timeout:
+        type: number
+        default: 15 # mins
+
+defaults:
+  run:
+    shell: bash
+
 jobs:
   lint:
-    defaults:
-      run:
-        shell: bash
-    timeout-minutes: 15
+    timeout-minutes: ${{inputs.timeout}}
     runs-on: ${{inputs.runner}}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
-
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v4
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-      - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: |
-          poetry config installer.max-workers 10
-          poetry install --no-interaction
+      - uses: ./.github/composites/python-setup
 
       - name: set PY for pre-commit
         run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
-
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
@@ -56,37 +32,13 @@ jobs:
         run: |
           ./scripts/lint.sh
   doctest:
-    defaults:
-      run:
-        shell: bash
-    timeout-minutes: 15
+    timeout-minutes: ${{inputs.timeout}}
     runs-on: ${{inputs.runner}}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
+      - uses: ./.github/composites/python-setup
 
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v4
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-      - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: |
-          poetry config installer.max-workers 10
-          poetry install --no-interaction
       - name: Run doctest
         env:
           AA_TOKEN: ${{ secrets.AA_TOKEN }}
@@ -95,10 +47,7 @@ jobs:
         run: ./scripts/doctest.sh
 
   test:
-    defaults:
-      run:
-        shell: bash
-    timeout-minutes: 15
+    timeout-minutes: ${{inputs.timeout}}
     runs-on: ${{inputs.runner}}
     services:
       argilla-elastic-search:
@@ -138,15 +87,16 @@ jobs:
           POSTGRES_USER: "il_sdk"
           POSTGRES_PASSWORD: "test"
       studio-backend:
-        image: registry.gitlab.aleph-alpha.de/product/studio/backend:latest
+        image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.65
         ports:
           - "8000:8000"
         env:
-          POSTGRES_HOST: "postgres"
-          POSTGRES_PORT: "5432"
           POSTGRES_DB: "il_sdk"
           POSTGRES_USER: "il_sdk"
           POSTGRES_PASSWORD: "test"
+          POSTGRES_HOST: "postgres"
+          POSTGRES_PORT: "5432"
+
           AUTHORIZATION_SERVICE_URL: ${{ secrets.AUTHORIZATION_SERVICE_URL }}
           AA_TOKEN: ${{ secrets.AA_TOKEN }}
           API_SCHEDULER_URL: ${{ secrets.CLIENT_URL }}
@@ -157,53 +107,31 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
-
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v4
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-      - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: |
-          poetry config installer.max-workers 10
-          poetry install --no-interaction --without docs
+      - uses: ./.github/composites/python-setup
 
       - name: Run pytest
         env:
+          POSTGRES_DB: "il_sdk"
+          POSTGRES_USER: "il_sdk"
+          POSTGRES_PASSWORD: "test"
+          POSTGRES_HOST: "localhost"
+          POSTGRES_PORT: "5433"
+
+          AUTHORIZATION_SERVICE_URL: ${{ secrets.AUTHORIZATION_SERVICE_URL }}
           AA_TOKEN: ${{ secrets.AA_TOKEN }}
+          DATA_SERVICE_URL: ${{secrets.DATA_SERVICE_URL}}
+
           HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
           ARGILLA_API_URL: "http://localhost:6900/"
           ARGILLA_API_KEY: "argilla.apikey"
           CLIENT_URL: ${{ secrets.CLIENT_URL }}
           STUDIO_URL: "http://localhost:8000/"
           DOCUMENT_INDEX_URL: ${{secrets.DOCUMENT_INDEX_URL}}
           PHARIA_KERNEL_URL: https://pharia-kernel.product.pharia.com
-          POSTGRES_HOST: "localhost"
-          POSTGRES_PORT: "5433"
-          POSTGRES_DB: "il_sdk"
-          POSTGRES_USER: "il_sdk"
-          POSTGRES_PASSWORD: "test"
         run: |
           ./scripts/test.sh
   run-notebooks:
-    defaults:
-      run:
-        shell: bash
-    timeout-minutes: 15
+    timeout-minutes: ${{inputs.timeout}}
     runs-on: ${{inputs.runner}}
     services:
       argilla-elastic-search:
@@ -228,15 +156,16 @@ jobs:
           POSTGRES_USER: "il_sdk"
           POSTGRES_PASSWORD: "test"
       studio-backend:
-        image: registry.gitlab.aleph-alpha.de/product/studio/backend:latest
+        image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.65
         ports:
           - "8001:8000"
         env:
-          POSTGRES_HOST: "postgres"
-          POSTGRES_PORT: "5432"
           POSTGRES_DB: "il_sdk"
           POSTGRES_USER: "il_sdk"
           POSTGRES_PASSWORD: "test"
+          POSTGRES_HOST: "postgres"
+          POSTGRES_PORT: "5432"
+
           AUTHORIZATION_SERVICE_URL: ${{ secrets.AUTHORIZATION_SERVICE_URL }}
           AA_TOKEN: ${{ secrets.AA_TOKEN }}
           API_SCHEDULER_URL: ${{ secrets.CLIENT_URL }}
@@ -247,28 +176,9 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
+      - uses: ./.github/composites/python-setup
 
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v4
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-      - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: |
-          poetry config installer.max-workers 10
-          poetry install --no-interaction
-      - name: Configure Poetry for notebooks and run
+      - name: Run Notebooks
         env:
           AA_TOKEN: ${{ secrets.AA_TOKEN }}
           HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}

diff --git a/.github/workflows/test-execution.yml b/.github/workflows/test-execution.yml
@@ -9,14 +9,10 @@ on:
   # manual trigger
   workflow_dispatch:
 
-
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
   python-tests:
     uses: ./.github/workflows/sdk-tests.yml
-    with:
-      runner: 'ubuntu-latest'
     secrets: inherit
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -61,7 +61,7 @@ services:
   # echo $GITLAB_TOKEN | docker login registry.gitlab.aleph-alpha.de -u your_email@for_gitlab --password-stdin
   # docker compose pull to update containers
   studio-backend:
-    image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.64
+    image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.65
     ports:
       - 8000:8000
     depends_on: