Upgrade to Python 3.12 / Snakemake 8 (#355)

* Upgrade to Python 3.12 / Snakemake 8; migrate from poetry to uv; improve pyrunner test strategy * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add ruff to dev-dependencies * Remove redundant import * Revert docs build to Python 3.11 * Update test-repo to use grapevne-helper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove absolute paths from test workflows * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add setuptools to github actions * Remove absolute path from pyinstaller test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Migrate install_miniforge script to .venv from poetry * Separate file and folder copying when packaging modules * Modify Dockerfile for Snakemake v8 and install grapevne-py * Fix bug that prevented email alerts from being sent * Remove sources from docs build * Setup conda using github action for postbuild tests * Switch payload-shell to payload-run for Windows testing * Disable workflow checks on Windows where modules connect together (known bug) * Update nightly and publish workflows to support action-based conda install * Update nightly and publish workflows to support action-based conda install * Reject non-native binaries introduced by PuLP * Remove .venv from build * Revert and update nightly_build github action * Disable colima/docker install on macos-12 as both very slow and inconsistent * Migrate to grapevne-py installer * Update grapevne_helper in tests --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
kraemer-lab · Sep 17, 2024 · 4d5f4f0 · 4d5f4f0
1 parent 20a6f75
commit 4d5f4f0
Show file tree

Hide file tree

Showing 88 changed files with 2,723 additions and 3,942 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -30,12 +30,12 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
+          python-version: "3.12"
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v2
       - name: install MacOS (aarch64) build dependencies
         shell: bash
         run: |
@@ -62,6 +62,11 @@ jobs:
       - name: Setup tmate session
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      # setuptools needs to be available globally for the yarn build script
+      - name: Install setuptools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install setuptools
       - name: Build GRAPEVNE
         working-directory: ./electron-app
         run: |

diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml
@@ -9,20 +9,14 @@ jobs:
     defaults:
       run:
         working-directory: builder
-    strategy:
-      matrix:
-        python-version: ["3.11"]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-      - name: Install dependencies
+          python-version: "3.12"
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v2
+      - name: Run tests
         run: |
-          poetry install
-      - name: Testing code with pytest
-        run: |
-          poetry run pytest
+          ./run_tests.sh
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -7,16 +7,13 @@ jobs:
   docs:
     name: Build documentation and check links
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.11"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.11"
       - name: Build documentation
         run: |
           python3 -m pip install --upgrade pip

diff --git a/.github/workflows/electron-app.yml b/.github/workflows/electron-app.yml
@@ -13,6 +13,18 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v2
+      # setuptools needs to be available globally for the yarn build script
+      - name: Install setuptools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install setuptools
       - name: Run tests
-        run: ./run_tests.sh
+        run: |
+          ./build_deps.sh
+          ./run_tests.sh
diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml
@@ -19,18 +19,24 @@ jobs:
       - uses: nanasess/setup-chromedriver@v2
         with:
           chromedriver-version: "126.0.6478.185"
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
+          python-version: "3.12"
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v2
       - name: install MacOS (M1) build dependencies
         shell: bash
         run: |
           if [ "$RUNNER_OS" == "macOS" ] && [ "$RUNNER_ARCH" == "ARM64" ]; then
             brew install pango
           fi
+      # setuptools needs to be available globally for the yarn build script
+      - name: Install setuptools
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install setuptools
       - name: build
         working-directory: ./electron-app
         shell: bash
@@ -41,10 +47,20 @@ jobs:
           yarn
           yarn build
           yarn package
+      # Install conda for post-build tests
+      - name: Set up Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: build
+          miniforge-version: latest
+          auto-update-conda: true
+          auto-activate-base: true
+          python-version: 3.12
+          use-mamba: true
       # only publish builds if they pass post-build tests
       - name: postbuild-tests
         working-directory: ./electron-app
-        shell: bash
+        shell: bash -el {0}  # required to sustain the conda environment
         run: |
           ./run_postbuild_tests.sh
       - name: Build distributables

diff --git a/.github/workflows/nodemapper.yml b/.github/workflows/nodemapper.yml
@@ -13,6 +13,5 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
       - name: Run tests
         run: ./run_tests.sh
diff --git a/.github/workflows/postbuildchecks.yml b/.github/workflows/postbuildchecks.yml
@@ -28,30 +28,44 @@ jobs:
       - uses: nanasess/setup-chromedriver@v2
         with:
           chromedriver-version: "126.0.6478.185"
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - name: Set up Miniconda
+        uses: conda-incubator/setup-miniconda@v3
         with:
-          python-version: "3.11"
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
+          activate-environment: build
+          miniforge-version: latest
+          auto-update-conda: true
+          auto-activate-base: true
+          python-version: 3.12
+          use-mamba: true
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v2
       - name: install MacOS (M1) build dependencies
-        shell: bash
+        shell: bash -el {0}  # required to sustain the conda environment
         run: |
           if [ "$RUNNER_OS" == "macOS" ] && [ "$RUNNER_ARCH" == "ARM64" ]; then
             brew install pango
           fi
+      # setuptools needs to be available globally for the yarn build script
+      - name: Install setuptools
+        shell: bash -el {0}  # required to sustain the conda environment
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install setuptools
       - name: build
         working-directory: ./electron-app
-        shell: bash
+        shell: bash -el {0}  # required to sustain the conda environment
         run: |
           corepack enable
           ./build_deps.sh
           yarn install
           yarn
           yarn build
           yarn package
+      # disabled: docker is very slow to install on macos-12, is unsupported on
+      #           macos-14, is available on windows and already tested under ubuntu
       - name: install docker MacOS (Intel chips)
-        shell: bash
+        shell: bash -el {0}  # required to sustain the conda environment
+        if: false
         run: |
           if [ "$RUNNER_OS" == "macOS" ] && [ "$RUNNER_ARCH" == "X64" ]; then
             brew install docker
@@ -63,6 +77,6 @@ jobs:
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
       - name: postbuild-tests
         working-directory: ./electron-app
-        shell: bash
+        shell: bash -el {0}  # required to sustain the conda environment
         run: |
           ./run_postbuild_tests.sh
diff --git a/.github/workflows/publish_build.yml b/.github/workflows/publish_build.yml
@@ -1,4 +1,5 @@
 name: Publish build
+
 on:
   workflow_call:
     inputs:
@@ -18,18 +19,24 @@ jobs:
       - uses: nanasess/setup-chromedriver@v2
         with:
           chromedriver-version: "126.0.6478.185"
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
+          python-version: "3.12"
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v2
       - name: install MacOS (M1) build dependencies
         shell: bash
         run: |
           if [ "$RUNNER_OS" == "macOS" ] && [ "$RUNNER_ARCH" == "ARM64" ]; then
             brew install pango
           fi
+      # setuptools needs to be available globally for the yarn build script
+      - name: Install setuptools
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install setuptools
       - name: build
         working-directory: ./electron-app
         shell: bash
@@ -43,10 +50,20 @@ jobs:
       - name: tmate debugging
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      # Install conda for post-build tests
+      - name: Set up Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: build
+          miniforge-version: latest
+          auto-update-conda: true
+          auto-activate-base: true
+          python-version: 3.12
+          use-mamba: true
       - name: postbuild-tests
         working-directory: ./electron-app
-        shell: bash
-        # 'yarn package' is only run to allow build_testsuite.sh to run before
+        shell: bash -el {0}  # required to sustain the conda environment
+        # 'yarn package' is only run to allow run_postbuild_tests.sh to run before
         # the package is pushed for publishing ('yarn run publish' builds and
         # publishes in one step)
         run: |

diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml
@@ -9,20 +9,14 @@ jobs:
     defaults:
       run:
         working-directory: runner
-    strategy:
-      matrix:
-        python-version: ["3.11"]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-      - name: Install dependencies
+          python-version: "3.12"
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v2
+      - name: Run tests
         run: |
-          poetry install
-      - name: Testing code with pytest
-        run: |
-          poetry run pytest
+          ./run_tests.sh
diff --git a/.lycheeignore b/.lycheeignore
@@ -0,0 +1,2 @@
+docs/_build/html/_static
+docs/_build/html/_sources
diff --git a/builder/builder/builder.py b/builder/builder/builder.py
@@ -407,19 +407,24 @@ def PackageModule_Local(build_path: str, node: Node) -> None:
         src = os.path.normpath(pathlib.Path(m_path, os.pardir))
         pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
         # Copy module to the build directory
-        ignore_in_root = ["results", "logs", "benchmarks"]
+        ignore_in_root = ["results", "logs", "benchmarks", ".test.sh"]
         ignore_anywhere = [".snakemake", "__pycache__"]
         folders_in_root = os.listdir(src)
         keep_folders = set(folders_in_root) - set(ignore_in_root)
         for folder in keep_folders:
-            shutil.copytree(
-                pathlib.Path(src, folder),
-                pathlib.Path(dest, folder),
-                dirs_exist_ok=True,
-                ignore=lambda directory, contents: contents
-                if any(map(directory.endswith, ignore_anywhere))
-                else set(),
-            )
+            src_folder = pathlib.Path(src, folder)
+            dest_folder = pathlib.Path(dest, folder)
+            if src_folder.is_file():
+                shutil.copy2(src_folder, dest_folder)
+            else:
+                shutil.copytree(
+                    src_folder,
+                    dest_folder,
+                    dirs_exist_ok=True,
+                    ignore=lambda directory, contents: contents
+                    if any(map(directory.endswith, ignore_anywhere))
+                    else set(),
+                )
         # Redirect snakefile location in config
         node.snakefile = str(
             pathlib.Path(