Merge branch 'main' into kra-vis

DendrouLab · Apr 29, 2024 · d508b63 · d508b63
2 parents 4a70208 + b7882bb
commit d508b63
Show file tree

Hide file tree

Showing 105 changed files with 4,963 additions and 1,286 deletions.
diff --git a/.github/workflows/clustering-ci.yml b/.github/workflows/clustering-ci.yml
@@ -0,0 +1,99 @@
+name: Run tutorials (clustering)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  clustering:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install -e .
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+      # Note: all three files are renamed during the download to trim the "subsample_" prefix
+      - name: Preparing the data
+        run: |
+          mkdir -p clustering && cd clustering
+          curl -L -o teaseq_corrected.h5mu https://figshare.com/ndownloader/files/45879852
+
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd clustering
+          panpipes clustering config
+
+      - name: Edit the submission file
+        run: |
+          cd clustering
+          curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/3895aa0ba60017b15ee1aa6531dc8c25/pipeline.yml
+
+      - name: Replace template contents in configuration file
+        run: |
+          cd clustering
+          sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_env+pipeline_env+g' pipeline.yml
+          sed -i 's+../teaseq_temp.h5mu+teaseq_corrected.h5mu+g' pipeline.yml
+
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree clustering
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd clustering
+          panpipes clustering show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd clustering
+          panpipes clustering make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree clustering
+
diff --git a/.github/workflows/integration01-ci.yml b/.github/workflows/integration01-ci.yml
@@ -0,0 +1,100 @@
+name: Run integration01
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  integration:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+# important: this patch is only to test if multivi integration works
+# issues are not related to panpipes https://discourse.scverse.org/t/error-when-training-model-on-m3-max-mps/1896/2
+# https://discourse.scverse.org/t/macbook-m1-m2-mps-acceleration-with-scvi/2075/4
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install '.[multivipatch]' 
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+      # Note: all three files are renamed during the download to trim the "subsample_" prefix
+      - name: Preparing the data
+        run: |
+          mkdir -p teaseq/integration && cd teaseq/integration
+          curl -L -o teaseq.h5mu https://figshare.com/ndownloader/files/44796985
+
+      # Note: we run the following to test that the commands works
+      # However, the following task will replacing the file anyway
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration config
+
+      - name: Edit the submission file
+        run: |
+          cd teaseq/integration
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes/main/tests/integration_1/pipeline.yml  
+
+      - name: Replace template contents in configuration file
+        run: |
+          cd teaseq/integration
+          sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_env+pipeline_env+g' pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree teaseq
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree teaseq
diff --git a/.github/workflows/integration02-ci.yml b/.github/workflows/integration02-ci.yml
@@ -0,0 +1,99 @@
+name: Run integration02
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  integration:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+# important: this patch is only to test if multivi integration works
+# issues are not related to panpipes https://discourse.scverse.org/t/error-when-training-model-on-m3-max-mps/1896/2
+# https://discourse.scverse.org/t/macbook-m1-m2-mps-acceleration-with-scvi/2075/4
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install '.[multivipatch]' 
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+      # Note: all three files are renamed during the download to trim the "subsample_" prefix
+      - name: Preparing the data
+        run: |
+          mkdir -p teaseq/integration && cd teaseq/integration
+          curl -L -o teaseq.h5mu https://figshare.com/ndownloader/files/44796985
+
+      # Note: we run the following to test that the commands works
+      # However, the following task will replacing the file anyway
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration config
+
+      - name: Edit the submission file
+        run: |
+          cd teaseq/integration
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes/main/tests/integration_2/pipeline.yml  
+      - name: Replace template contents in configuration file
+        run: |
+          cd teaseq/integration
+          sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_env+pipeline_env+g' pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree teaseq
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree teaseq