From e16c2b2daa44130deb40b119e2fe32e19b1f9ec3 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Wed, 21 Feb 2024 15:36:17 +0000 Subject: [PATCH 1/4] try out filtering ci workflow --- .github/workflows/filtering-ci.yml | 104 +++++++++++++++++++++++++++++ pipeline_env.yaml | 2 +- pyproject.toml | 36 +++++----- 3 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/filtering-ci.yml diff --git a/.github/workflows/filtering-ci.yml b/.github/workflows/filtering-ci.yml new file mode 100644 index 00000000..b3236680 --- /dev/null +++ b/.github/workflows/filtering-ci.yml @@ -0,0 +1,104 @@ +name: Run tutorials (preprocessing) + +on: + push: + branches: + - main + pull_request: + branches: + - main + +env: + debug: 'true' + +jobs: + preprocessing: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] # , "macos-latest", "windows-latest" + python-version: ["3.11"] + + steps: + - uses: actions/checkout@v4 + + - name: File tree + if: env.debug == 'true' + run: tree + + - uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + auto-activate-base: true + auto-update-conda: true + channels: conda-forge + channel-priority: strict + activate-environment: pipeline_env + environment-file: pipeline_env.yaml + + - name: Install Panpipes + shell: bash -el {0} + run: | + pip install -e . + conda list + + - name: Conda info + if: env.debug == 'true' + shell: bash -el {0} + run: conda info + + - name: Conda list + if: env.debug == 'true' + shell: pwsh + run: conda list + + # Note: all three files are renamed during the download to trim the "subsample_" prefix + - name: Preparing the data + run: | + mkdir -p teaseq/preprocessing && cd teaseq/preprocessing + # curl -L -o TODO https://figshare.com/ndownloader/files/TODO + + - name: Preparing the data (artifact) + uses: actions/download-artifact@v3 + with: + name: teaseq_unfilt.h5mu + path: teaseq/preprocessing/teaseq_unfilt.h5mu + + # Note: we run the following to test that the commands works + # However, the following task will replacing the file anyway + - name: Preparing the configuration file + shell: bash -el {0} + run: | + cd teaseq/preprocessing + panpipes preprocessing config + + - name: Edit the submission file + run: | + cd teaseq/preprocessing + curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/efb55f083a8aeb1bc12f18ac61f6637b/pipeline.yml + + # - name: Replace template contents in configuration file + # run: | + # cd teaseq/preprocessing + # sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_env+pipeline_env+g' pipeline.yml + + - name: File tree + if: env.debug == 'true' + run: tree teaseq + + # - name: Review pipeline tasks + # shell: bash -el {0} + # run: | + # cd teaseq/preprocessing + # panpipes preprocessing show full --local + + # - name: Run pipeline tasks + # shell: bash -el {0} + # run: | + # cd teaseq/preprocessing + # panpipes preprocessing make full --local + + - name: File tree + if: env.debug == 'true' + run: tree teaseq diff --git a/pipeline_env.yaml b/pipeline_env.yaml index b0e11027..fcea02c4 100644 --- a/pipeline_env.yaml +++ b/pipeline_env.yaml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: - - python=3.9 + - python=3.11 - r-base=4.3.0 - r-clustree - r-ggforce diff --git a/pyproject.toml b/pyproject.toml index e539d40b..409ff4c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,43 +26,43 @@ classifiers = [ "Programming Language :: R" ] -requires-python = ">= 3.9" +requires-python = ">= 3.11" dependencies = [ "anndata>=0.8.0", - "apsw", - "bbknn", - "cgatcore", - "drmaa", + "apsw", + "bbknn", + "cgatcore", + "drmaa", "gevent", "harmonypy", - "jax<=0.4.23", - "jaxlib<=0.4.23", - "leidenalg", + "jax", + "jaxlib", + "leidenalg", "louvain", "matplotlib<=3.7.3", "mofapy2", "mudata>=0.2.1", "muon", - "numpy==1.22.4", + "numpy>=1.22.4", "openpyxl", - "packaging", - "pandas>=1.0", - "paramiko", + "packaging", + "pandas>=1.0", + "paramiko", "pep8", "pysam", "pynndescent", "pytest", "pyyaml", - "ruffus", - "scanorama", - "scanpy>=1.9.1", + "ruffus", + "scanorama", + "scanpy>=1.9.1", "scib", "seaborn<=0.12.2", - "scikit-misc", + "scikit-misc", "scirpy", - "scrublet", - "scvi-tools>=0.16.3", + "scrublet", + "scvi-tools>=1.1.1", "sqlalchemy"] [project.optional-dependencies] From ee5f3036b0817669fd6ba843046b1cbb0642fc64 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Thu, 22 Feb 2024 14:39:14 +0000 Subject: [PATCH 2/4] download input file for filtering CI from figshare --- .github/workflows/filtering-ci.yml | 38 +++++++++++++----------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/.github/workflows/filtering-ci.yml b/.github/workflows/filtering-ci.yml index b3236680..9dc06cad 100644 --- a/.github/workflows/filtering-ci.yml +++ b/.github/workflows/filtering-ci.yml @@ -57,13 +57,7 @@ jobs: - name: Preparing the data run: | mkdir -p teaseq/preprocessing && cd teaseq/preprocessing - # curl -L -o TODO https://figshare.com/ndownloader/files/TODO - - - name: Preparing the data (artifact) - uses: actions/download-artifact@v3 - with: - name: teaseq_unfilt.h5mu - path: teaseq/preprocessing/teaseq_unfilt.h5mu + # curl -L -o teaseq_unfilt.h5mu https://figshare.com/ndownloader/files/44639302 # Note: we run the following to test that the commands works # However, the following task will replacing the file anyway @@ -78,26 +72,26 @@ jobs: cd teaseq/preprocessing curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/efb55f083a8aeb1bc12f18ac61f6637b/pipeline.yml - # - name: Replace template contents in configuration file - # run: | - # cd teaseq/preprocessing - # sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_env+pipeline_env+g' pipeline.yml + - name: Replace template contents in configuration file + run: | + cd teaseq/preprocessing + sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_bbknn+pipeline_env+g' pipeline.yml - name: File tree if: env.debug == 'true' run: tree teaseq - # - name: Review pipeline tasks - # shell: bash -el {0} - # run: | - # cd teaseq/preprocessing - # panpipes preprocessing show full --local - - # - name: Run pipeline tasks - # shell: bash -el {0} - # run: | - # cd teaseq/preprocessing - # panpipes preprocessing make full --local + - name: Review pipeline tasks + shell: bash -el {0} + run: | + cd teaseq/preprocessing + panpipes preprocess show full --local + + - name: Run pipeline tasks + shell: bash -el {0} + run: | + cd teaseq/preprocessing + panpipes preprocess make full --local - name: File tree if: env.debug == 'true' From 6583dcb34390fd51580feda965a30b3058aa6119 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Thu, 22 Feb 2024 14:39:26 +0000 Subject: [PATCH 3/4] uncomment line --- .github/workflows/filtering-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/filtering-ci.yml b/.github/workflows/filtering-ci.yml index 9dc06cad..123d3e39 100644 --- a/.github/workflows/filtering-ci.yml +++ b/.github/workflows/filtering-ci.yml @@ -57,7 +57,7 @@ jobs: - name: Preparing the data run: | mkdir -p teaseq/preprocessing && cd teaseq/preprocessing - # curl -L -o teaseq_unfilt.h5mu https://figshare.com/ndownloader/files/44639302 + curl -L -o teaseq_unfilt.h5mu https://figshare.com/ndownloader/files/44639302 # Note: we run the following to test that the commands works # However, the following task will replacing the file anyway From d58295f99fade26ca23366e82fbd97c67ea62177 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Thu, 22 Feb 2024 14:48:21 +0000 Subject: [PATCH 4/4] fix config step --- .github/workflows/filtering-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/filtering-ci.yml b/.github/workflows/filtering-ci.yml index 123d3e39..f28ba5bf 100644 --- a/.github/workflows/filtering-ci.yml +++ b/.github/workflows/filtering-ci.yml @@ -65,7 +65,7 @@ jobs: shell: bash -el {0} run: | cd teaseq/preprocessing - panpipes preprocessing config + panpipes preprocess config - name: Edit the submission file run: |