From a1f85cf5c9c715042398787b4e80aaeb44bc0f27 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 11:30:15 +0000 Subject: [PATCH 01/23] install miniconda --- ingestion-ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 ingestion-ci.yml diff --git a/ingestion-ci.yml b/ingestion-ci.yml new file mode 100644 index 00000000..983cff7d --- /dev/null +++ b/ingestion-ci.yml @@ -0,0 +1,40 @@ +name: Ingestion CI + +on: [push] + +env: + debug: 'true' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] # , "macos-latest", "windows-latest" + python-version: ["3.9"] + + steps: + - uses: goanpeca/setup-miniconda@v1 + with: + activate-environment: pipeline_env + auto-update-conda: true + python-version: ${{ matrix.python-version }} + auto-activate-base: false + + - name: Conda info + if: env.debug == 'true' + run: conda info + + - name: Conda list + if: env.debug == 'true' + run: conda list + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + run: | + python -m pytest tests/ From d590a4982c8c4bdc4515ba3823ef7e6f93c8d3c8 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 11:35:53 +0000 Subject: [PATCH 02/23] specify shell bash -l --- ingestion-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ingestion-ci.yml b/ingestion-ci.yml index 983cff7d..e3fe43b8 100644 --- a/ingestion-ci.yml +++ b/ingestion-ci.yml @@ -24,10 +24,12 @@ jobs: - name: Conda info if: env.debug == 'true' + shell: bash -l {0} run: conda info - name: Conda list if: env.debug == 'true' + shell: bash -l {0} run: conda list - name: Install dependencies From 760ac81703d5954b01119a801aef82649b75b5d0 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 11:49:59 +0000 Subject: [PATCH 03/23] add more trigger events --- ingestion-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ingestion-ci.yml b/ingestion-ci.yml index e3fe43b8..c315446d 100644 --- a/ingestion-ci.yml +++ b/ingestion-ci.yml @@ -1,6 +1,12 @@ name: Ingestion CI -on: [push] +on: + push: + branches: + - main + pull_request: + branches: + - main env: debug: 'true' From 909f3478d66817c8f0e7ad68a33b53578e706552 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 11:53:02 +0000 Subject: [PATCH 04/23] move workflow file in the right place --- ingestion-ci.yml => .github/workflows/ingestion-ci.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ingestion-ci.yml => .github/workflows/ingestion-ci.yml (100%) diff --git a/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml similarity index 100% rename from ingestion-ci.yml rename to .github/workflows/ingestion-ci.yml From d0ef13bb6b84b4685455e5aa308d3d17c75fb16e Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 12:03:55 +0000 Subject: [PATCH 05/23] use conda-incubator/setup-miniconda@v3 --- .github/workflows/ingestion-ci.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index c315446d..0a055d00 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -21,21 +21,26 @@ jobs: python-version: ["3.9"] steps: - - uses: goanpeca/setup-miniconda@v1 + - uses: conda-incubator/setup-miniconda@v3 with: + auto-activate-base: true activate-environment: pipeline_env auto-update-conda: true python-version: ${{ matrix.python-version }} - auto-activate-base: false + channels: conda-forge + channel-priority: strict + + - run: | + conda install r-base=4.3.0 - name: Conda info if: env.debug == 'true' - shell: bash -l {0} + shell: bash -el {0} run: conda info - name: Conda list if: env.debug == 'true' - shell: bash -l {0} + shell: pwsh run: conda list - name: Install dependencies From ce999299e9552e27153ebcb083288e13d34d5559 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 12:36:37 +0000 Subject: [PATCH 06/23] install r-base in pipeline environment --- .github/workflows/ingestion-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 0a055d00..3221b539 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -31,7 +31,7 @@ jobs: channel-priority: strict - run: | - conda install r-base=4.3.0 + conda install -n pipeline_env r-base=4.3.0 - name: Conda info if: env.debug == 'true' From ac6e1596449690a445f73a9a6ef8b326c5ee1459 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 15:45:20 +0000 Subject: [PATCH 07/23] checkout repo --- .github/workflows/ingestion-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 3221b539..2f761224 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -21,6 +21,8 @@ jobs: python-version: ["3.9"] steps: + - uses: actions/checkout@v3 + - uses: conda-incubator/setup-miniconda@v3 with: auto-activate-base: true From 3fc24052eec5d761b2131e850fbe74c1a0534d8c Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 16:15:16 +0000 Subject: [PATCH 08/23] install dependencies via yaml file --- .github/workflows/ingestion-ci.yml | 9 ++++----- pipeline_env.yaml | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 pipeline_env.yaml diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 2f761224..aae7c165 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -21,19 +21,19 @@ jobs: python-version: ["3.9"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v3 with: + miniforge-version: latest auto-activate-base: true - activate-environment: pipeline_env auto-update-conda: true python-version: ${{ matrix.python-version }} channels: conda-forge channel-priority: strict - run: | - conda install -n pipeline_env r-base=4.3.0 + mamba install -n pipeline_env r-base=4.3.0 - name: Conda info if: env.debug == 'true' @@ -47,8 +47,7 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + mamba env create -f requirements.txt - name: Run tests run: | diff --git a/pipeline_env.yaml b/pipeline_env.yaml new file mode 100644 index 00000000..0b765992 --- /dev/null +++ b/pipeline_env.yaml @@ -0,0 +1,18 @@ +name: pipeline_env +channels: + - conda-forge + - defaults +dependencies: + - python=3.9 + - r-base=4.3.0 + - r-clustree + - r-ggforce + - r-ggraph + - r-hdf5r + - r-optparse + - r-tidyverse + - r-xtable + - pip + - pip: + - panpipes[spatial] + - pytest From cb2b22c3f659d38a5fdf418e468338d91dca8e80 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 16:20:46 +0000 Subject: [PATCH 09/23] install dependencies via setup-miniconda --- .github/workflows/ingestion-ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index aae7c165..bc74adea 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -31,9 +31,8 @@ jobs: python-version: ${{ matrix.python-version }} channels: conda-forge channel-priority: strict - - - run: | - mamba install -n pipeline_env r-base=4.3.0 + activate-environment: pipeline_env + environment-file: pipeline_env.yaml - name: Conda info if: env.debug == 'true' @@ -45,10 +44,6 @@ jobs: shell: pwsh run: conda list - - name: Install dependencies - run: | - mamba env create -f requirements.txt - - name: Run tests run: | python -m pytest tests/ From c9aa60124156084c71d251f9af6a2281dbdedf64 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 16:30:49 +0000 Subject: [PATCH 10/23] use bash -el for all conda commands --- .github/workflows/ingestion-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index bc74adea..ba2bb073 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -45,5 +45,6 @@ jobs: run: conda list - name: Run tests + shell: bash -el {0} run: | python -m pytest tests/ From 19529d553e7c14c9244c671e0373503126e6eef0 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Fri, 2 Feb 2024 16:32:52 +0000 Subject: [PATCH 11/23] remove duplicate python spec --- .github/workflows/ingestion-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index ba2bb073..b63e1104 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -28,11 +28,10 @@ jobs: miniforge-version: latest auto-activate-base: true auto-update-conda: true - python-version: ${{ matrix.python-version }} channels: conda-forge channel-priority: strict activate-environment: pipeline_env - environment-file: pipeline_env.yaml + environment-file: pipeline_env.yaml # consider moving to etc/pipeline_env.yaml - name: Conda info if: env.debug == 'true' From d3c34169943226f2f083322ca1c87578d1a322a7 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 08:29:04 +0000 Subject: [PATCH 12/23] disable tests and prepare data and configuration --- .github/workflows/ingestion-ci.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index b63e1104..1c9ede98 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -43,7 +43,17 @@ jobs: shell: pwsh run: conda list - - name: Run tests - shell: bash -el {0} + #- name: Run tests + # shell: bash -el {0} + # run: | + # python -m pytest tests/ + + - name: Preparing the data run: | - python -m pytest tests/ + mkdir -p teaseq/ingest/data.dir + curl -o teaseq/ingest/data.dir/subsample_adt.h5ad https://figshare.com/ndownloader/files/41671551 + curl -o teaseq/ingest/data.dir/subsample_atac.h5ad https://figshare.com/ndownloader/files/41671554 + curl -o teaseq/ingest/data.dir/subsample_rna.h5ad https://figshare.com/ndownloader/files/41671557 + + - name: Preparing the configuration file + run: panpipes ingest config From 57b1eff2ab0dda4bb2bea3c2328aeca35a18ed5c Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 08:35:33 +0000 Subject: [PATCH 13/23] bash el to activate environment --- .github/workflows/ingestion-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 1c9ede98..8ca193e7 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -56,4 +56,5 @@ jobs: curl -o teaseq/ingest/data.dir/subsample_rna.h5ad https://figshare.com/ndownloader/files/41671557 - name: Preparing the configuration file + shell: bash -el {0} run: panpipes ingest config From a6db6727d6f029a61f1fbe18e4cbff8b5a188bca Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 08:37:47 +0000 Subject: [PATCH 14/23] rename workflow and jobs --- .github/workflows/ingestion-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 8ca193e7..ce5c5a66 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -1,4 +1,4 @@ -name: Ingestion CI +name: Tutorials CI on: push: @@ -12,7 +12,7 @@ env: debug: 'true' jobs: - build: + ingestion: runs-on: ubuntu-latest strategy: fail-fast: false From b8f2271d81e3ad3113d6de495940e8c5213d9887 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 08:53:33 +0000 Subject: [PATCH 15/23] set up config file and submission file --- .github/workflows/ingestion-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index ce5c5a66..3ac31c31 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -55,6 +55,18 @@ jobs: curl -o teaseq/ingest/data.dir/subsample_atac.h5ad https://figshare.com/ndownloader/files/41671554 curl -o teaseq/ingest/data.dir/subsample_rna.h5ad https://figshare.com/ndownloader/files/41671557 + # Note: we run the following to test that the commands works + # However, the following task will replacing the file anyway - name: Preparing the configuration file shell: bash -el {0} run: panpipes ingest config + + - name: Edit the submission file + run: curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/9a4fcbc3c0a4fdc8e6578d086a88730f/pipeline.yml + + - name: Preparing the submission file + run: curl -o sample_file_qc.txt https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/40ed291bf9a7ff73c4f396ec63cff2f7/sample_file_qc.txt + + - name: File tree + if: env.debug == 'true' + run: tree From 094bd9182485000ea20bf63803cc526c9ecea803 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 09:02:20 +0000 Subject: [PATCH 16/23] do not checkout the repo, download YAML file only --- .github/workflows/ingestion-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 3ac31c31..98fb3ecd 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -21,7 +21,9 @@ jobs: python-version: ["3.9"] steps: - - uses: actions/checkout@v4 + # - uses: actions/checkout@v4 + - name: Download Conda YAML file + run: curl -o pipeline_env.yaml https://raw.githubusercontent.com/DendrouLab/panpipes/kra-gha/pipeline_env.yaml - uses: conda-incubator/setup-miniconda@v3 with: From b5c554941b5373a4982f1936a61aeb0274fc74d3 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 14:44:41 +0000 Subject: [PATCH 17/23] final setup and run --- .github/workflows/ingestion-ci.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 98fb3ecd..cfdbdbe1 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -67,7 +67,21 @@ jobs: run: curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/9a4fcbc3c0a4fdc8e6578d086a88730f/pipeline.yml - name: Preparing the submission file - run: curl -o sample_file_qc.txt https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/40ed291bf9a7ff73c4f396ec63cff2f7/sample_file_qc.txt + run: curl -o sample_file_qc.txt https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/40ed291bf9a7ff73c4f396ec63cff2f7/sample_file_qc.txt + + - name: Preparing the QC gene lists + run: curl -o qc_genelist_1.0.csv https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/fd38f25644105ea357a26e78a59139bb/qc_genelist_1.0.csv + + - name: Replace template contents in configuration file + run: sed 's+panpipes-tutorials/tutorials/ingesting_data/qc_genelist_1.0.csv+qc_genelist_1.0.csv+g' pipeline.yml + + - name: Review pipeline tasks + shell: bash -el {0} + run: panpipes ingest show full --local + + - name: Run pipeline tasks + shell: bash -el {0} + run: panpipes ingest make full --local - name: File tree if: env.debug == 'true' From 4c775fe3d90ab2d952000f76557afcc2b6f86c77 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 14:53:49 +0000 Subject: [PATCH 18/23] show file tree at the end of setup --- .github/workflows/ingestion-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index cfdbdbe1..59e43cf4 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -75,6 +75,10 @@ jobs: - name: Replace template contents in configuration file run: sed 's+panpipes-tutorials/tutorials/ingesting_data/qc_genelist_1.0.csv+qc_genelist_1.0.csv+g' pipeline.yml + - name: File tree + if: env.debug == 'true' + run: tree + - name: Review pipeline tasks shell: bash -el {0} run: panpipes ingest show full --local From 185be66513d75aae81067d3ab63e52c50b2b98be Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 14:58:14 +0000 Subject: [PATCH 19/23] fix filenames --- .github/workflows/ingestion-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 59e43cf4..8c0dd311 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -50,12 +50,13 @@ jobs: # run: | # python -m pytest tests/ + # Note: all three files are renamed during the download to trim the "subsample_" prefix - name: Preparing the data run: | mkdir -p teaseq/ingest/data.dir - curl -o teaseq/ingest/data.dir/subsample_adt.h5ad https://figshare.com/ndownloader/files/41671551 - curl -o teaseq/ingest/data.dir/subsample_atac.h5ad https://figshare.com/ndownloader/files/41671554 - curl -o teaseq/ingest/data.dir/subsample_rna.h5ad https://figshare.com/ndownloader/files/41671557 + curl -o teaseq/ingest/data.dir/adt.h5ad https://figshare.com/ndownloader/files/41671551 + curl -o teaseq/ingest/data.dir/atac.h5ad https://figshare.com/ndownloader/files/41671554 + curl -o teaseq/ingest/data.dir/rna.h5ad https://figshare.com/ndownloader/files/41671557 # Note: we run the following to test that the commands works # However, the following task will replacing the file anyway From e0f56dc780af98d11e389adc82007b391dffdb6c Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 15:06:46 +0000 Subject: [PATCH 20/23] cd into the expected place --- .github/workflows/ingestion-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 8c0dd311..b4d2301f 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -53,10 +53,11 @@ jobs: # Note: all three files are renamed during the download to trim the "subsample_" prefix - name: Preparing the data run: | - mkdir -p teaseq/ingest/data.dir - curl -o teaseq/ingest/data.dir/adt.h5ad https://figshare.com/ndownloader/files/41671551 - curl -o teaseq/ingest/data.dir/atac.h5ad https://figshare.com/ndownloader/files/41671554 - curl -o teaseq/ingest/data.dir/rna.h5ad https://figshare.com/ndownloader/files/41671557 + mkdir -p teaseq/ingest && cd teaseq/ingest + mkdir data.dir + curl -o data.dir/adt.h5ad https://figshare.com/ndownloader/files/41671551 + curl -o data.dir/atac.h5ad https://figshare.com/ndownloader/files/41671554 + curl -o data.dir/rna.h5ad https://figshare.com/ndownloader/files/41671557 # Note: we run the following to test that the commands works # However, the following task will replacing the file anyway From a46a82216f7583cd8fcafaf335e36c0c09369c23 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 15:15:55 +0000 Subject: [PATCH 21/23] cd in all relevant steps of workflow --- .github/workflows/ingestion-ci.yml | 38 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index b4d2301f..8e2c39f9 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -53,29 +53,39 @@ jobs: # Note: all three files are renamed during the download to trim the "subsample_" prefix - name: Preparing the data run: | - mkdir -p teaseq/ingest && cd teaseq/ingest - mkdir data.dir - curl -o data.dir/adt.h5ad https://figshare.com/ndownloader/files/41671551 - curl -o data.dir/atac.h5ad https://figshare.com/ndownloader/files/41671554 - curl -o data.dir/rna.h5ad https://figshare.com/ndownloader/files/41671557 + mkdir -p teaseq/ingest/data.dir + cd teaseq/ingest/data.dir + curl -o adt.h5ad https://figshare.com/ndownloader/files/41671551 + curl -o atac.h5ad https://figshare.com/ndownloader/files/41671554 + curl -o rna.h5ad https://figshare.com/ndownloader/files/41671557 # Note: we run the following to test that the commands works # However, the following task will replacing the file anyway - name: Preparing the configuration file shell: bash -el {0} - run: panpipes ingest config + run: | + cd teaseq/ingest/data.dir + panpipes ingest config - name: Edit the submission file - run: curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/9a4fcbc3c0a4fdc8e6578d086a88730f/pipeline.yml + run: | + cd teaseq/ingest + curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/9a4fcbc3c0a4fdc8e6578d086a88730f/pipeline.yml - name: Preparing the submission file - run: curl -o sample_file_qc.txt https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/40ed291bf9a7ff73c4f396ec63cff2f7/sample_file_qc.txt + run: | + cd teaseq/ingest + curl -o sample_file_qc.txt https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/40ed291bf9a7ff73c4f396ec63cff2f7/sample_file_qc.txt - name: Preparing the QC gene lists - run: curl -o qc_genelist_1.0.csv https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/fd38f25644105ea357a26e78a59139bb/qc_genelist_1.0.csv + run: | + cd teaseq/ingest + curl -o qc_genelist_1.0.csv https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/fd38f25644105ea357a26e78a59139bb/qc_genelist_1.0.csv - name: Replace template contents in configuration file - run: sed 's+panpipes-tutorials/tutorials/ingesting_data/qc_genelist_1.0.csv+qc_genelist_1.0.csv+g' pipeline.yml + run: | + cd teaseq/ingest + sed 's+panpipes-tutorials/tutorials/ingesting_data/qc_genelist_1.0.csv+qc_genelist_1.0.csv+g' pipeline.yml - name: File tree if: env.debug == 'true' @@ -83,11 +93,15 @@ jobs: - name: Review pipeline tasks shell: bash -el {0} - run: panpipes ingest show full --local + run: | + cd teaseq/ingest + panpipes ingest show full --local - name: Run pipeline tasks shell: bash -el {0} - run: panpipes ingest make full --local + run: | + cd teaseq/ingest + panpipes ingest make full --local - name: File tree if: env.debug == 'true' From d91ee0bd1e5389e2ee81a653f55a5437b3831f11 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 16:14:47 +0000 Subject: [PATCH 22/23] curl -L to follow redirects --- .github/workflows/ingestion-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 8e2c39f9..a4774847 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -55,9 +55,9 @@ jobs: run: | mkdir -p teaseq/ingest/data.dir cd teaseq/ingest/data.dir - curl -o adt.h5ad https://figshare.com/ndownloader/files/41671551 - curl -o atac.h5ad https://figshare.com/ndownloader/files/41671554 - curl -o rna.h5ad https://figshare.com/ndownloader/files/41671557 + curl -L -o adt.h5ad https://figshare.com/ndownloader/files/41671551 + curl -L -o atac.h5ad https://figshare.com/ndownloader/files/41671554 + curl -L -o rna.h5ad https://figshare.com/ndownloader/files/41671557 # Note: we run the following to test that the commands works # However, the following task will replacing the file anyway @@ -85,7 +85,7 @@ jobs: - name: Replace template contents in configuration file run: | cd teaseq/ingest - sed 's+panpipes-tutorials/tutorials/ingesting_data/qc_genelist_1.0.csv+qc_genelist_1.0.csv+g' pipeline.yml + sed -i 's+panpipes-tutorials/tutorials/ingesting_data/qc_genelist_1.0.csv+qc_genelist_1.0.csv+g' pipeline.yml - name: File tree if: env.debug == 'true' From 95f0f8c1ed0c0debe8dd22b42508efb03d15a259 Mon Sep 17 00:00:00 2001 From: Kevin Rue-Albrecht Date: Mon, 5 Feb 2024 17:21:50 +0000 Subject: [PATCH 23/23] fix path of pipeline config --- .github/workflows/ingestion-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index a4774847..891c6ff5 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -64,7 +64,7 @@ jobs: - name: Preparing the configuration file shell: bash -el {0} run: | - cd teaseq/ingest/data.dir + cd teaseq/ingest panpipes ingest config - name: Edit the submission file