Skip to content

refactor: adjust final report; add full region scanning in CSM #229

refactor: adjust final report; add full region scanning in CSM

refactor: adjust final report; add full region scanning in CSM #229

Workflow file for this run

###############################################################################
#
# GitHub Actions CI configuration for integration tests
#
# AUTHOR: Maciej_Bak
# AFFILIATION: University_of_Basel
# AFFILIATION: Swiss_Institute_of_Bioinformatics
# CONTACT: [email protected]
# CREATED: 14-12-2021
# LICENSE: Apache_2.0
#
###############################################################################
---
name: integration
on:
push:
branches:
- '*'
pull_request:
branches:
- main
jobs:
PQA-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/PREPROCESSING/execution/run.sh \
-c .test/integration/module_PQA/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/PREPROCESSING/Snakefile \
--configfile=.test/integration/module_PQA/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/PREPROCESSING/Snakefile \
--configfile=.test/integration/module_PQA/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
ASE-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/EXTRACT_AS_EXONS/execution/run.sh \
-c .test/integration/module_ASE/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/EXTRACT_AS_EXONS/Snakefile \
--configfile=.test/integration/module_ASE/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/EXTRACT_AS_EXONS/Snakefile \
--configfile=.test/integration/module_ASE/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
TPA-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/PREPARE_TANDEM_PAS/execution/run.sh \
-c .test/integration/module_TPA/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/PREPARE_TANDEM_PAS/Snakefile \
--configfile=.test/integration/module_TPA/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/PREPARE_TANDEM_PAS/Snakefile \
--configfile=.test/integration/module_TPA/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
QEI-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/QUANTIFICATION/execution/run.sh \
-c .test/integration/module_QEI/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/QUANTIFICATION/Snakefile \
--configfile=.test/integration/module_QEI/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/QUANTIFICATION/Snakefile \
--configfile=.test/integration/module_QEI/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
PAQ-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/PAQR/execution/run.sh \
-c .test/integration/module_PAQ/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/PAQR/Snakefile \
--configfile=.test/integration/module_PAQ/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/PAQR/Snakefile \
--configfile=.test/integration/module_PAQ/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
CSM-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
# Test 1: kmers
- name: "Test 1: Create snakemake DAG"
run: |
bash modules/CREATE_SITECOUNT_MATRICES/execution/run.sh \
-c .test/integration/module_CSM/config1_kmers.yml \
-e local \
-t conda \
-g dag dag.svg
#
- name: "Test 1: Initiate snakemake dry run"
run: |
snakemake \
--snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \
--configfile=.test/integration/module_CSM/config1_kmers.yml \
--printshellcmds \
--dryrun \
--verbose
#
- name: "Test 1: Build conda envs for all the rules"
run: |
snakemake \
--snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \
--configfile=.test/integration/module_CSM/config1_kmers.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
# Test 2: pwms
- name: "Test 2: Create snakemake DAG"
run: |
bash modules/CREATE_SITECOUNT_MATRICES/execution/run.sh \
-c .test/integration/module_CSM/config2_pwms.yml \
-e local \
-t conda \
-g dag dag.svg
#
- name: "Test 2: Initiate snakemake dry run"
run: |
snakemake \
--snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \
--configfile=.test/integration/module_CSM/config2_pwms.yml \
--printshellcmds \
--dryrun \
--verbose
#
- name: "Test 2: Build conda envs for all the rules"
run: |
snakemake \
--snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \
--configfile=.test/integration/module_CSM/config2_pwms.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
MAE-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/MAEI/execution/run.sh \
-c .test/integration/module_MAE/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/MAEI/Snakefile \
--configfile=.test/integration/module_MAE/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/MAEI/Snakefile \
--configfile=.test/integration/module_MAE/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
KPC-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/KAPAC/execution/run.sh \
-c .test/integration/module_KPC/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/KAPAC/Snakefile \
--configfile=.test/integration/module_KPC/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/KAPAC/Snakefile \
--configfile=.test/integration/module_KPC/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
RES-module:
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Create snakemake DAG
run: |
bash modules/REPORT_RESULTS/execution/run.sh \
-c .test/integration/module_RES/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=modules/REPORT_RESULTS/Snakefile \
--configfile=.test/integration/module_RES/config.yml \
--printshellcmds \
--dryrun \
--verbose
- name: Build conda envs for all the rules
run: |
snakemake \
--snakefile=modules/REPORT_RESULTS/Snakefile \
--configfile=.test/integration/module_RES/config.yml \
--use-conda \
--conda-create-envs-only \
--cores 8
MAPP-test1:
needs:
- PQA-module
- ASE-module
- TPA-module
- QEI-module
- PAQ-module
- CSM-module
- MAE-module
- KPC-module
- RES-module
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Download genomic resources
run: |
bash scripts/download-ENSEMBL-resources.sh \
--species hsa \
--output-directory resources_ENSEMBL_hsa
- name: Download the atlas of PolyA sites
run: |
bash scripts/download-polyA-atlas.sh \
--species hsa \
--output-directory ATLAS2_hsa
- name: Download and parse ATtRACT motifs
run: |
bash scripts/download-and-parse-ATtRACT-motifs.sh \
--output-directory ATtRACT_hsa
- name: Create the main configfile
run: |
python scripts/create-main-config-file.py \
--config-template .test/integration/MAPP/config_template_1.yml \
--pipeline-configfile configs/config.yml
- name: Create snakemake rulegraph
run: |
bash execution/run.sh \
-c configs/config.yml \
-e local \
-t conda \
-g rulegraph rulegraph.svg
- name: Create snakemake DAG
run: |
bash execution/run.sh \
-c configs/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=Snakefile \
--configfile=configs/config.yml \
--printshellcmds \
--dryrun \
--verbose
MAPP-test2:
needs:
- PQA-module
- ASE-module
- TPA-module
- QEI-module
- PAQ-module
- CSM-module
- MAE-module
- KPC-module
- RES-module
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout mapp repository
uses: actions/checkout@v2
- name: Setup miniconda & mapp main environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: false
miniforge-variant: Mambaforge
channels: conda-forge,defaults
channel-priority: true
activate-environment: mapp
environment-file: env/environment.yml
auto-activate-base: false
- name: Display all miniconda & env info
run: |
conda info -a
conda list
- name: Download genomic resources
run: |
bash scripts/download-ENSEMBL-resources.sh \
--species hsa \
--output-directory resources_ENSEMBL_hsa
- name: Download the atlas of PolyA sites
run: |
bash scripts/download-polyA-atlas.sh \
--species hsa \
--output-directory ATLAS2_hsa
- name: Download and parse ATtRACT motifs
run: |
bash scripts/download-and-parse-ATtRACT-motifs.sh \
--output-directory ATtRACT_hsa
# for this test it is required that we set the
# modification date of resources files as older than
# the pipeline files (expected situation):
- name: Adjust modification date for resources
run: |
touch -a -m -t 200001010000.00 resources_ENSEMBL_hsa/Homo_sapiens.GRCh38.102.gtf
touch -a -m -t 200001010000.00 resources_ENSEMBL_hsa/Homo_sapiens.GRCh38.dna.primary_assembly.fa
touch -a -m -t 200001010000.00 ATLAS2_hsa/reformatted_filtered_atlas.bed
# according to the notes in the pipeline configuration:
# comment-out ASE, TPA and CSM modules inclusion from the main Snakefile:
- name: Comment-out ASE, TPA, CSM modules
run: |
sed '52 {s/^/#/}' -i Snakefile
sed '53 {s/^/#/}' -i Snakefile
sed '56 {s/^/#/}' -i Snakefile
- name: Create the main configfile
run: |
python scripts/create-main-config-file.py \
--config-template .test/integration/MAPP/config_template_2.yml \
--pipeline-configfile configs/config.yml
- name: Create snakemake rulegraph
run: |
bash execution/run.sh \
-c configs/config.yml \
-e local \
-t conda \
-g rulegraph rulegraph.svg
- name: Create snakemake DAG
run: |
bash execution/run.sh \
-c configs/config.yml \
-e local \
-t conda \
-g dag dag.svg
- name: Initiate snakemake dry run
run: |
snakemake \
--snakefile=Snakefile \
--configfile=configs/config.yml \
--printshellcmds \
--dryrun \
--verbose
...