Merge pull request #60 from CCBR/ci-build-test

ci: create simple E2E test for github actions

.github/environment.yml

@@ -0,0 +1,33 @@
+name: mimseq-dev
+channels:
+  - bioconda
+  - conda-forge
+dependencies: # from https://github.com/bioconda/bioconda-recipes/blob/e4f33b5c94119efef67e6da9955f458c29ab40d9/recipes/mimseq/meta.yaml
+  - r-base>=4.1
+  - biopython>=1.79
+  - matplotlib-base>=3.4.2
+  - numpy>=1.21.1
+  - pandas>=1.3.1
+  - requests>=2.26.0
+  - pybedtools>=0.8.2
+  - pyfiglet>=0.8.post1
+  - pysam>=0.16.0.1
+  - seaborn-base>=0.11.1
+  - statsmodels>=0.13.1
+  - infernal>=1.1.4
+  - blast>=2.10.1
+  - gmap>=2018.03.20,<=2019.02.26
+  - samtools>=1.11
+  - bedtools>=2.30.0
+  - r-ggplot2>=3.3.5
+  - r-ggpol>=0.0.7
+  - bioconductor-complexheatmap>=2.2.0
+  - bioconductor-deseq2>=1.26.0
+  - r-pheatmap>=1.0.12
+  - r-calibrate>=1.7.7
+  - r-gridextra>=2.3
+  - r-plyr>=1.8.6
+  - r-dplyr>=1.0.6
+  - r-tidyverse>=1.3.0
+  - r-devtools>=2.4.1
+  - r-reshape2>=1.4.4

.github/workflows/build.yml

@@ -0,0 +1,62 @@
+name: build
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  pull_request:
+    branches:
+      - main
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.7"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install most mimseq deps with conda
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: .github/environment.yml
+          cache-environment: false
+          cache-downloads: true
+          generate-run-shell: true
+          create-args: |
+            python=${{ matrix.python-version }}
+      - name: Install usearch
+        run: |
+          wget https://drive5.com/downloads/usearch10.0.240_i86linux32.gz
+          gunzip usearch10.0.240_i86linux32.gz
+          chmod +x usearch10.0.240_i86linux32
+          mv usearch10.0.240_i86linux32 /usr/local/bin/usearch
+        shell: micromamba-shell {0}
+      - name: Install mimseq
+        run: |
+          python -m pip install --upgrade pip setuptools
+          pip install .
+        shell: micromamba-shell {0}
+      - name: Test
+        run: |
+          pytest
+        shell: micromamba-shell {0}
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: mimseq-output
+          path: "test_*"
+
+  build-status: # for branch protection rule
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: always()
+    steps:
+      - name: Successful build
+        if: ${{ !(contains(needs.*.result, 'failure')) }}
+        run: exit 0
+      - name: Failing build
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1

setup.py

@@ -45,6 +45,7 @@ def package_files(directory):
 		"numpy",
 		"pyfiglet",
 		"pysam",
+        "pytest",
 		"seaborn",
 		"matplotlib",
 		"pybedtools",

tests/data/sampleData_subset_HEKvsK562.txt

@@ -0,0 +1,4 @@
+./tests/data/mimseq_hek_1.subset.fastq.gz	HEK293T
+./tests/data/mimseq_hek_2.subset.fastq.gz	HEK293T
+./tests/data/mimseq_k562_1.subset.fastq.gz	K562
+./tests/data/mimseq_k562_2.subset.fastq.gz	K562

tests/md5sums.yml

@@ -0,0 +1,34 @@
+test_cli:
+  - path: test_cli/counts/Anticodon_counts_raw.txt
+    md5sum: 3306c6c112a12e75c16d50748cff46e7
+  - path: test_cli/counts/Isodecoder_counts_raw.txt
+    md5sum: 6aedc13a33b837d5854649e705c098d6
+  - path: test_cli/CCAanalysis/CCAcounts.csv
+    md5sum: 99193120e5bb0f419b23b24ad9de56db
+  - path: test_cli/mods/allModsTable.csv
+    md5sum: 2fa08bd87e6587447faedd5110ffa75b
+  - path: test_cli/mods/mismatchTable.csv
+    md5sum: a0989a9e4bef3bd35c613f2319dc6801
+  - path: test_cli/mods/predictedMods.csv
+    md5sum: e559c490984fd0e17f8571c27ca5d443
+  - path: test_cli/DESeq2/cyto/anticodon/K562vsHEK293T_diffexpr-results.csv
+    md5sum: 61d2c83ae9722afb55582c18f0ff5563
+  - path: test_cli/DESeq2/cyto/isodecoder/K562vsHEK293T_diffexpr-results.csv
+    md5sum: b0fb1adea54c713dbff9a765f0a4d1d4
+test_cli_local_modomics:
+  - path: test_cli_local_modomics/counts/Anticodon_counts_raw.txt
+    md5sum: 8d6137502ea55c69d7e48bae402f6784
+  - path: test_cli_local_modomics/counts/Isodecoder_counts_raw.txt
+    md5sum: cc8ed25dccd0b062c73053dc8345ba24
+  - path: test_cli_local_modomics/CCAanalysis/CCAcounts.csv
+    md5sum: cc82e37e9815547ab8d63482b82488c1
+  - path: test_cli_local_modomics/mods/allModsTable.csv
+    md5sum: 23337325c55702d6098cb90ebce60d06
+  - path: test_cli_local_modomics/mods/mismatchTable.csv
+    md5sum: 8770ba48b83b6bb18a38351d224052f0
+  - path: test_cli_local_modomics/mods/predictedMods.csv
+    md5sum: c0ddaf02682ed2c0ec08e518740d0d02
+  - path: test_cli_local_modomics/DESeq2/cyto/anticodon/K562vsHEK293T_diffexpr-results.csv
+    md5sum: f6144b1ad7c075c3c18e999b8c3775ac
+  - path: test_cli_local_modomics/DESeq2/cyto/isodecoder/K562vsHEK293T_diffexpr-results.csv
+    md5sum: 1714c8e12a0fe9a1647d76d678d6e925

tests/subset.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import Bio.SeqIO
+import gzip
+import os
+
+def create_subset(in_fastq_path, out_fastq_path, n_seqs = 1000):
+    with gzip.open(in_fastq_path, 'rt') as file_handle:
+        records = list()
+        for seq_num, seq_record in enumerate(Bio.SeqIO.parse(file_handle, 'fastq')):
+            if seq_num >= n_seqs:
+                break
+            records.append(seq_record)
+    with gzip.open(out_fastq_path, 'wt') as file_handle:
+        Bio.SeqIO.write(records, file_handle, 'fastq')
+
+def main():
+    for fastq_filename in "mimseq_hek_1.fastq.gz  mimseq_hek_2.fastq.gz  mimseq_k562_1.fastq.gz  mimseq_k562_2.fastq.gz".split():
+        subset_filename = os.path.join('tests', 'data', fastq_filename.replace(".fastq.gz", ".subset.fastq.gz"))
+        create_subset(fastq_filename, subset_filename)
+
+if __name__ == "__main__":
+    main()

tests/test_cli.py

@@ -0,0 +1,52 @@
+import hashlib
+import os
+import subprocess
+import yaml
+
+def read_yaml(filepath = 'tests/md5sums.yml'):
+    with open(filepath, 'r') as infile:
+        yaml_dict = yaml.load(infile.read(), Loader=yaml.Loader)
+    return yaml_dict
+
+def get_md5(filepath, encoding = 'utf-8'):
+    with open(filepath, 'r') as file:
+        md5sum = hashlib.md5(file.read().encode(encoding)).hexdigest()
+    return md5sum
+
+def check_snapshot_md5sums(files):
+    all_equal = True
+    for snapshot in files:
+        observed_md5 = get_md5(snapshot['path'])
+        expected_md5 = snapshot['md5sum']
+        if observed_md5 != expected_md5:
+            all_equal = False
+            print("md5sum changed for", snapshot['path'], '\n\texpected:', expected_md5, "observed:", observed_md5)
+    return all_equal
+
+def run_cli(command):
+    out = subprocess.run(
+        command,
+        shell=True,
+        text=True,
+        capture_output=True,
+    )
+    if out.returncode:
+        print(out.stdout)
+        print(out.stderr)
+    return out
+
+def test_cli():
+    mimseq_out = run_cli(
+        "mimseq --species Hsap --cluster-id 0.97 --threads 2 --min-cov 0.0005 --max-mismatches 0.075 --control-condition HEK293T -n hg38_test --out-dir test_cli --max-multi 4 --remap --remap-mismatches 0.05 tests/data/sampleData_subset_HEKvsK562.txt"
+    )
+    is_success = mimseq_out.returncode == 0
+    snapshot_files = read_yaml('tests/md5sums.yml')['test_cli']
+    assert is_success and check_snapshot_md5sums(snapshot_files)
+
+# def test_cli_local_modomics():
+#     mimseq_out = run_cli(
+#         "mimseq --species Hsap --cluster-id 0.97 --threads 2 --min-cov 0.0005 --max-mismatches 0.075 --control-condition HEK293T -n hg38_test --out-dir test_cli_local_modomics --max-multi 4 --remap --remap-mismatches 0.05 --local-modomics tests/data/sampleData_subset_HEKvsK562.txt"
+#     )
+#     is_success = mimseq_out.returncode == 0
+#     snapshot_files = read_yaml('tests/md5sums.yml')['test_cli_local_modomics']
+#     assert is_success and check_snapshot_md5sums(snapshot_files)