Merge branch 'release/2.4.0'

Release of version 2.4.0

.dockerignore

@@ -8,3 +8,6 @@
 /.git
 /perl/docs
 /perl/docs.tar.gz
+/.travis.yml
+/cwls
+/examples

.gitignore

@@ -9,3 +9,16 @@
 /perl/docs/
 /perl/perltidy.LOG
 /env
+# python code related
+/run-cgprna/env/
+__pycache__/
+/run-cgprna/*.py[cod]
+/run-cgprna/*$py.class
+/run-cgprna/.coverage
+/run-cgprna/.cache
+/run-cgprna/htmlcov/
+/run-cgprna/.eggs
+/run-cgprna/build
+/run-cgprna/dist
+/run-cgprna/run_cgprna.egg-info
+.vscode

CHANGES.md

@@ -1,5 +1,21 @@
 # CHANGES
 
+## 2.4.0
+
+* Revised Dockerfile so that the builder stage is properly used and the image size is reduced. Resolved #28.
+* Added a CLI, so that user can complete a step in RNA-seq data workflow with just one command, which also eases the development of CWL files.
+   * CLI is written in Python
+   * currently 4 subcommands are implemented:
+      * `map`: uses star_mapping.pl to map and marks duplicates after mapping.
+      * `stats`: generates mapping stats using bam_stats and RSeQC.
+      * `bigwig`: generates bigwig file using bamToBw.pl
+      * `counts`: counts reads using htseq-count.
+* Built a new set of reference files for CLI to use. They're available on ftp://ftp.sanger.ac.uk/pub/cancer/support-files/cgpRna_container/.
+* Added CWL files:
+   * added a workflow to map sample by lanes, generate mapping stats for lanes, merge lane bams, generate bigwig file and count reads.
+   * added CWL tools/workflows for the workflow above to use.
+   * added example JSON for most of the CWL files.
+
 ## 2.3.4
 
 * RG tags are converted to shell safe strings before passing to Star. partially resolve #30.

Dockerfile

@@ -1,16 +1,26 @@
 FROM  quay.io/wtsicgp/dockstore-cgpmap:3.1.4 as builder
 USER  root
 
-# Tool version ENVs, some of them are also used in the second build stage, make sure version are consistent between the two stages.
+# Version of tools that are installed in both stages, make sure they are consistent.
+## VAGrENG dependcies
+ENV VER_BEDTOOLS "2.25.0-1"
+## cgpRna dependencies
+ENV VER_BOWTIE1 "1.1.2-3"
+ENV VER_BOWTIE2 "2.2.6-2"
+ENV VER_BLAST "2.2.31-4"
+ENV VER_GMAP "2015-12-31.v7-1"
+
+# Version of other tools
 ## VAGrENG dependcies
 ENV VER_VCFTOOLS "0.1.16"
 ENV VER_Set_IntervalTree "0.12"
-ENV VER_BEDTOOLS "2.25.0-1"
 ## CancerIT dependencies
 ENV VER_CGPVCF "v2.2.1"
 ENV VER_GRASS "v2.1.1"
 ENV VER_VAGRENT "v3.3.3"
 ## cgpRna dependencies
+ENV VER_RSEQC "3.0.0"
+ENV VER_HTSEQ "0.7.2"
 ENV VER_File_ShareDir_Install "0.13"
 ENV VER_Config_IniFiles "3.000002"
 ENV VER_STAR "2.5.0c"
@@ -20,6 +30,13 @@ ENV VER_DEFUSE "v0.8.2"
 ENV SOURCE_FATOTWOBIT "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit"
 ENV SOURCE_BLAT "https://hgwdev.gi.ucsc.edu/~kent/src/blatSrc35.zip"
 
+RUN apt-get -yq update
+
+RUN apt-get install -qy --no-install-recommends lsb-release
+
+RUN echo "deb http://cran.rstudio.com/bin/linux/ubuntu `lsb_release -cs`/" >> /etc/apt/sources.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
+
 RUN apt-get -yq update
 RUN apt-get install -yq --no-install-recommends \
 locales \
@@ -28,19 +45,34 @@ make \
 gcc \
 pkg-config \
 zlib1g-dev \
-software-properties-common \
 zip \
 unzip \
 libpng-dev \
-libboost-all-dev
+python3 python3-dev python3-pip python3-setuptools \
+r-base r-base-dev \
+libcurl4-gnutls-dev zlib1g-dev \
+bedtools=${VER_BEDTOOLS} \
+bowtie=${VER_BOWTIE1} \
+bowtie2=${VER_BOWTIE2} \
+ncbi-blast+=${VER_BLAST} \
+gmap=${VER_GMAP}
+
 # libboost-all-dev is required to compile defuse. Its installation installs python2 as well, which is required for building Bedtools
+RUN apt-get install -yq --no-install-recommends libboost-all-dev
+# for building matplotlib
+RUN apt-get install -yq --no-install-recommends libfreetype6-dev
+
+RUN apt-get install -yq --no-install-recommends liblzo2-dev
 
 RUN locale-gen en_US.UTF-8
 RUN update-locale LANG=en_US.UTF-8
 
 ENV OPT /opt/wtsi-cgp
 ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
 ENV PERL5LIB $OPT/lib/perl5
+ENV R_LIBS $OPT/R-lib
+ENV R_LIBS_USER $R_LIBS
+ENV PYTHONPATH $OPT/python-lib/lib/python3.5/site-packages
 ENV LD_LIBRARY_PATH $OPT/lib
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
@@ -52,14 +84,30 @@ RUN bash build/opt-build.sh $OPT
 # build the tools in this repo, separate to reduce build time on errors
 COPY . .
 RUN bash build/opt-build-local.sh $OPT
+RUN bash build/config-defuse.sh $OPT
 
 FROM ubuntu:16.04
 
 LABEL maintainer="[email protected]" \
       uk.ac.sanger.cgp="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" \
-      version="2.3.4" \
+      version="2.4.0" \
       description="cgpRna docker"
 
+# Version of tools that are installed in both stages, make sure they are consistent.
+## VAGrENG dependcies
+ENV VER_BEDTOOLS "2.25.0-1"
+## cgpRna dependencies
+ENV VER_BOWTIE1 "1.1.2-3"
+ENV VER_BOWTIE2 "2.2.6-2"
+ENV VER_BLAST "2.2.31-4"
+ENV VER_GMAP "2015-12-31.v7-1"
+
+RUN apt-get -yq update
+RUN apt-get install -qy --no-install-recommends lsb-release
+
+RUN echo "deb http://cran.rstudio.com/bin/linux/ubuntu `lsb_release -cs`/" >> /etc/apt/sources.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
+
 RUN apt-get -yq update
 RUN apt-get install -yq --no-install-recommends \
 apt-transport-https \
@@ -74,40 +122,36 @@ zlib1g \
 liblzma5 \
 libncurses5 \
 p11-kit \
-software-properties-common \
-unattended-upgrades && \
+unattended-upgrades \
+python3 \
+r-base \
+bedtools=${VER_BEDTOOLS} \
+bowtie=${VER_BOWTIE1} \
+bowtie2=${VER_BOWTIE2} \
+ncbi-blast+=${VER_BLAST} \
+gmap=${VER_GMAP} && \
 unattended-upgrade -d -v && \
 apt-get remove -yq unattended-upgrades && \
 apt-get autoremove -yq
 
 RUN locale-gen en_US.UTF-8
 RUN update-locale LANG=en_US.UTF-8
 
-# dependecy tool versions, some of them are also used in the first stage, make sure they are consistent between stages.
-## VAGrENG dependcies
-ENV VER_VCFTOOLS "0.1.16"
-ENV VER_BEDTOOLS "2.25.0-1"
-## cgpRna dependencies
-ENV VER_BOWTIE1 "1.1.2-3"
-ENV VER_BOWTIE2 "2.2.6-2"
-ENV VER_BLAST "2.2.31-4"
-ENV VER_GMAP "2015-12-31.v7-1"
-ENV VER_RSEQC "3.0.0"
-ENV VER_HTSEQ "0.7.2"
-
 ENV OPT /opt/wtsi-cgp
-ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$OPT/python-lib/bin/:$PATH
 ENV PERL5LIB $OPT/lib/perl5
+ENV R_LIBS $OPT/R-lib
+ENV R_LIBS_USER $R_LIBS
+ENV PYTHONPATH $OPT/python-lib/lib/python3.5/site-packages
 ENV LD_LIBRARY_PATH $OPT/lib
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 
 RUN mkdir -p $OPT
 COPY --from=builder $OPT $OPT
 
-COPY build/opt-build-sys-dependencies.sh ./
-COPY build/config-defuse.sh ./
-RUN bash opt-build-sys-dependencies.sh && rm -f opt-build-sys-dependencies.sh && bash config-defuse.sh $OPT && rm -f config-defuse.sh
+COPY run-cgprna $OPT/cgprna
+RUN cd $OPT/cgprna && python3 setup.py develop
 
 ## USER CONFIGURATION
 RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu

build/opt-build.sh

@@ -39,6 +39,8 @@ echo "Max compilation CPUs set to $CPU"
 SETUP_DIR=$INIT_DIR/install_tmp
 mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end
 mkdir -p $INST_PATH/bin
+mkdir -p $INST_PATH/R-lib
+mkdir -p $INST_PATH/python-lib
 cd $SETUP_DIR
 
 # make sure tools installed can see the install loc of libraries
@@ -49,6 +51,21 @@ export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/
 export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'`
 set -u
 
+
+# install R packages
+Rscript -e "install.packages(\"ada\", \"$INST_PATH/R-lib\")"  # required by Defuse
+
+# install python packages
+OPT_BK=$OPT # Somehow OPT affects compilation of numpy
+unset OPT
+pip3 install --install-option="--prefix=$INST_PATH/python-lib" --ignore-installed numpy  # for HTSeq installation.
+# matplotlib is required by HTSeq for plotting. Later version of matplotlib requires python3.6 or above.
+pip3 install --install-option="--prefix=$INST_PATH/python-lib" --ignore-installed \
+  RSeQC=="$VER_RSEQC" \
+  HTSeq=="$VER_HTSEQ" \
+  matplotlib==3.0
+OPT=$OPT_BK
+
 ## vcftools
 if [ ! -e $SETUP_DIR/vcftools.success ]; then
   curl -sSL --retry 10 https://github.com/vcftools/vcftools/releases/download/v${VER_VCFTOOLS}/vcftools-${VER_VCFTOOLS}.tar.gz > distro.tar.gz
@@ -63,9 +80,6 @@ if [ ! -e $SETUP_DIR/vcftools.success ]; then
   touch $SETUP_DIR/vcftools.success
 fi
 
-# install bedtools so that VAGrENT can be installed properly
-apt-get install -yq --no-install-recommends bedtools=${VER_BEDTOOLS}
-
 ## add File::ShareDir::Install for VAGrENT
 if [ ! -e $SETUP_DIR/File_ShareDir_Install.success ]; then
   cpanm -l $INST_PATH --mirror http://cpan.metacpan.org File::ShareDir::Install@$VER_File_ShareDir_Install