Merge branch 'release/2.3.0'

Release dockerised cgpRna

.dockerignore

@@ -0,0 +1,10 @@
+/LICENCE
+/prerelease.sh
+/README.md
+/INSTALL
+/LICENCE
+/CHANGES.md
+/.gitignore
+/.git
+/perl/docs
+/perl/docs.tar.gz

.travis.yml

@@ -0,0 +1,50 @@
+notifications:
+  slack: wtsi-cgpit:ptUMR1tkNyZJYd9TpGoss8WR
+  email: false
+
+sudo: false
+
+services:
+  - docker
+
+script:
+ - set -e
+ - echo 'Build and check docker image'
+ - docker build -t cgprna .
+ - docker images | grep -c cgprna
+ - echo 'Verify program(s) from each inherited base image - dockstore-cgpbigwig'
+ - docker run -t --rm cgprna bwjoin --version
+ - echo 'Verify program(s) from each inherited base image - dockstore-cgpmap'
+ - docker run -t --rm cgprna bwa_mem.pl -version
+ - docker run -t --rm cgprna bammarkduplicates2 --version
+ - docker run -t --rm cgprna samtools --version
+ - docker run -t --rm cgprna bash -c 'bwa 2>&1 | grep Version'
+ - echo 'Verify program(s) from this repo'
+ ## mapping
+ - docker run -t --rm cgprna star_mapping.pl --version
+ ## RSeQC
+ - docker run -t --rm cgprna split_bam.py --version
+ - docker run -t --rm cgprna geneBody_coverage.py --version
+ - docker run -t --rm cgprna read_distribution.py --version
+ ## BAM QC
+ - docker run -t --rm cgprna process_qcstats.pl --version
+ ## count
+ - docker run -t --rm cgprna which htseq-count
+ ## fusion
+ - docker run -t --rm cgprna tophat_fusion.pl --version
+ - docker run -t --rm cgprna star_fusion.pl --version
+ - docker run -t --rm cgprna defuse_fusion.pl --version
+ - docker run -t --rm cgprna which compare_overlapping_fusions.pl
+ ## some dependencies used by more than one tool
+ - docker run -t --rm cgprna bedtools --version
+ - docker run -t --rm cgprna R --version
+ - docker run -t --rm cgprna which blat
+ ### tophat dependencies
+ - docker run -t --rm cgprna bowtie --version
+ - docker run -t --rm cgprna bowtie2 --version
+ ### tophat-fusion dependencies
+ - docker run -t --rm cgprna blastn -version
+ ### Defuse dependencies
+ - docker run -t --rm cgprna which faToTwoBit
+ - docker run -t --rm cgprna gmap --version
+

CHANGES.md

@@ -1,5 +1,28 @@
 # CHANGES
 
+## 2.3.0
+
+* dockerised cgpRna. Within the docker container, version of some dependent tools have been changed:
+
+  1. Python3(**3.7.X**) is used in the container, thus:
+     - RSeQC is updated from version ***2.6.4*** to ***3.0.0***.
+     - However, version of HTSeq is not changed.
+
+  1. Defuse is updated from ***v0.7.0*** to ***v0.8.2*** due to "Possible precedence issue with control flow operator" warning with the version of Perl installed in the container. The fix is [here](https://bitbucket.org/dranew/defuse/commits/b979855999b8106f5dc9f9e54f86935c7bf4f62f). However it is not merged in any ***v0.7.x*** versions (also because there's only ONE ***v0.7.x*** version --***v0.7.0***), hence ***v0.8.2***, which is the latest version at the time, was chosen. 
+
+  1. To utilise apt packages for the ease of their installations, the following tool version changes were made:
+     - BedTools: ***2.21.0*** to ***2.25.0-1*** in apt,
+     - bowtie1: ***1.1.1*** to ***1.1.2-3*** in apt,
+     - bowtie2: ***2.2.3*** to ***2.2.6-2*** in apt,
+     - blast: ***2.2.30*** to ***2.2.31-4*** in apt,
+     - gmap: ***2015-09-10*** to ***2015-12-31.v7-1*** in apt. 
+
+* added an extra option "-updateconfig" to *defuse_fusion.pl*. It takes a file as input, which content is used to update *defuse-config.txt*. It'll search *defuse.ini* to find its default value. This addition shouldn't break any existing usage.
+
+* *defuse_fusion.pl* will always create a *defuse-config.txt* file in its temp folder and use it to run *defuse.pl*. In the temp file, **dataset_directory** is corrected using related command line input values. If "-updateconfig" presents, values in the file will be used to overwrite corresponding values in the temp file. **Note:** if ***dataset_directory*** key presents in the "-updateconfig" file, this will be the final value in the temp config file.
+
+* tophat-fusion-post now skips read dist step, which our infuse pipeline does not care and the `tophat-fusion-post.py` does not insert generated read distributions in final html report anyway neither.
+
 ## 2.2.2
 
 * Change tabix query call to query_full

Dockerfile

@@ -0,0 +1,118 @@
+FROM  quay.io/wtsicgp/dockstore-cgpmap:3.1.4 as builder
+USER  root
+
+# Tool version ENVs, some of them are also used in the second build stage, make sure version are consistent between the two stages.
+## VAGrENG dependcies
+ENV VER_VCFTOOLS "0.1.16"
+ENV VER_Set_IntervalTree "0.12"
+ENV VER_BEDTOOLS "2.25.0-1"
+## CancerIT dependencies
+ENV VER_CGPVCF "v2.2.1"
+ENV VER_GRASS "v2.1.1"
+ENV VER_VAGRENT "v3.3.3"
+## cgpRna dependencies
+ENV VER_File_ShareDir_Install "0.13"
+ENV VER_Config_IniFiles "3.000002"
+ENV VER_STAR "2.5.0c"
+ENV VER_STARFUSION "v0.1.1"
+ENV VER_TOPHAT "2.1.0"
+ENV VER_DEFUSE "v0.8.2"
+ENV SOURCE_FATOTWOBIT "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit"
+ENV SOURCE_BLAT "http://users.soe.ucsc.edu/~kent/src/blatSrc35.zip"
+
+RUN apt-get -yq update
+RUN apt-get install -yq --no-install-recommends \
+locales \
+g++ \
+make \
+gcc \
+pkg-config \
+zlib1g-dev \
+software-properties-common \
+zip \
+unzip \
+libpng-dev \
+libboost-all-dev
+# libboost-all-dev is required to compile defuse. Its installation installs python2 as well, which is required for building Bedtools
+
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8
+
+ENV OPT /opt/wtsi-cgp
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
+ENV PERL5LIB $OPT/lib/perl5
+ENV LD_LIBRARY_PATH $OPT/lib
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+# build tools from other repos
+ADD build/opt-build.sh build/
+RUN bash build/opt-build.sh $OPT
+
+# build the tools in this repo, separate to reduce build time on errors
+COPY . .
+RUN bash build/opt-build-local.sh $OPT
+
+FROM ubuntu:16.04
+
+LABEL maintainer="[email protected]" \
+      uk.ac.sanger.cgp="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" \
+      version="2.3.0" \
+      description="cgpRna docker"
+
+RUN apt-get -yq update
+RUN apt-get install -yq --no-install-recommends \
+apt-transport-https \
+locales \
+curl \
+ca-certificates \
+libperlio-gzip-perl \
+bzip2 \
+psmisc \
+time \
+zlib1g \
+liblzma5 \
+libncurses5 \
+p11-kit \
+software-properties-common \
+unattended-upgrades && \
+unattended-upgrade -d -v && \
+apt-get remove -yq unattended-upgrades && \
+apt-get autoremove -yq
+
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8
+
+# dependecy tool versions, some of them are also used in the first stage, make sure they are consistent between stages.
+## VAGrENG dependcies
+ENV VER_VCFTOOLS "0.1.16"
+ENV VER_BEDTOOLS "2.25.0-1"
+## cgpRna dependencies
+ENV VER_BOWTIE1 "1.1.2-3"
+ENV VER_BOWTIE2 "2.2.6-2"
+ENV VER_BLAST "2.2.31-4"
+ENV VER_GMAP "2015-12-31.v7-1"
+ENV VER_RSEQC "3.0.0"
+ENV VER_HTSEQ "0.7.2"
+
+ENV OPT /opt/wtsi-cgp
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
+ENV PERL5LIB $OPT/lib/perl5
+ENV LD_LIBRARY_PATH $OPT/lib
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+RUN mkdir -p $OPT
+COPY --from=builder $OPT $OPT
+
+COPY build/opt-build-sys-dependencies.sh ./
+COPY build/config-defuse.sh ./
+RUN bash opt-build-sys-dependencies.sh && rm -f opt-build-sys-dependencies.sh && bash config-defuse.sh $OPT && rm -f config-defuse.sh
+
+## USER CONFIGURATION
+RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu
+
+USER    ubuntu
+WORKDIR /home/ubuntu
+
+CMD ["/bin/bash"]

build/config-defuse.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+
+set -xe
+
+if [ "$#" -lt "1" ] ; then
+  echo "Please provide an installation path such as /opt/ICGC"
+  exit 1
+fi
+
+INST_PATH=$1
+INST_PATH=$(realpath $INST_PATH)
+
+if [ ! -d $INST_PATH/bin/defuse_install ]; then
+  echo "Could not find bin/defuse_install in $INST_PATH: Wrong installation path?"
+  exit 1
+fi
+
+if [ ! -d $INST_PATH/config ]; then
+  echo "Could not find folder 'config' in $INST_PATH: Wrong installation path?"
+  exit 1
+fi
+
+echo -e \
+"source_directory = $INST_PATH/bin/defuse_install\n"\
+"samtools_bin = samtools\n"\
+"bowtie_bin = bowtie\n"\
+"bowtie_build_bin = bowtie-build\n"\
+"blat_bin = blat\n"\
+"fatotwobit_bin = faToTwoBit\n"\
+"r_bin = R\n"\
+"rscript_bin = Rscript\n"\
+"gmap_bin = gmap\n"\
+"gmap_build_bin = gmap_build"\
+> $INST_PATH/config/defuse_running_env_config.ini
+echo "updateconfig=$(realpath $INST_PATH)/config/defuse_running_env_config.ini" >> $INST_PATH/config/defuse.ini

build/opt-build-local.sh

@@ -0,0 +1,60 @@
+#! /bin/bash
+
+set -xe
+
+if [[ -z "${TMPDIR}" ]]; then
+  TMPDIR=/tmp
+fi
+
+set -u
+
+if [ "$#" -lt "1" ] ; then
+  echo "Please provide an installation path such as /opt/ICGC"
+  exit 1
+fi
+
+# get path to this script
+SCRIPT_PATH=`dirname $0`;
+SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)`
+
+# get the location to install to
+INST_PATH=$1
+mkdir -p $1
+INST_PATH=`(cd $1 && pwd)`
+echo $INST_PATH
+
+# get current directory
+INIT_DIR=`pwd`
+
+CPU=`grep -c ^processor /proc/cpuinfo`
+if [ $? -eq 0 ]; then
+  if [ "$CPU" -gt "6" ]; then
+    CPU=6
+  fi
+else
+  CPU=1
+fi
+echo "Max compilation CPUs set to $CPU"
+
+SETUP_DIR=$INIT_DIR/install_tmp
+mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end
+mkdir -p $INST_PATH/bin
+cd $SETUP_DIR
+
+# make sure tools installed can see the install loc of libraries
+set +u
+export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'`
+export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'`
+export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'`
+export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'`
+set -u
+
+# Install cgpRna
+cd $INIT_DIR/perl
+perl Makefile.PL INSTALL_BASE=$INST_PATH
+make
+make test
+make install
+sed -i -e "/defuseversion/c defuseversion=$VER_DEFUSE" $INIT_DIR/perl/config/defuse.ini
+mkdir -p $INST_PATH/config
+cp $INIT_DIR/perl/config/*.ini $INST_PATH/config/

build/opt-build-sys-dependencies.sh

@@ -0,0 +1,48 @@
+#! /bin/bash
+
+set -xe
+
+# Add python ppa
+add-apt-repository -y ppa:deadsnakes/ppa
+# Add R key
+echo "deb http://cran.rstudio.com/bin/linux/ubuntu xenial/" | tee -a /etc/apt/sources.list
+# gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
+# gpg -a --export E084DAB9 | apt-key add -
+
+# install python3, R, bedtools, bowtie1, bowtie2, blast, gmap and other packages for RSeQC to install
+apt-get update
+
+# using --allow-unauthenticated because the added R key is not recognised, it worked fine without the option before 27 Jul 2019
+apt-get install -yq --no-install-recommends --allow-unauthenticated \
+python3.7 python3.7-dev \
+r-base r-base-dev \
+bedtools=${VER_BEDTOOLS} \
+bowtie=${VER_BOWTIE1} \
+bowtie2=${VER_BOWTIE2} \
+ncbi-blast+=${VER_BLAST} \
+gmap=${VER_GMAP} \
+libcurl4-gnutls-dev zlib1g-dev
+apt-get upgrade -yq gcc
+
+# install R packages:
+Rscript -e 'install.packages("ada")'  # required by Defuse
+
+# Replace python3
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+
+# install pip3
+curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py
+rm -f get-pip.py
+
+# for bx-python (one of RSeQC package dependencies)
+apt-get install -yq --no-install-recommends liblzo2-dev
+
+# install RSeQC and HTSeq
+pip3 install RSeQC=="$VER_RSEQC"
+pip3 install HTSeq=="$VER_HTSEQ"
+
+# if use HTSeq to plot
+pip3 install matplotlib
+
+# cleanning
+apt-get autoremove -yq