Merge remote-tracking branch 'wtsi/devel' into log-level2

CHANGES

@@ -1,5 +1,10 @@
 LIST OF CHANGES for npg_ranger project
 
+  - remove directly logging to file due to winston bug
+  - add flags to change logging level
+  - all logging output is now sent to stderr
+
+release 1.1.0
   - cwd before executing pipelines and cleaning after pipeline finishes
   - added more filters for /sample url
   - switch to [email protected] with curl support and [email protected]
@@ -12,9 +17,7 @@ LIST OF CHANGES for npg_ranger project
   - do not allow query strings containing multiple attribute-value pairs
     for the same attribute
   - upgrade Freebayes to 1.1.0 from 1.0.2-npg-Aug2016 in TravisCI and docs
-  - remove directly logging to file due to winston bug
-  - add flags to change logging level
-  - all logging output is now sent to stderr
+  - include docker configuration to deploy with containers
 
 release 1.0.0
   - show error in log when controller tests fail to cleanup

docker/README.md

@@ -0,0 +1,154 @@
+# Docker-compose containers
+This directory contains a collection of Dockerfiles, scripts, and a
+docker-compose.yml which will allow you to quickly set up a working ranger
+server.
+
+This configuration is locked at the 1.0.0 release in the ranger container. It
+includes proof of concept Dockerfiles and docker-compose.yml to bring up a
+mongodb container, an npg_ranger container and an apache container to act as a
+reverse proxy.
+
+## Preparing the instance
+Instance preparation may be covered by an existing image. Otherwise these steps
+should prepare the instance for the rest of the deployment.
+
+```
+export DEBIAN_FRONTEND=noninteractive
+sudo apt-get update
+
+sudo apt-get install htop s3cmd
+
+# To install docker packages signed with the key
+sudo apt-get -y install apt-transport-https ca-certificates
+sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+
+# cat /etc/issue to find ubuntu version for the correct ubuntu version in this case "ubuntu-xenial"
+echo "deb https://apt.dockerproject.org/repo ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
+sudo apt-get update
+sudo apt-get -y install linux-image-extra-$(uname -r) linux-image-extra-virtual
+sudo apt-get -y install docker-engine
+sudo apt-get -y upgrade
+
+# add the user (ubuntu) to the docker group so sudo is not required for every docker command
+sudo usermod -a -G docker $USER
+
+sudo service docker start
+
+# get docker compose binary
+sudo curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+```
+
+## References volume
+
+The deployment instructions assume there will be a **reference** root path to
+pass to the containers. If an existing volume with references is available then
+it may just need to be mounted.
+
+```
+# Find volume with references and attach to instance.
+# Then mount in the instance:
+sudo mkdir /references
+sudo chmod a+w /references
+# run lsblk to find references volume name then mount with something like:
+sudo mount /dev/xvdf /references
+```
+
+You may want to save these changes to the fstab file so they don't need to be
+repeated after stopping/starting the instance.
+
+**Warning:** Be specially careful when executing this instruction as it will
+prevent starting the instance in the future if wrong configuration is used.
+
+```
+# this line matches previous configuration, change if needed
+sudo echo -e "/dev/xvdf\t/references\text4\tdefaults\t0 2" | sudo tee --append /etc/fstab
+```
+
+## Configuration for s3cmd
+If data files are sourced from S3 (as in the sample data provided), s3cmd
+configuration needs to be set.
+
+```
+# interactive configuration to save s3cmd configuration at home
+s3cmd --configure
+```
+
+## Get a clone of the project
+
+```
+# use specific branch if needed
+git clone -b devel https://github.com/wtsi-npg/npg_ranger.git npg_ranger && pushd npg_ranger/docker
+```
+
+### Configuration for s3cmd
+Place the s3cmd configuration file generated in previous steps somewhere it can
+be used by docker. During container building phase this file will be passed to
+the container.
+
+```
+cp ~/.s3cfg ranger/s3cfg
+```
+
+## Working with temporary DNS hostname/IP
+If the instance gets a new IP/DNS hostname every time the it is started (e.g.
+AWS), set environment variables for containers to be aware if the current
+IP/hostname.
+
+```
+# Every time the intance is restarted and/or DNS/IP configuration changes
+
+git checkout ranger/config.json # Get original configuration file
+
+export PUBLIC_DNS=$(curl -s http://169.254.169.254/latest/meta-data/public-hostname) # To get public DNS hostname
+export PUBLIC_IP=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) # To get public IP
+
+sed -i "s/<%PUBLIC_DNS%>/$PUBLIC_DNS/g" ranger/config.json # Set dns hostname in configuration file
+sed -i "s/<%PUBLIC_IP%>/$PUBLIC_IP/g" ranger/config.json # Set public ip in configuration file
+```
+
+# Use docker-compose
+Use docker-compose to build and bring up containers.
+```
+docker-compose up -d --build
+```
+
+# Test data
+A sample data file is provided. It can be used to test the service but needs to
+be loaded.
+
+```
+# copy sample data to data mounting point
+cp ./rangerdb/fileinfo.json /data/mongo/fileinfo.json
+
+# Load some data (run only once! multiple times will create duplicate rows in database)
+docker exec docker_rangerdb_1 mongoimport --db imetacache --collection fileinfo --jsonArray --file /data/db/fileinfo.json --upsert --upsertFields data_object,filepath_by_host
+# This syntax will change in mongo 3.4 to something like --mode upsert --upsertFields data_object,filepath_by_host
+```
+
+# Stop and start apache server
+
+```
+# Every time the containers are brought down/up
+docker exec docker_rangerproxy_1 cp /appconf/docs/apache/httpd.conf /usr/local/apache2/httpd.conf
+docker exec docker_rangerproxy_1 perl docker_conf.pl
+docker exec docker_rangerproxy_1 mv /usr/local/apache2/httpd.conf /usr/local/apache2/conf/httpd.conf
+docker exec docker_rangerproxy_1 apachectl restart
+```
+
+# Test sample data
+
+```
+echo "03555f613ce1c9cba69a862137f13b76  temp.bam" >> test_data.md5
+echo "70b8f5f160e27210169ff50b013a75bb  temp.cram" >> test_data.md5
+echo "f03053c6a581f18f1eb41a259e31a9b0  temp.sam" >> test_data.md5
+
+curl "http://localhost:9090/npg_ranger/file?name=20818_1%23888.bam" -o temp.bam
+curl "http://localhost:9090/npg_ranger/sample?accession=NA12878&format=cram&region=chr22:16100000-16105000" -o temp.cram
+curl "http://localhost:9090/npg_ranger/sample?accession=NA12878&format=sam&region=chr22:16100000-16105000" -o temp.sam
+
+md5sum -c test_data.md5 && rm temp.bam temp.cram temp.sam test_data.md5
+```
+
+These dockerfiles are a proof of concept only; no security has been enabled on
+them, and are obviously not production-ready.

docker/docker-compose.yml

@@ -0,0 +1,37 @@
+version: "2"
+
+services:
+  rangerdb:
+    image: mongo:3.2.10
+    volumes:
+      - ../../../../data/mongo:/data/db
+      - ../:/appconf
+
+  ranger:
+    build: ./ranger
+    environment:
+      - REF_CACHE=/references/cram_cache/%2s/%2s/%s
+    volumes:
+      - ./ranger:/data/ranger
+      - ../../../../references:/references
+      - ../test/server/data:/seqdata
+    command: npg_ranger_server -sdp5050 -mmongodb://rangerdb/imetacache -r / -c /data/ranger/config.json
+    entrypoint: /wait-for-it.sh rangerdb:27017 --
+    links:
+      - rangerdb
+    depends_on:
+      - rangerdb
+    volumes_from:
+      - rangerdb
+
+  rangerproxy:
+    build: ./rangerproxy
+    volumes:
+      - ../../../../references:/references:ro
+    ports:
+      - "9090:9090"
+    links:
+      - ranger
+    volumes_from:
+      - rangerdb
+

docker/ranger/Dockerfile

@@ -0,0 +1,90 @@
+FROM ubuntu:14.04
+MAINTAINER Andrew Nowak <[email protected]>
+
+# Hold versions in environment variables.
+ENV NODE_VERSION=4.5.0 \
+    MONGODB_VERSION=3.2.4 \
+    HTSLIB_VERSION=1.3.2 \
+    SAMTOOLS_VERSION=1.3.1-npg-Sep2016 \
+    BIOBAMBAM_VERSION=2.0.50-release-20160705161609 \
+    FREEBAYES_VERSION=1.1.0 \
+    NPG_RANGER_VERSION=1.0.0 \
+    APT_DEPENDENCIES="wget build-essential git autoconf pkg-config cmake"
+
+RUN apt-get update -qq \
+  && apt-get install -qq zlib1g-dev libcurl4-openssl-dev libssl-dev libbz2-dev liblzma-dev s3cmd curl
+
+RUN apt-get update -qq \
+  && apt-get install -qq $APT_DEPENDENCIES \
+
+# Get Node.js
+  && wget -q "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" \
+  && tar xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 \
+  && rm "node-v$NODE_VERSION-linux-x64.tar.xz" \
+
+# Get Mongodb - only needed for running grunt tests
+# Use a dedicated mongodb container to host db in production
+  && wget -q "http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB_VERSION}.tgz" \
+  && tar xzf "mongodb-linux-x86_64-${MONGODB_VERSION}.tgz" -C /usr/local --strip-components=1 \
+  && rm "mongodb-linux-x86_64-${MONGODB_VERSION}.tgz" \
+
+# Get HTSlib
+  && git clone --branch $HTSLIB_VERSION --depth 1 "https://github.com/samtools/htslib.git" htslib \
+  && cd ./htslib \
+  && autoreconf -fi \
+  && git reset --hard \
+  && ./configure --enable-libcurl \
+  && make && make install \
+  && cd .. \
+
+# Get SAMtools
+  && git clone --branch $SAMTOOLS_VERSION --depth 1 "https://github.com/wtsi-npg/samtools.git" samtools \
+  && cd ./samtools \
+  && mkdir -p acinclude.m4 \
+  && wget -q -o /dev/null -O acinclude.m4/ax_with_htslib.m4 "https://github.com/samtools/samtools/files/62424/ax_with_htslib.m4.txt" \
+  && wget -q -o /dev/null -O acinclude.m4/ax_with_curses.m4 "http://git.savannah.gnu.org/gitweb/?p=autoconf-archive.git;a=blob_plain;f=m4/ax_with_curses.m4;hb=0351b066631215b4fdc3c672a8ef90b233687655" \
+  && aclocal -I acinclude.m4 && autoreconf -fi \
+  && git reset --hard \
+  && LIBS='-lcurl -lcrypto -lssl' ./configure --with-htslib=/htslib --without-curses \
+  && make && make install \
+  && cd .. \
+  && rm -rf samtools \
+
+# Get Biobambam
+  && wget -q "https://github.com/gt1/biobambam2/releases/download/${BIOBAMBAM_VERSION}/biobambam2-${BIOBAMBAM_VERSION}-x86_64-etch-linux-gnu.tar.gz" -O biobambam2.tar.gz \
+  && tar xzf biobambam2.tar.gz -C /usr/local --strip-components=1 \
+  && rm -r biobambam2.tar.gz \
+
+# Get Freebayes
+  && git clone --branch v$FREEBAYES_VERSION --depth 1 "https://github.com/ekg/freebayes.git" freebayes \
+  && cd ./freebayes \
+  && git submodule update --init --recursive \
+  && make \
+  && make install \
+  && cd .. \
+
+# Get Ranger
+  && npm install -g npg_ranger@${NPG_RANGER_VERSION}
+
+WORKDIR /npg_ranger
+
+RUN adduser --home /home/apprunner --disabled-password --gecos "" apprunner
+COPY ./config.json /npg_ranger/config.json
+COPY ./s3cfg /home/apprunner/.s3cfg
+RUN chown -R apprunner:apprunner /npg_ranger /home/apprunner
+
+# Remove build-time dependencies
+RUN apt-get remove -qq $APT_DEPENDENCIES \
+#  && apt-get autoremove --purge -qq \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+ADD https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh /wait-for-it.sh
+
+RUN chmod 0555 /wait-for-it.sh
+EXPOSE 5050
+
+USER apprunner
+
+CMD [ "/bin/bash" ]
+

docker/ranger/config.json

@@ -0,0 +1,10 @@
+{
+  "mongourl": "mongodb://localhost:27017/imetacache",
+  "protocol": "http:",
+  "originlist": ["http://ngs.sanger.ac.uk"],
+  "proxylist":  {
+    "http://localhost:9090": "http://localhost:9090/npg_ranger",
+    "http://<%PUBLIC_DNS%>:9090": "http://<%PUBLIC_DNS%>:9090/npg_ranger",
+    "http://<%PUBLIC_IP%>:9090": "http://<%PUBLIC_IP%>:9090/npg_ranger"
+  }
+}

docker/rangerdb/fileinfo.json

@@ -0,0 +1,11 @@
+[
+
+{"filepath_by_host" : { "*" : "/seqdata/20818_1#888.bam" }, "avh" : { "reference" : "PhiX/all/fasta/phix_unsnipped_short_no_N.fa", "sample_accession_number" : "ABC123456", "target" : "1", "manual_qc" : "1", "alignment" : "1" }, "collection" : "/seq/20818", "data_object" : "20818_1#888.bam" },
+
+{"filepath_by_host" : { "*" : "/seqdata/20907_1#888.bam" }, "avh" : { "reference" : "PhiX/all/fasta/phix_unsnipped_short_no_N.fa", "sample_accession_number" : "ABC123456", "target" : "1", "manual_qc" : "1", "alignment" : "1" }, "collection" : "/seq/20907", "data_object" : "20907_1#888.bam" },
+
+{"collection" : "CH", "data_object" : "NA12878.bam", "avh" : {"target" : "1","alignment" : "1", "reference" : "references/Homo_sapiens/GRCh38_full_analysis_set_plus_decoy_hla/all/fasta/Homo_sapiens.GRCh38_full_analysis_set_plus_decoy_hla.fa", "sample_accession_number" : "NA12878", "manual_qc": "1", "type": "cram"}, "filepath_by_host" : { "*" : "s3://ga4gh/grch38/NA12878.cram"}},
+{"collection" : "CH", "data_object" : "NA12891.bam", "avh" : {"target" : "1","alignment" : "1", "reference" : "references/Homo_sapiens/GRCh38_full_analysis_set_plus_decoy_hla/all/fasta/Homo_sapiens.GRCh38_full_analysis_set_plus_decoy_hla.fa", "sample_accession_number" : "NA12891", "manual_qc": "1", "type": "cram"}, "filepath_by_host" : { "*" : "s3://ga4gh/grch38/NA12891.cram"}},
+{"collection" : "CH", "data_object" : "NA12892.bam", "avh" : {"target" : "1","alignment" : "1", "reference" : "references/Homo_sapiens/GRCh38_full_analysis_set_plus_decoy_hla/all/fasta/Homo_sapiens.GRCh38_full_analysis_set_plus_decoy_hla.fa", "sample_accession_number" : "NA12892", "manual_qc": "1", "type": "cram"}, "filepath_by_host" : { "*" : "s3://ga4gh/grch38/NA12892.cram"}}
+
+]

docker/rangerdb/volume/.gitignore

@@ -0,0 +1,5 @@
+#http://stackoverflow.com/questions/115983/how-can-i-add-an-empty-directory-to-a-git-repository
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

docker/rangerproxy/Dockerfile

@@ -0,0 +1,32 @@
+FROM httpd:2.4.23
+MAINTAINER Andrew Nowak <[email protected]>
+
+ENV NODE_VERSION=4.5.0 \
+    NPG_RANGER_VERSION=1.0.0 \
+    APT_DEPENDENCIES="wget build-essential git autoconf pkg-config cmake"
+
+RUN apt-get update -qq \
+  && apt-get install -qq $APT_DEPENDENCIES \
+
+# Get Node.js
+  && wget -q "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" \
+  && tar xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 \
+  && rm "node-v$NODE_VERSION-linux-x64.tar.xz" \
+  && npm update -g npm
+
+COPY ./docker_conf.pl /usr/local/apache2/docker_conf.pl
+
+RUN git clone --branch npg_ranger_master --depth 1 https://github.com/wtsi-npg/dalliance.git dalliance \
+  && cd dalliance \
+  && npm install \
+  && ./node_modules/.bin/gulp \
+  && ln -s node_modules/npg_ranger/lib/uiclient/index.html . \
+  && cd ../htdocs \
+  && ln -s ../dalliance .
+
+RUN apt-get remove -qq $APT_DEPENDENCIES \
+#  && apt-get autoremove --purge -qq \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+EXPOSE 9090

docker/rangerproxy/docker_conf.pl

@@ -0,0 +1,26 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+local @ARGV = ('httpd.conf');
+local $^I = '.bak';
+while (<>) {
+    s{CUSTOM_PORT[\s]+80$}{CUSTOM_PORT  9090};
+    s{^LoadModule auth_openidc}{\#LoadModule auth_openidc};
+    s{^\#User(.*)$}{User www-data};
+    s{^\#Group(.*)$}{Group www-data};
+
+    s{^\#ServerName www.example.com:80}{ServerName localhost};
+
+    s{^LoadModule ssl_module}{\#LoadModule ssl_module};
+    s{^LoadModule proxy_connect_module}{\#LoadModule proxy_connect_module};
+
+    s{^Define REF_REPOSITORY(.*)$}{Define REF_REPOSITORY "/references"};
+    s{^Define RSOCKET_PATH(.*)$}{Define NPG_DOCKER 1};
+    next if (m{^<(.*?)Location(.*)}
+          or m{^( *)<(.*?)Limit(.*)}
+          or m{^( *)AuthType openid-connect}
+          or m{^( *)Require valid-user});
+    print;
+}