diff --git a/.travis.yml b/.travis.yml index 01967bc..85b13bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,29 +5,49 @@ services: install: scripts/travis-setup.sh cache: directories: - - "/home/travis/.conda/pkgs" + - "/opt/anaconda3/pkgs" jobs: include: - - stage: Linux+macOS + - stage: Linux if: "(type = push) AND ((branch IN (master, bulk)) OR (fork = true))" os: linux - script: scripts/travis-run.sh + script: travis_wait 30 scripts/travis-run-conda-build-all.sh - stage: Linux if: type = pull_request os: linux - script: scripts/travis-run.sh + script: travis_wait 30 scripts/travis-run-conda-build-all.sh env: global: - - SUBDAGS=1 - - BIOCONDA_UTILS_TAG=master - - BIOCONDA_UTILS_BUILD_ARGS="--loglevel=info" - - BIOCONDA_UTILS_LINT_ARGS= - - MINICONDA_VER="4.3.21" - - ANACONDA_USER='nyuad-cgsb' - - GITHUB_USER='nyuad-cgsb' - - MY_TRAVIS_REPO_SLUG='biosails/biostacks' - - CONTAINER_NAMESPACE='nyuad_cgsb' - - CONDA_PKGS_DIRS='/home/travis/.conda/pkgs' + - MINICONDA_VER: '4.3.21' + - ANACONDA_USER: 'nyuad-cgsb' + - GITHUB_USER: 'nyuad-cgsb' + - MY_TRAVIS_REPO_SLUG: 'biosails/biostacks' + - CONTAINER_NAMESPACE: 'nyuad_cgsb' + - CONDA_PKGS_DIRS: '/opt/anaconda3/pkgs' + - CONDA_BLD_PATH: '/opt/anaconda3/conda-bld' + - CONDA_R: '3.4.1' + - CONDA_PERL: '5.22.0' + - CONDA_HTSLIB: "1.6" + - CONDA_BOOST: "1.64" + - CONDA_NPY: "112" + - CONDA_NCURSES: "5.9" + - CONDA_GSL: "1.16" + - CONDA_GMP: "5.1" + - CONDA_HDF5: "1.8.17" + - CONDA_ZLIB: "1.2.8" + - CONDA_BZIP2: "1.0" + - CONDA_XZ: "5.2" + # - SUBDAGS=1 + # - BIOCONDA_UTILS_TAG=master + # - BIOCONDA_UTILS_BUILD_ARGS="--loglevel=debug" + # - BIOCONDA_UTILS_LINT_ARGS= + # - MINICONDA_VER="4.3.21" + # - ANACONDA_USER='nyuad-cgsb' + # - GITHUB_USER='nyuad-cgsb' + # - MY_TRAVIS_REPO_SLUG='biosails/biostacks' + # - CONTAINER_NAMESPACE='nyuad_cgsb' + # - CONDA_PKGS_DIRS='/opt/anaconda3/pkgs' + # - CONDA_BLD_PATH='/opt/anaconda3/conda-bld' - secure: JHMiExNBJUTCd/cSXb7cY6DK4MmxIPO3IBfQAD6fSNOEXJe2KY9b8qAjxPwzKzD2FzQOZ1dln1rJiF/3HeVUEu3QN96PfhuqSeIaICP+m8q5ZHgQIyvHwDc09ne/yip1l6BIDtXCuNw/tYUZh3tgpw+bTve3cGXFiqYopvqMlZ2NcW/thQbpWCsdTnJq1g+Sq6DO3oRZm3WGdfS2sh3mv6TErXhQJi8nqWoNyOl2S6JGHRNOHp5vA11D24cAyxY+f5ezhp6Z3QxWL2mnnEwnazEC1PgcE6nA443R2fQRnF5q+ze42g+lSE3FTelWgwY2Jq5hIZRcov2Wj9H+i5zFDmnf503YZ2YLQ52WXnkPtuYCCOi5Sl7/qALLnjyLYNGoBCTZaONeToNSBs/058teuDZCDSLoLFCLlKnpPf1/FnkpEE8cegedHjvCeo2ffehCCrXiAoelUeAOscq9f7GSGa0UrybQckAhIQpR/J1C+FrOaOoFUXhtpxFFwxT6l36Vnt8i0dW0BKve6WU+DeG84JC5NnoULVK40Rg/VgGaB0qq1rX7+TWPw2kGWGWuAUDy+d6jAa+q41VzXjErmgeh3ftCtqS4TLS8tGGuhXkFj3wn9Z7fHIC0r82OqAQNyzj07hNk0MM/btGlU//bquyTFq4gwT2+jS3t1UqENFvAOA0= - secure: BX03MYZvVxOOLCXezmYGytgnpAHuaNh8kwVq2bYShUAAPNECZVPtNiXF+TQSuTfnXMM+9fm9M6dXyvCvbc5S8I5BcWEjqJj0fWvxmayejElySxEnNdH6FCuV0vWIwTzDyTUjiZcio/NaUgKlbhKfCqo85pxwOErHxuJlUILDmzwFQa84UzpFaAK2fEn5yja2GJbWfOmHiormHUz+2itchNPVQKXjhw9uZqnDflrVjhNzmF4+zUCLwiM9V5NRKobVeovRNTNg6sQ8ur5aL97tkR8FQz2Y92uYLfUtly8tzJX3/ytQ6F6kSSlat3rJcLjw5x1+J/kE1jjXvQ0Wj1Uy7qCowTMZ6rsOY3WpUiEas0YsIPYfn382ZnhqKW6OspPIFw+3tYVOv2GZtJPaDjfXCYvC5UC2qIpc/Po25nPA8Z6JgTSw9q0Y1gsn6EA/Auf8RD8SUXgwrxqWe/rT7rrpbOAhlDvM+fsCf666yl4sTe+AeJ36PepJ1H5pMZ82oz+K5Gh6wjj74t4hta/u/bbkd6HK0DIFZ+uHoHLq3sEQc9hq/4ljvy6CCq1S6w3sLd4/PNtHhbYlKywLckID255lkkla+mZegMzVE1VjsKE+tQhbvWkitlhJwbIowIcTmXFBxBZJzyp/8qcVwOhUy8ghSACQ6MWXov0QKW0BXnvqNmQ= - secure: jwpTmy+EHGsvVBo4e2auBKrY17niCy7lTfmiNPSpWwVIBALKH8TehRKk3IBnGV0SeWCPgd5JUV6UMBIwJQFM/4vMF9BfJ1o9Z0Rsrw+w6OgV7A4kIDK4aDxohHbCouGEk47LDfGt/tBcD97jfWaHxB7Uj2wgwakVEk9hraBFhajf0IcmZ8FkAqZXDhfc0ePtZZlHhRcRSjsOV2nZOVsiLRPcdLEnR/7+5YgswUNwxnUKDqimt3d8tDYWwnOzbuh9wsiQZt90v8b/ELIsARjx5wDsm9ksIaDi7Wg3BdUZSVX2JZf8lIgH5u1TiLPM1kdCsEila0Qdi+O2oGm7lYv0Y3XiXOjqpJKmWdpaFFlU7cN7CagSBxUzgziEYlHGKoDWjwnl6e20JQunkkVBJBl04PedDOUR2bAy0BumhNfXH0OmZokPJ8VZ226C2NjIWRRWKLyBxYp+k7OSouGdNpr30h22w/rwMY+4BnsHpI80M+LFbtWEhOOXbQd6/0witz/gNpPuG72I96fZnnz0KTRFaUdp6fsC4w5ITMYgmuaFzazhCyU7isP8k3ghdJTNGeY6S9HnRyb3k1SsRZsCP0qB12NEb9dVVc+/Ip0gtju4SurH11bj6fhbaKl0LZogGQHqCnnoBNrHvGmutIHZDPFJC4JysbAtfRsC77p1c7jucGA= diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..880e846 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +from continuumio/miniconda3 + +ARG CONDA_PKG=local +ENV CONDA_PKG ${CONDA_PKG} + +RUN echo ${CONDA_PKG} +RUN which conda +RUN conda info +RUN mkdir -p /opt/conda/conda-bld/linux-64 + +## We have to add the pkgs and the build-system +COPY pkgs/* /opt/conda/pkgs/ +COPY conda-bld/linux-64/${CONDA_PKG} /opt/conda/conda-bld/linux-64 + +RUN conda config --set always_yes yes --set changeps1 no +RUN conda config --add channels r +RUN conda config --add channels conda-forge +RUN conda config --add channels bioconda +RUN conda config --add channels nyuad-cgsb + +RUN ls -lah /opt/conda/conda-bld/linux-64 +RUN conda config --add channels file://opt/conda/conda-bld/linux-64 +RUN conda index /opt/conda/conda-bld/linux-64 +RUN conda install -y --use-local gencore_rnaseq diff --git a/configs/bioconda_utils-requirements.txt b/configs/bioconda_utils-requirements.txt new file mode 100644 index 0000000..4feb63d --- /dev/null +++ b/configs/bioconda_utils-requirements.txt @@ -0,0 +1,23 @@ +anaconda-client=1.6.3 +argh=0.26.2 +beautifulsoup4=4.6.0 +conda=4.3.21 +conda-build=2.1.16 +galaxy-lib=17.9.7 +jinja2=2.9.6 +jsonschema=2.5.1 +networkx=1.11 +pyaml=15.8.2 +pydotplus=2.0.2 +python=3.5.2 +requests=2.12.4 +sphinx +docutils +sphinx_rtd_theme +involucro=1.1.2 +pandas=0.19.2 +numpy=1.12.1 +pygithub=1.29 +colorlog=2.10.* +six=1.10.0 +alabaster=0.7.9 diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml new file mode 100644 index 0000000..743c9db --- /dev/null +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -0,0 +1,54 @@ +{% set name = "gencore_rnaseq" %} +{% set version = "1.0" %} + +package: + name: {{ name }} + version: {{ version }} + +build: + number: 1 + skip: True # [osx] + +requirements: + run: + ##Alignment + - samtools ==1.3.1 + - kallisto ==0.43.0 + # - cufflinks-patch ==2.2.1 + # - bowtie2 ==2.2.8 + # - tophat ==2.1.0 + # - star ==2.5.2a + # - bcftools ==1.3.1 + # - blat ==35 + # - gatk ==3.6 + # - picard ==2.5.0 + # - prinseq ==0.20.4 + # - vcftools ==0.1.14 + # - bedtools ==2.25.0 + # - vcflib ==1.0.0_rc1 + # - bamtools ==2.4.0 + # - seqtk ==1.2 + ##Downstream processing + - r-base + # - r-essentials + # - r-cairo + # - bioconductor-deseq2 + # - bioconductor-edger >=3.14.0 + # - bioconductor-cummerbund >=2.16.0 + # - bioconductor-genomicranges >=1.26.1 + # - bioconductor-gviz >=1.16.5 + # - bioconductor-biovizbase >=1.18.0 + # - bioconductor-s4vectors >=0.12.0 + # - bioconductor-ballgown >=2.2.0 + # - htseq ==0.6.1.post1 + # - hisat2 ==2.0.4 + # - stringtie ==1.3.0 + +test: + commands: + - bowtie2 --help 2>&1 | grep "bowtie" + +about: + home: 'https://cgsb.abudhabi.nyu.edu/' + license: BSD + summary: 'Gencore RnaSeq Module' diff --git a/scripts/build_docs.py b/scripts/build_docs.py new file mode 100644 index 0000000..0db8ef2 --- /dev/null +++ b/scripts/build_docs.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +from conda_build.config import Config +from conda_build.metadata import MetaData +from conda_build.render import render_recipe + +recipe_dir = 'recipes/gencore_rnaseq/1.0/meta.yaml' + +config = Config(croot=recipe_dir, anaconda_upload=False, verbose=True, +activate=False, debug=False, variant=None) +metadata = render_recipe(recipe_dir, config=config) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh new file mode 100755 index 0000000..b077c0b --- /dev/null +++ b/scripts/travis-run-conda-build-all.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -euo pipefail +# Set some defaults +set +u +[[ -z $DOCKER_ARG ]] && DOCKER_ARG="" +[[ -z $TRAVIS ]] && TRAVIS="false" +[[ -z $BIOCONDA_UTILS_LINT_ARGS ]] && BIOCONDA_UTILS_LINT_ARGS="" +[[ -z $RANGE_ARG ]] && RANGE_ARG="--git-range master HEAD" +[[ -z $DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK ]] && DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK="false" +[[ -z $SKIP_LINTING ]] && SKIP_LINTING=false +[[ -z $UPLOAD_ARG ]] && UPLOAD_ARG="" +set -u + +if [[ $TRAVIS_BRANCH != "master" && $TRAVIS_BRANCH != "bulk" && $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] +then + echo "" + echo "Tests are skipped for pushes to the main bioconda-recipes repo." + echo "If you have opened a pull request, please see the full tests for that PR." + echo "See https://bioconda.github.io/build-system.html for details" + echo "" + exit 0 +fi + + +# TODO Add another upload +if [[ ( $TRAVIS_BRANCH == "master" || $TRAVIS_BRANCH == "bulk" ) && "$TRAVIS_PULL_REQUEST" == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] +then + UPLOAD_ARG="--upload-channels nyuad-cgsb " +fi + + +if [[ $DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK == "true" ]] +then + echo + echo "DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK is true." + echo "A comprehensive check will be performed to see what needs to be built." + RANGE_ARG="" +fi + +set -x + +export PATH=/home/travis/anaconda3/bin:$PATH +conda install -y conda-build-all +conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG +ls -lahR $CONDA_BLD_PATH + +echo "We should be building things now...." + +cp Dockerfile /opt/anaconda3 +cd /opt/anaconda3 +docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' -t 'gencore_rnaseq:1.0-r3.4.1_1' . + +docker images +set +x diff --git a/scripts/travis-run.sh b/scripts/travis-run.sh index 11e818c..cb297e4 100755 --- a/scripts/travis-run.sh +++ b/scripts/travis-run.sh @@ -8,7 +8,6 @@ set +u [[ -z $RANGE_ARG ]] && RANGE_ARG="--git-range master HEAD" [[ -z $DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK ]] && DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK="false" [[ -z $SKIP_LINTING ]] && SKIP_LINTING=false -# [[ -z $CONTAINER_NAMESPACE ]] && CONTAINER_NAMESPACE="nyuad_cgsb" set -u if [[ $TRAVIS_BRANCH != "master" && $TRAVIS_BRANCH != "bulk" && $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] @@ -74,7 +73,8 @@ export PATH=/anaconda/bin:$PATH # for local testing. if [[ $TRAVIS_OS_NAME == "linux" && $TRAVIS == "true" ]] then - DOCKER_ARG="--docker --mulled-test" + DOCKER_ARG="--mulled-test" + #DOCKER_ARG=" --mulled-test" fi # When building master or bulk, upload packages to anaconda and quay.io. @@ -111,6 +111,6 @@ fi set -x pip install --upgrade git+https://github.com/jerowe/bioconda-utils.git@develop -bioconda-utils build recipes config.yml $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" +bioconda-utils build recipes config.yml $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" docker images set +x diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 77b7d03..5be81e0 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -24,12 +24,20 @@ done SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) pip install pyyaml -sudo mkdir /anaconda -sudo chown -R $USER /anaconda -$SCRIPT_DIR/../simulate-travis.py --bootstrap /anaconda --overwrite -/anaconda/bin/conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 -/anaconda/bin/conda config --add channels file://anaconda/conda-bld -/anaconda/bin/conda install -y r-base r-essentials openjdk perl bioconductor-biobase - -/anaconda/bin/conda info -ls -lah /home/travis/.conda/pkgs + +sudo chown -R $USER /opt/anaconda3 + +wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh --quiet -O miniconda.sh +bash miniconda.sh -b -p /home/travis/anaconda3 +export PATH="/home/travis/anaconda3/bin:$PATH" +hash -r +conda config --set always_yes yes --set changeps1 no +conda config --add channels r +conda config --add channels conda-forge +conda config --add channels bioconda +conda config --add channels nyuad-cgsb + +# conda index /home/travis/anaconda3/conda-bld/linux-64 +# conda config --add channels file://home/travis/anaconda3/conda-bld +conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs +npm install -g marked-man