diff --git a/.ci/bears.dirs.prune.sh b/.ci/bears.dirs.prune.sh new file mode 100755 index 0000000000..dfd406b377 --- /dev/null +++ b/.ci/bears.dirs.prune.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# Delete empty bear directories +set -e + +dirs=$(find bears -depth -type d -and -not -name '__pycache__') + +for dir in $dirs; do + bears=$(ls $dir/*Bear.py 2>/dev/null || true) + subdirs=$(ls -d $dir/*/ 2>/dev/null || true) + if [[ -z "$bears""$subdirs" ]]; then + echo Removing $dir + rm -rf $dir + fi +done diff --git a/.ci/bears.prune.sh b/.ci/bears.prune.sh new file mode 100755 index 0000000000..34af336367 --- /dev/null +++ b/.ci/bears.prune.sh @@ -0,0 +1,164 @@ +#!/bin/sh + +# Delete bears +set -e -x + +if [[ "$BEARS" == "all" ]]; then + echo "No bears pruned." + exit 0 +fi + +if [[ -z "$BEARS" ]]; then + if [[ -n "$TRAVIS_LANGUAGE" ]]; then + BEARS=$TRAVIS_LANGUAGE + fi +fi + +if [[ "$BEARS" == "haskell" || "$BEARS" == "ghc" ]]; then + BEARS=cabal +elif [[ "$BEARS" == "ruby" ]]; then + BEARS=gem +elif [[ "${BEARS/node/}" != "$BEARS" ]]; then + BEARS=npm +elif [[ "$BEARS" == "r" ]]; then + BEARS=rscript +elif [[ "$BEARS" == "generic" ]]; then + BEARS=apt-get +fi + +bears=$(find bears -type f -and -name '*Bear.py' | sort) + +yield_result_bears=$(grep -m 1 -l 'yield Result' $bears) + +# unused +# non_yield_result_bears=$(comm -23 <(ls $bears) <(ls $yield_result_bears)) + +requirement_bears=$(grep -m 1 -l 'Requirement(' $bears) + +cabal_requirement_bears=$(grep -m 1 -l CabalRequirement $requirement_bears) +gem_requirement_bears=$(grep -m 1 -l GemRequirement $requirement_bears) +go_requirement_bears=$(grep -m 1 -l GoRequirement $requirement_bears) +npm_requirement_bears=$(grep -m 1 -l NpmRequirement $requirement_bears) +rscript_requirement_bears=$(grep -m 1 -l RscriptRequirement $requirement_bears) + +pip_requirement_bears=$(grep -m 1 -l PipRequirement $requirement_bears) + +non_pip_runtime_requirement_bears="$cabal_requirement_bears $gem_requirement_bears $go_requirement_bears $npm_requirement_bears $rscript_requirement_bears" + +other_requirement_bears=$(comm -23 <(ls $requirement_bears) <(ls $non_pip_runtime_requirement_bears $pip_requirement_bears)) + +distribution_requirement_bears=$(grep -m 1 -l 'DistributionRequirement(' $requirement_bears) + +apt_get_requirement_bears=$(grep -m 1 -l 'apt_get' $distribution_requirement_bears) + +# Verify that DistributionRequirement is the only other Requirement subclass used +unknown_requirement_bears=$(grep -m 1 -Pl '(? /dev/null + echo "deb http://archive.ubuntu.com/ubuntu/ $DIST-updates main universe" | sudo tee -a /etc/apt/sources.list.d/$DIST.list > /dev/null + echo "deb http://archive.ubuntu.com/ubuntu/ $DIST-backports main universe" | sudo tee -a /etc/apt/sources.list.d/$DIST.list > /dev/null + sudo apt-get -y update + sudo apt-get -y --no-install-recommends install chktex cppcheck default-jre flawfinder indent libperl-critic-perl libxml2-utils mono-mcs php7.0-cli php-codesniffer verilator +fi + +# Change environment for flawfinder from python to python2 +if [[ -n "$(which flawfinder)" ]]; then + sed -e '1s/.*/#!\/usr\/bin\/env python2/' $(which flawfinder) > $HOME/bin/flawfinder; + chmod a+x $HOME/bin/flawfinder; +fi diff --git a/.ci/deps.go.sh b/.ci/deps.go.sh index e303843d3a..857f2afdd0 100755 --- a/.ci/deps.go.sh +++ b/.ci/deps.go.sh @@ -9,3 +9,6 @@ gometalinter --install go get -u sourcegraph.com/sqs/goreturns go get -u github.com/BurntSushi/toml/cmd/tomlv + +# other approach +# grep ImportPath Godeps/Godeps.json | cut -d '"' -f 4 | grep '[a-z]' | xargs go get; diff --git a/.ci/deps.opam.sh b/.ci/deps.opam.sh index 3dc030cac0..447d9dd153 100755 --- a/.ci/deps.opam.sh +++ b/.ci/deps.opam.sh @@ -27,4 +27,6 @@ if [ ! -e ~/infer-linux64-v0.7.0/infer/bin ]; then opam pin add --yes --no-action infer . opam install --deps-only --yes infer ./build-infer.sh java + + cd - fi diff --git a/.ci/deps.perl.sh b/.ci/deps.perl.sh new file mode 100755 index 0000000000..363e822eae --- /dev/null +++ b/.ci/deps.perl.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e -x + +# VHDL Bakalint Installation +if [ ! -e ~/.local/bin/bakalint.pl ]; then + BAKALINT_VERSION=0.4.0 + wget "http://downloads.sourceforge.net/project/fpgalibre/bakalint/0.4.0/bakalint-0.4.0.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Ffpgalibre%2Ffiles%2Fbakalint%2F0.4.0%2F&ts=1461844926&use_mirror=netcologne" -O ~/bl.tar.gz + tar xf ~/bl.tar.gz -C ~/ + mv ~/bakalint-$BAKALINT_VERSION/bakalint.pl ~/.local/bin/ +fi diff --git a/.ci/deps.python.sh b/.ci/deps.python.sh new file mode 100755 index 0000000000..75fbf4f7a8 --- /dev/null +++ b/.ci/deps.python.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -e -x + +echo pip3.4 is $(which pip3.4) +echo python3.4 is $(which python3.4) +if [[ -n "$(which pip3.4)" ]]; then + ln -sf $(which pip3.4) $HOME/bin/pip3 +fi +if [[ -n "$(which python3.4)" ]]; then + ln -sf $(which python3.4) $HOME/bin/python3 +fi + +echo pip3.5 is $(which pip3.5) +echo python3.5 is $(which python3.5) +if [[ -n "$(which pip3.5)" ]]; then + ln -sf $(which pip3.5) $HOME/bin/pip3 +fi +if [[ -n "$(which python3.5)" ]]; then + ln -sf $(which python3.5) $HOME/bin/python3 +fi + +echo pip3 is $(which pip3) +echo python3 is $(which python3) +if [[ -n "$(which python3)" ]]; then + ln -sf $(which python3) $HOME/bin/python +fi + +python --version +# Needed as some hosts may be providing python 3.2 +wget https://bootstrap.pypa.io/get-pip.py +python get-pip.py --user +rm get-pip.py +cp $HOME/.local/bin/pip $HOME/.local/bin/pip3 +cp $HOME/.local/bin/pip $HOME/bin/pip +cp $HOME/.local/bin/pip $HOME/bin/pip3 +pip --version diff --git a/.ci/deps.sh b/.ci/deps.sh index 7722a69ea5..de0327f61e 100755 --- a/.ci/deps.sh +++ b/.ci/deps.sh @@ -24,14 +24,6 @@ if ! dartanalyzer -v &> /dev/null; then cp -rp ~/dart-sdk/* ~/.local/ fi -# VHDL Bakalint Installation -if [ ! -e ~/.local/bin/bakalint.pl ]; then - BAKALINT_VERSION=0.4.0 - wget "http://downloads.sourceforge.net/project/fpgalibre/bakalint/0.4.0/bakalint-0.4.0.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Ffpgalibre%2Ffiles%2Fbakalint%2F0.4.0%2F&ts=1461844926&use_mirror=netcologne" -O ~/bl.tar.gz - tar xf ~/bl.tar.gz -C ~/ - mv ~/bakalint-$BAKALINT_VERSION/bakalint.pl ~/.local/bin/ -fi - # elm-format Installation if [ ! -e ~/.local/bin/elm-format ]; then curl -fsSL -o elm-format.tgz https://github.com/avh4/elm-format/releases/download/0.5.2-alpha/elm-format-0.17-0.5.2-alpha-linux-x64.tgz diff --git a/.ci/tests.prune.sh b/.ci/tests.prune.sh new file mode 100755 index 0000000000..f5abf520c6 --- /dev/null +++ b/.ci/tests.prune.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# Delete tests for bears that have been removed +set -e + +bear_tests=$(find tests -type f -and -name '*Test.py') + +for test in tests/generate_packageTest.py $bear_tests; do + bear=${test/Test.py/.py} + bear=${bear/tests/bears} + dir=$(dirname $bear) + if [[ ! -d $dir ]]; then + echo Removing $test + rm -f $test + elif [[ ! -f $bear ]]; then + echo Removing $test + rm -f $test + fi +done diff --git a/.moban.dt/bears-setup.py.jj2 b/.moban.dt/bears-setup.py.jj2 index 98a6ae6473..e793bb6db7 100755 --- a/.moban.dt/bears-setup.py.jj2 +++ b/.moban.dt/bears-setup.py.jj2 @@ -16,4 +16,16 @@ extras_require = { required += [req for req in bear_required if not any(req.startswith(ignore) for ignore in ignore_requirements)] + +python_bears_deps = True + +config = configparser.ConfigParser() +config.read('setup.cfg') +if 'setup:custom' in config.sections(): + if config['setup:custom']['python-bears'] == 'False': + python_bears_deps = False + +if python_bears_deps: + required += [req for req in bear_required + if not req.startswith('language-check')] {% endblock %} diff --git a/.travis.yml b/.travis.yml index 165b44b7fd..f9ca2f729c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,6 +70,156 @@ jobs: after_failure: false if: branch = master AND type = push - *check_moban + + - language: python + python: 3.4 + addons: + apt: + packages: clang-3.4 + + - language: python + python: 3.5 + addons: + apt: + packages: clang-3.4 + + - language: node_js + node_js: 4 + - language: node_js + node_js: 5 + - language: node_js + node_js: 6 + + - language: ruby + rvm: 2.2 + - language: ruby + rvm: 2.1 + - language: ruby + rvm: 2.0 + + - language: haskell + ghc: 8.0.2 + dist: precise + addons: + apt: + sources: + - deadsnakes + - hvr-ghc + packages: + - cabal-install-1.24 + - ghc-8.0.2 + - happy-1.19.3 + - python3.4 + + - language: java + jdk: oraclejdk8 + + - language: java + jdk: openjdk7 + - language: java + jdk: openjdk6 + + - language: r + - language: dart + - language: julia + - language: php + - language: perl + # go 1.5 isnt supported by all go linters + - language: go + go: 1.6 + - language: go + go: 1.7 + + - language: bash + env: BEARS=lua + addons: + apt: + packages: + - luarocks + + - language: bash + env: BEARS=opam + addons: + apt: + sources: + - avsm + packages: + - ocaml + - opam + + - language: generic + env: DIST=precise + dist: precise + addons: + apt: + sources: + - deadsnakes + - hvr-ghc + packages: + - cabal-install-1.24 + - ghc-8.0.2 + - happy-1.19.3 + - chktex + - cppcheck + - flawfinder + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php5-cli + # php-codesniffer # linter error? + - python3.4 + - verilator + + - language: generic + env: DIST=trusty + dist: trusty + addons: + apt: + packages: + - chktex + - cppcheck + - flawfinder + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php5-cli + - php-codesniffer + - verilator + + - language: generic + env: DIST=debian-sid + dist: trusty + addons: + apt: + sources: + - debian-sid + packages: + - chktex + - cppcheck + - flawfinder + - indent + - libxml2-utils + - verilator + # Causes error during apt-get install: + # libperl-critic-perl + # mono-mcs + # php5-cli + # php-codesniffer + - language: generic + env: DIST=xenial + dist: trusty + sudo: required + - language: generic + env: DIST=yakkety + dist: trusty + sudo: required + - language: generic + env: DIST=zesty + dist: trusty + sudo: required + allow_failures: - *check_moban @@ -99,40 +249,9 @@ dist: trusty addons: apt: sources: *apt_sources - packages: - - aspcud - - astyle - - cabal-install-1.24 - - chktex - - clang-3.4 - - cppcheck - - devscripts - - flawfinder - - gfortran - - ghc - - golang-go - - happy - - indent - - julia - - libarpack2 - - libblas-dev - - libcolamd2.8.0 - - libfftw3-3 - - liblapack-dev - - libopenblas-base - - libpaper-utils - - libperl-critic-perl - - libumfpack5.6.2 - - libxml2-utils - - luarocks - - mercurial - - menhir - - mono-mcs - - ocaml - - opam - - php-codesniffer - - r-base - - verilator + +matrix: + include: cache: pip: true @@ -162,35 +281,115 @@ env: - CABAL_VERSION=1.24 - INFER_VERSION=0.7.0 - PATH="$HOME/.local/bin:/opt/cabal/$CABAL_VERSION/bin:$PATH:$TRAVIS_BUILD_DIR/node_modules/.bin:$TRAVIS_BUILD_DIR/vendor/bin:$HOME/.cabal/bin:$HOME/infer-linux64-v$INFER_VERSION/infer/bin:$HOME/.local/tailor/tailor-latest/bin:$HOME/.luarocks/bin" + - TRAVIS_GHC_ROOT=/opt/ghc before_install: - - nvm install 6.10.2 # Remove Ruby directive from Gemfile as this image has 2.2.5 - sed -i '/^ruby/d' Gemfile - - .ci/deps.sh - - .ci/deps.go.sh - - .ci/deps.cabal.sh - - .ci/deps.r.sh - - .ci/deps.opam.sh - - .ci/deps.java.sh + - mkdir $HOME/bin/ + - .ci/deps.apt-new.sh + - if [[ "$DIST" == "trusty" || "$DIST" == "debian-sid" ]]; then + ls /usr/bin/php $HOME/bin/; + fi + + - > + if [[ -z "$TRAVIS_PYTHON_VERSION" ]]; then + .ci/deps.python.sh + fi + + # Remove Ruby directive from Gemfile as this image has 2.2.5 + - sed -i '/^ruby/d' Gemfile + - if [[ "$BEARS" == "all" ]]; then + nvm install 4; nvm use 4; bash .ci/deps.sh; + fi + - if [[ "$BEARS" == "all" ]]; then bash .ci/deps.cabal.sh; fi # https://github.com/coala/coala/issues/3183 - cp requirements.txt requirements.orig - printf '%s\n%s\n%s\n' "$(cat test-requirements.txt)" "$(grep -v '^-r' docs-requirements.txt)" - "$(cat bear-requirements.txt requirements.txt)" + "$(cat requirements.txt)" > requirements.txt + - > + if [[ "$TRAVIS_LANGUAGE" == "python" ]]; then + cat bear-requirements.txt >> requirements.txt; + elif [[ "$TRAVIS_LANGUAGE" == "ruby" ]]; then + echo 'pyyaml' > bear-requirements.txt; + else + rm bear-requirements.txt; touch bear-requirements.txt; + fi + - sed -i '/^-r/d' requirements.txt + - if [[ -z "$TRAVIS_PYTHON_VERSION" ]]; then + pip install --user -r requirements.txt; + fi before_script: + - if [[ "$TRAVIS_LANGUAGE" == "go" ]]; then + .ci/deps.go.sh + fi + - if [[ "$TRAVIS_LANGUAGE" == "php" ]]; then + pear install pear/PHP_CodeSniffer; + phpenv rehash; + fi + # Force ghc-mod to resolve its Cabal version + - if [[ "$TRAVIS_LANGUAGE" == "haskell" ]]; then + ~/.cabal/bin/ghc-mod modules ; + fi + - if [[ "$TRAVIS_LANGUAGE" == "julia" ]]; then + julia -e "Pkg.add(\"Lint\")"; + fi + - if [[ "$TRAVIS_LANGUAGE" == "java" ]]; then + bash .ci/deps.java.sh; + fi + - if [[ "$BEARS" == "opam" ]]; then + bash .ci/deps.opam.sh; + fi + - if [[ "$BEARS" == "lua" ]]; then + luarocks install --local --deps-mode=none luacheck; + fi + - if [[ "$TRAVIS_LANGUAGE" == "perl" ]]; then + .ci/deps.perl.sh + fi - mv requirements.orig requirements.txt - .ci/deps.coala-bears.sh + # nltk 3.2.2 dropped support for Python 3.3 + - > + if [[ -n "$TRAVIS_PYTHON_VERSION" && "$TRAVIS_PYTHON_VERSION" != "3.3" ]]; + then + python -m nltk.downloader + punkt maxent_treebank_pos_tagger averaged_perceptron_tagger; + fi + # Only keep non-Python bears on Python 3.4 + - bash .ci/bears.prune.sh + # Delete tests for removed bears + - bash .ci/tests.prune.sh script: - python setup.py bdist_wheel - - pip install $(ls ./dist/*.whl)"[alldeps]" + - > + if [[ "$TRAVIS_LANGUAGE" == "python" ]]; then + pip install $(ls ./dist/*.whl)"[alldeps]" + else + pip install --user $(ls ./dist/*.whl) + fi - pytest # Ensure bear requirements are in sync with the bear PipRequirement - .ci/generate_bear_requirements.py --check --update + - > + if [[ "$TRAVIS_LANGUAGE" == "python" ]]; then + (cat bear-requirements.txt; echo "pyyaml~=3.12") + > | sort -u > bear-requirements.tmp + mv bear-requirements.tmp bear-requirements.txt + git diff --exit-code bear-requirements.txt + fi + - > + if [[ "$TRAVIS_LANGUAGE" == "ruby" ]]; then + git diff --exit-code Gemfile + fi + - > + if [[ "$TRAVIS_LANGUAGE" == "node_js" ]]; then + git diff --exit-code package.json + fi - coala --non-interactive - codecov - python setup.py docs diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json new file mode 100644 index 0000000000..ff7bcfea85 --- /dev/null +++ b/Godeps/Godeps.json @@ -0,0 +1,25 @@ +{ + "ImportPath": ".", + "GoVersion": "go1.5", + "GodepVersion": "v62", + "Deps": [ + { + "ImportPath": "github.com/golang/lint/golint" + }, + { + "ImportPath": "golang.org/x/tools/cmd/goimports" + }, + { + "ImportPath": "sourcegraph.com/sqs/goreturns" + }, + { + "ImportPath": "golang.org/x/tools/cmd/gotype" + }, + { + "ImportPath": "github.com/kisielk/gotool" + }, + { + "ImportPath": "github.com/kisielk/errcheck" + } + ] +} diff --git a/Makefile.PL b/Makefile.PL new file mode 100644 index 0000000000..e889d73de6 --- /dev/null +++ b/Makefile.PL @@ -0,0 +1,6 @@ +use ExtUtils::MakeMaker; +WriteMakefile( + NAME => 'Coala::Bears', + VERSION => '0.10', + PREREQ_PM => {Perl::Critic => 1.126}, +); diff --git a/makefile b/makefile new file mode 100644 index 0000000000..e69de29bb2 diff --git a/setup.cfg b/setup.cfg index d9a43ec6b7..098de2f397 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ addopts = --doctest-modules --doctest-ignore-import-error --cov + --cov-append --instafail --profile --reorder 'requirements.txt' 'test-requirements.txt' '*' diff --git a/setup.py b/setup.py index 0636a55722..3c1d3a1baf 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import configparser import locale import os import platform @@ -413,6 +414,18 @@ def read_requirements(filename): if not any(req.startswith(ignore) for ignore in ignore_requirements)] +python_bears_deps = True + +config = configparser.ConfigParser() +config.read('setup.cfg') +if 'setup:custom' in config.sections(): + if config['setup:custom']['python-bears'] == 'False': + python_bears_deps = False + +if python_bears_deps: + required += [req for req in bear_required + if not req.startswith('language-check')] + if extras_require: EXTRAS_REQUIRE = extras_require SETUP_COMMANDS.update({