Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Cleaning up build #1888

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ install:
# command to run tests
script:
- make test
- make pep8
- make style
- make doc
- make py-demos
- make cpp-demos
Expand Down
158 changes: 28 additions & 130 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -32,99 +32,68 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Contact: [email protected]

#
# `make help` for a summary of useful targets

#
# `SHELL=bash` Will break Titus's laptop, so don't use BASH-isms like
# `[[` conditional expressions.
#
PREFIX=/usr/local
CPPSOURCES=$(wildcard src/oxli/*.cc include/oxli/*.hh src/khmer/_cpy_*.cc include/khmer/_cpy_*.hh) setup.py
CYSOURCES=$(wildcard khmer/_oxli/*.pxd khmer/_oxli/*.pyx)
PYSOURCES=$(filter-out khmer/_version.py, \
$(wildcard khmer/*.py scripts/*.py oxli/*.py) )
PYSOURCES=$(filter-out khmer/_version.py, $(wildcard khmer/*.py scripts/*.py oxli/*.py) )
SOURCES=$(PYSOURCES) $(CPPSOURCES) $(CYSOURCES) setup.py

DEVPKGS=pep8==1.6.2 diff_cover autopep8 pylint coverage gcovr pytest pydocstyle

GCOVRURL=git+https://github.com/nschum/gcovr.git@never-executed-branches

VERSION=$(shell ./setup.py version | grep Version | awk '{print $$4}' \
| sed 's/+/-/')

# The following four variables are only used by cppcheck. If you want to
# change how things are compiled edit `setup.cfg` or `setup.py`.
DEFINES += -DNDEBUG -DVERSION=$(VERSION) -DSEQAN_HAS_BZIP2=1 \
-DSEQAN_HAS_ZLIB=1 -UNO_UNIQUE_RC

INCLUDESTRING=$(shell gcc -E -x c++ - -v < /dev/null 2>&1 >/dev/null \
| grep '^ /' | grep -v cc1plus)
INCLUDEOPTS=$(shell gcc -E -x c++ - -v < /dev/null 2>&1 >/dev/null \
| grep '^ /' | grep -v cc1plus | awk '{print "-I" $$1 " "}')

PYINCLUDE=$(shell python -c "import sysconfig; \
flags = ['-I' + sysconfig.get_path('include'), \
'-I' + sysconfig.get_path('platinclude')]; print(' '.join(flags))")

CPPCHECK_SOURCES=$(filter-out lib/test%, $(wildcard lib/*.cc khmer/_khmer.cc) )
CPPCHECK=cppcheck --enable=all \
--error-exitcode=1 \
--suppress='*:/Library/*' \
--suppress='*:*/include/python*/Python.h' \
--suppress='*:/usr/*' --platform=unix64 \
--std=c++11 --inline-suppr -Ilib -Ithird-party/bzip2 \
-Ithird-party/zlib -Ithird-party/smhasher -Ithird-party/rollinghash \
$(DEFINES) $(INCLUDEOPTS) $(PYINCLUDE) $(CPPCHECK_SOURCES) --quiet

UNAME := $(shell uname)
DEVPKGS=pycodestyle coverage gcovr pytest pydocstyle
VERSION=$(shell ./setup.py version | grep Version | awk '{print $$4}' | sed 's/+/-/')
INCLUDESTRING=$(shell gcc -E -x c++ - -v < /dev/null 2>&1 >/dev/null | grep '^ /' | grep -v cc1plus)
INCLUDEOPTS=$(shell gcc -E -x c++ - -v < /dev/null 2>&1 >/dev/null | grep '^ /' | grep -v cc1plus | awk '{print "-I" $$1 " "}')
PYINCLUDE=$(shell python -c "import sysconfig; flags = ['-I' + sysconfig.get_path('include'), '-I' + sysconfig.get_path('platinclude')]; print(' '.join(flags))")
UNAME=$(shell uname)
MODEXT=$(shell python -c "import sysconfig;print(sysconfig.get_config_var('SO'))")
EXTENSION_MODULE=khmer/_khmer$(MODEXT)
ifeq ($(UNAME),Linux)
TESTATTR ?= 'not known_failing and not jenkins and not huge'
else
TESTATTR ?= 'not known_failing and not jenkins and not huge and not linux'
endif

MODEXT=$(shell python -c \
"import sysconfig;print(sysconfig.get_config_var('SO'))")
EXTENSION_MODULE = khmer/_khmer$(MODEXT)
CY_MODULES = $($(wildcard khmer/_oxli/*.pyx): .pyx=.$(MODEXT))

PYLINT_TEMPLATE="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}"

## all : default task; compile C++ code, build shared object library
all: sharedobj


## help : print this help message and exit
help: Makefile
@sed -n 's/^##//p' $<


## install-dep : install most of the development dependencies via pip
install-dep: install-dependencies

install-dependencies:
pip install $(DEVPKGS)
pip install --requirement doc/requirements.txt


## sharedobj : build khmer shared object file
sharedobj: $(EXTENSION_MODULE)

$(EXTENSION_MODULE): $(CPPSOURCES) $(CYSOURCES)
./setup.py build_ext --inplace

coverage-debug: $(CPPSOURCES)
export CFLAGS="-pg -fprofile-arcs -ftest-coverage -O0"; ./setup.py \
build_ext --debug --inplace --libraries gcov
touch coverage-debug

## install : install the khmer module and scripts
install: FORCE
./setup.py build install


## dist : create a module package for distribution
dist: dist/khmer-$(VERSION).tar.gz

dist/khmer-$(VERSION).tar.gz: $(SOURCES)
./setup.py sdist


## clean : clean up all temporary / machine-generated files
clean: FORCE
cd src/oxli && $(MAKE) clean || true
Expand All @@ -150,6 +119,7 @@ debug: FORCE
-D_GLIBCXX_DEBUG -DDEBUG_ASSEMBLY=1 -DDEBUG_FILTERS=1"; python setup.py build_ext --debug \
--inplace


## doc : render documentation in HTML
doc: build/sphinx/html/index.html

Expand All @@ -159,6 +129,7 @@ build/sphinx/html/index.html: $(SOURCES) $(wildcard doc/*.rst) doc/conf.py all
@echo '--> docs in build/sphinx/html <--'
@echo ''


## pdf : render documentation as a PDF file
# packages needed include: texlive-latex-base texlive-latex-recommended
# texlive-fonts-recommended texlive-latex-extra
Expand All @@ -174,60 +145,28 @@ build/sphinx/latex/khmer.pdf: $(SOURCES) doc/conf.py $(wildcard doc/*.rst) \
cppcheck-result.xml: $(CPPSOURCES)
$(CPPCHECK) --xml-version=2 2> cppcheck-result.xml


## cppcheck : run static analysis on C++ code
cppcheck: FORCE
@$(CPPCHECK)

cppcheck-long: FORCE
@$(CPPCHECK) -Ithird-party/seqan/core/include

## pep8 : check Python code style
pep8: $(PYSOURCES) $(wildcard tests/*.py)
pep8 setup.py khmer/*.py scripts/*.py tests/*.py oxli/*.py examples/python-api/*.py
## style : check Python code style
style: $(PYSOURCES) $(wildcard tests/*.py)
pycodestyle setup.py khmer/*.py scripts/*.py tests/*.py oxli/*.py examples/python-api/*.py

pep8_report.txt: $(PYSOURCES) $(wildcard tests/*.py)
pep8 setup.py khmer/ scripts/ tests/ oxli/ \
> pep8_report.txt || true

diff_pep8_report: pep8_report.txt
diff-quality --violations=pep8 pep8_report.txt

## pydocstyle : check Python doc strings
pydocstyle: $(PYSOURCES) $(wildcard tests/*.py)
pydocstyle --ignore=D100,D101,D102,D103,D203 --match='(?!_version).*\.py' \
setup.py khmer/ scripts/ oxli/ || true

pydocstyle_report.txt: $(PYSOURCES) $(wildcard tests/*.py)
pydocstyle setup.py khmer/ scripts/ tests/ oxli/ \
> pydocstyle_report.txt 2>&1 || true
pydocstyle --ignore=D100,D101,D102,D103,D203 --match='(?!_version).*\.py' setup.py khmer/ scripts/ oxli/

diff_pydocstyle_report: pydocstyle_report.txt
diff-quality --violations=pep8 pydocstyle_report.txt

## astyle : fix most C++ code indentation and formatting
astyle: $(CPPSOURCES)
astyle -A10 --max-code-length=80 $(filter-out setup.py,$(CPPSOURCES))

## autopep8 : fix most Python code indentation and formatting
autopep8: $(PYSOURCES) $(wildcard tests/*.py)
autopep8 --recursive --in-place --exclude _version.py --ignore E309 \
setup.py khmer/*.py scripts/*.py tests/*.py oxli/*.py

## format : check/fix all code formatting (astyle and autopep8)
format: astyle autopep8
# Do nothing

## pylint : run static code analysis on Python code
pylint: $(PYSOURCES) $(wildcard tests/*.py)
pylint --msg-template=$(PYLINT_TEMPLATE) \
setup.py $(PYSOURCES) tests/*.py || true

pylint_report.txt: $(PYSOURCES) $(wildcard tests/*.py) $(wildcard sandbox/*.py)
pylint --msg-template=$(PYLINT_TEMPLATE) \
$(PYSOURCES) tests sandbox > pylint_report.txt || true

diff_pylint_report: pylint_report.txt
diff-quality --violations=pylint pylint_report.txt

# We need to get coverage to look at our scripts. Since they aren't in a
# python module we can't tell pytest to look for them (via an import
Expand Down Expand Up @@ -265,6 +204,7 @@ diff-cover.html: coverage-gcovr.xml coverage.xml
pytests.xml: FORCE
py.test --junitxml=$@ -m ${TESTATTR}


## doxygen : generate documentation of the C++ and Python code
# helpful packages: doxygen graphviz
# ignore warning re: _formulas.aux
Expand Down Expand Up @@ -305,6 +245,7 @@ libtest: FORCE
src/oxli/test-compile.cc -loxli
rm -rf install_target


## test : run the khmer test suite
test: FORCE
./setup.py develop
Expand All @@ -314,56 +255,11 @@ sloccount.sc: $(CPPSOURCES) $(PYSOURCES) $(wildcard tests/*.py) Makefile
sloccount --duplicates --wide --details include src khmer scripts tests \
setup.py Makefile > sloccount.sc


## sloccount : count lines of code
sloccount:
sloccount src include khmer scripts tests setup.py Makefile

coverity-build:
if [ -x "${cov_analysis_dir}/bin/cov-build" ]; \
then \
export PATH=${PATH}:${cov_analysis_dir}/bin; \
cov-build --dir cov-int --c-coverage gcov \
--disable-gcov-arg-injection make coverage-debug; \
cov-capture --dir cov-int --c-coverage gcov python -m pytest \
-m $(TESTATTR) ; \
cov-import-scm --dir cov-int --scm git 2>/dev/null; \
else echo 'bin/cov-build does not exist in $$cov_analysis_dir: '\
'${cov_analysis_dir}. Skipping coverity scan.'; \
fi

coverity-upload: cov-int
if [ -n "${COVERITY_TOKEN}" ]; \
then \
tar czf khmer-cov.tgz cov-int; \
curl --form token=${COVERITY_TOKEN} --form \
[email protected] --form [email protected] \
--form version=$(VERSION) \
https://scan.coverity.com/builds?project=ged-lab%2Fkhmer ; \
else echo 'Missing coverity credentials in $$COVERITY_TOKEN,'\
'skipping scan'; \
fi

coverity-clean-configuration:
rm -f ${cov_analysis_dir}/config/coverity_config.xml

coverity-configure:
if [[ -x ${cov_analysis_dir}/bin/cov-configure ]]; \
then \
export PATH=${PATH}:${cov_analysis_dir}/bin; \
for compiler in \
/usr/bin/gcc-4.8 /usr/bin/x86_64-linux-gnu-gcc; do \
cov-configure --comptype gcc \
--compiler $${compiler}; \
done; \
else echo 'bin/cov-configure does not exist in $$cov_analysis_dir: '\
'${cov_analysis_dir}. Skipping coverity configuration.'; \
fi

# may need to `sudo apt-get install bear`
compile_commands.json: clean
export PATH=$(shell echo $$PATH | sed 's=/usr/lib/ccache:==g') ; \
bear ./setup.py build_ext

convert-release-notes:
for file in doc/release-notes/*.md; do \
pandoc --from=markdown --to=rst $${file} > $${file%%.md}.rst; \
Expand All @@ -379,10 +275,12 @@ list-citation:
git log --format='%aN,%aE' | sort -u | grep -v -F -f author-skips.txt > authors.csv
python sort-authors-list.py


## cpp-demos : run programs demonstrating access to the (unstable) C++ API
cpp-demos: sharedobj
cd examples/c++-api/ && make all run


## py-demos : run programs demonstrating access to the Python API
py-demos: sharedobj
python examples/python-api/exact-counting.py
Expand Down
23 changes: 6 additions & 17 deletions setup.cfg
Original file line number Diff line number Diff line change
@@ -1,29 +1,17 @@
[build_ext]
define = SEQAN_HAS_BZIP2,SEQAN_HAS_ZLIB
undef = NO_UNIQUE_RC
libraries = z,bz2
include-dirs = include:third-party/seqan/core/include:third-party/smhasher:third-party/cqf:third-party/rollinghash

# When you update the include directories here also update the paths used in
# docker/Dockerfile
# libraries = z,bz2
## if using system libraries
include-dirs = include:third-party/zlib:third-party/bzip2:third-party/seqan/core/include:third-party/smhasher:third-party/cqf:third-party/rollinghash
# include-dirs = lib
## if using system libraries (broken)

# define = NDEBUG
# is not needed for most Linux installs
# see the OPT line in /usr/lib/python2.7/config/Makefile which gets included
# in distutils version of CFLAGS

[easy_install]

[aliases]
test=pytest


# See the docstring in versioneer.py for instructions. Note that you must
# re-run 'versioneer.py setup' after changing this section, and commit the
# resulting files.

[versioneer]
VCS = git
style = pep440
Expand All @@ -32,6 +20,7 @@ versionfile_build = khmer/_version.py
tag_prefix = v
parentdir_prefix = .

[pep8]

[pycodestyle]
exclude = _version.py
ignore = E309,E226,E402
ignore = E309,E226,E402,E722,W504
Loading