Add the ability to generate test coverage reports

The run_tests.sh script now has the ability to generate a coverage
report for the CPU code. It can be generated by passing the -l flag
to the buildAndRunTests function then the coverage report will be
available in the generated HTML website in
bin/html_coverage_report_${CHOLLA_MAKE_TYPE}

.gitignore

@@ -41,6 +41,8 @@ out.*
 o.*
 run
 disk.*
+*.gcno
+*.gcda
 
 # Logs and databases #
 ######################

Makefile

@@ -56,6 +56,10 @@ else
   GPUFILES := $(filter-out src/system_tests/% %_tests.cu,$(GPUFILES))
 endif
 
+ifeq ($(COVERAGE), true)
+  CXXFLAGS += --coverage
+endif
+
 OBJS     := $(subst .cpp,.o,$(CPPFILES)) \
             $(subst .cu,.o,$(GPUFILES))
 
@@ -214,6 +218,8 @@ clean:
 	rm -f $(CLEAN_OBJS)
 	rm -rf googletest
 	-find bin/ -type f -executable -name "cholla.*.$(MACHINE)*" -exec rm -f '{}' \;
+	-find src/ -type f -name "*.gcno" -delete
+	-find src/ -type f -name "*.gcda" -delete
 
 clobber: clean
 	-find bin/ -type f -executable -name "cholla*" -exec rm -f '{}' \;

builds/run_tests.sh

@@ -108,7 +108,7 @@ buildCholla ()
 {
   echo -e "\nBuilding Cholla...\n"
   builtin cd $CHOLLA_ROOT
-  make --jobs=$(nproc) TYPE=${CHOLLA_MAKE_TYPE} BUILD=${1}
+  make --jobs=$(nproc) TYPE=${CHOLLA_MAKE_TYPE} BUILD=${1} COVERAGE=${2}
 }
 # ==============================================================================
 
@@ -119,7 +119,7 @@ buildChollaTests ()
 {
   echo
   builtin cd $CHOLLA_ROOT
-  make --jobs=$(nproc) TYPE=${CHOLLA_MAKE_TYPE} TEST=true
+  make --jobs=$(nproc) TYPE=${CHOLLA_MAKE_TYPE} TEST=true COVERAGE=${1}
 }
 # ==============================================================================
 
@@ -203,6 +203,51 @@ runTests ()
 }
 # ==============================================================================
 
+# ==============================================================================
+# This function generates a coverage report after the tests have been run.
+# The final report is a website in bin/html_coverage_report_${CHOLLA_MAKE_TYPE}
+chollaCoverage ()
+{
+  # Setup the names of files that we will use
+  local base_file="bin/coverage_base_${CHOLLA_MAKE_TYPE}.info"
+  local test_file="bin/coverage_test_${CHOLLA_MAKE_TYPE}.info"
+  local combined_file="bin/coverage_combined_${CHOLLA_MAKE_TYPE}.info"
+
+  # Generate the initial report with no coverage info. This is needed so that
+  # lcov knows about all the files, not just the ones that are tested
+  lcov --capture --initial --directory ${CHOLLA_ROOT}/src --output-file ${base_file}
+
+  # Now we get the actual coverage information
+  lcov --capture  --directory ${CHOLLA_ROOT}/src --output-file ${test_file}
+
+  # Then combine the the two coverage files so we know what changed, i.e. which
+  # lines were actually covered
+  lcov --add-tracefile ${base_file} --add-tracefile ${test_file} --output-file ${combined_file}
+
+  # Extract data from only the files within CHOLLA_ROOT. This should exclude any
+  # system or external libraries
+  lcov --extract ${combined_file} "${CHOLLA_ROOT}/*" --output-file ${combined_file}
+
+  # exclude_patterns=('*-tests.cpp') # Remove traces of the tests themselves
+  # # --remove TRACEFILE PATTERN = remove all things associated with PATTERN in TRACEFILE
+  # lcov --remove ${combined_file} "${exclude_patterns[@]}" --output-file ${combined_file}
+
+  # List the contents
+  lcov --list ${combined_file}
+
+  # Generate HTML report
+  genhtml ${combined_file} --output-directory bin/html_coverage_report_${CHOLLA_MAKE_TYPE}
+
+  # Combine all tracefiles together. Define the different make types then add
+  # the appropriate prefixes and suffices.
+  # build_types=(cosmology disk dust gravity hydro mhd particles)
+  # build_types=("${build_types[@]/#/--add-trace bin/coverage_combined_}")
+  # build_types=("${build_types[@]/%/.info}")
+  # eval "build_types=(${build_types[@]})"
+  # lcov "${build_types[@]}" --output-file bin/full_coverage_report.info
+}
+# ==============================================================================
+
 # ==============================================================================
 # Call all the functions required for setting up, building, and running tests
 #
@@ -213,17 +258,19 @@ runTests ()
 # \param[in] -g (optional) If set then download and build a local version of
 # GoogleTest to use instead of the machine default
 # \param[in] -d (optional) Build Cholla in debug mode
+# \param[in] -l (optional) Generate coverage reports when building and running Cholla
 buildAndRunTests ()
 {
   # Unset BUILD_GTEST so that subsequent runs aren't tied to what previous runs
   # did
   unset BUILD_GTEST
 
   BUILD_MODE='OPTIMIZE'
+  CODE_COVERAGE='false'
 
   # Check arguments
   local OPTIND
-  while getopts "t:c:g:d" opt; do
+  while getopts "t:c:g:d:l" opt; do
     case $opt in
         t)  # Set the make type
             MAKE_TYPE_ARG="-t ${OPTARG}"
@@ -237,6 +284,9 @@ buildAndRunTests ()
         d)  # Build the debug version of Cholla?
             BUILD_MODE='DEBUG'
             ;;
+        l)  # Generate Code Coverage?
+            CODE_COVERAGE='true'
+            ;;
         \?)
             echo "Invalid option: -${OPTARG}" >&2
             return 1
@@ -263,8 +313,12 @@ buildAndRunTests ()
   if [[ -n $BUILD_GTEST ]]; then
     buildGoogleTest
   fi
-  buildCholla $BUILD_MODE && \
-  buildChollaTests && \
+  buildCholla $BUILD_MODE $CODE_COVERAGE && \
+  buildChollaTests $CODE_COVERAGE && \
   runTests
+
+  if [ $CODE_COVERAGE = "true" ]; then
+    chollaCoverage
+  fi
 }
 # ==============================================================================