From b3d8959590afa84ac9cdef845afe3ecb9adf92a7 Mon Sep 17 00:00:00 2001 From: Jonathan Woollett-Light Date: Fri, 21 Oct 2022 15:32:16 +0100 Subject: [PATCH] Update code coverage to grcov Signed-off-by: Jonathan Woollett-Light --- .gitignore | 1 + .../integration_tests/build/test_coverage.py | 173 ++++++++---------- .../integration_tests/build/test_unittests.py | 12 +- tools/devctr/Dockerfile.aarch64 | 4 +- tools/devctr/Dockerfile.x86_64 | 4 +- tools/devtool | 2 +- 6 files changed, 85 insertions(+), 111 deletions(-) diff --git a/.gitignore b/.gitignore index 4d5a5f7934b..76713f74350 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .vscode test_results/* *.core +*.profraw diff --git a/tests/integration_tests/build/test_coverage.py b/tests/integration_tests/build/test_coverage.py index 2922130a772..91e3f282d85 100644 --- a/tests/integration_tests/build/test_coverage.py +++ b/tests/integration_tests/build/test_coverage.py @@ -8,15 +8,10 @@ target should be put in `s3://spec.firecracker` and automatically updated. """ - import os -import platform -import re -import shutil import pytest from framework import utils -import host_tools.cargo_build as host # pylint: disable=import-error from host_tools import proc # We have different coverages based on the host kernel version. This is @@ -35,108 +30,92 @@ PROC_MODEL = proc.proc_type() -COVERAGE_MAX_DELTA = 0.05 - -CARGO_KCOV_REL_PATH = os.path.join(host.CARGO_BUILD_REL_PATH, "kcov") - -KCOV_COVERAGE_FILE = "index.js" -"""kcov will aggregate coverage data in this file.""" +# Toolchain target architecture. +if ("Intel" in PROC_MODEL) or ("AMD" in PROC_MODEL): + ARCH = "x86_64" +elif "ARM" in PROC_MODEL: + ARCH = "aarch64" +else: + raise Exception(f"Unsupported processor model ({PROC_MODEL})") -KCOV_COVERED_LINES_REGEX = r'"covered_lines":"(\d+)"' -"""Regex for extracting number of total covered lines found by kcov.""" +# Toolchain target. +# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see +# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and +# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains. +TARGET = f"{ARCH}-unknown-linux-gnu" -KCOV_TOTAL_LINES_REGEX = r'"total_lines" : "(\d+)"' -"""Regex for extracting number of total executable lines found by kcov.""" +# We allow coverage to have a max difference of `COVERAGE_MAX_DELTA` as percentage before failing +# the test. +COVERAGE_MAX_DELTA = 0.05 -SECCOMPILER_BUILD_DIR = "../build/seccompiler" +# grcov 0.8.* requires GLIBC >2.27, this is not present in ubuntu 18.04, when we update the docker +# container with a newer version of ubuntu we can also update this. +GRCOV_VERSION = "0.7.1" @pytest.mark.timeout(400) -def test_coverage(test_fc_session_root_path, test_session_tmp_path): - """Test line coverage for rust tests is within bounds. - - The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov - after a coverage run. +def test_coverage(): + """Test code coverage @type: build """ - proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL] - assert len(proc_model) == 1, "Could not get processor model!" - coverage_target_pct = COVERAGE_DICT[proc_model[0]] - exclude_pattern = ( - "${CARGO_HOME:-$HOME/.cargo/}," - "build/," - "tests/," - "usr/lib/gcc," - "lib/x86_64-linux-gnu/," - "test_utils.rs," - # The following files/directories are auto-generated - "bootparam.rs," - "elf.rs," - "mpspec.rs," - "msr_index.rs," - "bindings.rs," - "_gen" + # Get coverage target. + processor_model = [item for item in COVERAGE_DICT if item in PROC_MODEL] + assert len(processor_model) == 1, "Could not get processor model!" + coverage_target = COVERAGE_DICT[processor_model[0]] + + # Re-direct to repository root. + os.chdir("..") + + # Generate test profiles. + utils.run_cmd( + f'\ + env RUSTFLAGS="-Cinstrument-coverage" \ + LLVM_PROFILE_FILE="coverage-%p-%m.profraw" \ + cargo test --all --target={TARGET} -- --test-threads=1 \ + ' ) - exclude_region = "'mod tests {'" - target = "{}-unknown-linux-musl".format(platform.machine()) - - cmd = ( - 'CARGO_WRAPPER="kcov" RUSTFLAGS="{}" CARGO_TARGET_DIR={} ' - "cargo kcov --all " - "--target {} --output {} -- " - "--exclude-pattern={} " - "--exclude-region={} --verify" - ).format( - host.get_rustflags(), - os.path.join(test_fc_session_root_path, CARGO_KCOV_REL_PATH), - target, - test_session_tmp_path, - exclude_pattern, - exclude_region, - ) - # We remove the seccompiler custom build directory, created by the - # vmm-level `build.rs`. - # If we don't delete it before and after running the kcov command, we will - # run into linker errors. - shutil.rmtree(SECCOMPILER_BUILD_DIR, ignore_errors=True) - # By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify` - # to kcov. To pass others arguments, we need to include the defaults. - utils.run_cmd(cmd) - - shutil.rmtree(SECCOMPILER_BUILD_DIR) - - coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE) - with open(coverage_file, encoding="utf-8") as cov_output: - contents = cov_output.read() - covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0]) - total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0]) - coverage = covered_lines / total_lines * 100 - print("Number of executable lines: {}".format(total_lines)) - print("Number of covered lines: {}".format(covered_lines)) - print("Thus, coverage is: {:.2f}%".format(coverage)) - - coverage_low_msg = ( - "Current code coverage ({:.2f}%) is >{:.2f}% below the target ({}%).".format( - coverage, COVERAGE_MAX_DELTA, coverage_target_pct - ) - ) - - assert coverage >= coverage_target_pct - COVERAGE_MAX_DELTA, coverage_low_msg - # Get the name of the variable that needs updating. - namespace = globals() - cov_target_name = [name for name in namespace if namespace[name] is COVERAGE_DICT][ - 0 - ] - - coverage_high_msg = ( - "Current code coverage ({:.2f}%) is >{:.2f}% above the target ({}%).\n" - "Please update the value of {}.".format( - coverage, COVERAGE_MAX_DELTA, coverage_target_pct, cov_target_name - ) + # Generate coverage report. + utils.run_cmd( + f'\ + cargo install --version {GRCOV_VERSION} grcov \ + && grcov . \ + -s . \ + --binary-path ./build/cargo_target/{TARGET}/debug/ \ + --excl-start "mod tests" \ + --ignore "build/*" \ + -t html \ + --branch \ + --ignore-not-existing \ + -o ./build/cargo_target/{TARGET}/debug/coverage \ + ' ) - assert coverage <= coverage_target_pct + COVERAGE_MAX_DELTA, coverage_high_msg - - return (f"{coverage}%", f"{coverage_target_pct}% +/- {COVERAGE_MAX_DELTA * 100}%") + # Extract coverage from html report. + # + # The line looks like `90.83 %

` and is the first + # occurrence of the `` element in the file. + # + # When we update grcov to 0.8.* we can update this to pull the coverage from a generated .json + # file. + index = open( + f"./build/cargo_target/{TARGET}/debug/coverage/index.html", encoding="utf-8" + ) + index_contents = index.read() + end = index_contents.find(" %

") + start = index_contents[:end].rfind(">") + coverage_str = index_contents[start + 1 : end] + coverage = float(coverage_str) + + # Compare coverage. + high = coverage_target * (1.0 + COVERAGE_MAX_DELTA) + low = coverage_target * (1.0 - COVERAGE_MAX_DELTA) + assert ( + coverage >= low + ), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% below \ + the target ({coverage_target:.2f}%)" + assert ( + coverage <= high + ), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% above \ + the target ({coverage_target:.2f}%)" diff --git a/tests/integration_tests/build/test_unittests.py b/tests/integration_tests/build/test_unittests.py index 5ba0be585c3..5477e728689 100644 --- a/tests/integration_tests/build/test_unittests.py +++ b/tests/integration_tests/build/test_unittests.py @@ -7,9 +7,10 @@ import host_tools.cargo_build as host # pylint:disable=import-error MACHINE = platform.machine() -# No need to run unittests for musl since -# we run coverage with musl for all platforms. -TARGET = "{}-unknown-linux-gnu".format(MACHINE) +# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see +# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and +# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains. +TARGET = "{}-unknown-linux-musl".format(MACHINE) def test_unittests(test_fc_session_root_path): @@ -20,7 +21,4 @@ def test_unittests(test_fc_session_root_path): """ extra_args = "--release --target {} ".format(TARGET) - host.cargo_test( - test_fc_session_root_path, - extra_args=extra_args - ) + host.cargo_test(test_fc_session_root_path, extra_args=extra_args) diff --git a/tools/devctr/Dockerfile.aarch64 b/tools/devctr/Dockerfile.aarch64 index cb7a9b76b3d..bd248667bf8 100644 --- a/tools/devctr/Dockerfile.aarch64 +++ b/tools/devctr/Dockerfile.aarch64 @@ -100,10 +100,8 @@ RUN cd "$TMP_POETRY_DIR" \ RUN mkdir "$TMP_BUILD_DIR" \ && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \ && rustup target add aarch64-unknown-linux-musl \ - && rustup component add clippy \ + && rustup component add clippy llvm-tools-preview \ && cd "$TMP_BUILD_DIR" \ - && cargo install cargo-kcov \ - && cargo kcov --print-install-kcov-sh | sh \ && rm -rf "$CARGO_HOME/registry" \ && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \ && rm -rf "$CARGO_HOME/git" \ diff --git a/tools/devctr/Dockerfile.x86_64 b/tools/devctr/Dockerfile.x86_64 index 25aaa8b9eb5..c4192ffbfc7 100644 --- a/tools/devctr/Dockerfile.x86_64 +++ b/tools/devctr/Dockerfile.x86_64 @@ -109,14 +109,12 @@ RUN (curl -sL https://deb.nodesource.com/setup_14.x | bash) \ RUN mkdir "$TMP_BUILD_DIR" \ && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \ && rustup target add x86_64-unknown-linux-musl \ - && rustup component add rustfmt clippy clippy-preview \ + && rustup component add rustfmt clippy clippy-preview llvm-tools-preview \ && rustup install --profile minimal "stable" \ && cd "$TMP_BUILD_DIR" \ - && cargo install cargo-kcov \ && cargo +"stable" install cargo-audit \ # Fix a version that does not require cargo edition 2021. && cargo install --locked cargo-deny --version '^0.9.1' \ - && cargo kcov --print-install-kcov-sh | sh \ && rm -rf "$CARGO_HOME/registry" \ && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \ && rm -rf "$CARGO_HOME/git" \ diff --git a/tools/devtool b/tools/devtool index 3d2c8ba790b..f141710c99b 100755 --- a/tools/devtool +++ b/tools/devtool @@ -72,7 +72,7 @@ DEVCTR_IMAGE_NO_TAG="public.ecr.aws/firecracker/fcuvm" # Development container tag -DEVCTR_IMAGE_TAG="v44" +DEVCTR_IMAGE_TAG="cpuid-grcov" # Development container image (name:tag) # This should be updated whenever we upgrade the development container.