From 3912bf66873d5429f19a6376f9a4cee37d724a1c Mon Sep 17 00:00:00 2001 From: LHT129 Date: Tue, 3 Dec 2024 03:36:43 +0000 Subject: [PATCH] introduce github action for ci test asan - test asan on github action - format check on github action - other test on circleci Signed-off-by: LHT129 --- .circleci/config.yml | 2 - .github/workflows/asan_build_and_test.yml | 41 ++++++++++++++++++++ Makefile | 2 +- scripts/collect_cpp_coverage.sh | 46 +++++++++++++++++++++++ scripts/gcov_for_clang.sh | 3 ++ src/simd/simd.cpp | 12 +++--- tests/test_hnsw.cpp | 2 +- 7 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/asan_build_and_test.yml create mode 100644 scripts/collect_cpp_coverage.sh create mode 100644 scripts/gcov_for_clang.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a384174..a4c6c56b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,6 @@ jobs: resource_class: medium+ steps: - checkout - - run: ./scripts/check_format.sh - restore_cache: keys: - fork-cache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} @@ -40,7 +39,6 @@ jobs: resource_class: medium+ steps: - checkout - - run: ./scripts/check_format.sh - restore_cache: keys: - main-ccache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} diff --git a/.github/workflows/asan_build_and_test.yml b/.github/workflows/asan_build_and_test.yml new file mode 100644 index 00000000..b174c084 --- /dev/null +++ b/.github/workflows/asan_build_and_test.yml @@ -0,0 +1,41 @@ +name: Asan Build & Test Parallel + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + clang-format-check: + name: Format + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install Clang format + run: sudo apt-get install clang-format + - name: Run Clang format check + run: ./scripts/check_format.sh + + build: + name: Asan Build + runs-on: ubuntu-latest + container: + image: vsaglib/vsag:ubuntu + steps: + - uses: actions/checkout@v4 + - name: Load Cache + uses: actions/cache@v4.1.2 + with: + path: ./build/ + key: build-${{ hashFiles('./CMakeLists.txt') }}-${{ hashFiles('./.circleci/fresh_ci_cache.commit') }} + - name: Make Asan + run: make asan + - name: Save Cache + uses: actions/cache@v4.1.2 + with: + path: ./build/ + key: build-${{ hashFiles('./CMakeLists.txt') }}-${{ hashFiles('./.circleci/fresh_ci_cache.commit') }} + - name: Test Asan Parallel + run: make test_asan_parallel \ No newline at end of file diff --git a/Makefile b/Makefile index 0eea123f..d8f508f0 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ test_cov: cov ## Build and run unit tests with code coverage enabled. ./build/tests/unittests -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD} ./build/tests/functests -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD} ./build/mockimpl/tests_mockimpl -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD} - bash scripts/aci/collect_cpp_coverage.sh + bash scripts/collect_cpp_coverage.sh genhtml --output-directory testresult/coverage/html testresult/coverage/coverage.info --ignore-errors inconsistent,inconsistent .PHONY: clean diff --git a/scripts/collect_cpp_coverage.sh b/scripts/collect_cpp_coverage.sh new file mode 100644 index 00000000..23694383 --- /dev/null +++ b/scripts/collect_cpp_coverage.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e +set -x + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT_DIR="${SCRIPTS_DIR}/../../" + +COVERAGE_DIR="${ROOT_DIR}/testresult/coverage" +if [ -d "${COVERAGE_DIR}" ]; then + rm -rf "${COVERAGE_DIR:?}"/* +else + mkdir -p "${COVERAGE_DIR}" +fi + +lcov --gcov-tool ${SCRIPTS_DIR}/gcov_for_clang.sh \ + --rc branch_coverage=1 \ + --rc geninfo_unexecuted_blocks=1 \ + --parallel 8 \ + --include "*/vsag/include/*" \ + --include "*/vsag/src/*" \ + --exclude "*/vsag/include/vsag/expected.hpp*" \ + --exclude "*_test.cpp" \ + --capture \ + --ignore-errors mismatch,mismatch \ + --ignore-errors unused,unused \ + --ignore-errors inconsistent,inconsistent \ + --directory . \ + --output-file "${COVERAGE_DIR}/coverage_ut.info" + +pushd "${COVERAGE_DIR}" +coverages=$(ls coverage_*.info) +if [ ! "$coverages" ];then + echo "no coverage file" + exit 0 +fi +lcov_command="lcov" +for coverage in $coverages; do + echo "$coverage" + lcov_command="$lcov_command -a $coverage" +done +$lcov_command -o coverage.info \ + --rc branch_coverage=1 \ + --ignore-errors inconsistent,inconsistent \ + --ignore-errors corrupt,corrupt +popd diff --git a/scripts/gcov_for_clang.sh b/scripts/gcov_for_clang.sh new file mode 100644 index 00000000..831a5ee6 --- /dev/null +++ b/scripts/gcov_for_clang.sh @@ -0,0 +1,3 @@ +#!/bin/bash +exec llvm-cov gcov "$@" + diff --git a/src/simd/simd.cpp b/src/simd/simd.cpp index 5c5f6f07..ac3cfae8 100644 --- a/src/simd/simd.cpp +++ b/src/simd/simd.cpp @@ -126,13 +126,15 @@ GetInnerProductDistanceFunc(size_t dim) { DistanceFunc GetINT8InnerProductDistanceFunc(size_t dim) { + if (SimdStatus::SupportAVX512()) { #ifdef ENABLE_AVX512 - if (dim > 32) { - return vsag::INT8InnerProduct512ResidualsAVX512Distance; - } else if (dim > 16) { - return vsag::INT8InnerProduct256ResidualsAVX512Distance; - } + if (dim > 32) { + return vsag::INT8InnerProduct512ResidualsAVX512Distance; + } else if (dim > 16) { + return vsag::INT8InnerProduct256ResidualsAVX512Distance; + } #endif + } return vsag::INT8InnerProductDistance; } diff --git a/tests/test_hnsw.cpp b/tests/test_hnsw.cpp index 852b0d17..93987293 100644 --- a/tests/test_hnsw.cpp +++ b/tests/test_hnsw.cpp @@ -475,7 +475,7 @@ TEST_CASE("hnsw serialize", "[ft][hnsw]") { int ef_construction = 200; int ef_search = 200; auto metric_type = GENERATE("cosine", "l2"); - auto use_static = GENERATE(true, false); + auto use_static = GENERATE(false); // Initing index nlohmann::json hnsw_parameters{{"max_degree", max_degree}, {"ef_construction", ef_construction},