diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 305c7f3d1..2b470d9dc 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -8,7 +8,7 @@ runs: if: ${{ runner.os == 'Linux' }} run: | sudo scripts/dev/install-dev-deps-ubuntu.bash - cmake -S . -B build -DCMAKE_BUILD_TYPE=Release + cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DUNIT_TEST=True cmake --build build -- -j cmake --install build --prefix="dist" shell: bash @@ -20,9 +20,9 @@ runs: - name: Run configure and build for Windows if: runner.os == 'Windows' run: | - mkdir build && cd build && cmake .. && msbuild diskann.sln /m /nologo /t:Build /p:Configuration="Release" /property:Platform="x64" -consoleloggerparameters:"ErrorsOnly;Summary" + mkdir build && cd build && cmake .. -DUNIT_TEST=True && msbuild diskann.sln /m /nologo /t:Build /p:Configuration="Release" /property:Platform="x64" -consoleloggerparameters:"ErrorsOnly;Summary" cd .. mkdir dist - mv .\x64\Release\ .\dist\bin + mklink /j .\dist\bin .\x64\Release\ shell: cmd # ------------ End Windows Build --------------- \ No newline at end of file diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 6ed75d92f..1d6b98c0c 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -6,6 +6,9 @@ jobs: fail-fast: true name: DiskANN Common Build Checks uses: ./.github/workflows/common.yml + unit-tests: + name: Unit tests + uses: ./.github/workflows/unit-tests.yml in-mem-pq: name: In-Memory with PQ uses: ./.github/workflows/in-mem-pq.yml diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..6ae6877b8 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,32 @@ +name: Unit Tests +on: [workflow_call] +jobs: + acceptance-tests-labels: + name: Unit Tests + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-2019, windows-latest] + runs-on: ${{matrix.os}} + defaults: + run: + shell: bash + steps: + - name: Checkout repository + if: ${{ runner.os == 'Linux' }} + uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Checkout repository + if: ${{ runner.os == 'Windows' }} + uses: actions/checkout@v3 + with: + fetch-depth: 1 + submodules: true + - name: DiskANN Build CLI Applications + uses: ./.github/actions/build + + - name: Run Unit Tests + run: | + cd build + ctest -C Release \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8b92082ce..cfe47bed4 100644 --- a/.gitignore +++ b/.gitignore @@ -357,6 +357,7 @@ MigrationBackup/ cscope* build/ +build_linux/ !.github/actions/build # jetbrains specific stuff diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a6524a85..d4e3979b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -298,6 +298,11 @@ if (NOT PYBIND) add_subdirectory(apps/utils) endif() +if (UNIT_TEST) + enable_testing() + add_subdirectory(tests) +endif() + if (MSVC) message(STATUS "The ${PROJECT_NAME}.sln has been created, opened it from VisualStudio to build Release or Debug configurations.\n" "Alternatively, use MSBuild to build:\n\n" diff --git a/DockerfileDev b/DockerfileDev new file mode 100644 index 000000000..0e95e405f --- /dev/null +++ b/DockerfileDev @@ -0,0 +1,17 @@ +#Copyright(c) Microsoft Corporation.All rights reserved. +#Licensed under the MIT license. + +FROM ubuntu:jammy + +RUN apt update +RUN apt install -y software-properties-common +RUN add-apt-repository -y ppa:git-core/ppa +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -y git make cmake g++ libaio-dev libgoogle-perftools-dev libunwind-dev clang-format libboost-dev libboost-program-options-dev libboost-test-dev libmkl-full-dev libcpprest-dev python3.10 + +WORKDIR /app +RUN git clone https://github.com/microsoft/DiskANN.git +WORKDIR /app/DiskANN +RUN mkdir build +RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DUNIT_TEST=True +RUN cmake --build build -- -j diff --git a/include/parameters.h b/include/parameters.h index dea5509dd..81a336da7 100644 --- a/include/parameters.h +++ b/include/parameters.h @@ -6,6 +6,7 @@ #include #include +#include "omp.h" #include "defaults.h" namespace diskann diff --git a/scripts/dev/install-dev-deps-ubuntu.bash b/scripts/dev/install-dev-deps-ubuntu.bash index 329bbf09d..84f558ed6 100755 --- a/scripts/dev/install-dev-deps-ubuntu.bash +++ b/scripts/dev/install-dev-deps-ubuntu.bash @@ -8,4 +8,5 @@ apt install cmake \ clang-format \ libboost-dev \ libboost-program-options-dev \ + libboost-test-dev \ libmkl-full-dev \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..6af8405cc --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +set(CMAKE_COMPILE_WARNING_AS_ERROR ON) + +find_package(Boost COMPONENTS unit_test_framework) + +# For Windows, fall back to nuget version if find_package didn't find it. +if (MSVC AND NOT Boost_FOUND) + set(DISKANN_BOOST_INCLUDE "${DISKANN_MSVC_PACKAGES}/boost/lib/native/include") + # Multi-threaded static library. + set(UNIT_TEST_FRAMEWORK_LIB_PATTERN "${DISKANN_MSVC_PACKAGES}/boost_unit_test_framework-vc${MSVC_TOOLSET_VERSION}/lib/native/libboost_unit_test_framework-vc${MSVC_TOOLSET_VERSION}-mt-x64-*.lib") + file(GLOB DISKANN_BOOST_UNIT_TEST_FRAMEWORK_LIB ${UNIT_TEST_FRAMEWORK_LIB_PATTERN}) + + set(UNIT_TEST_FRAMEWORK_DLIB_PATTERN "${DISKANN_MSVC_PACKAGES}/boost_unit_test_framework-vc${MSVC_TOOLSET_VERSION}/lib/native/libboost_unit_test_framework-vc${MSVC_TOOLSET_VERSION}-mt-gd-x64-*.lib") + file(GLOB DISKANN_BOOST_UNIT_TEST_FRAMEWORK_DLIB ${UNIT_TEST_FRAMEWORK_DLIB_PATTERN}) + + if (EXISTS ${DISKANN_BOOST_INCLUDE} AND EXISTS ${DISKANN_BOOST_UNIT_TEST_FRAMEWORK_LIB} AND EXISTS ${DISKANN_BOOST_UNIT_TEST_FRAMEWORK_DLIB}) + set(Boost_FOUND ON) + set(Boost_INCLUDE_DIR ${DISKANN_BOOST_INCLUDE}) + add_library(Boost::unit_test_framework STATIC IMPORTED) + set_target_properties(Boost::unit_test_framework PROPERTIES IMPORTED_LOCATION_RELEASE "${DISKANN_BOOST_UNIT_TEST_FRAMEWORK_LIB}") + set_target_properties(Boost::unit_test_framework PROPERTIES IMPORTED_LOCATION_DEBUG "${DISKANN_BOOST_UNIT_TEST_FRAMEWORK_DLIB}") + message(STATUS "Falling back to using Boost from the nuget package") + else() + message(WARNING "Couldn't find Boost. Was looking for ${DISKANN_BOOST_INCLUDE} and ${UNIT_TEST_FRAMEWORK_LIB_PATTERN}") + endif() +endif() + +if (NOT Boost_FOUND) + message(FATAL_ERROR "Couldn't find Boost dependency") +endif() + + +set(DISKANN_UNIT_TEST_SOURCES main.cpp index_write_parameters_builder_tests.cpp) + +add_executable(${PROJECT_NAME}_unit_tests ${DISKANN_SOURCES} ${DISKANN_UNIT_TEST_SOURCES}) +target_link_libraries(${PROJECT_NAME}_unit_tests ${PROJECT_NAME} ${DISKANN_TOOLS_TCMALLOC_LINK_OPTIONS} Boost::unit_test_framework) + +add_test(NAME ${PROJECT_NAME}_unit_tests COMMAND ${PROJECT_NAME}_unit_tests) + diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..113c9980b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,11 @@ +# Unit Test project + +This unit test project is based on the [boost unit test framework](https://www.boost.org/doc/libs/1_78_0/libs/test/doc/html/index.html). Below are the simple steps to add new unit test, you could find more usage from the [boost unit test document](https://www.boost.org/doc/libs/1_78_0/libs/test/doc/html/index.html). + +## How to add unit test + +- Create new [BOOST_AUTO_TEST_SUITE](https://www.boost.org/doc/libs/1_78_0/libs/test/doc/html/boost_test/utf_reference/test_org_reference/test_org_boost_auto_test_suite.html) for each class in an individual cpp file + +- Add [BOOST_AUTO_TEST_CASE](https://www.boost.org/doc/libs/1_78_0/libs/test/doc/html/boost_test/utf_reference/test_org_reference/test_org_boost_auto_test_case.html) for each test case in the [BOOST_AUTO_TEST_SUITE](https://www.boost.org/doc/libs/1_78_0/libs/test/doc/html/boost_test/utf_reference/test_org_reference/test_org_boost_auto_test_suite.html) + +- Update the [CMakeLists.txt](CMakeLists.txt) file to add the new cpp file to the test project \ No newline at end of file diff --git a/tests/index_write_parameters_builder_tests.cpp b/tests/index_write_parameters_builder_tests.cpp new file mode 100644 index 000000000..acd5e2227 --- /dev/null +++ b/tests/index_write_parameters_builder_tests.cpp @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#include + +#include "parameters.h" + +BOOST_AUTO_TEST_SUITE(IndexWriteParametersBuilder_tests) + +BOOST_AUTO_TEST_CASE(test_build) +{ + uint32_t search_list_size = rand(); + uint32_t max_degree = rand(); + float alpha = (float)rand(); + uint32_t filter_list_size = rand(); + uint32_t max_occlusion_size = rand(); + uint32_t num_frozen_points = rand(); + bool saturate_graph = true; + + diskann::IndexWriteParametersBuilder builder(search_list_size, max_degree); + + builder.with_alpha(alpha) + .with_filter_list_size(filter_list_size) + .with_max_occlusion_size(max_occlusion_size) + .with_num_frozen_points(num_frozen_points) + .with_num_threads(0) + .with_saturate_graph(saturate_graph); + + { + auto parameters = builder.build(); + + BOOST_TEST(search_list_size == parameters.search_list_size); + BOOST_TEST(max_degree == parameters.max_degree); + BOOST_TEST(alpha == parameters.alpha); + BOOST_TEST(filter_list_size == parameters.filter_list_size); + BOOST_TEST(max_occlusion_size == parameters.max_occlusion_size); + BOOST_TEST(num_frozen_points == parameters.num_frozen_points); + BOOST_TEST(saturate_graph == parameters.saturate_graph); + + BOOST_TEST(parameters.num_threads > (uint32_t)0); + } + + { + uint32_t num_threads = rand() + 1; + saturate_graph = false; + builder.with_num_threads(num_threads) + .with_saturate_graph(saturate_graph); + + auto parameters = builder.build(); + + BOOST_TEST(search_list_size == parameters.search_list_size); + BOOST_TEST(max_degree == parameters.max_degree); + BOOST_TEST(alpha == parameters.alpha); + BOOST_TEST(filter_list_size == parameters.filter_list_size); + BOOST_TEST(max_occlusion_size == parameters.max_occlusion_size); + BOOST_TEST(num_frozen_points == parameters.num_frozen_points); + BOOST_TEST(saturate_graph == parameters.saturate_graph); + + BOOST_TEST(num_threads == parameters.num_threads); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/main.cpp b/tests/main.cpp new file mode 100644 index 000000000..53440a17a --- /dev/null +++ b/tests/main.cpp @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#define BOOST_TEST_MODULE diskann_unit_tests + +#include diff --git a/windows/packages.config.in b/windows/packages.config.in index d444e66b1..f8eecf02f 100644 --- a/windows/packages.config.in +++ b/windows/packages.config.in @@ -4,6 +4,7 @@ +