Skip to content

Commit

Permalink
add debug lines, remove old debug lines
Browse files Browse the repository at this point in the history
  • Loading branch information
jparismorgan committed Jul 1, 2024
1 parent 2d8e8ec commit fbe2596
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 7 deletions.
2 changes: 2 additions & 0 deletions src/include/index/ivf_pq_index.h
Original file line number Diff line number Diff line change
Expand Up @@ -1200,6 +1200,8 @@ class ivf_pq_index {
::num_vectors(*partitioned_pq_vectors_) == 0) {
read_index_infinite();
}
debug_partitioned_matrix(
*partitioned_pq_vectors_, "partitioned_pq_vectors_");
auto&& [active_partitions, active_queries] =
detail::ivf::partition_ivf_flat_index<indices_type>(
flat_ivf_centroids_, query_vectors, nprobe, num_threads_);
Expand Down
2 changes: 0 additions & 2 deletions src/include/test/unit_api_ivf_pq_index.cc
Original file line number Diff line number Diff line change
Expand Up @@ -860,8 +860,6 @@ TEST_CASE("write and load index with timestamps", "[api_ivf_pq_index]") {
scores_vector_array.num_vectors());
auto ids = std::span<uint32_t>(
(uint32_t*)ids_vector_array.data(), ids_vector_array.num_vectors());
debug_vector(scores, "scores");
debug_vector(ids, "ids");

CHECK(std::equal(
scores.begin(),
Expand Down
28 changes: 23 additions & 5 deletions src/include/test/unit_ivf_pq_index.cc
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,8 @@ TEST_CASE(
TEMPLATE_TEST_CASE(
"query stacked hypercube", "[flativf_index]", float, uint8_t) {
size_t k_dist = GENERATE(0, 32);
std::cout << "k_dist: " << k_dist
<< " -----------------------------------------" << std::endl;
size_t k_near = k_dist;
size_t k_far = k_dist;

Expand All @@ -399,6 +401,11 @@ TEMPLATE_TEST_CASE(
hypercube4(j + 9, i) = hypercube1(j, i);
}
}
debug_matrix(hypercube0, "hypercube0");
debug_matrix(hypercube1, "hypercube1");
debug_matrix(hypercube2, "hypercube2");
debug_matrix(hypercube4, "hypercube4");
debug_vector(ids, "ids");

size_t k_nn = 6;
size_t nlist = 1;
Expand All @@ -418,25 +425,31 @@ TEMPLATE_TEST_CASE(
auto query2 = ColMajorMatrix<TestType>();
auto query4 = ColMajorMatrix<TestType>();
SECTION("query with all 0's") {
std::cout << "query with all 0's ------------" << std::endl;
query2 = ColMajorMatrix<TestType>{{0, 0, 0, 0, 0, 0}};
query4 = ColMajorMatrix<TestType>{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
}
SECTION("query with all 127's") {
std::cout << "query with all 127's ------------" << std::endl;
query2 = ColMajorMatrix<TestType>{{127, 127, 127, 127, 127, 127}};
query4 = ColMajorMatrix<TestType>{
{127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127}};
}
SECTION("query with all 0's and then 127's") {
std::cout << "query with all 0's and then 127's ------------" << std::endl;
query2 = ColMajorMatrix<TestType>{{0, 0, 0, 127, 127, 127}};
query4 = ColMajorMatrix<TestType>{
{0, 0, 0, 0, 0, 0, 127, 127, 127, 127, 127, 127}};
}
SECTION("query with all 127's and then 0's") {
std::cout << "query with all 127's and then 0's ------------" << std::endl;
query2 = ColMajorMatrix<TestType>{{127, 127, 127, 0, 0, 0}};
query4 = ColMajorMatrix<TestType>{
{127, 127, 127, 127, 127, 127, 0, 0, 0, 0, 0, 0}};
}
SECTION("query with alternating 127's and 0's") {
std::cout << "query with alternating 127's and 0's ------------"
<< std::endl;
query2 = ColMajorMatrix<TestType>{
{127, 0, 127, 0, 127, 0}, {0, 127, 0, 127, 0, 127}};
query4 = ColMajorMatrix<TestType>{
Expand All @@ -449,19 +462,29 @@ TEMPLATE_TEST_CASE(
auto top_k_ivf_scores = ColMajorMatrix<float>();
auto top_k_ivf_ids = ColMajorMatrix<uint32_t>();

std::cout << "query2 ~~~~~~" << std::endl;
std::tie(top_k_scores, top_k_ids) = detail::flat::qv_query_heap(
hypercube2, query2, k_nn, 1, sum_of_squares_distance{});
debug_matrix(top_k_ids, "top_k_ids");
debug_matrix(top_k_scores, "top_k_scores");
std::tie(top_k_ivf_scores, top_k_ivf_ids) =
ivf_idx2.query_infinite_ram(query2, k_nn, nlist);
debug_matrix(top_k_ivf_ids, "top_k_ivf_ids");
debug_matrix(top_k_ivf_scores, "top_k_ivf_scores");
size_t intersections0 = count_intersections(top_k_ivf_ids, top_k_ids, k_nn);
double recall0 = intersections0 / ((double)top_k_ids.num_cols() * k_nn);
CHECK(intersections0 == k_nn * num_vectors(query2));
CHECK(recall0 == 1.0);

std::cout << "query4 ~~~~~~" << std::endl;
std::tie(top_k_scores, top_k_ids) = detail::flat::qv_query_heap(
hypercube4, query4, k_nn, 1, sum_of_squares_distance{});
debug_matrix(top_k_ids, "top_k_ids");
debug_matrix(top_k_scores, "top_k_scores");
std::tie(top_k_ivf_scores, top_k_ivf_ids) =
ivf_idx4.query_infinite_ram(query4, k_nn, nlist);
debug_matrix(top_k_ivf_ids, "top_k_ivf_ids");
debug_matrix(top_k_ivf_scores, "top_k_ivf_scores");
size_t intersections1 =
(long)count_intersections(top_k_ivf_ids, top_k_ids, k_nn);
double recall1 = intersections1 / ((double)top_k_ids.num_cols() * k_nn);
Expand Down Expand Up @@ -586,7 +609,6 @@ TEST_CASE("query empty index", "[ivf_pq_index]") {
{
auto data =
ColMajorMatrixWithIds<siftsmall_feature_type>(dimensions, num_vectors);
debug_matrix_with_ids(data, "data");
index.train(data, data.raveled_ids());
index.add(data, data.raveled_ids());
}
Expand Down Expand Up @@ -661,8 +683,6 @@ TEST_CASE("query simple", "[ivf_pq_index]") {
auto value = static_cast<feature_type>(i);
auto queries = ColMajorMatrix<feature_type>{{value, value, value, value}};
auto&& [scores, ids] = index.query_infinite_ram(queries, k_nn, nprobe);
debug_matrix(scores, "scores");
debug_matrix(ids, "ids");
CHECK(scores(0, 0) == 0);
CHECK(ids(0, 0) == i * 11);
}
Expand All @@ -682,8 +702,6 @@ TEST_CASE("query simple", "[ivf_pq_index]") {
auto value = static_cast<feature_type>(i);
auto queries = ColMajorMatrix<feature_type>{{value, value, value, value}};
auto&& [scores, ids] = index.query_infinite_ram(queries, k_nn, nprobe);
debug_matrix(scores, "scores");
debug_matrix(ids, "ids");
CHECK(scores(0, 0) == 0);
CHECK(ids(0, 0) == i * 11);
}
Expand Down

0 comments on commit fbe2596

Please sign in to comment.