Skip to content

Commit

Permalink
add more results from linux
Browse files Browse the repository at this point in the history
  • Loading branch information
ravishankar committed Oct 30, 2024
1 parent 7f99c6a commit 38e3c1a
Show file tree
Hide file tree
Showing 11 changed files with 266 additions and 7 deletions.
6 changes: 3 additions & 3 deletions apps/build_memory_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ namespace po = boost::program_options;
int main(int argc, char **argv)
{
std::string data_type, dist_fn, data_path, index_path_prefix, label_file, universal_label, label_type;
uint32_t num_threads, R, L, Lf, build_PQ_bytes, reduce_prune;
uint32_t num_threads, R, L, Lf, build_PQ_bytes;
float alpha;
bool use_pq_build, use_opq;

Expand Down Expand Up @@ -70,9 +70,9 @@ int main(int argc, char **argv)
program_options_utils::FILTERED_LBUILD);
optional_configs.add_options()("label_type", po::value<std::string>(&label_type)->default_value("uint"),
program_options_utils::LABEL_TYPE_DESCRIPTION);
optional_configs.add_options()("reduce_prune", po::value<uint32_t>(&reduce_prune)->default_value(0),"reduce_prune");
//optional_configs.add_options()("reduce_prune", po::bool_switch(&reduce_prune)->default_value(false),"reduce_prune");

std::cout<<"reduce_prune: "<<reduce_prune<<std::endl;
//std::cout<<"reduce_prune: "<<reduce_prune<<std::endl;

// Merge required and optional parameters
desc.add(required_configs).add(optional_configs);
Expand Down
2 changes: 1 addition & 1 deletion include/index.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
#define EXPAND_IF_FULL 0
#define DEFAULT_MAXC 750

//extern uint32_t reduce_prune;
//extern bool reduce_prune;
namespace diskann
{

Expand Down
56 changes: 56 additions & 0 deletions linux_results/baseline/r32_k50.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries.fbin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 20375040 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/prec1M_gt100.bin, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/prec1M_gt100.bin ...
Metadata: #pts = 13265, #dims = 100...
L2: Using AVX2 distance computation DistanceL2Float
Resizing took: 0.0399196s
From graph header, expected_file_size: 102834512, _max_observed_degree: 32, _start: 121334, file_frozen_pts: 0
Loading vamana graph /home/rakri/avarhade/Dump/prec_memory_index_r32_l100...done. Index has 1010812 nodes and 24697810 out-edges, _start is set to 121334
Num frozen points:0 _nd: 1010812 _start: 121334 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 1010812
Index loaded
Using 48 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50
=================================================================================
50 11674.91 1795.90 4087.20 12787.71 47.27
60 11893.21 2037.37 4020.38 10736.31 50.39
70 12110.38 2275.12 3949.26 9390.59 53.10
80 12264.40 2503.44 3900.47 8996.04 55.32
90 12326.87 2730.00 3881.30 8868.14 57.24
100 12305.29 2955.04 3886.87 10258.41 58.99
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
9 changes: 9 additions & 0 deletions linux_results/baseline/r32_l100.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Starting index build with R: 32 Lbuild: 100 alpha: 1.2 #threads: 48
L2: Using AVX2 distance computation DistanceL2Float
Using only first 1010812 from file..
Starting index build with 1010812 points...
0% of index build completed.9.89304% of index build completed.19.7861% of index build completed.29.6791% of index build completed.39.5721% of index build completed.49.4652% of index build completed.59.3582% of index build completed.69.2513% of index build completed.79.1443% of index build completed.89.0373% of index build completed.98.9304% of index build completed.Starting final cleanup..done. Link time: 60.7324s
Index built with degree: max:32 avg:24.4336 min:1 count(deg<2):34623
Indexing time: 61.4367
Not saving tags as they are not enabled.
Time taken for save: 2.03361s.
Expand Down
56 changes: 56 additions & 0 deletions linux_results/baseline/r64_k50.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries.fbin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 20375040 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/prec1M_gt100.bin, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/prec1M_gt100.bin ...
Metadata: #pts = 13265, #dims = 100...
L2: Using AVX2 distance computation DistanceL2Float
Resizing took: 0.0357637s
From graph header, expected_file_size: 187075060, _max_observed_degree: 64, _start: 121334, file_frozen_pts: 0
Loading vamana graph /home/rakri/avarhade/Dump/prec_memory_index_r64_l100...done. Index has 1010812 nodes and 45757947 out-edges, _start is set to 121334
Num frozen points:0 _nd: 1010812 _start: 121334 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 1010812
Index loaded
Using 48 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50
=================================================================================
50 6963.86 3189.04 6866.28 20730.67 63.48
60 7561.50 3642.34 6331.25 14469.44 66.48
70 7903.99 4090.72 6055.25 12422.36 69.00
80 8168.08 4533.11 5859.89 12460.41 71.06
90 7786.78 4970.30 6144.71 13538.97 72.82
100 7335.64 5404.81 6516.13 23234.51 74.38
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
9 changes: 9 additions & 0 deletions linux_results/baseline/r64_l100.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Starting index build with R: 64 Lbuild: 100 alpha: 1.2 #threads: 48
L2: Using AVX2 distance computation DistanceL2Float
Using only first 1010812 from file..
Starting index build with 1010812 points...
0% of index build completed.9.89304% of index build completed.19.7861% of index build completed.29.6791% of index build completed.39.5721% of index build completed.49.4652% of index build completed.59.3582% of index build completed.69.2513% of index build completed.79.1443% of index build completed.89.0373% of index build completed.98.9304% of index build completed.Starting final cleanup..done. Link time: 99.1661s
Index built with degree: max:64 avg:45.2685 min:1 count(deg<2):32781
Indexing time: 99.8501
Not saving tags as they are not enabled.
Time taken for save: 2.12821s.
Expand Down
56 changes: 56 additions & 0 deletions linux_results/reduce_prune/r32_k50.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries.fbin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 20375040 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/prec1M_gt100.bin, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/prec1M_gt100.bin ...
Metadata: #pts = 13265, #dims = 100...
L2: Using AVX2 distance computation DistanceL2Float
Resizing took: 0.0397607s
From graph header, expected_file_size: 104455332, _max_observed_degree: 32, _start: 121334, file_frozen_pts: 0
Loading vamana graph /home/rakri/avarhade/Dump/prec_memory_index_reduce_prune_r32_l100...done. Index has 1010812 nodes and 25103015 out-edges, _start is set to 121334
Num frozen points:0 _nd: 1010812 _start: 121334 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 1010812
Index loaded
Using 48 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50
=================================================================================
50 13183.78 1623.59 3618.12 12777.66 45.36
60 13067.15 1838.40 3657.08 9479.67 48.62
70 13165.96 2047.06 3631.82 8503.63 51.32
80 13209.41 2252.19 3619.46 8429.98 53.67
90 13249.98 2452.17 3609.54 8649.75 55.66
100 13320.25 2649.74 3589.92 8799.18 57.41
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
9 changes: 9 additions & 0 deletions linux_results/reduce_prune/r32_l100.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Starting index build with R: 32 Lbuild: 100 alpha: 1.2 #threads: 48
L2: Using AVX2 distance computation DistanceL2Float
Using only first 1010812 from file..
Starting index build with 1010812 points...
0% of index build completed.9.89304% of index build completed.19.7861% of index build completed.29.6791% of index build completed.39.5721% of index build completed.49.4652% of index build completed.59.3582% of index build completed.69.2513% of index build completed.79.1443% of index build completed.89.0373% of index build completed.98.9304% of index build completed.Starting final cleanup..done. Link time: 57.5361s
Index built with degree: max:32 avg:24.8345 min:4 count(deg<2):0
Indexing time: 58.2334
Not saving tags as they are not enabled.
Time taken for save: 2.0286s.
Expand Down
56 changes: 56 additions & 0 deletions linux_results/reduce_prune/r64_k50.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries.fbin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 20375040 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/prec1M_gt100.bin, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/prec1M_gt100.bin ...
Metadata: #pts = 13265, #dims = 100...
L2: Using AVX2 distance computation DistanceL2Float
Resizing took: 0.0388856s
From graph header, expected_file_size: 201452296, _max_observed_degree: 64, _start: 121334, file_frozen_pts: 0
Loading vamana graph /home/rakri/avarhade/Dump/prec_memory_index_reduce_prune_r64_l100...done. Index has 1010812 nodes and 49352256 out-edges, _start is set to 121334
Num frozen points:0 _nd: 1010812 _start: 121334 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 1010812
Index loaded
Using 48 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50
=================================================================================
50 7914.20 2791.77 6040.13 16719.30 61.43
60 8342.94 3192.20 5736.71 12783.12 64.61
70 8596.86 3584.60 5568.84 12889.46 67.19
80 8866.31 3972.00 5397.79 11050.67 69.36
90 8810.74 4355.74 5433.07 10118.80 71.23
100 8195.36 4735.37 5840.12 10296.32 72.82
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
9 changes: 9 additions & 0 deletions linux_results/reduce_prune/r64_l100.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Starting index build with R: 64 Lbuild: 100 alpha: 1.2 #threads: 48
L2: Using AVX2 distance computation DistanceL2Float
Using only first 1010812 from file..
Starting index build with 1010812 points...
0% of index build completed.9.89304% of index build completed.19.7861% of index build completed.29.6791% of index build completed.39.5721% of index build completed.49.4652% of index build completed.59.3582% of index build completed.69.2513% of index build completed.79.1443% of index build completed.89.0373% of index build completed.98.9304% of index build completed.Starting final cleanup..done. Link time: 89.0615s
Index built with degree: max:64 avg:48.8244 min:4 count(deg<2):0
Indexing time: 89.7581
Not saving tags as they are not enabled.
Time taken for save: 2.33791s.
Expand Down
5 changes: 2 additions & 3 deletions src/index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

#define MAX_POINTS_FOR_USING_BITSET 10000000

//uint32_t reduce_prune = 0;
// bool reduce_prune = false;

namespace diskann
{
Expand Down Expand Up @@ -1329,9 +1329,8 @@ template <typename T, typename TagT, typename LabelT> void Index<T, TagT, LabelT
LockGuard guard(_locks[node]);
std::vector<uint32_t> reduced_pruned_list = pruned_list;

bool reduce_prune = true;
bool reduce_prune = false;
if (reduce_prune){
//std::cout<<"reduce prune"<<reduce_prune<<std::endl;
reduced_pruned_list.resize((uint32_t)_indexingRange/2);
}
_graph_store->set_neighbours(node, reduced_pruned_list);
Expand Down

0 comments on commit 38e3c1a

Please sign in to comment.