Fixing dummy map file name to be consistent with PQFlashIndex and del… (

#588) * Fixing dummy map file name to be consistent with PQFlashIndex and deleted unnecessary files * Fixed code formatting * Upgrading upload-artifact to v4 --------- Co-authored-by: Gopal Srinivasa <[email protected]>
microsoft · Oct 6, 2024 · bae427e · bae427e
1 parent 54bdd93
commit bae427e
Show file tree

Hide file tree

Showing 10 changed files with 14 additions and 13 deletions.
diff --git a/.github/workflows/build-python-pdoc.yml b/.github/workflows/build-python-pdoc.yml
@@ -43,13 +43,13 @@ jobs:
           EOF
           )" >> $GITHUB_ENV
       - name: Archive documentation version artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: dependencies
           path: |
             dependencies_documentation.txt
       - name: Archive documentation artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: documentation-site
           path: |

diff --git a/.github/workflows/dynamic-labels.yml b/.github/workflows/dynamic-labels.yml
@@ -94,7 +94,7 @@ jobs:
           dist/bin/search_memory_index --data_type float --dist_fn l2 --fail_if_recall_below 70 --index_path_prefix data/index_rand_ins_del.after-concurrent-delete-del2500-7500 --result_path res_stream --query_file data/rand_float_10D_1K_norm1.0.bin --gt_file data/gt100_rand_random10D_1K -K 10 -L 20 40 60 80 100 -T 64
 
       - name: upload data and bin
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: dynamic
           path: |

diff --git a/.github/workflows/dynamic.yml b/.github/workflows/dynamic.yml
@@ -67,7 +67,7 @@ jobs:
           dist/bin/search_memory_index --data_type uint8 --dist_fn l2 --fail_if_recall_below 70 --index_path_prefix data/index_ins_del.after-concurrent-delete-del2500-7500 --result_path data/res_ins_del --query_file data/rand_uint8_10D_1K_norm50.0.bin --gt_file data/gt100_random10D_10K-conc-2500-7500 -K 10 -L 20 40 60 80 100 -T 8 --dynamic true --tags 1
 
       - name: upload data and bin
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: dynamic
           path: |

diff --git a/.github/workflows/in-mem-no-pq.yml b/.github/workflows/in-mem-no-pq.yml
@@ -73,7 +73,7 @@ jobs:
           dist/bin/search_memory_index --data_type uint8 --dist_fn cosine --fail_if_recall_below 70 --index_path_prefix data/index_l2_rand_uint8_10D_10K_norm50.0 --query_file data/rand_uint8_10D_1K_norm50.0.bin --recall_at 10 --result_path temp --gt_file data/cosine_rand_uint8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 -L  16 32
 
       - name: upload data and bin
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: in-memory-no-pq
           path: |

diff --git a/.github/workflows/in-mem-pq.yml b/.github/workflows/in-mem-pq.yml
@@ -48,7 +48,7 @@ jobs:
           dist/bin/search_memory_index --data_type uint8 --dist_fn l2 --fail_if_recall_below 70 --index_path_prefix data/index_l2_rand_uint8_10D_10K_norm50.0_buildpq5 --query_file data/rand_uint8_10D_1K_norm50.0.bin --recall_at 10 --result_path temp --gt_file data/l2_rand_uint8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 -L  16 32
 
       - name: upload data and bin
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: in-memory-pq
           path: |

diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml
@@ -111,7 +111,7 @@ jobs:
           dist/bin/search_memory_index --num_threads 48 --data_type uint8 --dist_fn l2 --filter_label 5 --index_path_prefix data/stit_zipf_32_100_64_new --query_file data/rand_uint8_10D_1K_norm50.0.bin --result_path data/zipf_stit_96_10_90_new --gt_file data/l2_zipf_uint8_10D_10K_norm50.0_10D_1K_norm50.0_gt100_wlabel -K 10 -L 16 32 150
 
       - name: upload data and bin
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: labels
           path: |

diff --git a/.github/workflows/multi-sector-disk-pq.yml b/.github/workflows/multi-sector-disk-pq.yml
@@ -52,7 +52,7 @@ jobs:
           dist/bin/search_disk_index --data_type int8 --dist_fn l2 --fail_if_recall_below 70 --index_path_prefix data/disk_index_l2_rand_int8_4096D_5K_norm1.0_diskfull_oneshot --result_path /tmp/res --query_file data/rand_int8_4096D_1K_norm1.0.bin --gt_file data/l2_rand_int8_4096D_5K_norm1.0_4096D_1K_norm1.0_gt100 --recall_at 5 -L 250 -W 2 --num_nodes_to_cache 100 -T 16
 
       - name: upload data and bin
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: multi-sector-disk-pq
           path: |

diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml
@@ -19,7 +19,7 @@ jobs:
           mkdir metrics
           docker run -v ./metrics:/app/logs perf &> ./metrics/combined_stdouterr.log
       - name: Upload Metrics Logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: metrics
           path: |

diff --git a/.github/workflows/push-test.yml b/.github/workflows/push-test.yml
@@ -41,7 +41,7 @@ jobs:
           echo "dependencies" > dependencies_${{ matrix.os }}.txt
           pipdeptree >> dependencies_${{ matrix.os }}.txt
       - name: Archive dispannpy dependencies artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: dependencies
           path: |

diff --git a/src/disk_utils.cpp b/src/disk_utils.cpp
@@ -1188,8 +1188,8 @@ int build_disk_index(const char *dataFilePath, const char *indexFilePath, const
     std::string mem_univ_label_file = mem_index_path + "_universal_label.txt";
     std::string disk_univ_label_file = disk_index_path + "_universal_label.txt";
     std::string disk_labels_int_map_file = disk_index_path + "_labels_map.txt";
-    std::string dummy_remap_file = disk_index_path + "_dummy_remap.txt"; // remap will be used if we break-up points of
-                                                                         // high label-density to create copies
+    std::string dummy_remap_file = disk_index_path + "_dummy_map.txt"; // remap will be used if we break-up points of
+                                                                       // high label-density to create copies
 
     std::string sample_base_prefix = index_prefix_path + "_sample";
     // optional, used if disk index file must store pq data
@@ -1274,7 +1274,6 @@ int build_disk_index(const char *dataFilePath, const char *indexFilePath, const
         augmented_labels_file = index_prefix_path + "_augmented_labels.txt";
         if (filter_threshold != 0)
         {
-            dummy_remap_file = index_prefix_path + "_dummy_remap.txt";
             breakup_dense_points<T>(data_file_to_use, labels_file_to_use, filter_threshold, augmented_data_file,
                                     augmented_labels_file,
                                     dummy_remap_file); // RKNOTE: This has large memory footprint,
@@ -1365,6 +1364,8 @@ int build_disk_index(const char *dataFilePath, const char *indexFilePath, const
     if (created_temp_file_for_processed_data)
         std::remove(prepped_base.c_str());
     std::remove(mem_index_path.c_str());
+    std::remove((mem_index_path + ".data").c_str());
+    std::remove((mem_index_path + ".tags").c_str());
     if (use_disk_pq)
         std::remove(disk_pq_compressed_vectors_path.c_str());