Skip to content

Commit 318908d

Browse files
ljcuiljcui
committed
update
1 parent a621462 commit 318908d

15 files changed

+286
-78
lines changed

‎src/core/index_manager.cpp

+5-7
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,6 @@ IndexManager::IndexManager(KvTransaction& txn, SchemaManager* v_schema_manager,
109109
new HNSW(idx.label, idx.field, idx.distance_type, idx.index_type,
110110
idx.dimension, {idx.hnsw_m, idx.hnsw_ef_construction})));
111111
uint64_t count = 0;
112-
std::vector<std::vector<float>> floatvector;
113-
std::vector<int64_t> vids;
114112
auto kv_iter = schema->GetPropertyTable().GetIterator(txn);
115113
for (kv_iter->GotoFirstKey(); kv_iter->IsValid(); kv_iter->Next()) {
116114
auto prop = kv_iter->GetValue();
@@ -119,14 +117,14 @@ IndexManager::IndexManager(KvTransaction& txn, SchemaManager* v_schema_manager,
119117
}
120118
auto vid = graph::KeyPacker::GetVidFromPropertyTableKey(kv_iter->GetKey());
121119
auto vector = (extractor->GetConstRef(prop)).AsType<std::vector<float>>();
122-
floatvector.emplace_back(vector);
123-
vids.emplace_back(vid);
120+
vsag_index->Add({std::move(vector)}, {vid});
124121
count++;
122+
if ((count % 10000) == 0) {
123+
LOG_INFO() << "vector index count: " << count;
124+
}
125125
}
126-
vsag_index->Build();
127-
vsag_index->Add(floatvector, vids, count);
128126
kv_iter.reset();
129-
LOG_DEBUG() << "index count: " << count;
127+
LOG_INFO() << "vector index count: " << count;
130128
schema->MarkVectorIndexed(extractor->GetFieldId(), vsag_index.release());
131129
LOG_INFO() << FMA_FMT("end building vertex vector index for {}:{} in detached model",
132130
idx.label, idx.field);

‎src/core/index_manager.h

+20-4
Original file line numberDiff line numberDiff line change
@@ -289,43 +289,59 @@ class IndexManager {
289289
std::vector<VectorIndexSpec> ListVectorIndex(KvTransaction& txn);
290290

291291
// vertex index
292-
std::pair<std::vector<IndexSpec>, std::vector<CompositeIndexSpec>> ListAllIndexes(
292+
std::tuple<std::vector<IndexSpec>, std::vector<CompositeIndexSpec>,
293+
std::vector<VectorIndexSpec>> ListAllIndexes(
293294
KvTransaction& txn) {
294295
std::vector<IndexSpec> indexes;
295296
std::vector<CompositeIndexSpec> compositeIndexes;
296-
IndexSpec is;
297-
CompositeIndexSpec cis;
297+
std::vector<VectorIndexSpec> vectorIndexes;
298298
size_t v_index_len = strlen(_detail::VERTEX_INDEX);
299299
size_t e_index_len = strlen(_detail::EDGE_INDEX);
300300
size_t c_index_len = strlen(_detail::COMPOSITE_INDEX);
301+
size_t ve_index_len = strlen(_detail::VERTEX_VECTOR_INDEX);
301302
auto it = index_list_table_->GetIterator(txn);
302303
for (it->GotoFirstKey(); it->IsValid(); it->Next()) {
303304
std::string index_name = it->GetKey().AsString();
304305
if (index_name.size() > v_index_len &&
305306
index_name.substr(index_name.size() - v_index_len) == _detail::VERTEX_INDEX) {
306307
_detail::IndexEntry ent = LoadIndex(it->GetValue());
308+
IndexSpec is;
307309
is.label = ent.label;
308310
is.field = ent.field;
309311
is.type = ent.type;
310312
indexes.emplace_back(std::move(is));
311313
} else if (index_name.size() > e_index_len &&
312314
index_name.substr(index_name.size() - e_index_len) == _detail::EDGE_INDEX) {
313315
_detail::IndexEntry ent = LoadIndex(it->GetValue());
316+
IndexSpec is;
314317
is.label = ent.label;
315318
is.field = ent.field;
316319
is.type = ent.type;
317320
indexes.emplace_back(std::move(is));
321+
} else if (index_name.size() > ve_index_len &&
322+
index_name.substr(index_name.size() - ve_index_len) == _detail::VERTEX_VECTOR_INDEX) {
323+
_detail::VectorIndexEntry ent = LoadVectorIndex(it->GetValue());
324+
VectorIndexSpec vis;
325+
vis.label = ent.label;
326+
vis.field = ent.field;
327+
vis.distance_type = ent.distance_type;
328+
vis.dimension = ent.dimension;
329+
vis.hnsw_ef_construction = ent.hnsw_ef_construction;
330+
vis.hnsw_m = ent.hnsw_m;
331+
vis.index_type = ent.index_type;
332+
vectorIndexes.emplace_back(vis);
318333
} else if (index_name.size() > c_index_len &&
319334
index_name.substr(index_name.size() - c_index_len) ==
320335
_detail::COMPOSITE_INDEX) {
321336
_detail::CompositeIndexEntry idx = LoadCompositeIndex(it->GetValue());
337+
CompositeIndexSpec cis;
322338
cis.label = idx.label;
323339
cis.fields = idx.field_names;
324340
cis.type = idx.index_type;
325341
compositeIndexes.emplace_back(std::move(cis));
326342
}
327343
}
328-
return {indexes, compositeIndexes};
344+
return {std::move(indexes), std::move(compositeIndexes), std::move(vectorIndexes)};
329345
}
330346
};
331347
} // namespace lgraph

‎src/core/lightning_graph.cpp

+24-9
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ void LightningGraph::DropAllVertex() {
6666
Transaction txn = CreateWriteTxn(false);
6767
ScopedRef<SchemaInfo> curr_schema = schema_.GetScopedRef();
6868
// clear indexes
69-
auto [indexes, composite_indexes] = index_manager_->ListAllIndexes(txn.GetTxn());
69+
auto [indexes, composite_indexes, vector_indexes] = index_manager_->ListAllIndexes(txn.GetTxn());
7070
for (auto& idx : indexes) {
7171
auto v_schema = curr_schema->v_schema_manager.GetSchema(idx.label);
7272
auto e_schema = curr_schema->e_schema_manager.GetSchema(idx.label);
@@ -89,6 +89,15 @@ void LightningGraph::DropAllVertex() {
8989
v_schema->GetCompositeIndex(idx.fields)->Clear(txn.GetTxn());
9090
}
9191
}
92+
for (auto& idx : vector_indexes) {
93+
auto v_schema = curr_schema->v_schema_manager.GetSchema(idx.label);
94+
FMA_DBG_ASSERT(v_schema);
95+
if (v_schema) {
96+
auto ext = v_schema->GetFieldExtractor(idx.field);
97+
FMA_DBG_ASSERT(ext);
98+
ext->GetVectorIndex()->Clear();
99+
}
100+
}
92101
// clear detached property data
93102
for (auto& name : curr_schema->v_schema_manager.GetAllLabels()) {
94103
auto s = curr_schema->v_schema_manager.GetSchema(name);
@@ -2219,8 +2228,6 @@ bool LightningGraph::BlockingAddVectorIndex(bool is_vertex, const std::string& l
22192228
label, field);
22202229
VectorIndex* index = extractor->GetVectorIndex();
22212230
uint64_t count = 0;
2222-
std::vector<std::vector<float>> floatvector;
2223-
std::vector<int64_t> vids;
22242231
auto dim = index->GetVecDimension();
22252232
auto kv_iter = schema->GetPropertyTable().GetIterator(txn.GetTxn());
22262233
for (kv_iter->GotoFirstKey(); kv_iter->IsValid(); kv_iter->Next()) {
@@ -2234,13 +2241,13 @@ bool LightningGraph::BlockingAddVectorIndex(bool is_vertex, const std::string& l
22342241
THROW_CODE(VectorIndexException,
22352242
"vector size error, size:{}, dim:{}", vector.size(), dim);
22362243
}
2237-
floatvector.emplace_back(std::move(vector));
2238-
vids.emplace_back(vid);
2244+
index->Add({std::move(vector)}, {vid});
22392245
count++;
2246+
if ((count % 10000) == 0) {
2247+
LOG_INFO() << "vector index count: " << count;
2248+
}
22402249
}
2241-
index->Build();
2242-
index->Add(floatvector, vids, count);
2243-
LOG_INFO() << "index count: " << count;
2250+
LOG_INFO() << "vector index count: " << count;
22442251
LOG_INFO() << FMA_FMT("end building vertex vector index for {}:{} in detached model",
22452252
label, field);
22462253
kv_iter.reset();
@@ -2809,7 +2816,7 @@ void LightningGraph::DropAllIndex() {
28092816
ScopedRef<SchemaInfo> curr_schema = schema_.GetScopedRef();
28102817
std::unique_ptr<SchemaInfo> new_schema(new SchemaInfo(*curr_schema.Get()));
28112818
std::unique_ptr<SchemaInfo> backup_schema(new SchemaInfo(*curr_schema.Get()));
2812-
auto [indexes, composite_indexes] = index_manager_->ListAllIndexes(txn.GetTxn());
2819+
auto [indexes, composite_indexes, vector_indexes] = index_manager_->ListAllIndexes(txn.GetTxn());
28132820

28142821
bool success = true;
28152822
for (auto& idx : indexes) {
@@ -2840,6 +2847,14 @@ void LightningGraph::DropAllIndex() {
28402847
}
28412848
v_schema->UnVertexCompositeIndex(idx.fields);
28422849
}
2850+
for (auto& idx : vector_indexes) {
2851+
auto v_schema = new_schema->v_schema_manager.GetSchema(idx.label);
2852+
FMA_DBG_ASSERT(v_schema);
2853+
auto ret = index_manager_->DeleteVectorIndex(txn.GetTxn(), idx.label, idx.field);
2854+
FMA_DBG_ASSERT(ret);
2855+
auto ext = v_schema->GetFieldExtractor(idx.field);
2856+
v_schema->UnVectorIndex(ext->GetFieldId());
2857+
}
28432858
if (success) {
28442859
schema_.Assign(new_schema.release());
28452860
AutoCleanupAction revert_assign_new_schema(

‎src/core/schema.cpp

+2-4
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ void Schema::AddVectorToVectorIndex(KvTransaction& txn, VertexId vid, const Valu
314314
"vector index dimension mismatch, vector size:{}, dim:{}",
315315
floatvector.back().size(), dim);
316316
}
317-
index->Add(floatvector, vids, 1);
317+
index->Add(floatvector, vids);
318318
}
319319
}
320320

@@ -323,9 +323,7 @@ void Schema::DeleteVectorIndex(KvTransaction& txn, VertexId vid, const Value& re
323323
auto& fe = fields_[idx];
324324
if (fe.GetIsNull(record)) continue;
325325
VectorIndex* index = fe.GetVectorIndex();
326-
std::vector<int64_t> vids;
327-
vids.push_back(vid);
328-
index->Add({}, vids, 0);
326+
index->Remove({vid});
329327
}
330328
}
331329

‎src/core/transaction.cpp

+40-2
Original file line numberDiff line numberDiff line change
@@ -968,8 +968,46 @@ Transaction::SetVertexProperty(VertexIterator& it, size_t n_fields, const FieldT
968968
// no need to update index since blob cannot be indexed
969969
} else if (fe->Type() == FieldType::FLOAT_VECTOR) {
970970
fe->ParseAndSet(new_prop, values[i]);
971-
schema->DeleteVectorIndex(*txn_, vid, old_prop);
972-
schema->AddVectorToVectorIndex(*txn_, vid, new_prop);
971+
VectorIndex* index = fe->GetVectorIndex();
972+
if (index) {
973+
bool oldnull = fe->GetIsNull(old_prop);
974+
bool newnull = fe->GetIsNull(new_prop);
975+
std::vector<int64_t> vids {vid};
976+
if (!oldnull && !newnull) {
977+
const auto& old_v = fe->GetConstRef(old_prop);
978+
const auto& new_v = fe->GetConstRef(new_prop);
979+
if (old_v == new_v) {
980+
continue;
981+
}
982+
// delete
983+
index->Remove(vids);
984+
// add
985+
auto dim = index->GetVecDimension();
986+
std::vector<std::vector<float>> floatvector;
987+
floatvector.emplace_back(new_v.AsFloatVector());
988+
if (floatvector.back().size() != (size_t)dim) {
989+
THROW_CODE(InputError,
990+
"vector index dimension mismatch, vector size:{}, dim:{}",
991+
floatvector.back().size(), dim);
992+
}
993+
index->Add(floatvector, vids);
994+
} else if (oldnull && !newnull) {
995+
// add
996+
const auto& new_v = fe->GetConstRef(new_prop);
997+
auto dim = index->GetVecDimension();
998+
std::vector<std::vector<float>> floatvector;
999+
floatvector.emplace_back(new_v.AsFloatVector());
1000+
if (floatvector.back().size() != (size_t)dim) {
1001+
THROW_CODE(InputError,
1002+
"vector index dimension mismatch, vector size:{}, dim:{}",
1003+
floatvector.back().size(), dim);
1004+
}
1005+
index->Add(floatvector, vids);
1006+
} else if (!oldnull && newnull) {
1007+
// delete
1008+
index->Remove(vids);
1009+
}
1010+
}
9731011
} else {
9741012
fe->ParseAndSet(new_prop, values[i]);
9751013
// update index if there is no error

‎src/core/transaction.h

+1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ class Transaction {
8181
std::vector<IteratorBase*> iterators_;
8282
FullTextIndex* fulltext_index_;
8383
std::vector<FTIndexEntry> fulltext_buffers_;
84+
std::vector<VectorIndexEntry> vector_buffers_;
8485
std::unordered_map<LabelId, int64_t> vertex_delta_count_;
8586
std::unordered_map<LabelId, int64_t> edge_delta_count_;
8687
std::set<LabelId> vertex_label_delete_;

‎src/core/value.h

+2
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,8 @@ class Value {
522522
*/
523523
std::string AsString() const { return AsType<std::string>(); }
524524

525+
std::vector<float> AsFloatVector() const { return AsType<std::vector<float>>(); }
526+
525527
/**
526528
* Create a Value that is a const reference to the object t
527529
*

‎src/core/vector_index.h

+17-4
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
namespace lgraph {
2222

23+
2324
class VectorIndex {
2425
friend class Schema;
2526
friend class LightningGraph;
@@ -66,12 +67,13 @@ class VectorIndex {
6667

6768
// add vector to index and build index
6869
virtual void Add(const std::vector<std::vector<float>>& vectors,
69-
const std::vector<int64_t>& vids, int64_t num_vectors) = 0;
70+
const std::vector<int64_t>& vids) = 0;
71+
72+
virtual void Remove(const std::vector<int64_t>& vids) = 0;
7073

71-
// build index
72-
virtual void Build() = 0;
74+
virtual void Clear() = 0;
7375

74-
// serialize index
76+
// serialize index
7577
virtual std::vector<uint8_t> Save() = 0;
7678

7779
// load index form serialization
@@ -83,5 +85,16 @@ class VectorIndex {
8385

8486
virtual std::vector<std::pair<int64_t, float>>
8587
RangeSearch(const std::vector<float>& query, float radius, int ef_search, int limit) = 0;
88+
89+
virtual int64_t GetElementsNum() = 0;
90+
virtual int64_t GetMemoryUsage() = 0;
8691
};
92+
93+
struct VectorIndexEntry {
94+
VectorIndex* index;
95+
bool add;
96+
std::vector<int64_t> vids;
97+
std::vector<std::vector<float>> vectors;
98+
};
99+
87100
} // namespace lgraph

0 commit comments

Comments
 (0)