Skip to content

Commit

Permalink
add uts and fix update old data.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ColinLeeo
ColinLeeo committed Nov 28, 2024
1 parent c3b56e3 commit 02a7b72
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 11 deletions.
6 changes: 3 additions & 3 deletions src/core/field_extractor_base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,12 +189,12 @@ class FieldExtractorBase {

// Get copy from record.
ENABLE_IF_FIXED_FIELD(T, void) GetCopy(const Value& record, T& data) const {
FMA_DBG_ASSERT(field_data_helper::FieldTypeSize(def_.type) == sizeof(T));
FMA_DBG_ASSERT(field_data_helper::FieldTypeSize(def_.type) == sizeof(data));
size_t offset = GetFieldOffset(record);
size_t size = GetDataSize(record);
// for Field_extractor_v1, size always equals sizeof(T)
if (size == sizeof(T)) {
memcpy(&data, (char*)record.Data() + offset, sizeof(T));
if (size == sizeof(data)) {
memcpy(&data, (char*)record.Data() + offset, sizeof(data));
} else {
// For FieldExtractorV2, even with fixed-length data, there may be cases
// where the data length in the record does not match the defined length,
Expand Down
69 changes: 68 additions & 1 deletion src/core/schema.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/**
/**
* Copyright 2022 AntGroup CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -504,6 +504,35 @@ void Schema::ParseAndSet(Value& record, const FieldData& data,
GetFieldExtractorV1(extractor)->ParseAndSet(record, data);
return;
}
FieldId count = GetFieldExtractorV2(extractor)->GetRecordCount(record);
if (count <= extractor->GetFieldId()) {
Value new_prop = CreateEmptyRecord();
for (const auto& field : name_to_idx_) {
_detail::FieldExtractorV2* extr = GetFieldExtractorV2(GetFieldExtractor(field.first));
extr->SetIsNull(new_prop, false);
if (extr->IsFixedType()) {
if (extr->GetFieldId() >= count && extr->HasInitedValue()) {
SetFixedSizeValue(new_prop,
field_data_helper::FieldDataToValueOfFieldType(
extr->GetInitedValue(), extr->Type()),
extr);
} else if (extr->GetFieldId() < count) {
SetFixedSizeValue(new_prop, extr->GetConstRef(record), extr);
}
} else {
if (extr->GetFieldId() >= count && extr->HasInitedValue()) {
_SetVariableLengthValue(new_prop,
field_data_helper::FieldDataToValueOfFieldType(
extr->GetInitedValue(), extr->Type()),
extr);
} else if (extr->GetFieldId() < count) {
_SetVariableLengthValue(new_prop, extr->GetConstRef(record), extr);
}
}
}
record = new_prop;
}

bool data_is_null = data.type == FieldType::NUL;
extractor->SetIsNull(record, data_is_null);
if (data_is_null) return;
Expand Down Expand Up @@ -849,6 +878,44 @@ void Schema::CopyFieldsRaw(Value& dst, const std::vector<size_t> fids_in_dst,
}
}

void Schema::SetFixedSizeValue(Value& record, const Value& data,
::lgraph::_detail::FieldExtractorV2* extractor) const {
#define _SET_FIXED_FIELD(ft) \
do { \
typename field_data_helper::FieldType2StorageType<FieldType::ft>::type sd; \
extractor->ConvertData(&sd, data.Data(), sizeof(sd)); \
memcpy(ptr, &sd, sizeof(sd)); \
} while (0)
FMA_DBG_ASSERT(extractor->IsFixedType());
auto* ptr = static_cast<char*>(extractor->GetFieldPointer(record));
if (data.Size() == extractor->TypeSize()) {
memcpy(ptr, data.Data(), data.Size());
} else {
switch (extractor->Type()) {
case FieldType::INT8:
_SET_FIXED_FIELD(INT8);
break;
case FieldType::INT16:
_SET_FIXED_FIELD(INT16);
break;
case FieldType::INT32:
_SET_FIXED_FIELD(INT32);
break;
case FieldType::INT64:
_SET_FIXED_FIELD(INT64);
break;
case FieldType::FLOAT:
_SET_FIXED_FIELD(FLOAT);
break;
case FieldType::DOUBLE:
_SET_FIXED_FIELD(DOUBLE);
break;
default:
LOG_ERROR() << "Error here";
}
}
}

void Schema::RefreshLayout() {
if (fast_alter_schema) {
RefreshLayoutForFastSchema();
Expand Down
2 changes: 2 additions & 0 deletions src/core/schema.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,8 @@ class Schema {
}
}

void SetFixedSizeValue(Value& record, const Value& data,
::lgraph::_detail::FieldExtractorV2* extractor) const;
// copy field values from src to dst
// dst must be a record created with this schema
// fields are assumed to have the same type
Expand Down
54 changes: 47 additions & 7 deletions test/test_schema_change.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "gtest/gtest.h"

#include "core/lightning_graph.h"
#include "core/schema_common.h"
#include "./graph_factory.h"
#include "./test_tools.h"
#include "./random_port.h"
Expand Down Expand Up @@ -682,7 +683,7 @@ TEST_F(TestSchemaChange, UpdateSchemaAndData) {
CreateSampleDB(dir, true);
LightningGraph graph(conf);
size_t n_changed = 0;
graph.AlterLabelDelFields("person", std::vector<std::string>{"desc"}, true, &n_changed);
graph.AlterLabelDelFields("person", std::vector<std::string>{"img2"}, true, &n_changed);
auto txn = graph.CreateReadTxn();
auto vit = txn.GetVertexIterator(0);
auto field_data = txn.GetVertexFields(vit);
Expand All @@ -702,13 +703,11 @@ TEST_F(TestSchemaChange, UpdateSchemaAndData) {
LightningGraph graph(conf);
auto txn = graph.CreateWriteTxn();
txn.AddVertex(std::string("person"),
std::vector<std::string>({"id", "name", "age", "img", "img2", "cond"}),
std::vector<std::string>(
{"3", "p1", "11.5", "", lgraph_api::base64::Encode("img2"), "20"}));
std::vector<std::string>({"id", "name", "age", "img", "desc", "cond"}),
std::vector<std::string>({"3", "p1", "11.5", "", "simple desc", "20"}));
txn.AddVertex(std::string("person"),
std::vector<std::string>({"id", "name", "age", "img", "img2", "cond"}),
std::vector<std::string>(
{"4", "p1", "11.5", "", lgraph_api::base64::Encode("img2"), "40"}));
std::vector<std::string>({"id", "name", "age", "img", "desc", "cond"}),
std::vector<std::string>({"4", "p1", "11.5", "", "desc", "40"}));
txn.Commit();
txn = graph.CreateReadTxn();
auto vit = txn.GetVertexIterator(0);
Expand All @@ -721,4 +720,45 @@ TEST_F(TestSchemaChange, UpdateSchemaAndData) {
field_data = txn.GetVertexFields(vit);
UT_EXPECT_EQ(field_data.size(), 6);
}

UT_LOG() << "Test Update data with new Schema";
{
LightningGraph graph(conf);
auto txn = graph.CreateWriteTxn();
auto itr = txn.GetVertexIterator(0);
try {
txn.SetVertexProperty(itr, std::vector<std::string>{"desc"},
std::vector<FieldData>{FieldData::String("test")});
} catch (const FieldNotFoundException& e) {
UT_EXPECT_TRUE(true);
} catch (const LgraphException& e) {
UT_EXPECT_TRUE(false);
}
// shorter than orig
txn.SetVertexProperty(itr, std::vector<std::string>{"name"},
std::vector<FieldData>{FieldData::String("p")});
// larger than orig
txn.SetVertexProperty(
itr, std::vector<std::string>{"desc"},
std::vector<FieldData>{FieldData::String("this is a desc larger than before")});
// set value that not exists in record
txn.SetVertexProperty(itr, std::vector<std::string>{"cond"},
std::vector<FieldData>{FieldData::Int32(100)});
txn.Commit();
auto read_txn = graph.CreateReadTxn();
auto read_itr = read_txn.GetVertexIterator(0);
auto field_data = read_txn.GetVertexFields(read_itr);
std::unordered_map<std::string, FieldData> ret;
for (const auto& pair : field_data) {
ret.insert(pair);
std::cout << pair.first << std::endl;
std::cout << pair.second.ToString("") << std::endl;
}
UT_EXPECT_EQ(ret["id"], FieldData::Int32(1));
UT_EXPECT_EQ(ret["name"], FieldData::String("p"));
UT_EXPECT_EQ(ret["age"], FieldData::Float(11.5));
UT_EXPECT_EQ(ret["desc"], FieldData::String("this is a desc larger than before"));
UT_EXPECT_EQ(ret["cond"], FieldData::Int32(100));
UT_EXPECT_EQ(ret.size(), 6);
}
}

0 comments on commit 02a7b72

Please sign in to comment.