Skip to content

Commit

Permalink
Changes in validation
Browse files Browse the repository at this point in the history 
* Configure collator list in validator-console
* Remove "lite-validator" flags
* Better compatibility in validate-query.cpp
  • Loading branch information
@SpyCheese
SpyCheese committed Jun 11, 2024
1 parent f91ed6c commit 38ab70c
Show file tree
Hide file tree
Showing 16 changed files with 368 additions and 107 deletions.
9 changes: 9 additions & 0 deletions tl/generate/scheme/ton_api.tl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,12 @@ engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_pri
engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay;
engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig;

engine.validator.collatorsList.collator adnl_id:int256 trusted:Bool = engine.validator.collatorsList.Collator;
engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator)
= engine.validator.collatorsList.Shard;
engine.validator.collatorsList self_collate:Bool use_config_41:Bool shards:(vector engine.validator.collatorsList.shard)
= engine.validator.CollatorsList;

---functions---
---types---

Expand Down Expand Up @@ -752,6 +758,9 @@ engine.validator.addShard shard:tonNode.shardId = engine.validator.Success;
engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success;
engine.validator.delShard shard:tonNode.shardId = engine.validator.Success;

engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success;
engine.validator.showCollatorsList = engine.validator.CollatorsList;

---types---

storage.pong = storage.Pong;
Expand Down
Binary file modified tl/generate/scheme/ton_api.tlo
View file
Binary file not shown.
77 changes: 77 additions & 0 deletions validator-engine-console/validator-engine-console-query.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1316,3 +1316,80 @@ td::Status DelShardQuery::receive(td::BufferSlice data) {
td::TerminalIO::out() << "successfully removed shard\n";
return td::Status::OK();
}

td::Status SetCollatorsListQuery::run() {
TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token<std::string>());
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();
}

td::Status SetCollatorsListQuery::send() {
TRY_RESULT(data, td::read_file(file_name_));
TRY_RESULT(json, td::json_decode(data.as_slice()));
auto list = ton::create_tl_object<ton::ton_api::engine_validator_collatorsList>();
TRY_STATUS(ton::ton_api::from_json(*list, json.get_object()));
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_setCollatorsList>(std::move(list));
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}

td::Status SetCollatorsListQuery::receive(td::BufferSlice data) {
TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
"received incorrect answer: ");
td::TerminalIO::out() << "success\n";
return td::Status::OK();
}

td::Status ClearCollatorsListQuery::run() {
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();
}

td::Status ClearCollatorsListQuery::send() {
auto list = ton::create_tl_object<ton::ton_api::engine_validator_collatorsList>();
list->self_collate_ = true;
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_setCollatorsList>(std::move(list));
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}

td::Status ClearCollatorsListQuery::receive(td::BufferSlice data) {
TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
"received incorrect answer: ");
td::TerminalIO::out() << "success\n";
return td::Status::OK();
}

td::Status ShowCollatorsListQuery::run() {
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();
}

td::Status ShowCollatorsListQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_showCollatorsList>();
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}

td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) {
TRY_RESULT_PREFIX(list, ton::fetch_tl_object<ton::ton_api::engine_validator_collatorsList>(data.as_slice(), true),
"received incorrect answer: ");
td::TerminalIO::out() << "Collators list:\n";
if (list->self_collate_) {
td::TerminalIO::out() << "self_collate = true\n";
}
if (list->use_config_41_) {
td::TerminalIO::out() << "use_config_41 = true\n";
}
if (list->shards_.empty()) {
td::TerminalIO::out() << "Shard list is empty\n";
return td::Status::OK();
}
for (const auto &shard : list->shards_) {
td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n";
for (const auto &collator : shard->collators_) {
td::TerminalIO::out() << " Collator " << collator->adnl_id_ << (collator->trusted_ ? " (trusted)" : "") << "\n";
}
}
return td::Status::OK();
}
60 changes: 60 additions & 0 deletions validator-engine-console/validator-engine-console-query.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1342,3 +1342,63 @@ class DelShardQuery : public Query {
td::int32 wc_;
td::int64 shard_;
};

class SetCollatorsListQuery : public Query {
public:
SetCollatorsListQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
: Query(console, std::move(tokenizer)) {
}
td::Status run() override;
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "setcollatorslist";
}
static std::string get_help() {
return "setcollatorslist <filename>\tset list of collators from file <filename>";
}
std::string name() const override {
return get_name();
}

private:
std::string file_name_;
};

class ClearCollatorsListQuery : public Query {
public:
ClearCollatorsListQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
: Query(console, std::move(tokenizer)) {
}
td::Status run() override;
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "clearcollatorslist";
}
static std::string get_help() {
return "clearcollatorslist\tclear list of collators";
}
std::string name() const override {
return get_name();
}
};

class ShowCollatorsListQuery : public Query {
public:
ShowCollatorsListQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
: Query(console, std::move(tokenizer)) {
}
td::Status run() override;
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "showcollatorslist";
}
static std::string get_help() {
return "showcollatorslist\tshow list of collators";
}
std::string name() const override {
return get_name();
}
};
3 changes: 3 additions & 0 deletions validator-engine-console/validator-engine-console.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,9 @@ void ValidatorEngineConsole::run() {
add_query_runner(std::make_unique<QueryRunnerImpl<AddShardQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<DelShardQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<DelCollatorQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<SetCollatorsListQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<ClearCollatorsListQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<ShowCollatorsListQuery>>());
}

bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {
Expand Down
87 changes: 72 additions & 15 deletions validator-engine/validator-engine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1472,7 +1472,6 @@ td::Status ValidatorEngine::load_global_config() {
}
validator_options_.write().set_hardforks(std::move(h));
validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled);
validator_options_.write().set_validator_mode(validator_mode_);

return td::Status::OK();
}
Expand Down Expand Up @@ -1502,6 +1501,31 @@ void ValidatorEngine::set_shard_check_function() {
}
}

void ValidatorEngine::load_collators_list() {
collators_list_ = {};
auto data_R = td::read_file(collators_list_file());
if (data_R.is_error()) {
return;
}
auto data = data_R.move_as_ok();
auto json_R = td::json_decode(data.as_slice());
if (json_R.is_error()) {
LOG(ERROR) << "Failed to parse collators list: " << json_R.move_as_error();
return;
}
auto json = json_R.move_as_ok();
collators_list_ = ton::create_tl_object<ton::ton_api::engine_validator_collatorsList>();
auto S = ton::ton_api::from_json(*collators_list_, json.get_object());
if (S.is_error()) {
LOG(ERROR) << "Failed to parse collators list: " << S;
collators_list_ = {};
return;
}
td::Ref<ton::validator::CollatorsList> list{true};
list.write().unpack(*collators_list_);
validator_options_.write().set_collators_list(std::move(list));
}

void ValidatorEngine::load_empty_local_config(td::Promise<td::Unit> promise) {
auto ret_promise = td::PromiseCreator::lambda(
[SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
Expand Down Expand Up @@ -1794,6 +1818,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) {

void ValidatorEngine::start() {
set_shard_check_function();
load_collators_list();
read_config_ = true;
start_adnl();
}
Expand Down Expand Up @@ -3792,6 +3817,52 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateS
});
}

void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
if (!(perm & ValidatorEnginePermissions::vep_modify)) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
return;
}
if (!started_) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
return;
}

auto s = td::json_encode<std::string>(td::ToJson(*query.list_), true);
auto S = td::write_file(collators_list_file(), s);
if (S.is_error()) {
promise.set_value(create_control_query_error(std::move(S)));
return;
}

collators_list_ = std::move(query.list_);
td::Ref<ton::validator::CollatorsList> list{true};
list.write().unpack(*collators_list_);
validator_options_.write().set_collators_list(std::move(list));
td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options,
validator_options_);
promise.set_value(ton::serialize_tl_object(ton::create_tl_object<ton::ton_api::engine_validator_success>(), true));
}

void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
if (!(perm & ValidatorEnginePermissions::vep_default)) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
return;
}
if (!started_) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
return;
}
if (collators_list_) {
promise.set_value(ton::serialize_tl_object(collators_list_, true));
} else {
auto list = ton::create_tl_object<ton::ton_api::engine_validator_collatorsList>();
list->self_collate_ = true;
promise.set_value(ton::serialize_tl_object(list, true));
}
}

void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query,
td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
td::Promise<td::BufferSlice> promise) {
Expand Down Expand Up @@ -4260,20 +4331,6 @@ int main(int argc, char *argv[]) {
p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() {
acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); });
});
p.add_option('\0', "lite-validator-shards",
"lite-mode validator for shard blocks (don't collate blocks, use collator nodes instead)", [&]() {
acts.push_back([&x]() {
td::actor::send_closure(x, &ValidatorEngine::set_validator_mode,
ton::validator::ValidatorManagerOptions::validator_lite_shards);
});
});
p.add_option('\0', "lite-validator-all",
"lite-mode validator for all blocks (don't collate blocks, use collator nodes instead)", [&]() {
acts.push_back([&x]() {
td::actor::send_closure(x, &ValidatorEngine::set_validator_mode,
ton::validator::ValidatorManagerOptions::validator_lite_all);
});
});
td::uint32 threads = 7;
p.add_checked_option(
't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) {
Expand Down
14 changes: 9 additions & 5 deletions validator-engine/validator-engine.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ class ValidatorEngine : public td::actor::Actor {
td::Ref<ton::validator::ValidatorManagerOptions> validator_options_;
Config config_;
ton::tl_object_ptr<ton::ton_api::engine_validator_customOverlaysConfig> custom_overlays_config_;
ton::tl_object_ptr<ton::ton_api::engine_validator_collatorsList> collators_list_;

std::set<ton::PublicKeyHash> running_gc_;

Expand Down Expand Up @@ -235,8 +236,6 @@ class ValidatorEngine : public td::actor::Actor {
ton::BlockSeqno truncate_seqno_{0};
std::string session_logs_file_;
bool not_all_shards_ = false;
ton::validator::ValidatorManagerOptions::ValidatorMode validator_mode_ =
ton::validator::ValidatorManagerOptions::validator_normal;

std::set<ton::CatchainSeqno> unsafe_catchains_;
std::map<ton::BlockSeqno, std::pair<ton::CatchainSeqno, td::uint32>> unsafe_catchain_rotations_;
Expand Down Expand Up @@ -319,9 +318,6 @@ class ValidatorEngine : public td::actor::Actor {
void set_not_all_shards() {
not_all_shards_ = true;
}
void set_validator_mode(ton::validator::ValidatorManagerOptions::ValidatorMode mode) {
validator_mode_ = mode;
}

void start_up() override;
ValidatorEngine() {
Expand All @@ -333,6 +329,7 @@ class ValidatorEngine : public td::actor::Actor {
void load_local_config(td::Promise<td::Unit> promise);
void load_config(td::Promise<td::Unit> promise);
void set_shard_check_function();
void load_collators_list();

void start();

Expand Down Expand Up @@ -411,6 +408,9 @@ class ValidatorEngine : public td::actor::Actor {
std::string custom_overlays_config_file() const {
return db_root_ + "/custom-overlays.json";
}
std::string collators_list_file() const {
return db_root_ + "/collators-list.json";
}

void load_custom_overlays_config();
td::Status write_custom_overlays_config();
Expand Down Expand Up @@ -514,6 +514,10 @@ class ValidatorEngine : public td::actor::Actor {
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
template <class T>
void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
td::Promise<td::BufferSlice> promise) {
Expand Down
2 changes: 1 addition & 1 deletion validator/fabric.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ namespace ton {

namespace validator {

enum ValidateMode { fake = 1, full_collated_data = 2 };
enum ValidateMode { fake = 1 };
enum CollateMode { skip_store_candidate = 1 };

td::actor::ActorOwn<Db> create_db_actor(td::actor::ActorId<ValidatorManager> manager, std::string db_root_,
Expand Down
Loading

0 comments on commit 38ab70c

Please sign in to comment.