diff --git a/CHANGELOG.md b/CHANGELOG.md index e47d94ab..9009b039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.3] - 2018-10-09 +### Changed +- Modified DB rollback mechanism to support starting from backup + +### Fixed +- Accounts database snapshots during dividend block +- Further stability improvements for peer communication +- Node debug logs cleanup + ## [1.0.2] - 2018-09-27 ### Fixed - Node did not stop when there was not enough signatures @@ -54,8 +63,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update default parameters - Creating a dev version with reference to the last tag -[Unreleased]: https://github.com/adshares/ads/compare/v1.0.2...HEAD +[Unreleased]: https://github.com/adshares/ads/compare/v1.0.3...HEAD +[1.0.3]: https://github.com/adshares/ads/compare/v1.0.2...v1.0.3 [1.0.2]: https://github.com/adshares/ads/compare/v1.0.1...v1.0.2 [1.0.1]: https://github.com/adshares/ads/compare/v1.0.0...v1.0.1 [1.0.0]: https://github.com/adshares/ads/compare/v0.0.6...v1.0.0 diff --git a/src/common/command/getbroadcastmsg.cpp b/src/common/command/getbroadcastmsg.cpp index 245e19e3..6f6154a6 100644 --- a/src/common/command/getbroadcastmsg.cpp +++ b/src/common/command/getbroadcastmsg.cpp @@ -124,7 +124,7 @@ bool GetBroadcastMsg::send(INetworkClient& netClient) { return false; } - ELOG("size %ud %08X\n", m_header.fileSize, m_data.info.block); + DLOG("size %ud %08X\n", m_header.fileSize, m_data.info.block); unsigned char *readBuffer = new unsigned char[m_header.fileSize]; if (!netClient.readData(readBuffer, m_header.fileSize)) { diff --git a/src/common/helper/blocks.cpp b/src/common/helper/blocks.cpp index 06f1730a..13006ac1 100644 --- a/src/common/helper/blocks.cpp +++ b/src/common/helper/blocks.cpp @@ -14,6 +14,7 @@ #include "default.h" #include "libarchive.h" +#include "servers.h" namespace Helper { @@ -96,79 +97,110 @@ uint32_t get_users_count(uint16_t bank) { return (st.st_size/sizeof(user_t)); } -void db_backup(uint32_t block_path, uint16_t nodes) { +void db_backup(uint32_t block_path, uint16_t nodes) +{ // used to temporary name for snapshot, resolve names colision between sparse local und/* file and currently creating db snapshot. const char* const snapshot_postfix = "tmp"; + char backupFilePath[64]; if (!is_snapshot_directory(block_path)) { return; } - char previousSnapshotPath[64]; - char backupFilePath[64]; - unsigned int usert_size = sizeof(user_t); + Helper::FileName::getName(backupFilePath, block_path, "servers.srv"); + Helper::Servers servers(backupFilePath); + servers.load(); - // for each node for (uint16_t bank = 1; bank < nodes; ++bank) { - // get undo files, youngest at begin + + // open snapshot file + Helper::FileName::getUndo(backupFilePath, block_path, bank); + strncat(backupFilePath, snapshot_postfix, sizeof(backupFilePath) - strlen(backupFilePath) - 1); + Helper::FileWrapper snapshotFile; + if (!snapshotFile.open(backupFilePath, O_WRONLY | O_CREAT, 0644)) + { + WLOG("Can't open snapshot file: %s\n" , backupFilePath); + return; + } + + // open bank file + char filePath[64]; + Helper::FileName::getUsr(filePath, bank); + Helper::FileWrapper bankFile; + if (!bankFile.open(filePath)) + { + WLOG("Can't open bank file: %s\n", filePath); + return; + } + + // open undos from now to block_path + uint32_t current_block = time(NULL); + current_block -= current_block%BLOCKSEC; std::vector> undoFiles; std::shared_ptr undo_file; - for (int i=0; iisOpen()) { undoFiles.push_back(undo_file); } } - // load previous snapshot or usr/* for initial run - uint32_t users = get_users_count(bank); - Helper::FileName::getUndo(previousSnapshotPath, (block_path - (BLOCKDIV*BLOCKSEC)), bank); - if (!boost::filesystem::exists(previousSnapshotPath)) { - Helper::FileName::getUsr(previousSnapshotPath, bank); - } - - Helper::FileWrapper last_snapshot(previousSnapshotPath, O_RDONLY, false); - if (!last_snapshot.isOpen()) { - return; - } - - user_t u; - Helper::FileName::getUndo(backupFilePath, block_path, bank); - strncat(backupFilePath, snapshot_postfix, sizeof(backupFilePath) - strlen(backupFilePath) - 1); - - int current_snapshot = open(backupFilePath, O_WRONLY | O_CREAT, 0644); - if (current_snapshot < 0) { - return; - } - // for each user + uint32_t users = servers.getNode(bank).accountCount; for (uint32_t user = 0; user < users; ++user) { - u = {}; + user_t usr = {}; + // read undos and find user (user->msid != 0) for (auto it=undoFiles.begin(); it != undoFiles.end(); ++it) { - (*it)->seek(user * usert_size, SEEK_SET); - (*it)->read((char*)&u, usert_size); - if (u.msid != 0) // user found, get from undo + (*it)->seek(user * sizeof(user_t), SEEK_SET); + (*it)->read((char*)&usr, sizeof(user_t)); + if (usr.msid != 0) { - last_snapshot.seek(usert_size, SEEK_CUR); - if (!write(current_snapshot, (char*)&u, usert_size)) break; + // user found + bankFile.seek(sizeof(user_t), SEEK_CUR); break; } } // end of undo - if (u.msid != 0) continue; // user already found - // not found in undo, get from last snapshot - last_snapshot.read((char*)&u, usert_size); - if (!write(current_snapshot, (char*)&u, usert_size)) break; - } // foreach user - close(current_snapshot); - } // foreach bank + if (usr.msid == 0) + { + // if not found read user from bank file + bankFile.read((char*)&usr, sizeof(user_t)); + + if (!undoFiles.empty()) + { + user_t latest_record = {}; + auto latest_undo = undoFiles.back(); + latest_undo->seek(user * sizeof(user_t), SEEK_SET); + latest_undo->read((char*)&latest_record, sizeof(user_t)); + if (latest_record.msid != 0) + { + memcpy((char*)&usr, &latest_record, sizeof(user_t)); + } + } + } + + // write user to snapshot file + if (!snapshotFile.write((char*)&usr, sizeof(user_t))) + { + WLOG("Can't write to bank file\n"); + return; + } + + } // end of for each user + } // end of for each node // change snapshot temp file name to correct one for (uint16_t bank = 1; bank < nodes; ++bank) @@ -178,8 +210,6 @@ void db_backup(uint32_t block_path, uint16_t nodes) { std::rename(backupFilePath, std::string(backupFilePath).substr(0, Helper::FileName::kUndoNameFixedLength).c_str()); } - - return; } } diff --git a/src/common/helper/filewrapper.cpp b/src/common/helper/filewrapper.cpp index 0dce180d..cf1abf88 100644 --- a/src/common/helper/filewrapper.cpp +++ b/src/common/helper/filewrapper.cpp @@ -12,10 +12,10 @@ FileWrapper::FileWrapper() : { } -FileWrapper::FileWrapper(const std::string filepath, int mask, bool removeOnClose) : +FileWrapper::FileWrapper(const std::string filepath, int mask, int mode, bool removeOnClose) : m_file_descriptor(-1), m_filepath(filepath), m_remove_on_close(removeOnClose) { - m_file_descriptor = ::open(m_filepath.c_str(), mask); + m_file_descriptor = ::open(m_filepath.c_str(), mask, mode); } FileWrapper::~FileWrapper() diff --git a/src/common/helper/filewrapper.h b/src/common/helper/filewrapper.h index ccb51bff..61c86869 100644 --- a/src/common/helper/filewrapper.h +++ b/src/common/helper/filewrapper.h @@ -13,7 +13,7 @@ class FileWrapper { public: FileWrapper(); - FileWrapper(const std::string filepath, int mask, bool removeOnClose = false); + FileWrapper(const std::string filepath, int mask, int mode = 0644, bool removeOnClose = false); ~FileWrapper(); bool isOpen(); diff --git a/src/common/message.hpp b/src/common/message.hpp index a09556ac..8049bc70 100644 --- a/src/common/message.hpp +++ b/src/common/message.hpp @@ -133,8 +133,6 @@ class message : memcpy(data+4+64+6,&now,4); memcpy(data+4+64+10,text,text_len); - DLOG("INI:%016lX\n",*(uint64_t*)mypk); - if(text_type==MSGTYPE_BLK) { if(mysk==NULL) { // creating message from network memcpy(data+4,mypk,64); @@ -475,13 +473,13 @@ class message : char hash[2*SHA256_DIGEST_LENGTH]; ed25519_key2text(hash,nhash,SHA256_DIGEST_LENGTH); - ELOG("nhash %.*s\n", 2*SHA256_DIGEST_LENGTH,hash); + ILOG("nhash %.*s\n", 2*SHA256_DIGEST_LENGTH,hash); ed25519_key2text(hash, mhash,SHA256_DIGEST_LENGTH); - ELOG("mhash %.*s\n", 2*SHA256_DIGEST_LENGTH,hash); + ILOG("mhash %.*s\n", 2*SHA256_DIGEST_LENGTH,hash); if(memcmp(mhash,nhash,32)){ - DLOG("HASHTREE failed (path len:%d)\n",(int)hashes.size()); + ELOG("HASHTREE failed (path len:%d)\n",(int)hashes.size()); return(false);} return(true); } @@ -797,7 +795,6 @@ class message : return(ed25519_sign_open(data+4+64,10+sizeof(hash_t),svpk,data+4)); } if(data[0]==MSGTYPE_INI) { - DLOG("INI:%016lX\n",*(uint64_t*)svpk); return(ed25519_sign_open(data+4+64,len-4-64,svpk,data+4)); } assert(0); diff --git a/src/common/servers.hpp b/src/common/servers.hpp index afadd4df..a575d16e 100644 --- a/src/common/servers.hpp +++ b/src/common/servers.hpp @@ -121,7 +121,7 @@ class servers { // also a block void create_genesis_block(const std::string genesis_file) { assert(nodes.size() == 0); - ELOG("INIT: using genesis file %s\n", genesis_file.c_str()); + ILOG("INIT: using genesis file %s\n", genesis_file.c_str()); boost::property_tree::ptree data; boost::property_tree::read_json(genesis_file, data); @@ -137,7 +137,7 @@ class servers { // also a block char hash_text[2 * SHA256_DIGEST_LENGTH]; ed25519_key2text(hash_text, genesis_hash, SHA256_DIGEST_LENGTH); - ELOG("genesis hash: %.*s\n", 2 * SHA256_DIGEST_LENGTH, hash_text); + ILOG("genesis hash: %.*s\n", 2 * SHA256_DIGEST_LENGTH, hash_text); boost::property_tree::ptree dataConfig = data.get_child("config"); @@ -147,7 +147,7 @@ class servers { // also a block ELOG("Invalid genesis start time: %d, must be divisible by %d\n", startTimeOpt.get(), BLOCKSEC); exit(-1); } else { - ELOG("Genesis start time: %d\n", startTimeOpt.get()); + ILOG("Genesis start time: %d\n", startTimeOpt.get()); } @@ -158,7 +158,7 @@ class servers { // also a block uint64_t clockNow = time(NULL); while (clockNow < waitUntil) { boost::this_thread::sleep(boost::posix_time::seconds(1)); - ELOG("Awaiting for genesis block time: %lu s\n", waitUntil - clockNow); + ILOG("Awaiting for genesis block time: %lu s\n", waitUntil - clockNow); clockNow = time(NULL); RETURN_ON_SHUTDOWN() ; @@ -285,7 +285,7 @@ class servers { // also a block } } - ELOG("INIT: weight diff: %016lX\n", TOTALMASS-sum); + ILOG("INIT: weight diff: %016lX\n", TOTALMASS-sum); //nodes.begin()->weight=TOTALMASS-sum; assert(num > 0); //vtot=(uint16_t)(num>20,now&0xFFFFF); @@ -1636,7 +1636,7 @@ class servers { // also a block tree.finish(tmphash); char hash[2*SHA256_DIGEST_LENGTH]; ed25519_key2text(hash,tmphash,SHA256_DIGEST_LENGTH); - ELOG("NODHASH sync %.*s\n",2*SHA256_DIGEST_LENGTH,hash); + ILOG("NODHASH sync %.*s\n",2*SHA256_DIGEST_LENGTH,hash); return(memcmp(tmphash,peer_nodehash,SHA256_DIGEST_LENGTH)); } @@ -1689,7 +1689,7 @@ class servers { // also a block div=0; } //ELOG("NEW DIVIDEND %08X (%.8f)\n",div,(float)(div)/0xFFFFFFFF); - ELOG("NEW DIVIDEND %08X (%.8f) (diff:%016lX,div:%.8lf)\n", + ILOG("NEW DIVIDEND %08X (%.8f) (diff:%016lX,div:%.8lf)\n", div,(float)(div)/0xFFFF,TOTALMASS-sum,(double)(TOTALMASS-sum)/(double)sum); } blockdir(); @@ -1750,7 +1750,7 @@ class servers { // also a block void clean_old(uint16_t svid) { if (MAX_UNDO > 0) { char pat[8]; - ELOG("CLEANING by %04X\n",svid); + ILOG("CLEANING by %04X\n",svid); sprintf(pat,"%04X",svid); for(int i=MAX_UNDO; i peer) @@ -124,8 +134,10 @@ void PeerConnectManager::addActivePeer(uint16_t svid, boost::shared_ptr pe m_ioService.dispatch(boost::bind(&PeerConnectManager::addActivePeerImpl, this, svid, peer)); } -void PeerConnectManager::addActivePeerImpl(uint16_t svid , boost::shared_ptr peer) +void PeerConnectManager::addActivePeerImpl(uint16_t svid, boost::shared_ptr peer) { + DLOG("ENTERED addActivePeerImpl, peer svid:%u\n", svid); + try{ boost::upgrade_lock< boost::shared_mutex > lock(m_peerMx); DLOG("Add active peer svid: %ud\n", svid); @@ -139,6 +151,8 @@ void PeerConnectManager::addActivePeerImpl(uint16_t svid , boost::shared_ptrdata != nullptr) { + DLOG("msg type:%u\n", msg->data[0]); + } + boost::shared_lock< boost::shared_mutex > lock(m_peerMx); auto svidPeer = m_activePeers.find(svid); if(svidPeer != m_activePeers.end()){ svidPeer->second->deliver(msg); + DLOG("LEFT deliverImpl 1\n"); return; } msg->sent_erase(svid); + DLOG("LEFT deliverImpl 2\n"); } void PeerConnectManager::deliverToAll(message_ptr msg) @@ -460,11 +481,17 @@ void PeerConnectManager::deliverToAll(message_ptr msg) void PeerConnectManager::deliverToAllImpl(message_ptr msg) { + DLOG("ENTERED deliverToAllImpl\n"); + if (msg->data != nullptr) { + DLOG("msg type:%u\n", msg->data[0]); + } boost::shared_lock< boost::shared_mutex > lock(m_peerMx); for(auto& peer: m_activePeers){ peer.second->deliver(msg); } + + DLOG("LEFT deliverToAllImpl\n"); } void PeerConnectManager::update(message_ptr msg, uint16_t svid) @@ -474,12 +501,18 @@ void PeerConnectManager::update(message_ptr msg, uint16_t svid) void PeerConnectManager::updateImpl(message_ptr msg, uint16_t svid) { + DLOG("ENTERED updateImpl, peer svid:%u\n", svid); + if (msg->data != nullptr) { + DLOG("msg type:%u\n", msg->data[0]); + } boost::shared_lock< boost::shared_mutex > lock(m_peerMx); auto svidPeer = m_activePeers.find(svid); if(svidPeer != m_activePeers.end()){ svidPeer->second->update(msg); } + + DLOG("LEFT updateImpl\n"); } void PeerConnectManager::updateAll(message_ptr msg) @@ -489,12 +522,18 @@ void PeerConnectManager::updateAll(message_ptr msg) void PeerConnectManager::updateAllImpl(message_ptr msg) { + DLOG("ENTERED updateAllImpl\n"); + if (msg->data != nullptr) { + DLOG("msg type:%u\n", msg->data[0]); + } boost::shared_lock< boost::shared_mutex > lock(m_peerMx); for(auto& peer: m_activePeers) { peer.second->update(msg); } + + DLOG("LEFT updateAllImpl\n"); } void PeerConnectManager::getReadyPeers(std::set& ready) diff --git a/src/escd/office.cpp b/src/escd/office.cpp index b4d65878..1d9cf594 100644 --- a/src/escd/office.cpp +++ b/src/escd/office.cpp @@ -42,7 +42,7 @@ office::~office() { if(offifd_>=0) { close(offifd_); } - ELOG("Office down\n"); + ILOG("Office down\n"); } void office::iorun_client(int i) { @@ -802,7 +802,7 @@ bool office::add_msg(IBlockCommand& utxs, uint32_t& msid, uint32_t& mpos) { std::unique_lock lock(file_); if(message_tnum>=MESSAGE_TNUM_MAX) { - ELOG("MESSAGE busy, delaying message addition\n"); + ILOG("MESSAGE busy, delaying message addition\n"); } while(message_tnum>=MESSAGE_TNUM_MAX) { @@ -862,7 +862,7 @@ bool office::add_msg(uint8_t* msg, uint32_t len, uint32_t& msid, uint32_t& mpos) std::unique_lock lock(file_); if(message_tnum>=MESSAGE_TNUM_MAX) { - ELOG("MESSAGE busy, delaying message addition\n"); + ILOG("MESSAGE busy, delaying message addition\n"); } while(message_tnum>=MESSAGE_TNUM_MAX) { @@ -906,7 +906,7 @@ bool office::add_msg(uint8_t* msg, usertxs& utxs, uint32_t& msid, uint32_t& mpos int len=utxs.size; file_.lock(); if(message_tnum>=MESSAGE_TNUM_MAX) { - ELOG("MESSAGE busy, delaying message addition\n"); + ILOG("MESSAGE busy, delaying message addition\n"); } while(message_tnum>=MESSAGE_TNUM_MAX) { file_.unlock(); @@ -1289,7 +1289,7 @@ void office::handle_accept(client_ptr c, const boost::system::error_code& error) DLOG("OFFICE new ticket (total open:%ld)\n",clients_.size()); #endif if(clients_.size()>=MAXCLIENTS || srv_.do_sync || message.length()>MESSAGE_TOO_LONG) { - ELOG("OFFICE busy, delaying connection\n"); + ILOG("OFFICE busy, delaying connection\n"); } while(clients_.size()>=MAXCLIENTS || srv_.do_sync || message.length()>MESSAGE_TOO_LONG) { diff --git a/src/escd/peer.hpp b/src/escd/peer.hpp index 4c124f47..ee353be8 100644 --- a/src/escd/peer.hpp +++ b/src/escd/peer.hpp @@ -95,14 +95,15 @@ class peer : public boost::enable_shared_from_this { DLOG("%04X PEER IORUN START\n",svid); try { peer_io_service_.run(); + DLOG("%04X PEER IORUN END\n",svid); } //Now we know the server is down. catch (std::exception& e) { //FIXME, stop peer after Broken pipe (now does not stop if peer ends with 'assert') //FIXME, wipe out inactive peers (better solution) ELOG("%04X CATCH IORUN Service.Run error:%s\n",svid,e.what()); leave(); - } - } + } + } void tryAsyncConnect(boost::asio::ip::tcp::resolver::iterator& connIt, int timeout) { m_netclient.asyncConnect(connIt, boost::bind(&peer::connect, this, boost::asio::placeholders::error), timeout); @@ -590,7 +591,7 @@ class peer : public boost::enable_shared_from_this { } else if(read_msg_->data[0]==MSGTYPE_SOK) { uint32_t now; memcpy(&now,read_msg_->data+1,4); - ELOG("%04X Authenticated, peer in sync at %08X\n",svid,now); + ILOG("%04X Authenticated, peer in sync at %08X\n",svid,now); update_sync(); do_sync=0; setState(ST_SYNCD); @@ -844,9 +845,9 @@ class peer : public boost::enable_shared_from_this { ELOG("%04X ERROR, hashing header chain :-(\n",svid); char hash[2*SHA256_DIGEST_LENGTH]; ed25519_key2text(hash,headers[num-2].nowhash,SHA256_DIGEST_LENGTH); - ELOG("%04X NOWHASH nowhash %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X NOWHASH nowhash %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); ed25519_key2text(hash,peer_hs.head.oldhash,SHA256_DIGEST_LENGTH); - ELOG("%04X NOWHASH oldhash %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X NOWHASH oldhash %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); leave(); return; } @@ -856,10 +857,10 @@ class peer : public boost::enable_shared_from_this { ELOG("%04X ERROR, initial oldhash mismatch :-(\n",svid); char hash[2*SHA256_DIGEST_LENGTH]; ed25519_key2text(hash,headers[0].oldhash,SHA256_DIGEST_LENGTH); - ELOG("%04X NOWHASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X NOWHASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); ed25519_key2text(hash,sync_ls.nowhash,SHA256_DIGEST_LENGTH); - ELOG("%04X NOWHASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); - ELOG("%04X Maybe start syncing from an older block (peer will disconnect)\n\n",svid); + ILOG("%04X NOWHASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X Maybe start syncing from an older block (peer will disconnect)\n\n",svid); leave(); return; } @@ -943,9 +944,9 @@ class peer : public boost::enable_shared_from_this { ELOG("%04X ERROR got wrong msglist msghash\n",svid); // consider updating server char hash[2*SHA256_DIGEST_LENGTH]; ed25519_key2text(hash,read_msg_->data+12+header.vok*sizeof(svsi_t),SHA256_DIGEST_LENGTH); - ELOG("%04X MSGHASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X MSGHASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); ed25519_key2text(hash,header.msghash,SHA256_DIGEST_LENGTH); - ELOG("%04X MSGHASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X MSGHASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); leave(); return; } @@ -1021,7 +1022,7 @@ class peer : public boost::enable_shared_from_this { } for(uint32_t block=path+BLOCKSEC; block<=srvs_.now; block++) { Helper::FileName::getUndo(filename, block, bank); - int uf = open(filename, O_RDONLY); + int uf = open(filename, O_RDONLY|O_CREAT, 0644); if(uf<0) { continue; } @@ -1288,7 +1289,7 @@ class peer : public boost::enable_shared_from_this { ELOG("%04X SERVERS incompatible with hash\n",svid); char hash[2*SHA256_DIGEST_LENGTH]; ed25519_key2text(hash,sync_hs.head.nodhash,SHA256_DIGEST_LENGTH); - ELOG("%04X NODHASH peer %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X NODHASH peer %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); free(peer_svsi); free(peer_nods); throw std::runtime_error("CATCH asio error (handle_read_servers)\n"); @@ -1351,9 +1352,9 @@ class peer : public boost::enable_shared_from_this { char hash[2*SHA256_DIGEST_LENGTH]; ELOG("%04X WARNING, last message hash mismatch, should run full resync\n",svid); ed25519_key2text(hash,srvs_.nodes[opts_.svid].msha,SHA256_DIGEST_LENGTH); - ELOG("%04X HASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X HASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); ed25519_key2text(hash,peer_hs.msha,SHA256_DIGEST_LENGTH); - ELOG("%04X HASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X HASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); } if(incoming_) { DLOG("%04X INCOMING HANDSHAKE \n",svid); @@ -1367,18 +1368,18 @@ class peer : public boost::enable_shared_from_this { char hash[2*SHA256_DIGEST_LENGTH]; ELOG("%04X ERROR oldhash mismatch, FIXME, move back more blocks to sync\n",svid); ed25519_key2text(hash,sync_hs.head.oldhash,SHA256_DIGEST_LENGTH); - ELOG("%04X HASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X HASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); ed25519_key2text(hash,peer_hs.head.oldhash,SHA256_DIGEST_LENGTH); - ELOG("%04X HASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X HASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); return(0); } if(memcmp(peer_hs.head.nowhash,sync_hs.head.nowhash,SHA256_DIGEST_LENGTH)) { char hash[2*SHA256_DIGEST_LENGTH]; ELOG("%04X WARNING nowhash mismatch, not tested :-( move back one block to sync\n",svid); ed25519_key2text(hash,sync_hs.head.nowhash,SHA256_DIGEST_LENGTH); - ELOG("%04X HASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X HASH have %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); ed25519_key2text(hash,peer_hs.head.nowhash,SHA256_DIGEST_LENGTH); - ELOG("%04X HASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); + ILOG("%04X HASH got %.*s\n",svid,2*SHA256_DIGEST_LENGTH,hash); return(0); } } @@ -1390,7 +1391,7 @@ class peer : public boost::enable_shared_from_this { } //if(peer_hs.head.now { setState(ST_SYNCD); return(1); } else { - ELOG("%04X Authenticated, peer in sync\n",svid); + ILOG("%04X Authenticated, peer in sync\n",svid); update_sync(); last_active=now; // protect from disconnect do_sync=0; @@ -1464,7 +1465,7 @@ class peer : public boost::enable_shared_from_this { last_active=now; // protect from disconnect do_sync=0; // set peer in sync, we are not in sync (server_.do_sync==1) - ELOG("%04X ADD active peer !!!!!!!!!!!\n",svid); + ILOG("%04X ADD active peer !!!!!!!!!!!\n",svid); setState(ST_SYNCD); return(1); @@ -1487,7 +1488,7 @@ class peer : public boost::enable_shared_from_this { } busy=0; // make peer available for download traffic - ELOG("%04X CONTINUE after authentication1\n",svid); + ILOG("%04X CONTINUE after authentication1\n",svid); asyncWaitForNewMessageHeader(); return; } @@ -1561,7 +1562,7 @@ class peer : public boost::enable_shared_from_this { } busy=0; // make peer available for download traffic - ELOG("%04X CONTINUE after authentication2\n",svid); + ILOG("%04X CONTINUE after authentication2\n",svid); asyncWaitForNewMessageHeader(); return; } @@ -1979,7 +1980,7 @@ class peer : public boost::enable_shared_from_this { } void sync_finish() { - ELOG("%04X SYNC OK\n",svid); + ILOG("%04X SYNC OK\n",svid); hash_s* hash_p=(hash_s*)last_message_hash; server_.save_candidate(BLOCK_MODE,*hash_p,PEER_block_add,PEER_block_del,svid); boost::lock_guard lock(pio_); diff --git a/src/escd/server.cpp b/src/escd/server.cpp index 4b7771b6..f0a113ca 100644 --- a/src/escd/server.cpp +++ b/src/escd/server.cpp @@ -21,11 +21,11 @@ server::~server() { //threadpool.join_all(); //clock_thread->interrupt(); //clock_thread->join(); - ELOG("Server down\n"); + ILOG("Server down\n"); } void server::start() { - ELOG("SERVER start point\n"); + ILOG("SERVER start point\n"); mkdir("usr",0755); // create dir for bank accounts mkdir("inx",0755); // create dir for bank message indeces @@ -50,7 +50,6 @@ void server::start() { } memcpy(pkey, last_srvs_.nodes[opts_.svid].pk, sizeof(hash_t)); - //DLOG("INI:%016lX\n",*(uint64_t*)pkey); if (!last_srvs_.find_key(pkey, skey)) { char pktext[2 * 32 + 1]; pktext[2 * 32] = '\0'; @@ -66,7 +65,7 @@ void server::start() { srvs_.now -= BLOCKSEC; period_start = srvs_.nextblock(); - ELOG("MAKE BLOCKCHAIN\n"); + ILOG("MAKE BLOCKCHAIN\n"); message_map empty; // srvs_.now = last_srvs_.nodes[0].mtim; srvs_.msg = 0; @@ -87,12 +86,12 @@ void server::start() { // create empty servers so sync can start and download updated servers last_srvs_.find_pkey(pkey); // get this node public key last_srvs_.init_fast(opts_.svid, pkey); - ELOG("CREATING nodes for fast sync\n"); + ILOG("CREATING nodes for fast sync\n"); } else if(!opts_.init) { ELOG("ERROR reading servers (size:%d)\n",(int)last_srvs_.nodes.size()); exit(-1); } - ELOG("CREATING first node\n"); + ILOG("CREATING first node\n"); } else if( (int)opts_.svid >(int)last_srvs_.nodes.size()) { ELOG("ERROR reading servers (size:%d)\n",(int)last_srvs_.nodes.size()); exit(-1); @@ -110,7 +109,7 @@ void server::start() { if(opts_.back && !opts_.comm) { if(undo_bank(false)) { - ELOG("DATABASE check passed, run again with --comm (-c) to commit database change and proceed\n"); + ILOG("DATABASE check passed, run again with --comm (-c) to commit database change and proceed\n"); exit(0); } else { ELOG("DATABASE check failed\n"); @@ -127,7 +126,7 @@ void server::start() { ELOG("ERROR reading servers for path %08X\n",lastpath); exit(-1); } - ELOG("INIT from last state @ %08X with MSID: %08X (file usr/0001.dat found)\n",lastpath,msid_); + ILOG("INIT from last state @ %08X with MSID: %08X (file usr/0001.dat found)\n",lastpath,msid_); if(!undo_bank(true)) { ELOG("ERROR loading initial database, fatal\n"); exit(-1); @@ -140,7 +139,7 @@ void server::start() { srvs_=last_srvs_; memcpy(srvs_.oldhash,last_srvs_.nowhash,SHA256_DIGEST_LENGTH); period_start=srvs_.nextblock(); - ELOG("MAKE BLOCKCHAIN\n"); + ILOG("MAKE BLOCKCHAIN\n"); for(; srvs_.nowmsid_) { - ELOG("ERROR initial msid lower than on network, fatal (%08X<%08X)\n",msid_,firstmsid); - if(!do_fast) { + + if(msid_ > 0 && !do_fast) { + ELOG("Initial msid lower than on network, fatal (%08X<%08X)\n",msid_,firstmsid); exit(-1); } + WLOG("Setting initial msid from network (%08X)\n",firstmsid); msid_=firstmsid; return; } @@ -323,19 +327,19 @@ void server::recyclemsid(uint32_t lastpath) { ntime=time(NULL); assert(ntime>=srvs_.now); } - ELOG("RECYCLED message %04X:%08X from %08X/ signing with new time %08X [len:%d]\n",opts_.svid,lastmsid,lastpath,ntime,msg->len); + ILOG("RECYCLED message %04X:%08X from %08X/ signing with new time %08X [len:%d]\n",opts_.svid,lastmsid,lastpath,ntime,msg->len); msg->signnewtime(ntime,skey,pkey,msha); ntime++; } memcpy(msha,msg->sigh,sizeof(hash_t)); if(txs_insert(msg)) { - ELOG("RECYCLED message %04X:%08X from %08X/ inserted\n",opts_.svid,lastmsid,lastpath); + ILOG("RECYCLED message %04X:%08X from %08X/ inserted\n",opts_.svid,lastmsid,lastpath); if(srvs_.now!=lastpath) { - ELOG("MOVE message %04X:%08X from %08X/ to %08X/\n",opts_.svid,lastmsid,lastpath,srvs_.now); + ILOG("MOVE message %04X:%08X from %08X/ to %08X/\n",opts_.svid,lastmsid,lastpath,srvs_.now); msg->move(srvs_.now); } } else { - ELOG("RECYCLED message %04X:%08X from %08X/ known\n",opts_.svid,lastmsid,lastpath); + ILOG("RECYCLED message %04X:%08X from %08X/ known\n",opts_.svid,lastmsid,lastpath); } } if(msid_!=start_msid) { @@ -473,7 +477,7 @@ int server::undo_bank(bool commit) { //will undo database changes and check if t } void server::load_banks() { - ELOG("LOAD BANKS\n"); + ILOG("LOAD BANKS\n"); //create missing bank messages uint16_t end=last_srvs_.nodes.size(); @@ -520,7 +524,7 @@ void server::load_banks() { } } missing_.unlock(); - ELOG("WAITING for banks (peers:%d)\n",(int)ready.size()); + ILOG("WAITING for banks (peers:%d)\n",(int)ready.size()); //TODO sleep much shorter !!! boost::this_thread::sleep(boost::posix_time::seconds(1)); //yes, yes, use futur/promise instead RETURN_ON_SHUTDOWN(); @@ -570,14 +574,14 @@ void server::load_chain() { for(; !n;) { //boost::this_thread::sleep(boost::posix_time::seconds(1)); m_peerManager.getMoreHeaders(srvs_.now); // try getting more headers - ELOG("\nWAITING 1s (%08X<%08X)\n",srvs_.now,now); + ILOG("\nWAITING 1s (%08X<%08X)\n",srvs_.now,now); boost::this_thread::sleep(boost::posix_time::seconds(1)); RETURN_ON_SHUTDOWN(); n=headers.size(); } auto block=headers.begin(); for(;;) { - ELOG("START syncing header %08X\n",block->now); + ILOG("START syncing header %08X\n",block->now); if(srvs_.now!=block->now) { ELOG("ERROR, got strange block numbers %08X<>%08X\n",srvs_.now,block->now); exit(-1); @@ -746,7 +750,7 @@ void server::load_chain() { for(block++; block==headers.end(); block++) { // wait for peers to load more blocks block--; headers_.unlock(); - ELOG("WAITING at block end (headers:%d) (srvs_.now:%08X;now:%08X) \n", + ILOG("WAITING at block end (headers:%d) (srvs_.now:%08X;now:%08X) \n", (int)headers.size(),srvs_.now,now); //FIXME, insecure !!! better to ask more peers / wait for block with enough votes m_peerManager.getMoreHeaders(block->now+BLOCKSEC); @@ -978,7 +982,7 @@ void server::LAST_block_final(hash_s& cand) { it->second->status|=MSGSTAT_VAL; if(it->second->msid==0xFFFFFFFF) { uint16_t svid=it->second->svid; - ELOG("DOUBLE setting dbl status for node %04X\n",svid); + ILOG("DOUBLE setting dbl status for node %04X\n",svid); dbls_.lock(); dbl_srvs_.insert(svid); //FIXME, pointless dbls_.unlock(); @@ -994,7 +998,7 @@ void server::LAST_block_final(hash_s& cand) { for(auto am=winner->msg_add.begin(); am!=winner->msg_add.end(); am++) { if(*(uint32_t*)(((uint8_t*)&am->first)+2)==0xFFFFFFFF) { uint16_t svid=*(uint16_t*)(((uint8_t*)&am->first)+6); - ELOG("DOUBLE setting dbl status for node %04X\n",svid); + ILOG("DOUBLE setting dbl status for node %04X\n",svid); dbls_.lock(); dbl_srvs_.insert(svid); //FIXME, pointless dbls_.unlock(); @@ -1136,7 +1140,7 @@ void server::LAST_block_final(hash_s& cand) { continue; } if((tm->second->status & MSGSTAT_BAD)) { - ELOG("REMOVE bad message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); + ILOG("REMOVE bad message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); if((tm->second->status & MSGSTAT_COM)) { undo_message(tm->second); } @@ -1153,11 +1157,11 @@ void server::LAST_block_final(hash_s& cand) { continue; } if(!(tm->second->status & MSGSTAT_VAL) && (tm->second->status & MSGSTAT_COM)) { - ELOG("UNDO message %04X:%08X [min:%08X len:%d status:%X]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len,tm->second->status); + ILOG("UNDO message %04X:%08X [min:%08X len:%d status:%X]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len,tm->second->status); undo_message(tm->second); //if(tm->second->nowsecond->nowsecond->svid,tm->second->msid,minmsid,tm->second->len); + ILOG("REMOVE late message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); message_ptr msg=tm->second; bad_insert(tm->second); //remove_message(tm->second); @@ -1167,7 +1171,7 @@ void server::LAST_block_final(hash_s& cand) { txs_msgs_.erase(tm); } } else { - ELOG("INVALIDATE message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); + ILOG("INVALIDATE message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); tm->second->move(LAST_block+BLOCKSEC); wait_.lock(); wait_msgs_.push_back(tm->second); @@ -1178,7 +1182,7 @@ void server::LAST_block_final(hash_s& cand) { if(!(tm->second->status & MSGSTAT_VAL) && tm->second->path && tm->second->path<=LAST_block) { //if(tm->second->nowsecond->nowsecond->svid,tm->second->msid,minmsid,tm->second->len); + ILOG("REMOVE late message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); message_ptr msg=tm->second; bad_insert(tm->second); //remove_message(tm->second); @@ -1189,19 +1193,19 @@ void server::LAST_block_final(hash_s& cand) { txs_msgs_.erase(tm); } } else { - ELOG("MOVE message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); + ILOG("MOVE message %04X:%08X [min:%08X len:%d]\n",tm->second->svid,tm->second->msid,minmsid,tm->second->len); tm->second->move(LAST_block+BLOCKSEC); } } if((tm->second->status & (MSGSTAT_VAL | MSGSTAT_COM)) == MSGSTAT_VAL ) { if(tm->second->msid==0xFFFFFFFF) { - ELOG("COMMIT dbl message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); + ILOG("COMMIT dbl message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); tm->second->status|=MSGSTAT_COM; // the only place to commit dbl messages assert(srvs_.nodes[tm->second->svid].status&SERVER_DBL); continue; } if(tm->second->status & MSGSTAT_DAT) { - ELOG("QUEUE message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); + ILOG("QUEUE message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); wait_.lock(); for(auto wm=wait_msgs_.begin(); wm!=wait_msgs_.end(); wm++) { if((*wm)==tm->second) { @@ -1222,7 +1226,7 @@ void server::LAST_block_final(hash_s& cand) { //auto it=bad_msgs_.find(*(hash_s*)tm->second->sigh); if(it!=bad_msgs_.end()) { if(it->second->hash.num==tm->second->hash.num) { - ELOG("RECOVER message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); + ILOG("RECOVER message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); tm->second=it->second; assert(tm->second->status & MSGSTAT_DAT); bad_recover(tm->second); @@ -1243,7 +1247,7 @@ void server::LAST_block_final(hash_s& cand) { } } //FIXME, check info about peer inventory !!! find out who knows about the message !!! - ELOG("MISSING message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); + WLOG("MISSING message %04X:%08X [len:%d]\n",tm->second->svid,tm->second->msid,tm->second->len); missing_msgs_[tm->second->hash.num]=tm->second; } } @@ -1330,20 +1334,20 @@ void server::count_votes(uint32_t now,hash_s& cand) { if(now>srvs_.now+BLOCKSEC+(BLOCKSEC/2)) { panic=true; if(!ofip_isreadonly()) { - ELOG("LATE CANDIDATE !!!, set office readonly\n"); + WLOG("LATE CANDIDATE !!!, set office readonly\n"); ofip_readonly(); } - ELOG("\n\nCANDIDATE SELECTED:%016lX second:%016lX max:%016lX counted:%016lX BECAUSE OF TIMEOUT!!!\n\n\n", + WLOG("\n\nCANDIDATE SELECTED:%016lX second:%016lX max:%016lX counted:%016lX BECAUSE OF TIMEOUT!!!\n\n\n", cnd1->score,x,votes_max,votes_counted); } else { - ELOG("CANDIDATE ELECTED:%016lX second:%016lX max:%016lX counted:%016lX\n", + ILOG("CANDIDATE ELECTED:%016lX second:%016lX max:%016lX counted:%016lX\n", cnd1->score,x,votes_max,votes_counted); } do_block=2; winner=cnd1; char text[2*SHA256_DIGEST_LENGTH]; ed25519_key2text(text,best.hash,SHA256_DIGEST_LENGTH); - ELOG("CAND %.*s elected\n",2*SHA256_DIGEST_LENGTH,text); + ILOG("CAND %.*s elected\n",2*SHA256_DIGEST_LENGTH,text); if(winner->failed) { ELOG("BAD CANDIDATE elected :-(\n"); } @@ -1362,7 +1366,7 @@ void server::count_votes(uint32_t now,hash_s& cand) { } } if(do_block==2 && winner->elected_accept()) { - ELOG("CANDIDATE winner accepted\n"); + ILOG("CANDIDATE winner accepted\n"); do_block=3; if(do_vote) { write_candidate(best); @@ -1379,12 +1383,12 @@ void server::count_votes(uint32_t now,hash_s& cand) { } } if(do_vote && cnd1->accept() && cnd1->peers.size()>1) { - ELOG("CANDIDATE proposal accepted\n"); + ILOG("CANDIDATE proposal accepted\n"); write_candidate(best); return; } if(do_vote && now>srvs_.now+BLOCKSEC+(do_vote-1)*VOTE_DELAY) { - ELOG("CANDIDATE proposing\n"); + ILOG("CANDIDATE proposing\n"); write_candidate(cand); } } @@ -1468,13 +1472,13 @@ void server::prepare_poll() { } //uint64_t score=last_srvs_.nodes[svid_rank[j]].weight/2+TOTALMASS/(2*(VIP_MAX+1)); uint64_t score=(last_srvs_.nodes[svid_rank[j]].weight+TOTALMASS)/(2*(VIP_MAX+1)); - ELOG("ELECTOR[%d]=%016lX\n",svid_rank[j],score); + ILOG("ELECTOR[%d]=%016lX\n",svid_rank[j],score); electors[svid_rank[j]]=score; votes_max+=score; } extern candidate_ptr nullcnd; winner=nullcnd; - ELOG("ELECTOR max:%016lX\n",votes_max); + ILOG("ELECTOR max:%016lX\n",votes_max); // READNLY ? if readonly server and not enough electors ... resync ! #ifdef DEBUG if(electors.size()mtim=busy_msg->now; memcpy(nod->msha,busy_msg->sigh,sizeof(hash_t)); if(busy_msg->svid==opts_.svid && msid_msid) { - ELOG("WARNING !!! increasing local msid by network !!!\n"); + WLOG("WARNING !!! increasing local msid by network !!!\n"); msid_=nod->msid; } uint32_t now=time(NULL); @@ -3065,7 +3069,7 @@ bool server::process_message(message_ptr msg) { } else { if(node==opts_.svid) { ofip_change_pkey((uint8_t*)utxs.key(p)); - ELOG("WARNING, changing my node key !\n"); + WLOG("WARNING, changing my node key !\n"); } uint64_t ppb=make_ppi(tmpos,omsid,msg->msid,msg->svid,node); txs_bky[ppb]=*(hash_s*)utxs.key(p); @@ -3360,7 +3364,7 @@ bool server::process_message(message_ptr msg) { for(auto it=txs_bky.begin(); it!=txs_bky.end(); it++) { uint16_t node=ppi_bbank(it->first); if(last_srvs_.nodes[node].status & SERVER_DBL) { - ELOG("WARNING schedule resetting DBL status for node %04X!\n",node); + WLOG("WARNING schedule resetting DBL status for node %04X!\n",node); //blk_bky[ppi].push_back(it->first); blk_bky[it->first]=node; } @@ -3670,16 +3674,16 @@ void server::commit_block(std::set& update) { //assume single thread auto bi=new_bnk.begin(); peer=(*bi)&0xffff; new_bnk.erase(bi); - ELOG("BANK, overwrite %04X\n",peer); + ILOG("BANK, overwrite %04X\n",peer); srvs_.put_node(u,peer); } //save_undo() in put_node() !!! else if(srvs_.nodes.size()>16) & 0xFFFFFFFFL; uint16_t svid=(key>>48); if(msid==0xFFFFFFFF) { - ELOG("%04X WARNING peer removed DBL spend from%04X\n",peer,svid); + WLOG("%04X WARNING peer removed DBL spend from%04X\n",peer,svid); failed=true; break; } message_ptr pm=message_svidmsid(svid,msid); //LOCK: txs_ if(pm!=nullmsg && (pm->status & (MSGSTAT_DAT|MSGSTAT_VAL)) && pm->got(srvs_.now+((BLOCKSEC*3)/4)) && (last_srvs_.vokdata+1,SHA256_DIGEST_LENGTH); - ELOG("LAST HASH put %.*s\n",(int)(2*SHA256_DIGEST_LENGTH),hash); + ILOG("LAST HASH put %.*s\n",(int)(2*SHA256_DIGEST_LENGTH),hash); m_peerManager.deliverToAll(put_msg); // sets BLOCK_MODE for peers } do_block=1; //must be before save_candidate diff --git a/src/tools/performance_test/perf_test.py b/src/tools/performance_test/perf_test.py new file mode 100644 index 00000000..92aa346f --- /dev/null +++ b/src/tools/performance_test/perf_test.py @@ -0,0 +1,244 @@ +""" +This test should be run using python3 from directory containing esc binary (or symbolic link) +Node 1 must be running + +1) create 10k accounts +2) accounts 0..7 receive 1M esc, other accounts 1 esc +3) SEND_ONE TEST: accounts 0..7 send 10 esc to accounts 6001..7250 (10k transactions in total) +4) create token accounts for all users (10k) for token 0 +5) account 0 adds 10M own tokens and moves 1M own tokens to accounts 1..7 +6) MOVE_TOKENS TEST: accounts 0..7 move 10 tokens to accounts 6001..7250 (10k transactions in total) +""" + +from time import * +from multiprocessing import Process +import os + + +def get_account_address(account_id): + return '0001-' + str(hex(account_id)[2:]).rjust(8, '0') + '-XXXX' + + +def prepare_accounts(num_accounts, num_esc): + create_account_directories(num_esc) + create_accounts(num_esc, num_accounts) + + +def create_account_directories(num_accounts): + for i in range(num_accounts): + path = 'user_' + str(i) + if not os.path.exists(path): + os.makedirs(path) + os.chdir(path) + os.symlink('../esc', 'esc') + with open('settings.cfg', 'w') as settings: + settings.write('port=9091\n') + settings.write('host=127.0.0.1\n') + settings.write('address=%s\n' % get_account_address(i)) + settings.write('secret=14B183205CA661F589AD83809952A692DFA48F5D490B10FD120DA7BF10F2F4A0\n') + + os.chdir('..') + + +def create_accounts(num_esc, num_accounts): + print('create_accounts') + os.chdir("user_0") + p = os.popen('./esc -w . > /dev/null 2>&1', 'w') + + p.write('{"run":"get_me"}\n') + for i in range(1, num_accounts+1): + amount = '1000000' if i < num_esc else '1' + p.write('{"run":"create_account"}\n') + p.write('{"run":"send_one","address":"' + get_account_address(i) + '","amount":"' + amount + '"}\n') + + p.close() + os.chdir('..') + + +def prepare_token_accounts(num_accounts, num_esc): + create_token_accounts(num_accounts) + add_tokens_to_first_account() + move_tokens_to_first_accounts(num_esc) + + +def send_one_from_client(client_id, send_cmd): + os.chdir('user_' + str(client_id)) + print('send_one from: ' + str(os.getcwd())) + + p = os.popen('./esc -w . > /dev/null 2>&1', 'w') + p.write('{"run":"get_me"}\n') + + for cmd in send_cmd: + p.write(cmd) + + p.close() + + +def print_results(transaction_type, delta, num_transactions): + print(transaction_type + ': %.2f s' % delta) + print(transaction_type + ': %.2f transactions/s\n' % (num_transactions/delta)) + + +def get_me_from_client(client_id, txs_per_client): + os.chdir('user_' + str(client_id)) + print('get_me from: ', str(os.getcwd())) + p = os.popen('./esc -w . > /dev/null 2>&1', 'w') + + for _ in range(txs_per_client): + p.write('{"run":"get_me"}\n') + + p.close() + + +def test_get_me(num_transactions, num_clients, num_exec): + txs_per_client = num_transactions//num_clients + print("per client: ", txs_per_client) + + for _ in range(num_exec): + threads = [] + + for i in range(num_clients): + threads.append(Process(target=get_me_from_client, args=(i, txs_per_client,))) + + start = time() + + for i in threads: + i.start() + + for i in threads: + i.join() + + end = time() + delta = end - start + print_results('get_me', delta, num_transactions) + + +def build_send_one(txs_per_client): + send_cmd = [] + + offset = 1000 + + for i in range(offset, offset + txs_per_client): + send_cmd.append('{"run":"send_one","address":"' + get_account_address(i) + '","amount":1}\n') + + return send_cmd + + +def test_send_one(num_transactions, num_esc, num_exec): + send_cmd = build_send_one(num_transactions//num_esc) + + for _ in range(num_exec): + threads = [] + + for i in range(num_esc): + threads.append(Process(target=send_one_from_client, args=(i, send_cmd))) + + start = time() + + for i in threads: + i.start() + + for i in threads: + i.join() + + end = time() + delta = end - start + print_results('send_one', delta, num_transactions) + + +def create_token_accounts(num_accounts): + print('create_token_accounts') + os.chdir('user_0') + + for i in range(num_accounts+1): + p = os.popen('./esc -w . --address ' + get_account_address(i) + ' > /dev/null 2>&1', 'w') + p.write('{"run":"get_me"}\n') + p.write('{"run":"create_token_account", "token":"00000000"}\n') + p.close() + + os.chdir('..') + + +def add_tokens_to_first_account(): + os.chdir("user_0") + + p = os.popen('./esc -w . > /dev/null 2>&1', 'w') + p.write('{"run":"get_me"}\n') + p.write('{"run":"add_tokens", "amount":"20000000"}\n') + p.close() + + os.chdir("..") + + +def move_tokens_to_first_accounts(num_esc): + os.chdir('user_0') + + p = os.popen('./esc -w . > /dev/null 2>&1', 'w') + p.write('{"run":"get_me"}\n') + + for i in range(1, num_esc): + p.write('{"run":"move_tokens", "token":"00000000", "to":"' + get_account_address(i) + '", "amount":"1000000"}\n') + + p.close() + os.chdir('..') + + +def build_move_tokens(txs_per_client): + move_cmd = [] + + offset = 6000 + + for i in range(offset, offset + txs_per_client): + move_cmd.append('{"run":"move_tokens", "token":"00000000", "to":"' + get_account_address(i) + '","amount":1}\n') + + return move_cmd + + +def move_tokens_from_client(client_id, move_cmd): + os.chdir('user_' + str(client_id)) + print('move_tokens from: ' + str(os.getcwd())) + + p = os.popen('./esc -w . > /dev/null 2>&1', 'w') + p.write('{"run":"get_me"}\n') + + for cmd in move_cmd: + p.write(cmd) + + p.close() + + +def test_move_tokens(num_transactions, num_esc, num_exec): + move_cmd = build_move_tokens(num_transactions//num_esc) + + for _ in range(num_exec): + threads = [] + for i in range(num_esc): + threads.append(Process(target=move_tokens_from_client, args=(i, move_cmd))) + + start = time() + + for i in threads: + i.start() + + for i in threads: + i.join() + + end = time() + delta = end - start + print_results('move_tokens', delta, num_transactions) + + +def run_tests(): + num_accounts = 10000 + num_transactions = 100000 + num_esc = 16 + + prepare_accounts(num_accounts, num_esc) + test_get_me(num_transactions, num_esc, num_exec=1) + test_send_one(num_transactions, num_esc, num_exec=10) + + # prepare_token_accounts(num_accounts, num_esc) + # test_move_tokens(num_transactions, num_esc, num_exec=1) + + +run_tests() diff --git a/src/unittest/CMakeLists.txt b/src/unittest/CMakeLists.txt index f128c709..3fdff9b8 100644 --- a/src/unittest/CMakeLists.txt +++ b/src/unittest/CMakeLists.txt @@ -3,6 +3,8 @@ set (UNITTEST_APP unittest) SET (SRC_LIST unittest.cpp blocks_helper_test.cpp + snapshot_test.cpp + ../../external/ed25519/ed25519.c ) SET (HEADERS_LIST @@ -14,5 +16,5 @@ include_directories(${GTEST_INCLUDE_DIRS}) add_executable(${UNITTEST_APP} ${SRC_LIST}) -target_link_libraries(${UNITTEST_APP} LINK_PUBLIC common gtest gtest_main) +target_link_libraries(${UNITTEST_APP} LINK_PUBLIC common gtest gtest_main common crypto) link_directories(${UNITTEST_APP} ${GTEST_LIBRARY_DIRS}) diff --git a/src/unittest/snapshot_test.cpp b/src/unittest/snapshot_test.cpp new file mode 100644 index 00000000..1adce294 --- /dev/null +++ b/src/unittest/snapshot_test.cpp @@ -0,0 +1,245 @@ +#include "gtest/gtest.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "default.hpp" +#include "helper/blocks.h" +#include "helper/servers.h" + +const int kNodeId = 1; +const std::string kMainDirectory = "blk"; +const std::string kDBDirectory = "usr/"; +const std::string kSnapshotDirectory = "blk/5AB/79000/und/"; +const int kStartBlockDec = 1521979360; +const int kSnapshotBlock = kStartBlockDec + BLOCKSEC; + +/* + * + * | | | | | + * | 1 | | 2 | | 3 + * |____| |____| |__x <-- interrupt point + * ^ + * |____ snapshot directory + * + * und/0001.dat files + * | block| user | msid | weight | + * |------|---------|---------|---------| + * | 1 | 0 | 11 | 10 | + * | | 1 | 1 | 1 | + * -------------------------------------- + * | 2 | 0 | 12 | 10 | + * | | - | - | - | <-- sparse file, later replaced by snapshot + * | | 2 | 1 | 1 | + * -------------------------------------- + * | 3 | 0 | 13 | 3 | + * | | 1 | 2 | 5 | + * | | 2 | 2 | 4 | + * -------------------------------------- + * + * usr/0001.dat + * | | | | | + * | | 0 | 14 | 4 | + * | | 1 | 3 | 4 | + * | | 2 | 3 | 4 | + * | | 3 | 1 | 1 | + * -------------------------------------- + * + * snapshot file + * | | | | | + * | | 0 | 12 | 10 | + * | | 1 | 2 | 5 | + * | | 2 | 1 | 1 | + * | | | | | + * -------------------------------------- + */ + +user_t snapshot_usrs[3]; + +void clean() +{ + boost::filesystem::remove_all("blk"); + boost::filesystem::remove_all("usr"); +} + +TEST(SnapshotTest, prepareData) +{ + + clean(); + + snapshot_usrs[0].msid = 12; + snapshot_usrs[0].weight = 10; + snapshot_usrs[1].msid = 2; + snapshot_usrs[1].weight = 5; + snapshot_usrs[2].msid = 1; + snapshot_usrs[2].weight = 1; + + char filename[64]; + + boost::filesystem::create_directories("usr"); + + uint32_t block_time = kStartBlockDec; + Helper::FileName::getBlk(filename, block_time, "und"); + boost::filesystem::create_directories(filename); + block_time += BLOCKSEC; + Helper::FileName::getBlk(filename, block_time, "und"); + boost::filesystem::create_directories(filename); + block_time += BLOCKSEC; + Helper::FileName::getBlk(filename, block_time, "und"); + boost::filesystem::create_directories(filename); + + block_time = kStartBlockDec; + std::ofstream file; + user_t usr; + + // 1st block + Helper::FileName::getUndo(filename, block_time, kNodeId); + file.open(filename, std::ofstream::out | std::ofstream::binary); + usr = {}; + usr.msid = 11; + usr.weight = 10; + file.write((char*)&usr, sizeof(user_t)); + usr.msid = 1; + usr.weight = 1; + file.write((char*)&usr, sizeof(user_t)); + file.close(); + + // 2nd block + block_time += BLOCKSEC; + Helper::FileName::getUndo(filename, block_time, kNodeId); + file.open(filename, std::ofstream::out | std::ofstream::binary); + usr = {}; + usr.msid = 12; + usr.weight = 10; + file.write((char*)&usr, sizeof(user_t)); + file.seekp(2 * sizeof(user_t)); + usr.msid = 1; + usr.weight = 1; + file.write((char*)&usr, sizeof(user_t)); + file.close(); + + // 3rd block + block_time += BLOCKSEC; + Helper::FileName::getUndo(filename, block_time, kNodeId); + file.open(filename, std::ofstream::out | std::ofstream::binary); + usr = {}; + usr.msid = 13; + usr.weight = 3; + file.write((char*)&usr, sizeof(user_t)); + usr.msid = 2; + usr.weight = 5; + file.write((char*)&usr, sizeof(user_t)); + usr.msid = 2; + usr.weight = 4; + file.write((char*)&usr, sizeof(user_t)); + file.close(); + + // create usr file + Helper::FileName::getUsr(filename, kNodeId); + file.open(filename, std::ofstream::out | std::ofstream::binary); + usr = {}; + usr.msid = 14; + usr.weight = 4; + file.write((char*)&usr, sizeof(user_t)); + usr = {}; + usr.msid = 3; + usr.weight = 4; + file.write((char*)&usr, sizeof(user_t)); + usr = {}; + usr.msid = 3; + usr.weight = 4; + file.write((char*)&usr, sizeof(user_t)); + usr = {}; + usr.msid = 1; + usr.weight = 1; + file.write((char*)&usr, sizeof(user_t)); + file.close(); + + // servers.srv in snapshot directory + Helper::FileName::getName(filename, kSnapshotBlock, "servers.srv"); + Helper::ServersHeader header{}; + header.nodesCount = 2; + Helper::ServersNode node{}; + node.accountCount = 3; + file.open(filename, std::ofstream::out | std::ofstream::binary); + file.write((char*)&header, sizeof(header)); + file.write((char*)&node, sizeof(node)); + file.write((char*)&node, sizeof(node)); + file.close(); +} + +bool compareUsr(const user_t& usr1, const user_t& usr2) +{ + if (usr1.msid != usr2.msid ) return false; + if (usr1.time != usr2.time ) return false; + if (usr1.lpath != usr2.lpath ) return false; + if (usr1.user != usr2.user ) return false; + if (usr1.node != usr2.node ) return false; + if (usr1.stat != usr2.stat ) return false; + if (usr1.rpath != usr2.rpath ) return false; + if (usr1.weight != usr2.weight) return false; + + return true; +} + +TEST(SnapshotTest, createSnapshot) +{ + char filename[64]; + Helper::FileName::getUndo(filename, kSnapshotBlock, kNodeId); + std::stringstream src_data{}, dst_data{}; + std::ifstream file(filename, std::ifstream::in | std::ifstream::binary); + src_data << file.rdbuf(); + file.close(); + + Helper::db_backup(kSnapshotBlock, 2); // 2 nodes incl. 0 + + file.open(filename, std::ifstream::in | std::ifstream::binary); + dst_data << file.rdbuf(); + + EXPECT_NE(src_data.str(), dst_data.str()); + + file.seekg(0, std::ios_base::beg); + + user_t usr{}; + file.read((char*)&usr, sizeof(user_t)); + EXPECT_TRUE(compareUsr(usr, snapshot_usrs[0])); + + usr = {}; + file.read((char*)&usr, sizeof(user_t)); + EXPECT_TRUE(compareUsr(usr, snapshot_usrs[1])); + + usr = {}; + file.read((char*)&usr, sizeof(user_t)); + EXPECT_TRUE(compareUsr(usr, snapshot_usrs[2])); + +} + +TEST(SnapshotTest, createSnapshotNotDividendBlock) +{ + char filename[64]; + uint32_t block_time = kSnapshotBlock + BLOCKSEC; + Helper::FileName::getUndo(filename, block_time, kNodeId); + std::stringstream src_data{}, dst_data{}; + std::ifstream file; + file.open(filename, std::ifstream::in | std::ifstream::binary); + src_data << file.rdbuf(); + file.close(); + + Helper::db_backup(block_time, 2); + + file.open(filename, std::ifstream::in | std::ifstream::binary); + dst_data << file.rdbuf(); + file.close(); + + EXPECT_EQ(src_data.str(), dst_data.str()); +} + +TEST(SnapshotTest, clean) +{ + clean(); +}