From 553fa01fca2065c8da2b0dcdefff8dcdcfc4b5cc Mon Sep 17 00:00:00 2001 From: krypdkat Date: Thu, 9 May 2024 23:46:13 +0700 Subject: [PATCH 1/9] upgrade mining params --- src/public_settings.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/public_settings.h b/src/public_settings.h index fde81059..b7e40da4 100644 --- a/src/public_settings.h +++ b/src/public_settings.h @@ -53,8 +53,8 @@ static unsigned short CONTRACT_FILE_NAME[] = L"contract????.???"; #define DATA_LENGTH 256 #define INFO_LENGTH 128 -#define NUMBER_OF_INPUT_NEURONS 8192 -#define NUMBER_OF_OUTPUT_NEURONS 8192 +#define NUMBER_OF_INPUT_NEURONS 16384 +#define NUMBER_OF_OUTPUT_NEURONS 16384 #define MAX_INPUT_DURATION 256 #define MAX_OUTPUT_DURATION 256 #define NEURON_VALUE_LIMIT 1LL From 3264521773fac74e7b9254d6304ba2d5f6eb89c4 Mon Sep 17 00:00:00 2001 From: krypdkat Date: Thu, 9 May 2024 23:47:26 +0700 Subject: [PATCH 2/9] dynamic alloc score buffer --- src/qubic.cpp | 3 ++- src/score.h | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/qubic.cpp b/src/qubic.cpp index d60fa4a8..d0010a1c 100644 --- a/src/qubic.cpp +++ b/src/qubic.cpp @@ -2704,7 +2704,7 @@ static void beginEpoch1of2() CONTRACT_FILE_NAME[sizeof(CONTRACT_FILE_NAME) / sizeof(CONTRACT_FILE_NAME[0]) - 3] = (system.epoch % 100) / 10 + L'0'; CONTRACT_FILE_NAME[sizeof(CONTRACT_FILE_NAME) / sizeof(CONTRACT_FILE_NAME[0]) - 2] = system.epoch % 10 + L'0'; - bs->SetMem(score, sizeof(*score), 0); + score->initMemory(); score->resetTaskQueue(); bs->SetMem(minerSolutionFlags, NUMBER_OF_MINER_SOLUTION_FLAGS / 8, 0); bs->SetMem((void*)minerPublicKeys, sizeof(minerPublicKeys), 0); @@ -3948,6 +3948,7 @@ static bool initialize() logStatusToConsole(L"EFI_BOOT_SERVICES.AllocatePool() fails", status, __LINE__); return false; } + setMem(score, sizeof(*score), 0); bs->SetMem(solutionThreshold, sizeof(int) * MAX_NUMBER_EPOCH, 0); if (status = bs->AllocatePool(EfiRuntimeServicesData, NUMBER_OF_MINER_SOLUTION_FLAGS / 8, (void**)&minerSolutionFlags)) diff --git a/src/score.h b/src/score.h index 05f9baf9..7bad7fbf 100644 --- a/src/score.h +++ b/src/score.h @@ -36,11 +36,11 @@ struct ScoreFunction static constexpr unsigned int maxAllNeuronLength = dataLength + numberOfNeuronsMaxInputOutput + infoLength; long long miningData[dataLength]; - struct + struct synapseStruct { char inputLength[(numberOfInputNeurons + infoLength) * (dataLength + numberOfInputNeurons + infoLength)]; char outputLength[(numberOfOutputNeurons + dataLength) * (infoLength + numberOfOutputNeurons + dataLength)]; - } _synapses[solutionBufferCount]; + } * _synapses; struct queueItem { short tick; @@ -74,7 +74,7 @@ struct ScoreFunction unsigned char _maxIndexBuffer[allParamsCount * 2][32]; static_assert(maxInputDuration <= 256 && maxOutputDuration <= 256, "Need to increase size of _maxIndexBuffer"); short buffer[256]; - } _computeBuffer[solutionBufferCount]; + } * _computeBuffer; static_assert(maxInputDuration <= 256 && maxOutputDuration <= 256, "Need to regenerate mod num table"); // _totalModNum[i]: total of divisible numbers of i unsigned char _totalModNum[257]; @@ -111,6 +111,33 @@ struct ScoreFunction } } + bool initMemory() + { + if (_synapses == nullptr) { + EFI_STATUS status; + if (status = bs->AllocatePool(EfiRuntimeServicesData, sizeof(synapseStruct) * solutionBufferCount, (void**)&_synapses)) + { + logStatusToConsole(L"EFI_BOOT_SERVICES.AllocatePool() fails", status, __LINE__); + return false; + } + } + if (_computeBuffer == nullptr) { + EFI_STATUS status; + if (status = bs->AllocatePool(EfiRuntimeServicesData, sizeof(computeBuffer) * solutionBufferCount, (void**)&_computeBuffer)) + { + logStatusToConsole(L"EFI_BOOT_SERVICES.AllocatePool() fails", status, __LINE__); + return false; + } + } + for (int i = 0; i < solutionBufferCount; i++) { + setMem(&_synapses[i], sizeof(synapseStruct), 0); + } + for (int i = 0; i < solutionBufferCount; i++) { + setMem(&_computeBuffer[i], sizeof(computeBuffer), 0); + } + return true; + } + // Save score cache to SCORE_CACHE_FILE_NAME void saveScoreCache(int epoch) { From 4a33c55bb37d8b07e36191f380955e5c2a0c9799 Mon Sep 17 00:00:00 2001 From: cyber-pc Date: Wed, 8 May 2024 21:07:35 +0700 Subject: [PATCH 3/9] Support get mining score ranking. --- src/network_messages/special_command.h | 18 +++++++++++++++ src/qubic.cpp | 32 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/network_messages/special_command.h b/src/network_messages/special_command.h index 16951f58..14980d86 100644 --- a/src/network_messages/special_command.h +++ b/src/network_messages/special_command.h @@ -100,8 +100,26 @@ struct UtcTime #define SPECIAL_COMMAND_QUERY_TIME 12ULL // send this to node to query time, responds with time read from clock #define SPECIAL_COMMAND_SEND_TIME 13ULL // send this to node to set time, responds with time read from clock after setting + struct SpecialCommandSendTime { unsigned long long everIncreasingNonceAndCommandType; UtcTime utcTime; }; + +#define SPECIAL_COMMAND_GET_MINING_SCORE_RANKING 14ULL +#pragma pack( push, 1) +template +struct SpecialCommandGetMiningScoreRanking +{ + struct ScoreRankingEntry { + m256i minerPublicKey; + unsigned int minerScore; + }; + + unsigned long long everIncreasingNonceAndCommandType; + unsigned int numberOfRankings; + ScoreRankingEntry rankings[maxNumberOfMiners]; +}; + +#pragma pack(pop) diff --git a/src/qubic.cpp b/src/qubic.cpp index d60fa4a8..98d43a29 100644 --- a/src/qubic.cpp +++ b/src/qubic.cpp @@ -171,6 +171,8 @@ static bool competitorComputorStatuses[(NUMBER_OF_COMPUTORS - QUORUM) * 2]; static unsigned int minimumComputorScore = 0, minimumCandidateScore = 0; static int solutionThreshold[MAX_NUMBER_EPOCH] = { -1 }; static unsigned long long solutionTotalExecutionTicks = 0; +static volatile char minerScoreArrayLock = 0; +static SpecialCommandGetMiningScoreRanking requestMiningScoreRanking; BroadcastFutureTickData broadcastedFutureTickData; @@ -603,6 +605,7 @@ static void processBroadcastComputors(Peer* peer, RequestResponseHeader* header) if (request->computors.epoch == system.epoch) { numberOfOwnComputorIndices = 0; + ACQUIRE(minerScoreArrayLock); for (unsigned int i = 0; i < NUMBER_OF_COMPUTORS; i++) { minerPublicKeys[i] = request->computors.publicKeys[i]; @@ -618,6 +621,7 @@ static void processBroadcastComputors(Peer* peer, RequestResponseHeader* header) } } } + RELEASE(minerScoreArrayLock); } } } @@ -1215,6 +1219,29 @@ static void processSpecialCommand(Peer* peer, RequestResponseHeader* header) enqueueResponse(peer, sizeof(SpecialCommandSendTime), SpecialCommand::type, header->dejavu(), &response); } break; + case SPECIAL_COMMAND_GET_MINING_SCORE_RANKING: + { + requestMiningScoreRanking.everIncreasingNonceAndCommandType = + (request->everIncreasingNonceAndCommandType & 0xFFFFFFFFFFFFFF) | (SPECIAL_COMMAND_GET_MINING_SCORE_RANKING << 56); + + ACQUIRE(minerScoreArrayLock); + requestMiningScoreRanking.numberOfRankings = numberOfMiners; + for (int i = 0; i < requestMiningScoreRanking.numberOfRankings; ++i) + { + requestMiningScoreRanking.rankings[i].minerPublicKey = minerPublicKeys[i]; + requestMiningScoreRanking.rankings[i].minerScore = minerScores[i]; + + } + RELEASE(minerScoreArrayLock); + enqueueResponse(peer, + sizeof(requestMiningScoreRanking.everIncreasingNonceAndCommandType) + + sizeof(requestMiningScoreRanking.numberOfRankings) + + sizeof(requestMiningScoreRanking.rankings[0]) * requestMiningScoreRanking.numberOfRankings, + SpecialCommand::type, + header->dejavu(), + &requestMiningScoreRanking); + } + break; } } } @@ -2319,6 +2346,7 @@ static void processTick(unsigned long long processorNumber) } } + ACQUIRE(minerScoreArrayLock); unsigned int minerIndex; for (minerIndex = 0; minerIndex < numberOfMiners; minerIndex++) { @@ -2365,6 +2393,7 @@ static void processTick(unsigned long long processorNumber) } competitorComputorStatuses[i + (NUMBER_OF_COMPUTORS - QUORUM)] = false; } + RELEASE(minerScoreArrayLock); // bubble sorting -> top 225 from competitorPublicKeys have computors and candidates which are the best from that subset for (unsigned int i = NUMBER_OF_COMPUTORS - QUORUM; i < (NUMBER_OF_COMPUTORS - QUORUM) * 2; i++) @@ -2401,10 +2430,13 @@ static void processTick(unsigned long long processorNumber) minimumCandidateScore = minimumComputorScore; } + ACQUIRE(minerScoreArrayLock); for (unsigned int i = 0; i < QUORUM; i++) { system.futureComputors[i] = minerPublicKeys[i]; } + RELEASE(minerScoreArrayLock); + for (unsigned int i = QUORUM; i < NUMBER_OF_COMPUTORS; i++) { system.futureComputors[i] = competitorPublicKeys[i - QUORUM]; From 0c25925391bb16dd60a48342877957557b88b8f2 Mon Sep 17 00:00:00 2001 From: krypdkat Date: Fri, 10 May 2024 10:01:42 +0700 Subject: [PATCH 4/9] add reset score cache to init mem --- src/score.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/score.h b/src/score.h index 7bad7fbf..5cf564d7 100644 --- a/src/score.h +++ b/src/score.h @@ -135,6 +135,7 @@ struct ScoreFunction for (int i = 0; i < solutionBufferCount; i++) { setMem(&_computeBuffer[i], sizeof(computeBuffer), 0); } + setMem(&scoreCache, sizeof(scoreCache), 0); return true; } From 83886955af467f5042e3008ec8ac64a29f82dbfb Mon Sep 17 00:00:00 2001 From: Philipp Werner <22914157+philippwerner@users.noreply.github.com> Date: Mon, 13 May 2024 08:26:13 +0200 Subject: [PATCH 5/9] Some docs --- src/network_core/peers.h | 5 +++++ src/qubic.cpp | 1 + 2 files changed, 6 insertions(+) diff --git a/src/network_core/peers.h b/src/network_core/peers.h index 23289dad..969c73bf 100644 --- a/src/network_core/peers.h +++ b/src/network_core/peers.h @@ -128,6 +128,7 @@ static void closePeer(Peer* peer) } } +// Add message to sending buffer of specific peer, can only called from main thread (not thread-safe). static void push(Peer* peer, RequestResponseHeader* requestResponseHeader) { // The sending buffer may queue multiple messages, each of which may need to transmitted in many small packets. @@ -149,6 +150,7 @@ static void push(Peer* peer, RequestResponseHeader* requestResponseHeader) } } +// Add message to sending buffer of random peer, can only called from main thread (not thread-safe). static void pushToAny(RequestResponseHeader* requestResponseHeader) { unsigned short suitablePeerIndices[NUMBER_OF_OUTGOING_CONNECTIONS + NUMBER_OF_INCOMING_CONNECTIONS]; @@ -166,6 +168,7 @@ static void pushToAny(RequestResponseHeader* requestResponseHeader) } } +// Add message to sending buffer of some random peers, can only called from main thread (not thread-safe). static void pushToSeveral(RequestResponseHeader* requestResponseHeader) { unsigned short suitablePeerIndices[NUMBER_OF_OUTGOING_CONNECTIONS + NUMBER_OF_INCOMING_CONNECTIONS]; @@ -186,6 +189,7 @@ static void pushToSeveral(RequestResponseHeader* requestResponseHeader) } } +// Add message to response queue of specific peer. If peer is NULL, it will be sent to random peers. Can be called from any thread. static void enqueueResponse(Peer* peer, RequestResponseHeader* responseHeader) { ACQUIRE(responseQueueHeadLock); @@ -207,6 +211,7 @@ static void enqueueResponse(Peer* peer, RequestResponseHeader* responseHeader) RELEASE(responseQueueHeadLock); } +// Add message to response queue of specific peer. If peer is NULL, it will be sent to random peers. Can be called from any thread. static void enqueueResponse(Peer* peer, unsigned int dataSize, unsigned char type, unsigned int dejavu, const void* data) { ACQUIRE(responseQueueHeadLock); diff --git a/src/qubic.cpp b/src/qubic.cpp index 519ded5f..8e0a1670 100644 --- a/src/qubic.cpp +++ b/src/qubic.cpp @@ -5172,6 +5172,7 @@ EFI_STATUS efi_main(EFI_HANDLE imageHandle, EFI_SYSTEM_TABLE* systemTable) } } + // Add messages from response queue to sending buffer const unsigned short responseQueueElementHead = ::responseQueueElementHead; if (responseQueueElementTail != responseQueueElementHead) { From 44da1ec548cafc191e634dea6f556f6d19b36984 Mon Sep 17 00:00:00 2001 From: Philipp Werner <22914157+philippwerner@users.noreply.github.com> Date: Tue, 14 May 2024 14:13:13 +0200 Subject: [PATCH 6/9] Update params for epoch 109 (v1.203.0) --- src/public_settings.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/public_settings.h b/src/public_settings.h index b7e40da4..4f4c8130 100644 --- a/src/public_settings.h +++ b/src/public_settings.h @@ -35,12 +35,12 @@ // Config options that should NOT be changed by operators #define VERSION_A 1 -#define VERSION_B 202 +#define VERSION_B 203 #define VERSION_C 0 // Epoch and initial tick for node startup -#define EPOCH 108 -#define TICK 13820000 +#define EPOCH 109 +#define TICK 13960000 #define ARBITRATOR "AFZPUAIYVPNUYGJRQVLUKOPPVLHAZQTGLYAAUUNBXFTVTAMSBKQBLEIEPCVJ" From 1cc7adfcef55d84cf1c67e974e2c8795861e412e Mon Sep 17 00:00:00 2001 From: Philipp Werner <22914157+philippwerner@users.noreply.github.com> Date: Tue, 14 May 2024 14:27:06 +0200 Subject: [PATCH 7/9] Fix score unit tests --- src/score.h | 11 +++++------ test/score.cpp | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/score.h b/src/score.h index 5cf564d7..d7792d12 100644 --- a/src/score.h +++ b/src/score.h @@ -113,19 +113,18 @@ struct ScoreFunction bool initMemory() { + // TODO: call freePool() for buffers allocated below if (_synapses == nullptr) { - EFI_STATUS status; - if (status = bs->AllocatePool(EfiRuntimeServicesData, sizeof(synapseStruct) * solutionBufferCount, (void**)&_synapses)) + if (!allocatePool(sizeof(synapseStruct) * solutionBufferCount, (void**)&_synapses)) { - logStatusToConsole(L"EFI_BOOT_SERVICES.AllocatePool() fails", status, __LINE__); + logToConsole(L"Failed to allocate memory for score solution buffer!"); return false; } } if (_computeBuffer == nullptr) { - EFI_STATUS status; - if (status = bs->AllocatePool(EfiRuntimeServicesData, sizeof(computeBuffer) * solutionBufferCount, (void**)&_computeBuffer)) + if (!allocatePool(sizeof(computeBuffer) * solutionBufferCount, (void**)&_computeBuffer)) { - logStatusToConsole(L"EFI_BOOT_SERVICES.AllocatePool() fails", status, __LINE__); + logToConsole(L"Failed to allocate memory for score solution buffer!"); return false; } } diff --git a/test/score.cpp b/test/score.cpp index 3e17c629..dd4c46f5 100644 --- a/test/score.cpp +++ b/test/score.cpp @@ -46,6 +46,7 @@ struct ScoreTester score_ref_impl = new ScoreFuncRef; memset(score, 0, sizeof(ScoreFuncOpt)); memset(score_ref_impl, 0, sizeof(ScoreFuncRef)); + EXPECT_TRUE(score->initMemory()); score->initMiningData(_mm256_setzero_si256()); score_ref_impl->initMiningData(); } From 4882e3111aefd2dd217d022140b3b0b62deca7d1 Mon Sep 17 00:00:00 2001 From: Philipp Werner <22914157+philippwerner@users.noreply.github.com> Date: Tue, 14 May 2024 14:34:06 +0200 Subject: [PATCH 8/9] Fix compiler warning --- src/qubic.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/qubic.cpp b/src/qubic.cpp index 8e0a1670..128ea294 100644 --- a/src/qubic.cpp +++ b/src/qubic.cpp @@ -1226,11 +1226,10 @@ static void processSpecialCommand(Peer* peer, RequestResponseHeader* header) ACQUIRE(minerScoreArrayLock); requestMiningScoreRanking.numberOfRankings = numberOfMiners; - for (int i = 0; i < requestMiningScoreRanking.numberOfRankings; ++i) + for (unsigned int i = 0; i < requestMiningScoreRanking.numberOfRankings; ++i) { requestMiningScoreRanking.rankings[i].minerPublicKey = minerPublicKeys[i]; requestMiningScoreRanking.rankings[i].minerScore = minerScores[i]; - } RELEASE(minerScoreArrayLock); enqueueResponse(peer, From b6ad419430786c9b9bcd25a12f536cb3af07beb9 Mon Sep 17 00:00:00 2001 From: Philipp Werner <22914157+philippwerner@users.noreply.github.com> Date: Tue, 14 May 2024 14:49:06 +0200 Subject: [PATCH 9/9] Make sure score locks are initialized with 0 --- src/score.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/score.h b/src/score.h index d7792d12..b4aad1ce 100644 --- a/src/score.h +++ b/src/score.h @@ -130,10 +130,10 @@ struct ScoreFunction } for (int i = 0; i < solutionBufferCount; i++) { setMem(&_synapses[i], sizeof(synapseStruct), 0); - } - for (int i = 0; i < solutionBufferCount; i++) { setMem(&_computeBuffer[i], sizeof(computeBuffer), 0); + solutionEngineLock[i] = 0; } + scoreCacheLock = 0; setMem(&scoreCache, sizeof(scoreCache), 0); return true; }