From 4d0c76ef35d6a9c636bac3f86c7d79e83230aa22 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Thu, 19 Sep 2024 21:22:43 +0900 Subject: [PATCH 1/5] fix: break from infinite loop when txn receipt not retrieved Signed-off-by: Ji Hwan --- cmd/loadtest/output.go | 5 +++++ util/util.go | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/loadtest/output.go b/cmd/loadtest/output.go index 6d175acf..629f8b4d 100644 --- a/cmd/loadtest/output.go +++ b/cmd/loadtest/output.go @@ -97,6 +97,11 @@ func printBlockSummary(c *ethclient.Client, bs map[uint64]blockSummary, startNon successfulTx, totalTx := getSuccessfulTransactionCount(bs) meanBlocktime, medianBlocktime, minBlocktime, maxBlocktime, stddevBlocktime, varianceBlocktime := getTimestampBlockSummary(bs) + if len(bs[0].Receipts) == 0 { + log.Error().Msg("Transaction receipts could not be retrieved") + return + } + if summaryOutputMode == "text" { p.Printf("Successful Tx: %v\tTotal Tx: %v\n", number.Decimal(successfulTx), number.Decimal(totalTx)) p.Printf("Total Mining Time: %s\n", totalMiningTime) diff --git a/util/util.go b/util/util.go index 2f3de79c..0491811b 100644 --- a/util/util.go +++ b/util/util.go @@ -165,7 +165,7 @@ func GetReceipts(ctx context.Context, rawBlocks []*json.RawMessage, c *ethrpc.Cl err := c.BatchCallContext(ctx, blms[start:end]) if err != nil { log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") - return nil, err + break } start = end if last { @@ -182,6 +182,10 @@ func GetReceipts(ctx context.Context, rawBlocks []*json.RawMessage, c *ethrpc.Cl } receipts = append(receipts, b.Result.(*json.RawMessage)) } + if len(receipts) == 0 { + log.Error().Msg("No receipts have been fetched") + return nil, nil + } log.Info().Int("hashes", len(txHashes)).Int("receipts", len(receipts)).Msg("Fetched tx receipts") return receipts, nil } From 743b894d37adf82a5b22b5dd621d7291fe47c73f Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Fri, 20 Sep 2024 13:52:11 +0900 Subject: [PATCH 2/5] fix: json unmarshal issue when txreceipt batch call has more than 100txns Signed-off-by: Ji Hwan --- cmd/loadtest/output.go | 20 +++++++++----------- util/util.go | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/cmd/loadtest/output.go b/cmd/loadtest/output.go index 629f8b4d..2be17c5d 100644 --- a/cmd/loadtest/output.go +++ b/cmd/loadtest/output.go @@ -97,11 +97,6 @@ func printBlockSummary(c *ethclient.Client, bs map[uint64]blockSummary, startNon successfulTx, totalTx := getSuccessfulTransactionCount(bs) meanBlocktime, medianBlocktime, minBlocktime, maxBlocktime, stddevBlocktime, varianceBlocktime := getTimestampBlockSummary(bs) - if len(bs[0].Receipts) == 0 { - log.Error().Msg("Transaction receipts could not be retrieved") - return - } - if summaryOutputMode == "text" { p.Printf("Successful Tx: %v\tTotal Tx: %v\n", number.Decimal(successfulTx), number.Decimal(totalTx)) p.Printf("Total Mining Time: %s\n", totalMiningTime) @@ -110,12 +105,15 @@ func printBlockSummary(c *ethclient.Client, bs map[uint64]blockSummary, startNon p.Printf("Transactions per sec: %v\n", number.Decimal(tps)) p.Printf("Gas Per Second: %v\n", number.Decimal(gaspersec)) p.Printf("Latencies - Min: %v\tMedian: %v\tMax: %v\n", number.Decimal(minLatency.Seconds()), number.Decimal(medianLatency.Seconds()), number.Decimal(maxLatency.Seconds())) - p.Printf("Mean Blocktime: %vs\n", number.Decimal(meanBlocktime)) - p.Printf("Median Blocktime: %vs\n", number.Decimal(medianBlocktime)) - p.Printf("Minimum Blocktime: %vs\n", number.Decimal(minBlocktime)) - p.Printf("Maximum Blocktime: %vs\n", number.Decimal(maxBlocktime)) - p.Printf("Blocktime Standard Deviation: %vs\n", number.Decimal(stddevBlocktime)) - p.Printf("Blocktime Variance: %vs\n", number.Decimal(varianceBlocktime)) + // Blocktime related metrics can only be calculated when there are at least two blocks + if len(bs) > 1 { + p.Printf("Mean Blocktime: %vs\n", number.Decimal(meanBlocktime)) + p.Printf("Median Blocktime: %vs\n", number.Decimal(medianBlocktime)) + p.Printf("Minimum Blocktime: %vs\n", number.Decimal(minBlocktime)) + p.Printf("Maximum Blocktime: %vs\n", number.Decimal(maxBlocktime)) + p.Printf("Blocktime Standard Deviation: %vs\n", number.Decimal(stddevBlocktime)) + p.Printf("Blocktime Variance: %vs\n", number.Decimal(varianceBlocktime)) + } } else if summaryOutputMode == "json" { summaryOutput := SummaryOutput{} summaryOutput.Summaries = jsonSummaryList diff --git a/util/util.go b/util/util.go index 0491811b..c424927c 100644 --- a/util/util.go +++ b/util/util.go @@ -161,14 +161,36 @@ func GetReceipts(ctx context.Context, rawBlocks []*json.RawMessage, c *ethrpc.Cl } break } - - err := c.BatchCallContext(ctx, blms[start:end]) - if err != nil { - log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") - break - } - start = end - if last { + // go-ethereum's BatchCallContext fails when the number of txns are over 100, because it attempts to unmarshal a single json source object into a slice if the number of txns go over 100. + // A workaround could be to implement our own unmarshal function for batch calls to unmarshal the single json object into a custom type (non-slice), but the below method has been chosen. + // When the number of txns go over 100 and forms a slice, this logic will batch call in units of 100 txns. This will successfully unmarshal the json source object. + if end <= 100 { + err := c.BatchCallContext(ctx, blms[start:end]) + if err != nil { + log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") + break + } + start = end + if last { + break + } + } else { + for index := uint64(0); index <= end; index = index + 100 { + // Check for the final section of the slice, and also check end is not equal to index, because this will send another already sent BatchCall otherwise. + if end < index+100 && end != index { + err := c.BatchCallContext(ctx, blms[index:end]) + if err != nil { + log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") + break + } + break + } + err := c.BatchCallContext(ctx, blms[index:index+100]) + if err != nil { + log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") + break + } + } break } } From 906e90309ba8fe2c0a6cb9332854a4a457a9589e Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Mon, 23 Sep 2024 20:29:46 +0900 Subject: [PATCH 3/5] fix: remove batch logic in favour of --batch-size flag, and add checks for empty receipts Signed-off-by: Ji Hwan --- cmd/loadtest/output.go | 8 ++++++++ util/util.go | 42 ++++++++++++------------------------------ 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/cmd/loadtest/output.go b/cmd/loadtest/output.go index 2be17c5d..56d6a049 100644 --- a/cmd/loadtest/output.go +++ b/cmd/loadtest/output.go @@ -98,6 +98,14 @@ func printBlockSummary(c *ethclient.Client, bs map[uint64]blockSummary, startNon meanBlocktime, medianBlocktime, minBlocktime, maxBlocktime, stddevBlocktime, varianceBlocktime := getTimestampBlockSummary(bs) if summaryOutputMode == "text" { + // In the case where no transaction receipts could be retrieved, return. + if successfulTx == 0 { + log.Error().Msg("No transaction could be retrieved from the receipts") + return + } + if len(bs) == 0 { + log.Debug().Int("Length of blockSummary", len(bs)).Msg("blockSummary is empty") + } p.Printf("Successful Tx: %v\tTotal Tx: %v\n", number.Decimal(successfulTx), number.Decimal(totalTx)) p.Printf("Total Mining Time: %s\n", totalMiningTime) p.Printf("Total Transactions: %v\n", number.Decimal(totalTransactions)) diff --git a/util/util.go b/util/util.go index c424927c..2aa17e70 100644 --- a/util/util.go +++ b/util/util.go @@ -137,6 +137,10 @@ func GetReceipts(ctx context.Context, rawBlocks []*json.RawMessage, c *ethrpc.Cl var start uint64 = 0 for { + // if len(blms) == 0 { + // log.Debug().Int("Length of BatchElem", len(blms)).Msg("BatchElem is empty") + // return nil, nil + // } last := false end := start + batchSize if int(end) > len(blms) { @@ -161,36 +165,14 @@ func GetReceipts(ctx context.Context, rawBlocks []*json.RawMessage, c *ethrpc.Cl } break } - // go-ethereum's BatchCallContext fails when the number of txns are over 100, because it attempts to unmarshal a single json source object into a slice if the number of txns go over 100. - // A workaround could be to implement our own unmarshal function for batch calls to unmarshal the single json object into a custom type (non-slice), but the below method has been chosen. - // When the number of txns go over 100 and forms a slice, this logic will batch call in units of 100 txns. This will successfully unmarshal the json source object. - if end <= 100 { - err := c.BatchCallContext(ctx, blms[start:end]) - if err != nil { - log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") - break - } - start = end - if last { - break - } - } else { - for index := uint64(0); index <= end; index = index + 100 { - // Check for the final section of the slice, and also check end is not equal to index, because this will send another already sent BatchCall otherwise. - if end < index+100 && end != index { - err := c.BatchCallContext(ctx, blms[index:end]) - if err != nil { - log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") - break - } - break - } - err := c.BatchCallContext(ctx, blms[index:index+100]) - if err != nil { - log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") - break - } - } + + err := c.BatchCallContext(ctx, blms[start:end]) + if err != nil { + log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") + break + } + start = end + if last { break } } From 32772ea0e679b3be00c9abf03212f6f700bd4558 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Mon, 23 Sep 2024 23:02:55 +0900 Subject: [PATCH 4/5] chore: cleanup Signed-off-by: Ji Hwan --- cmd/loadtest/output.go | 6 +++--- util/util.go | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/loadtest/output.go b/cmd/loadtest/output.go index 56d6a049..60902541 100644 --- a/cmd/loadtest/output.go +++ b/cmd/loadtest/output.go @@ -103,9 +103,6 @@ func printBlockSummary(c *ethclient.Client, bs map[uint64]blockSummary, startNon log.Error().Msg("No transaction could be retrieved from the receipts") return } - if len(bs) == 0 { - log.Debug().Int("Length of blockSummary", len(bs)).Msg("blockSummary is empty") - } p.Printf("Successful Tx: %v\tTotal Tx: %v\n", number.Decimal(successfulTx), number.Decimal(totalTx)) p.Printf("Total Mining Time: %s\n", totalMiningTime) p.Printf("Total Transactions: %v\n", number.Decimal(totalTransactions)) @@ -122,6 +119,9 @@ func printBlockSummary(c *ethclient.Client, bs map[uint64]blockSummary, startNon p.Printf("Blocktime Standard Deviation: %vs\n", number.Decimal(stddevBlocktime)) p.Printf("Blocktime Variance: %vs\n", number.Decimal(varianceBlocktime)) } + if len(bs) == 0 { + log.Debug().Int("Length of blockSummary", len(bs)).Msg("blockSummary is empty") + } } else if summaryOutputMode == "json" { summaryOutput := SummaryOutput{} summaryOutput.Summaries = jsonSummaryList diff --git a/util/util.go b/util/util.go index 859398ff..f063015a 100644 --- a/util/util.go +++ b/util/util.go @@ -135,12 +135,13 @@ func GetReceipts(ctx context.Context, rawBlocks []*json.RawMessage, c *ethrpc.Cl blmsBlockMap[i] = txHashMap[tx] } + if len(blms) == 0 { + log.Debug().Int("Length of BatchElem", len(blms)).Msg("BatchElem is empty") + return nil, nil + } + var start uint64 = 0 for { - if len(blms) == 0 { - log.Debug().Int("Length of BatchElem", len(blms)).Msg("BatchElem is empty") - return nil, nil - } last := false end := start + batchSize if int(end) > len(blms) { @@ -168,7 +169,7 @@ func GetReceipts(ctx context.Context, rawBlocks []*json.RawMessage, c *ethrpc.Cl err := c.BatchCallContext(ctx, blms[start:end]) if err != nil { - log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts") + log.Error().Err(err).Str("randtx", txHashes[0]).Uint64("start", start).Uint64("end", end).Msg("RPC issue fetching receipts, have you checked the batch size limit of the RPC endpoint and adjusted the --batch-size flag?") break } start = end From 1a60ba137aeae207f410e09ac27fed1bfdbd203e Mon Sep 17 00:00:00 2001 From: Ji Hwan KIM <125336262+jhkimqd@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:16:48 +0900 Subject: [PATCH 5/5] Update cmd/loadtest/output.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Léo Vincent <28714795+leovct@users.noreply.github.com> --- cmd/loadtest/output.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/loadtest/output.go b/cmd/loadtest/output.go index 60902541..2437e3ef 100644 --- a/cmd/loadtest/output.go +++ b/cmd/loadtest/output.go @@ -118,8 +118,7 @@ func printBlockSummary(c *ethclient.Client, bs map[uint64]blockSummary, startNon p.Printf("Maximum Blocktime: %vs\n", number.Decimal(maxBlocktime)) p.Printf("Blocktime Standard Deviation: %vs\n", number.Decimal(stddevBlocktime)) p.Printf("Blocktime Variance: %vs\n", number.Decimal(varianceBlocktime)) - } - if len(bs) == 0 { + } else { log.Debug().Int("Length of blockSummary", len(bs)).Msg("blockSummary is empty") } } else if summaryOutputMode == "json" {