From 322c81120463941c02bad9d3a070eadf8cc02b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Buko=C5=A1ek?= Date: Fri, 2 Oct 2020 16:52:44 +0200 Subject: [PATCH] w1p: Add no_wait mode to conbench --- go/extra/conbench/cmd/conbench.go | 81 ++++++++++++++++++++++++------ go/extra/conbench/conbench-plot.sh | 24 ++++++++- 2 files changed, 87 insertions(+), 18 deletions(-) diff --git a/go/extra/conbench/cmd/conbench.go b/go/extra/conbench/cmd/conbench.go index 00dcda1d7cf..38327d372d2 100644 --- a/go/extra/conbench/cmd/conbench.go +++ b/go/extra/conbench/cmd/conbench.go @@ -46,6 +46,10 @@ const ( // Use test entity for funding? CfgUseTestEntity = "use_test_entity" + // CfgNoWait uses SubmitTxNoWait instead of SubmitTx, submits txns for the + // given amount of time, then stops. + CfgNoWait = "no_wait" + // Gas price (should be set to the minimum gas price of validators). CfgGasPrice = "gas_price" @@ -84,7 +88,7 @@ type localAccount struct { cachedGas uint64 } -func transfer(ctx context.Context, cc consensus.ClientBackend, from *localAccount, toAddr staking.Address, amount uint64, noCache bool) error { +func transfer(ctx context.Context, cc consensus.ClientBackend, from *localAccount, toAddr staking.Address, amount uint64, noCache, noWait bool) error { var err error // Get sender's nonce if not yet cached (or if we're ignoring cache). @@ -138,16 +142,21 @@ func transfer(ctx context.Context, cc consensus.ClientBackend, from *localAccoun // Increment cached nonce. atomic.AddUint64(&from.cachedNonce, 1) - // Submit with timeout to avoid blocking forever if the client node - // is skipping CheckTx checks. The timeout should be set large enough - // for the network to handle the submission. - timeout := viper.GetDuration(CfgSubmitTxTimeout) - submissionCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - if err = cc.SubmitTx(submissionCtx, signedTx); err != nil { - return err + if noWait { + // Submit transaction, but don't wait for it to be included in a block. + return cc.SubmitTxNoWait(ctx, signedTx) + } else { + // Submit with timeout to avoid blocking forever if the client node + // is skipping CheckTx checks. The timeout should be set large enough + // for the network to handle the submission. + timeout := viper.GetDuration(CfgSubmitTxTimeout) + submissionCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + if err = cc.SubmitTx(submissionCtx, signedTx); err != nil { + return err + } + return nil } - return nil } func refund(ctx context.Context, cc consensus.ClientBackend, sc staking.Backend, from *localAccount, toAddr staking.Address) error { @@ -169,7 +178,7 @@ func refund(ctx context.Context, cc consensus.ClientBackend, sc staking.Backend, } // We don't want refunds to fail, so disable caching. - if err = transfer(ctx, cc, from, toAddr, amount, true); err != nil { + if err = transfer(ctx, cc, from, toAddr, amount, true, false); err != nil { return fmt.Errorf("unable to refund from account: %w", err) } @@ -367,7 +376,7 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo account[a].cachedGas = uint64(estGas) // Each account gets perAccountFunds tokens. - if errr := transfer(ctx, cc, &fundingAcct, account[a].addr, perAccountFunds, true); errr != nil { + if errr := transfer(ctx, cc, &fundingAcct, account[a].addr, perAccountFunds, true, false); errr != nil { // An error has happened while funding, make sure to refund the // funding account from the accounts funded until this point. logger.Error("error while funding, attempting to refund account") @@ -380,6 +389,9 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo } } + noWait := viper.IsSet(CfgNoWait) + noWaitDuration := viper.GetDuration(CfgNoWait) + logger.Info("starting benchmark", "num_accounts", numAccounts) startStatus, err := cc.GetStatus(ctx) if err != nil { @@ -399,6 +411,7 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo totalSubmitTimeNs uint64 numSubmitSamples uint64 numSubmitErrors uint64 + gottaStopFast uint32 ) // Perform benchmark in parallel, one goroutine per account. @@ -407,12 +420,17 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo go func(idx uint64) { var noCache bool for s := uint64(0); s < numSamples; s++ { + if atomic.LoadUint32(&gottaStopFast) > 0 { + // Terminate. + return + } + fromIdx := idx toIdx := idx toAddr := account[toIdx].addr startT := time.Now() - if err = transfer(ctx, cc, &account[fromIdx], toAddr, 1, noCache); err != nil { + if err = transfer(ctx, cc, &account[fromIdx], toAddr, 1, noCache, noWait); err != nil { atomic.AddUint64(&numSubmitErrors, 1) // Disable cache for the next sample, just in case // we messed up the nonce or if the gas cost changed. @@ -428,9 +446,14 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo }(uint64(a)) } - // Wait for all goroutines to finish. - for i := uint64(0); i < numAccounts*numSamples; i++ { - <-doneCh + if !noWait { + // Wait for all goroutines to finish. + for i := uint64(0); i < numAccounts*numSamples; i++ { + <-doneCh + } + } else { + time.Sleep(noWaitDuration) + atomic.StoreUint32(&gottaStopFast, 1) } benchmarkDuration := time.Since(benchmarkStartT) @@ -450,13 +473,17 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo // transactions per second and other stats. // Note that we count all transactions, not just the ones made // by this benchmark. + // + // In addition, do a sliding window for the max avg tps. var totalTxs uint64 var maxTxs uint64 minTxs := uint64(18446744073709551615) txsPerBlock := make([]uint64, 0) txBytesPerBlock := make([]uint64, 0) blockDeltaT := make([]float64, 0) + blockT := make([]time.Time, 0) var prevBlockT time.Time + for height := benchmarkStartHeight; height <= benchmarkStopHeight; height++ { // Count number of transactions. txs, grr := cc.GetTransactions(ctx, height) @@ -492,6 +519,7 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo } blockDeltaT = append(blockDeltaT, blk.Time.Sub(prevBlockT).Seconds()) prevBlockT = blk.Time + blockT = append(blockT, blk.Time) } tps := float64(totalTxs) / benchmarkDuration.Seconds() @@ -502,6 +530,24 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo avgSubmitTimeNs := float64(totalSubmitTimeNs) / float64(numSubmitSamples) + // Do a sliding window over the block size array to get the max avg tps. + var bestAvgTps float64 + for slidingWindowSize := 1; slidingWindowSize <= 32; slidingWindowSize++ { + for i := range txsPerBlock { + var curAvgTps float64 + j := i + for ; j < i+slidingWindowSize && j < len(txsPerBlock); j++ { + curAvgTps += float64(txsPerBlock[j]) + } + curAvgTps /= blockT[j-1].Sub(blockT[i]).Seconds() + // Can divide by zero above if blocks are too close together. + // XXX: Find a workaround for this. + if curAvgTps > bestAvgTps && !math.IsInf(curAvgTps, 0) { + bestAvgTps = curAvgTps + } + } + } + logger.Info("benchmark finished", // Number of accounts involved in benchmark (level of parallelism). "num_accounts", numAccounts, @@ -535,6 +581,8 @@ func doRun(cmd *cobra.Command, args []string) error { // nolint: gocyclo "block_sizes_bytes", strings.Trim(fmt.Sprint(txBytesPerBlock), "[]"), // Time delta between blocks (in seconds). "block_delta_t_s", strings.Trim(fmt.Sprint(blockDeltaT), "[]"), + // Maximum average tps over a sliding window. + "max_avg_tps", bestAvgTps, ) // Refund money into original funding account. @@ -557,6 +605,7 @@ func init() { fs.Uint64(CfgNumAccounts, 10, "Number of accounts to create for benchmarking (also level of parallelism)") fs.Uint64(CfgNumSamples, 30, "Number of samples (transfers) per account") fs.Duration(CfgSubmitTxTimeout, 10*time.Second, "Timeout for SubmitTx (set this based on network parameters)") + fs.Duration(CfgNoWait, 10*time.Second, "Use SubmitTxNoWait instead of SubmitTx (spam transactions) for given amount of time") fs.Bool(CfgUseTestEntity, false, "Use test entity for funding (only for testing)") fs.Uint64(CfgGasPrice, 1, "Gas price (should be set to the minimum gas price of validators)") fs.Bool(CfgFundAndExit, false, "Only fund accounts and exit") diff --git a/go/extra/conbench/conbench-plot.sh b/go/extra/conbench/conbench-plot.sh index 11e1b2ab79a..97960c8486f 100755 --- a/go/extra/conbench/conbench-plot.sh +++ b/go/extra/conbench/conbench-plot.sh @@ -50,6 +50,7 @@ BS_PLOT="conbench-block-size.${NOW}.png" BSS_PLOT="conbench-block-sizes.${NOW}.png" BSSB_PLOT="conbench-block-sizes-bytes.${NOW}.png" BTS_PLOT="conbench-block-times.${NOW}.png" +MATPS_PLOT="conbench-max-avg-tps.${NOW}.png" # Get the root directory of the repository. ROOT="$(cd $(dirname $0)/../../../; pwd -P)" @@ -77,6 +78,7 @@ BS_DATA_FILE="$(mktemp -t oasis-conbench-bs-plot-XXXXXXXXXX)" BSS_DATA_FILE="$(mktemp -t oasis-conbench-bss-plot-XXXXXXXXXX)" BSSB_DATA_FILE="$(mktemp -t oasis-conbench-bssb-plot-XXXXXXXXXX)" BTS_DATA_FILE="$(mktemp -t oasis-conbench-bts-plot-XXXXXXXXXX)" +MATPS_DATA_FILE="$(mktemp -t oasis-conbench-max-avg-tps-plot-XXXXXXXXXX)" ARGS="$@" @@ -99,7 +101,7 @@ run_bench() { # Run benchmark. printf "${GRN}*** Running benchmark for ${num_accounts} accounts...${OFF}\n" - conbench --skip_funding --num_accounts ${num_accounts} > "${output}" + conbench --skip_funding --no_wait=60s --num_samples=100 --num_accounts ${num_accounts} > "${output}" local results=$(fgrep 'msg="benchmark finished"' "${output}") echo "${results}" | tee -a "${RAW_DATA}" @@ -118,6 +120,8 @@ run_bench() { local bts=$(echo "${results}" | sed -r 's/[[:alnum:]_]+=/\n&/g' | awk -F= '$1=="block_delta_t_s"{print $2}' | tr -d '"') + local matps=$(echo "${results}" | sed -r 's/[[:alnum:]_]+=/\n&/g' | awk -F= '$1=="max_avg_tps"{print $2}') + rm "${output}" if [[ "${no_plot}" == "no_plot" ]]; then @@ -148,6 +152,8 @@ run_bench() { echo "${num_accounts} ${blk} ${bt}" >> "${BTS_DATA_FILE}" blk=$((blk+1)) done + + echo "${num_accounts} ${matps}" >> "${MATPS_DATA_FILE}" } ACCT="10, 50, 100, 175, 250, 325, 425, 500, 650, 800, 900" @@ -304,7 +310,21 @@ set palette defined (0 "dark-violet", 1 "blue", 2 "cyan", 3 "yellow", 4 "red") splot '${BTS_DATA_FILE}' with impulses lw 2 lc palette notitle EOF -rm "${TPS_DATA_FILE}" "${ST_DATA_FILE}" "${BS_DATA_FILE}" "${BSS_DATA_FILE}" "${BSSB_DATA_FILE}" "${BTS_DATA_FILE}" +# Plot max avg TPS graph. +gnuplot <<- EOF +set title "Maximum average transactions per second" +set xlabel "Number of parallel accounts" +set xtics (${ACCT}) +set ylabel "transactions/s" textcolor lt 1 +set autoscale y +set grid +set term png +set output "${MATPS_PLOT}" +plot '${MATPS_DATA_FILE}' using 1:2 with linespoint notitle +EOF + + +rm "${TPS_DATA_FILE}" "${ST_DATA_FILE}" "${BS_DATA_FILE}" "${BSS_DATA_FILE}" "${BSSB_DATA_FILE}" "${BTS_DATA_FILE}" "${MATPS_DATA_FILE}" printf "${GRN}*** Refunding original funding account...${OFF}\n" conbench --num_accounts ${MAX_ACCTS} --refund_and_exit