Merge pull request hyperledger#76 from kaleido-io/no-wait-sub

No wait submission for all test cases

README.md

@@ -134,15 +134,15 @@ There are various options for creating your own customized tests. A full list of
   - The test will allow at most `startRate` actions to happen per second. Over the period of `rateRampUpTime` seconds the allowed rate will increase linearly until `endRate` actions per seconds are reached. At this point the test will continue at `endRate` actions per second until the test finishes.
   - If `startRate` is the only value that is set, the test will run at that rate for the entire test.
 - Waiting for mint transactions to be confirmed before doing the next one
-  - See `skipMintConfirmations` (defaults to `false`).
+  - See `noWaitSubmission` (defaults to `false`).
   - When set to `true` each worker routine will perform its action (e.g. minting a token) and wait for confirmation of that event before doing its next action.
 - Setting the features of a token being tested
   - See `supportsData` and `supportsURI` attributes of a test instance.
   - `supportsData` defaults to `true` since the sample token contract used by FireFly supports minting tokens with data. When set to `true` the message included in the mint transaction will include the ID of the worker routine and used to correlate received confirmation events.
   - `supportsURI` defaults to `true` for nonfungible tokens. This attribute is ignored for fungible token tests. If set to `true` the ID of a worker routine will be set in the URI and used to correlate received confirmation events.
-  - If neither attribute is set to true any received confirmation events cannot be correlated with mint transactions. In this case the test behaves as if `skipMintConfirmations` is set to `true`.
+  - If neither attribute is set to true any received confirmation events cannot be correlated with mint transactions. In this case the test behaves as if `noWaitSubmission` is set to `true`.
 - Waiting at the end of the test for the minted token balance of the `mintRecipient` address to equal the expected value. Since a test might be run several times with the same address the test gets the balance at the beginning of the test, and then again at the end. The difference is expected to equal the value of `maxActions`. To enable this check set the `maxTokenBalanceWait` token option the length of time to wait for the balance to be reached. If `maxTokenBalanceWait` is not set the test will not check balances.
-- Having a worker loop submit more than 1 action per loop by setting `actionsPerLoop` for the test. This can be helpful when you want to scale the number of actions done in parallel without having to scale the number of workers. The default value is `1` for this attribute. If setting to a value > `1` it is recommended to have `skipMintConfirmations` to set `false`.
+- Having a worker loop submit more than 1 action per loop by setting `actionsPerLoop` for the test. This can be helpful when you want to scale the number of actions done in parallel without having to scale the number of workers. The default value is `1` for this attribute. If setting to a value > `1` it is recommended to have `noWaitSubmission` to set `false`.
 
 ## Distributed Deployment
 

cmd/run.go

@@ -215,6 +215,7 @@ func generateRunnerConfigFromInstance(instance *conf.InstanceConfig, perfConfig
 	// Common configuration regardless of running with manually defined nodes or a local stack
 	runnerConfig.LogLevel = perfConfig.LogLevel
 	runnerConfig.SkipMintConfirmations = instance.SkipMintConfirmations
+	runnerConfig.NoWaitSubmission = instance.NoWaitSubmission
 	runnerConfig.Length = instance.Length
 	runnerConfig.Daemon = perfConfig.Daemon
 	runnerConfig.LogEvents = perfConfig.LogEvents

config/example-remote-node-instances-fungible.yaml

@@ -21,10 +21,8 @@ instances:
         workers: 4
     fireflyNamespace: default
     maxTimePerAction: 300s
-    skipMintConfirmations: true
-    startRate: 1
-    endRate: 20
-    rateRampUpTime: 60m
+    noWaitSubmission: true
+    rampLength: 60m
     tokenOptions:
       tokenType: nonfungible
       mintRecipient: 0xcd833ed165a7b26b68a3cb97daede71691dcaf97

internal/conf/conf.go

@@ -51,7 +51,8 @@ type RunnerConfig struct {
 	MaxTimePerAction          time.Duration
 	MaxActions                int64
 	RampLength                time.Duration
-	SkipMintConfirmations     bool
+	SkipMintConfirmations     bool // deprecated
+	NoWaitSubmission          bool
 }
 
 type PerformanceTestConfig struct {
@@ -82,7 +83,8 @@ type InstanceConfig struct {
 	MaxTimePerAction          time.Duration    `json:"maxTimePerAction,omitempty" yaml:"maxTimePerAction,omitempty"`
 	MaxActions                int64            `json:"maxActions,omitempty" yaml:"maxActions,omitempty"`
 	RampLength                time.Duration    `json:"rampLength,omitempty" yaml:"rampLength,omitempty"`
-	SkipMintConfirmations     bool             `json:"skipMintConfirmations" yaml:"skipMintConfirmations"`
+	SkipMintConfirmations     bool             `json:"skipMintConfirmations" yaml:"skipMintConfirmations"` // deprecated
+	NoWaitSubmission          bool             `json:"noWaitSubmission" yaml:"noWaitSubmission"`
 	DelinquentAction          string           `json:"delinquentAction,omitempty" yaml:"delinquentAction,omitempty"`
 	PerWorkerSigningKeyPrefix string           `json:"perWorkerSigningKeyPrefix,omitempty" yaml:"perWorkerSigningKeyPrefix,omitempty"`
 }

internal/perf/perf.go

@@ -519,6 +519,32 @@ perfLoop:
 	}
 
 	pr.stopping = true
+
+	tallyStart := time.Now()
+
+	if pr.cfg.NoWaitSubmission {
+		eventsCount := getMetricVal(receivedEventsCounter)
+		submissionCount := getMetricVal(totalActionsCounter)
+		log.Infof("<No wait submission mode> Wait for the event count %f to reach request sent count %f, within 30s", eventsCount, submissionCount)
+		for {
+			if eventsCount == submissionCount {
+				break
+			} else if eventsCount > submissionCount {
+				log.Warnf("The number of events received %f is greater than the number of requests sent %f.", eventsCount, submissionCount)
+				break
+			}
+
+			// Check if more than 1 minute has passed
+			if time.Since(tallyStart) > 30*time.Second {
+				log.Errorf("The number of events received %f doesn't tally up to the number of requests sent %f after %s.", eventsCount, submissionCount, time.Since(time.Unix(pr.startTime, 0)))
+				break
+			}
+
+			time.Sleep(time.Second * 1)
+			log.Infof("<No wait submission mode> Wait for the event count %f to reach request sent count %f", eventsCount, submissionCount)
+		}
+	}
+
 	measuredActions := pr.totalSummary
 	measuredTime := time.Since(time.Unix(pr.startTime, 0))
 
@@ -713,7 +739,7 @@ func (pr *perfRunner) eventLoop(nodeURL string, wsconn wsclient.WSClient) (err e
 			wsconn.Send(context.Background(), ackJSON)
 			pr.recordCompletedAction()
 			// Release worker so it can continue to its next task
-			if !pr.stopping && !pr.cfg.SkipMintConfirmations {
+			if !pr.stopping && !pr.cfg.NoWaitSubmission && !pr.cfg.SkipMintConfirmations {
 				if workerID >= 0 {
 					pr.wsReceivers[workerID] <- nodeURL
 				}
@@ -811,7 +837,7 @@ func (pr *perfRunner) runLoop(tc TestCase) error {
 					break
 				}
 			}
-			if testName == conf.PerfTestTokenMint.String() && pr.cfg.SkipMintConfirmations {
+			if pr.cfg.NoWaitSubmission || (testName == conf.PerfTestTokenMint.String() && pr.cfg.SkipMintConfirmations) {
 				// For minting tests a worker can (if configured) skip waiting for a matching response event
 				// before making itself available for the next job
 				confirmationsPerAction = 0
@@ -844,14 +870,19 @@ func (pr *perfRunner) runLoop(tc TestCase) error {
 			pr.totalTime.Record(totalDurationPerLoop)
 			secondsPerLoop := totalDurationPerLoop.Seconds()
 
-			eventReceivingDurationPerLoop := time.Since(sentTime)
-			eventReceivingSecondsPerLoop = eventReceivingDurationPerLoop.Seconds()
-			pr.receiveTime.Record(totalDurationPerLoop)
+			if pr.cfg.NoWaitSubmission {
+				log.Infof("%d <-- %s Finished (loop=%d) after %f seconds", workerID, testName, loop, secondsPerLoop)
 
-			total := submissionSecondsPerLoop + eventReceivingSecondsPerLoop
-			subPortion := int((submissionSecondsPerLoop / total) * 100)
-			envPortion := int((eventReceivingSecondsPerLoop / total) * 100)
-			log.Infof("%d <-- %s Finished (loop=%d), submission time: %f s, event receive time: %f s. Ratio (%d/%d) after %f seconds", workerID, testName, loop, submissionSecondsPerLoop, eventReceivingSecondsPerLoop, subPortion, envPortion, secondsPerLoop)
+			} else {
+				eventReceivingDurationPerLoop := time.Since(sentTime)
+				eventReceivingSecondsPerLoop = eventReceivingDurationPerLoop.Seconds()
+				pr.receiveTime.Record(totalDurationPerLoop)
+
+				total := submissionSecondsPerLoop + eventReceivingSecondsPerLoop
+				subPortion := int((submissionSecondsPerLoop / total) * 100)
+				envPortion := int((eventReceivingSecondsPerLoop / total) * 100)
+				log.Infof("%d <-- %s Finished (loop=%d), submission time: %f s, event receive time: %f s. Ratio (%d/%d) after %f seconds", workerID, testName, loop, submissionSecondsPerLoop, eventReceivingSecondsPerLoop, subPortion, envPortion, secondsPerLoop)
+			}
 
 			if histErr == nil {
 				log.Infof("%d <-- %s Emmiting (loop=%d) after %f seconds", workerID, testName, loop, secondsPerLoop)

scripts/prepForRemote.sh

@@ -69,7 +69,7 @@ instances:
     apiPrefix: ${REMOTE_ENDPOINT_API_PREFIX}
     signingAddress: ${SIGNING_KEY}
     maxTimePerAction: 60s
-    skipMintConfirmations: true
+    noWaitSubmission: true
     delinquentAction: log
     length: 500h
     tokenOptions: