From 9aab6b68cc47704026fa4c3ab3d75434bb4083f6 Mon Sep 17 00:00:00 2001 From: nicholasguoalgorand <67928479+nicholasguoalgorand@users.noreply.github.com> Date: Thu, 29 Oct 2020 07:33:23 -0700 Subject: [PATCH 01/34] Avoid printing control characters for displaying assets (#1660) Asset names can have control characters, which will be displayed to the terminal when run commands such as goal asset info. This allows malicious asset names to delete terminal input and / or print arbitrary input. --- cmd/goal/account.go | 4 ++ cmd/goal/accountsList.go | 3 +- cmd/goal/asset.go | 4 +- test/e2e-go/cli/goal/expect/goalAssetTest.exp | 54 +++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 test/e2e-go/cli/goal/expect/goalAssetTest.exp diff --git a/cmd/goal/account.go b/cmd/goal/account.go index e9e928dae6..7d740f14cb 100644 --- a/cmd/goal/account.go +++ b/cmd/goal/account.go @@ -570,10 +570,12 @@ func printAccountInfo(client libgoal.Client, address string, account v1.Account) if len(name) == 0 { name = "" } + _, name = unicodePrintable(name) units := assetParams.UnitName if len(units) == 0 { units = "units" } + _, units = unicodePrintable(units) total := assetDecimalsFmt(assetParams.Total, assetParams.Decimals) url := "" if len(assetParams.URL) != 0 { @@ -602,11 +604,13 @@ func printAccountInfo(client libgoal.Client, address string, account v1.Account) if len(assetName) == 0 { assetName = "" } + _, assetName = unicodePrintable(assetName) unitName := assetParams.UnitName if len(unitName) == 0 { unitName = "units" } + _, unitName = unicodePrintable(unitName) frozen := "" if assetHolding.Frozen { diff --git a/cmd/goal/accountsList.go b/cmd/goal/accountsList.go index 20485c2762..ef89bdd415 100644 --- a/cmd/goal/accountsList.go +++ b/cmd/goal/accountsList.go @@ -228,7 +228,8 @@ func (accountList *AccountsList) outputAccount(addr string, acctInfo v1.Account, if len(acctInfo.AssetParams) > 0 { var out []string for curid, params := range acctInfo.AssetParams { - out = append(out, fmt.Sprintf("%d (%d %s)", curid, params.Total, params.UnitName)) + _, unitName := unicodePrintable(params.UnitName) + out = append(out, fmt.Sprintf("%d (%d %s)", curid, params.Total, unitName)) } fmt.Printf("\t[created asset IDs: %s]", strings.Join(out, ", ")) } diff --git a/cmd/goal/asset.go b/cmd/goal/asset.go index 4ca454c80f..074e00bfbb 100644 --- a/cmd/goal/asset.go +++ b/cmd/goal/asset.go @@ -603,8 +603,8 @@ var infoAssetCmd = &cobra.Command{ fmt.Printf("Asset ID: %d\n", assetID) fmt.Printf("Creator: %s\n", params.Creator) - fmt.Printf("Asset name: %s\n", params.AssetName) - fmt.Printf("Unit name: %s\n", params.UnitName) + reportInfof("Asset name: %s\n", params.AssetName) + reportInfof("Unit name: %s\n", params.UnitName) fmt.Printf("Maximum issue: %s %s\n", assetDecimalsFmt(params.Total, params.Decimals), params.UnitName) fmt.Printf("Reserve amount: %s %s\n", assetDecimalsFmt(res.Amount, params.Decimals), params.UnitName) fmt.Printf("Issued: %s %s\n", assetDecimalsFmt(params.Total-res.Amount, params.Decimals), params.UnitName) diff --git a/test/e2e-go/cli/goal/expect/goalAssetTest.exp b/test/e2e-go/cli/goal/expect/goalAssetTest.exp new file mode 100644 index 0000000000..d9384b8f23 --- /dev/null +++ b/test/e2e-go/cli/goal/expect/goalAssetTest.exp @@ -0,0 +1,54 @@ +#!/usr/bin/expect -f +set err 0 +log_user 1 + +if { [catch { + + source goalExpectCommon.exp + set TEST_ALGO_DIR [lindex $argv 0] + set TEST_DATA_DIR [lindex $argv 1] + + puts "TEST_ALGO_DIR: $TEST_ALGO_DIR" + puts "TEST_DATA_DIR: $TEST_DATA_DIR" + + set TIME_STAMP [clock seconds] + + set TEST_ROOT_DIR $TEST_ALGO_DIR/root + set TEST_PRIMARY_NODE_DIR $TEST_ROOT_DIR/Primary/ + set NETWORK_NAME test_net_expect_$TIME_STAMP + set NETWORK_TEMPLATE "$TEST_DATA_DIR/nettemplates/TwoNodes50EachFuture.json" + + exec cp $TEST_DATA_DIR/../../gen/devnet/genesis.json $TEST_ALGO_DIR + + # Create network + ::AlgorandGoal::CreateNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + ::AlgorandGoal::StartNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + set PRIMARY_NODE_ADDRESS [ ::AlgorandGoal::GetAlgodNetworkAddress $TEST_PRIMARY_NODE_DIR ] + puts "Primary Node Address: $PRIMARY_NODE_ADDRESS" + + set PRIMARY_WALLET_NAME unencrypted-default-wallet + + # Determine primary account + set PRIMARY_ACCOUNT_ADDRESS [::AlgorandGoal::GetHighestFundedAccountForWallet $PRIMARY_WALLET_NAME $TEST_PRIMARY_NODE_DIR] + + exec goal asset create --creator $PRIMARY_ACCOUNT_ADDRESS --total 90000 --name 'testassetname\b\b\b' --unitname 'u' --datadir $TEST_PRIMARY_NODE_DIR + + spawn goal asset info --creator $PRIMARY_ACCOUNT_ADDRESS --unitname 'u' --datadir $TEST_PRIMARY_NODE_DIR + expect { + timeout { close; ::AlgorandGoal::Abort "Asset Info Failed" } + "Asset name: 'testassetname'" { puts "Successfully displayed asset"; close;} + eof { close; ::AlgorandGoal::Abort "Asset Info Unexpected Result" } + } + + spawn goal account info --address $PRIMARY_ACCOUNT_ADDRESS --datadir $TEST_PRIMARY_NODE_DIR + expect { + timeout { close; ::AlgorandGoal::Abort "Account Info Failed" } + "'testassetname', supply" { puts "Successfully displayed account asset"; close;} + eof { close; ::AlgorandGoal::Abort "Account Info Unexpected Result" } + } +} EXCEPTION] } { + puts "ERROR in goalFormattingTest: $EXCEPTION" + exit 1 +} From fae2bfac18380c4a49fcb9a84192b8485de3312a Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 30 Oct 2020 17:32:04 -0400 Subject: [PATCH 02/34] Update mainnet-model for scale and distribution (#1669) Increasing the number of nodes from 25 to 50 so that it would be closed to the actual mainnet 180 participating nodes, and adjust for geographic distribtuion --- .../recipes/mainnet-model/Makefile | 2 +- .../recipes/mainnet-model/generated/net.json | 861 ++++++++++++++---- .../mainnet-model/generated/topology.json | 203 ++++- .../recipes/mainnet-model/network-tpl.json | 34 +- 4 files changed, 850 insertions(+), 250 deletions(-) diff --git a/test/testdata/deployednettemplates/recipes/mainnet-model/Makefile b/test/testdata/deployednettemplates/recipes/mainnet-model/Makefile index 780b5931d9..bebe8dd2cd 100644 --- a/test/testdata/deployednettemplates/recipes/mainnet-model/Makefile +++ b/test/testdata/deployednettemplates/recipes/mainnet-model/Makefile @@ -1,6 +1,6 @@ all: network_performance_rules -network_performance_rules: generate_network_rules.js data/bandwidth.json data/countries.json data/latency.json configs/node.json configs/nonPartNode.json configs/relay.json +network_performance_rules: generate_network_rules.js data/bandwidth.json data/countries.json data/latency.json configs/node.json configs/nonPartNode.json configs/relay.json network-tpl.json node generate_network_rules.js ../../generate-recipe/generate_network.py --template ./network-tpl.json diff --git a/test/testdata/deployednettemplates/recipes/mainnet-model/generated/net.json b/test/testdata/deployednettemplates/recipes/mainnet-model/generated/net.json index 45e48856b5..32dc430096 100644 --- a/test/testdata/deployednettemplates/recipes/mainnet-model/generated/net.json +++ b/test/testdata/deployednettemplates/recipes/mainnet-model/generated/net.json @@ -511,17 +511,9 @@ "Name": "Wallet1", "ParticipationOnly": false }, - { - "Name": "Wallet26", - "ParticipationOnly": false - }, { "Name": "Wallet51", "ParticipationOnly": false - }, - { - "Name": "Wallet76", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -546,17 +538,9 @@ "Name": "Wallet2", "ParticipationOnly": false }, - { - "Name": "Wallet27", - "ParticipationOnly": false - }, { "Name": "Wallet52", "ParticipationOnly": false - }, - { - "Name": "Wallet77", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -581,17 +565,9 @@ "Name": "Wallet3", "ParticipationOnly": false }, - { - "Name": "Wallet28", - "ParticipationOnly": false - }, { "Name": "Wallet53", "ParticipationOnly": false - }, - { - "Name": "Wallet78", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -616,17 +592,9 @@ "Name": "Wallet4", "ParticipationOnly": false }, - { - "Name": "Wallet29", - "ParticipationOnly": false - }, { "Name": "Wallet54", "ParticipationOnly": false - }, - { - "Name": "Wallet79", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -651,17 +619,9 @@ "Name": "Wallet5", "ParticipationOnly": false }, - { - "Name": "Wallet30", - "ParticipationOnly": false - }, { "Name": "Wallet55", "ParticipationOnly": false - }, - { - "Name": "Wallet80", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -686,17 +646,9 @@ "Name": "Wallet6", "ParticipationOnly": false }, - { - "Name": "Wallet31", - "ParticipationOnly": false - }, { "Name": "Wallet56", "ParticipationOnly": false - }, - { - "Name": "Wallet81", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -721,17 +673,9 @@ "Name": "Wallet7", "ParticipationOnly": false }, - { - "Name": "Wallet32", - "ParticipationOnly": false - }, { "Name": "Wallet57", "ParticipationOnly": false - }, - { - "Name": "Wallet82", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -756,17 +700,9 @@ "Name": "Wallet8", "ParticipationOnly": false }, - { - "Name": "Wallet33", - "ParticipationOnly": false - }, { "Name": "Wallet58", "ParticipationOnly": false - }, - { - "Name": "Wallet83", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -791,17 +727,9 @@ "Name": "Wallet9", "ParticipationOnly": false }, - { - "Name": "Wallet34", - "ParticipationOnly": false - }, { "Name": "Wallet59", "ParticipationOnly": false - }, - { - "Name": "Wallet84", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -826,17 +754,9 @@ "Name": "Wallet10", "ParticipationOnly": false }, - { - "Name": "Wallet35", - "ParticipationOnly": false - }, { "Name": "Wallet60", "ParticipationOnly": false - }, - { - "Name": "Wallet85", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -861,17 +781,9 @@ "Name": "Wallet11", "ParticipationOnly": false }, - { - "Name": "Wallet36", - "ParticipationOnly": false - }, { "Name": "Wallet61", "ParticipationOnly": false - }, - { - "Name": "Wallet86", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -896,17 +808,9 @@ "Name": "Wallet12", "ParticipationOnly": false }, - { - "Name": "Wallet37", - "ParticipationOnly": false - }, { "Name": "Wallet62", "ParticipationOnly": false - }, - { - "Name": "Wallet87", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -931,17 +835,9 @@ "Name": "Wallet13", "ParticipationOnly": false }, - { - "Name": "Wallet38", - "ParticipationOnly": false - }, { "Name": "Wallet63", "ParticipationOnly": false - }, - { - "Name": "Wallet88", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -966,17 +862,9 @@ "Name": "Wallet14", "ParticipationOnly": false }, - { - "Name": "Wallet39", - "ParticipationOnly": false - }, { "Name": "Wallet64", "ParticipationOnly": false - }, - { - "Name": "Wallet89", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1001,17 +889,9 @@ "Name": "Wallet15", "ParticipationOnly": false }, - { - "Name": "Wallet40", - "ParticipationOnly": false - }, { "Name": "Wallet65", "ParticipationOnly": false - }, - { - "Name": "Wallet90", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1036,17 +916,9 @@ "Name": "Wallet16", "ParticipationOnly": false }, - { - "Name": "Wallet41", - "ParticipationOnly": false - }, { "Name": "Wallet66", "ParticipationOnly": false - }, - { - "Name": "Wallet91", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1071,17 +943,9 @@ "Name": "Wallet17", "ParticipationOnly": false }, - { - "Name": "Wallet42", - "ParticipationOnly": false - }, { "Name": "Wallet67", "ParticipationOnly": false - }, - { - "Name": "Wallet92", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1106,17 +970,9 @@ "Name": "Wallet18", "ParticipationOnly": false }, - { - "Name": "Wallet43", - "ParticipationOnly": false - }, { "Name": "Wallet68", "ParticipationOnly": false - }, - { - "Name": "Wallet93", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1141,17 +997,9 @@ "Name": "Wallet19", "ParticipationOnly": false }, - { - "Name": "Wallet44", - "ParticipationOnly": false - }, { "Name": "Wallet69", "ParticipationOnly": false - }, - { - "Name": "Wallet94", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1176,17 +1024,9 @@ "Name": "Wallet20", "ParticipationOnly": false }, - { - "Name": "Wallet45", - "ParticipationOnly": false - }, { "Name": "Wallet70", "ParticipationOnly": false - }, - { - "Name": "Wallet95", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1211,17 +1051,9 @@ "Name": "Wallet21", "ParticipationOnly": false }, - { - "Name": "Wallet46", - "ParticipationOnly": false - }, { "Name": "Wallet71", "ParticipationOnly": false - }, - { - "Name": "Wallet96", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1246,17 +1078,9 @@ "Name": "Wallet22", "ParticipationOnly": false }, - { - "Name": "Wallet47", - "ParticipationOnly": false - }, { "Name": "Wallet72", "ParticipationOnly": false - }, - { - "Name": "Wallet97", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1281,17 +1105,9 @@ "Name": "Wallet23", "ParticipationOnly": false }, - { - "Name": "Wallet48", - "ParticipationOnly": false - }, { "Name": "Wallet73", "ParticipationOnly": false - }, - { - "Name": "Wallet98", - "ParticipationOnly": false } ], "APIToken": "{{APIToken}}", @@ -1317,15 +1133,34 @@ "ParticipationOnly": false }, { - "Name": "Wallet49", + "Name": "Wallet74", "ParticipationOnly": false - }, + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N25", + "Group": "", + "Nodes": [ + { + "Name": "node25", + "Wallets": [ { - "Name": "Wallet74", + "Name": "Wallet25", "ParticipationOnly": false }, { - "Name": "Wallet99", + "Name": "Wallet75", "ParticipationOnly": false } ], @@ -1341,22 +1176,662 @@ ] }, { - "Name": "N25", + "Name": "N26", "Group": "", "Nodes": [ { - "Name": "node25", + "Name": "node26", "Wallets": [ { - "Name": "Wallet25", + "Name": "Wallet26", "ParticipationOnly": false }, { - "Name": "Wallet50", + "Name": "Wallet76", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N27", + "Group": "", + "Nodes": [ + { + "Name": "node27", + "Wallets": [ + { + "Name": "Wallet27", "ParticipationOnly": false }, { - "Name": "Wallet75", + "Name": "Wallet77", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N28", + "Group": "", + "Nodes": [ + { + "Name": "node28", + "Wallets": [ + { + "Name": "Wallet28", + "ParticipationOnly": false + }, + { + "Name": "Wallet78", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N29", + "Group": "", + "Nodes": [ + { + "Name": "node29", + "Wallets": [ + { + "Name": "Wallet29", + "ParticipationOnly": false + }, + { + "Name": "Wallet79", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N30", + "Group": "", + "Nodes": [ + { + "Name": "node30", + "Wallets": [ + { + "Name": "Wallet30", + "ParticipationOnly": false + }, + { + "Name": "Wallet80", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N31", + "Group": "", + "Nodes": [ + { + "Name": "node31", + "Wallets": [ + { + "Name": "Wallet31", + "ParticipationOnly": false + }, + { + "Name": "Wallet81", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N32", + "Group": "", + "Nodes": [ + { + "Name": "node32", + "Wallets": [ + { + "Name": "Wallet32", + "ParticipationOnly": false + }, + { + "Name": "Wallet82", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N33", + "Group": "", + "Nodes": [ + { + "Name": "node33", + "Wallets": [ + { + "Name": "Wallet33", + "ParticipationOnly": false + }, + { + "Name": "Wallet83", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N34", + "Group": "", + "Nodes": [ + { + "Name": "node34", + "Wallets": [ + { + "Name": "Wallet34", + "ParticipationOnly": false + }, + { + "Name": "Wallet84", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N35", + "Group": "", + "Nodes": [ + { + "Name": "node35", + "Wallets": [ + { + "Name": "Wallet35", + "ParticipationOnly": false + }, + { + "Name": "Wallet85", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N36", + "Group": "", + "Nodes": [ + { + "Name": "node36", + "Wallets": [ + { + "Name": "Wallet36", + "ParticipationOnly": false + }, + { + "Name": "Wallet86", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N37", + "Group": "", + "Nodes": [ + { + "Name": "node37", + "Wallets": [ + { + "Name": "Wallet37", + "ParticipationOnly": false + }, + { + "Name": "Wallet87", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N38", + "Group": "", + "Nodes": [ + { + "Name": "node38", + "Wallets": [ + { + "Name": "Wallet38", + "ParticipationOnly": false + }, + { + "Name": "Wallet88", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N39", + "Group": "", + "Nodes": [ + { + "Name": "node39", + "Wallets": [ + { + "Name": "Wallet39", + "ParticipationOnly": false + }, + { + "Name": "Wallet89", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N40", + "Group": "", + "Nodes": [ + { + "Name": "node40", + "Wallets": [ + { + "Name": "Wallet40", + "ParticipationOnly": false + }, + { + "Name": "Wallet90", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N41", + "Group": "", + "Nodes": [ + { + "Name": "node41", + "Wallets": [ + { + "Name": "Wallet41", + "ParticipationOnly": false + }, + { + "Name": "Wallet91", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N42", + "Group": "", + "Nodes": [ + { + "Name": "node42", + "Wallets": [ + { + "Name": "Wallet42", + "ParticipationOnly": false + }, + { + "Name": "Wallet92", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N43", + "Group": "", + "Nodes": [ + { + "Name": "node43", + "Wallets": [ + { + "Name": "Wallet43", + "ParticipationOnly": false + }, + { + "Name": "Wallet93", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N44", + "Group": "", + "Nodes": [ + { + "Name": "node44", + "Wallets": [ + { + "Name": "Wallet44", + "ParticipationOnly": false + }, + { + "Name": "Wallet94", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N45", + "Group": "", + "Nodes": [ + { + "Name": "node45", + "Wallets": [ + { + "Name": "Wallet45", + "ParticipationOnly": false + }, + { + "Name": "Wallet95", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N46", + "Group": "", + "Nodes": [ + { + "Name": "node46", + "Wallets": [ + { + "Name": "Wallet46", + "ParticipationOnly": false + }, + { + "Name": "Wallet96", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N47", + "Group": "", + "Nodes": [ + { + "Name": "node47", + "Wallets": [ + { + "Name": "Wallet47", + "ParticipationOnly": false + }, + { + "Name": "Wallet97", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N48", + "Group": "", + "Nodes": [ + { + "Name": "node48", + "Wallets": [ + { + "Name": "Wallet48", + "ParticipationOnly": false + }, + { + "Name": "Wallet98", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N49", + "Group": "", + "Nodes": [ + { + "Name": "node49", + "Wallets": [ + { + "Name": "Wallet49", + "ParticipationOnly": false + }, + { + "Name": "Wallet99", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N50", + "Group": "", + "Nodes": [ + { + "Name": "node50", + "Wallets": [ + { + "Name": "Wallet50", "ParticipationOnly": false }, { diff --git a/test/testdata/deployednettemplates/recipes/mainnet-model/generated/topology.json b/test/testdata/deployednettemplates/recipes/mainnet-model/generated/topology.json index db766f1233..c27dd1710b 100644 --- a/test/testdata/deployednettemplates/recipes/mainnet-model/generated/topology.json +++ b/test/testdata/deployednettemplates/recipes/mainnet-model/generated/topology.json @@ -27,22 +27,22 @@ }, { "Name": "R6", - "Group": "eu-r", + "Group": "us-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R7", - "Group": "eu-r", + "Group": "us-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R8", - "Group": "eu-r", + "Group": "us-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R9", - "Group": "eu-r", + "Group": "us-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { @@ -52,72 +52,72 @@ }, { "Name": "R11", - "Group": "ap-r", + "Group": "eu-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R12", - "Group": "ap-r", + "Group": "eu-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R13", - "Group": "ap-r", + "Group": "eu-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R14", - "Group": "ap-r", + "Group": "eu-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R15", - "Group": "ap-r", + "Group": "eu-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R16", - "Group": "af-r", + "Group": "eu-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R17", - "Group": "af-r", + "Group": "ap-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R18", - "Group": "af-r", + "Group": "ap-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R19", - "Group": "af-r", + "Group": "ap-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R20", - "Group": "af-r", + "Group": "ap-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R21", - "Group": "au-r", + "Group": "ap-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R22", - "Group": "au-r", + "Group": "ap-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R23", - "Group": "au-r", + "Group": "ap-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "R24", - "Group": "au-r", + "Group": "af-r", "Template": "AWS-US-WEST-1-c5.xlarge" }, { @@ -152,12 +152,12 @@ }, { "Name": "N6", - "Group": "eu-n", + "Group": "us-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N7", - "Group": "eu-n", + "Group": "us-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { @@ -177,76 +177,201 @@ }, { "Name": "N11", - "Group": "ap-n", + "Group": "eu-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N12", - "Group": "ap-n", + "Group": "eu-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N13", - "Group": "ap-n", + "Group": "eu-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N14", - "Group": "ap-n", + "Group": "eu-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N15", - "Group": "ap-n", + "Group": "eu-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N16", - "Group": "af-n", + "Group": "eu-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N17", - "Group": "af-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N18", - "Group": "af-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N19", - "Group": "af-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N20", - "Group": "af-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N21", - "Group": "au-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N22", - "Group": "au-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N23", - "Group": "au-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N24", - "Group": "au-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "N25", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N26", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N27", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N28", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N29", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N30", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N31", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N32", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N33", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N34", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N35", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N36", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N37", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N38", + "Group": "ap-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N39", + "Group": "af-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N40", + "Group": "af-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N41", + "Group": "af-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N42", + "Group": "af-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N43", + "Group": "af-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N44", + "Group": "af-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N45", + "Group": "af-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N46", + "Group": "au-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N47", + "Group": "au-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N48", + "Group": "au-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N49", + "Group": "au-n", + "Template": "AWS-US-WEST-1-c5.xlarge" + }, + { + "Name": "N50", "Group": "au-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, @@ -262,7 +387,7 @@ }, { "Name": "NPN3", - "Group": "eu-n", + "Group": "us-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { @@ -272,7 +397,7 @@ }, { "Name": "NPN5", - "Group": "ap-n", + "Group": "eu-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { @@ -282,17 +407,17 @@ }, { "Name": "NPN7", - "Group": "af-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "NPN8", - "Group": "af-n", + "Group": "ap-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { "Name": "NPN9", - "Group": "au-n", + "Group": "af-n", "Template": "AWS-US-WEST-1-c5.xlarge" }, { diff --git a/test/testdata/deployednettemplates/recipes/mainnet-model/network-tpl.json b/test/testdata/deployednettemplates/recipes/mainnet-model/network-tpl.json index 3ab40e1ce1..38d6fa96e1 100644 --- a/test/testdata/deployednettemplates/recipes/mainnet-model/network-tpl.json +++ b/test/testdata/deployednettemplates/recipes/mainnet-model/network-tpl.json @@ -1,7 +1,7 @@ { "network": { "wallets": 100, - "nodes": 25 + "nodes": 50 }, "instances": { "relays": { @@ -12,7 +12,7 @@ "participatingNodes": { "config": "./configs/node.json", "type": "c5.xlarge", - "count": 25 + "count": 50 }, "nonParticipatingNodes": { "config": "./configs/nonPartNode.json", @@ -24,7 +24,7 @@ { "name": "us-r", "percent": { - "relays": 20, + "relays": 36, "participatingNodes": 0, "nonParticipatingNodes": 0 }, @@ -34,15 +34,15 @@ "name": "us-n", "percent": { "relays": 0, - "participatingNodes": 20, - "nonParticipatingNodes": 20 + "participatingNodes": 14, + "nonParticipatingNodes": 35 }, "region": "us-west-1" }, { "name": "eu-r", "percent": { - "relays": 20, + "relays": 28, "participatingNodes": 0, "nonParticipatingNodes": 0 }, @@ -52,15 +52,15 @@ "name": "eu-n", "percent": { "relays": 0, - "participatingNodes": 20, - "nonParticipatingNodes": 20 + "participatingNodes": 18, + "nonParticipatingNodes": 25 }, "region": "us-west-1" }, { "name": "ap-r", "percent": { - "relays": 20, + "relays": 28, "participatingNodes": 0, "nonParticipatingNodes": 0 }, @@ -70,15 +70,15 @@ "name": "ap-n", "percent": { "relays": 0, - "participatingNodes": 20, - "nonParticipatingNodes": 20 + "participatingNodes": 44, + "nonParticipatingNodes": 30 }, "region": "us-west-1" }, { "name": "af-r", "percent": { - "relays": 20, + "relays": 4, "participatingNodes": 0, "nonParticipatingNodes": 0 }, @@ -88,15 +88,15 @@ "name": "af-n", "percent": { "relays": 0, - "participatingNodes": 20, - "nonParticipatingNodes": 20 + "participatingNodes": 14, + "nonParticipatingNodes": 5 }, "region": "us-west-1" }, { "name": "au-r", "percent": { - "relays": 20, + "relays": 4, "participatingNodes": 0, "nonParticipatingNodes": 0 }, @@ -106,8 +106,8 @@ "name": "au-n", "percent": { "relays": 0, - "participatingNodes": 20, - "nonParticipatingNodes": 20 + "participatingNodes": 10, + "nonParticipatingNodes": 5 }, "region": "us-west-1" } From 94994c15fe4930f2d083b92aee4ed1dd5bf665d5 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 30 Oct 2020 17:32:38 -0400 Subject: [PATCH 03/34] Separate the error message for duplicate txn id and lease (#1658) Convert isDup to checkDup and move error object generation into checkDup so that we can differentiate between txid duplicate and txlease collision. --- ledger/acctupdates.go | 6 +- ledger/cow.go | 10 +-- ledger/cow_test.go | 4 +- ledger/error.go | 13 ++++ ledger/eval.go | 16 ++-- ledger/ledger.go | 4 +- ledger/txtail.go | 24 ++++-- ledger/txtail_test.go | 92 +++++++++++++++++++++++ test/scripts/e2e_subs/keyreg-teal-test.sh | 2 +- 9 files changed, 142 insertions(+), 29 deletions(-) create mode 100644 ledger/txtail_test.go diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index b7a10f9b4a..24c51e6d86 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -837,10 +837,10 @@ func (aul *accountUpdatesLedgerEvaluator) Totals(rnd basics.Round) (AccountTotal return aul.au.totalsImpl(rnd) } -// isDup return whether a transaction is a duplicate one. It's not needed by the accountUpdatesLedgerEvaluator and implemented as a stub. -func (aul *accountUpdatesLedgerEvaluator) isDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, txlease) (bool, error) { +// checkDup test to see if the given transaction id/lease already exists. It's not needed by the accountUpdatesLedgerEvaluator and implemented as a stub. +func (aul *accountUpdatesLedgerEvaluator) checkDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, txlease) error { // this is a non-issue since this call will never be made on non-validating evaluation - return false, fmt.Errorf("accountUpdatesLedgerEvaluator: tried to check for dup during accountUpdates initialization ") + return fmt.Errorf("accountUpdatesLedgerEvaluator: tried to check for dup during accountUpdates initialization ") } // LookupWithoutRewards returns the account balance for a given address at a given round, without the reward diff --git a/ledger/cow.go b/ledger/cow.go index 10b4c69166..273155f5da 100644 --- a/ledger/cow.go +++ b/ledger/cow.go @@ -34,7 +34,7 @@ import ( type roundCowParent interface { lookup(basics.Address) (basics.AccountData, error) - isDup(basics.Round, basics.Round, transactions.Txid, txlease) (bool, error) + checkDup(basics.Round, basics.Round, transactions.Txid, txlease) error txnCounter() uint64 getCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) compactCertLast() basics.Round @@ -109,20 +109,20 @@ func (cb *roundCowState) lookup(addr basics.Address) (data basics.AccountData, e return cb.lookupParent.lookup(addr) } -func (cb *roundCowState) isDup(firstValid, lastValid basics.Round, txid transactions.Txid, txl txlease) (bool, error) { +func (cb *roundCowState) checkDup(firstValid, lastValid basics.Round, txid transactions.Txid, txl txlease) error { _, present := cb.mods.Txids[txid] if present { - return true, nil + return &TransactionInLedgerError{Txid: txid} } if cb.proto.SupportTransactionLeases && (txl.lease != [32]byte{}) { expires, ok := cb.mods.txleases[txl] if ok && cb.mods.hdr.Round <= expires { - return true, nil + return &LeaseInLedgerError{txid: txid, lease: txl} } } - return cb.lookupParent.isDup(firstValid, lastValid, txid, txl) + return cb.lookupParent.checkDup(firstValid, lastValid, txid, txl) } func (cb *roundCowState) txnCounter() uint64 { diff --git a/ledger/cow_test.go b/ledger/cow_test.go index 4bb21c545f..5cc50efc7a 100644 --- a/ledger/cow_test.go +++ b/ledger/cow_test.go @@ -34,8 +34,8 @@ func (ml *mockLedger) lookup(addr basics.Address) (basics.AccountData, error) { return ml.balanceMap[addr], nil } -func (ml *mockLedger) isDup(firstValid, lastValid basics.Round, txn transactions.Txid, txl txlease) (bool, error) { - return false, nil +func (ml *mockLedger) checkDup(firstValid, lastValid basics.Round, txn transactions.Txid, txl txlease) error { + return nil } func (ml *mockLedger) getAssetCreator(assetIdx basics.AssetIndex) (basics.Address, bool, error) { diff --git a/ledger/error.go b/ledger/error.go index 010d201d17..60eda510cd 100644 --- a/ledger/error.go +++ b/ledger/error.go @@ -33,6 +33,19 @@ func (tile TransactionInLedgerError) Error() string { return fmt.Sprintf("transaction already in ledger: %v", tile.Txid) } +// LeaseInLedgerError is returned when a transaction cannot be added because it has a lease that already being used in the relavant rounds +type LeaseInLedgerError struct { + txid transactions.Txid + lease txlease +} + +// Error implements the error interface for the LeaseInLedgerError stuct +func (lile *LeaseInLedgerError) Error() string { + // format the lease as address. + addr := basics.Address(lile.lease.lease) + return fmt.Sprintf("transaction %v using an overlapping lease %s", lile.txid, addr.String()) +} + // BlockInLedgerError is returned when a block cannot be added because it has already been done type BlockInLedgerError struct { LastRound basics.Round diff --git a/ledger/eval.go b/ledger/eval.go index ce8f5a2364..256003bfdf 100644 --- a/ledger/eval.go +++ b/ledger/eval.go @@ -76,8 +76,8 @@ func (x *roundCowBase) lookup(addr basics.Address) (acctData basics.AccountData, return acctData, err } -func (x *roundCowBase) isDup(firstValid, lastValid basics.Round, txid transactions.Txid, txl txlease) (bool, error) { - return x.l.isDup(x.proto, x.rnd+1, firstValid, lastValid, txid, txl) +func (x *roundCowBase) checkDup(firstValid, lastValid basics.Round, txid transactions.Txid, txl txlease) error { + return x.l.checkDup(x.proto, x.rnd+1, firstValid, lastValid, txid, txl) } func (x *roundCowBase) txnCounter() uint64 { @@ -221,7 +221,7 @@ type ledgerForEvaluator interface { GenesisHash() crypto.Digest BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) Totals(basics.Round) (AccountTotals, error) - isDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, txlease) (bool, error) + checkDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, txlease) error LookupWithoutRewards(basics.Round, basics.Address) (basics.AccountData, basics.Round, error) GetCreatorForRound(basics.Round, basics.CreatableIndex, basics.CreatableType) (basics.Address, bool, error) CompactCertVoters(basics.Round) (*VotersForRound, error) @@ -477,13 +477,10 @@ func (eval *BlockEvaluator) testTransaction(txn transactions.SignedTxn, cow *rou // Transaction already in the ledger? txid := txn.ID() - dup, err := cow.isDup(txn.Txn.First(), txn.Txn.Last(), txid, txlease{sender: txn.Txn.Sender, lease: txn.Txn.Lease}) + err = cow.checkDup(txn.Txn.First(), txn.Txn.Last(), txid, txlease{sender: txn.Txn.Sender, lease: txn.Txn.Lease}) if err != nil { return err } - if dup { - return TransactionInLedgerError{txn.ID()} - } return nil } @@ -637,13 +634,10 @@ func (eval *BlockEvaluator) transaction(txn transactions.SignedTxn, appEval *app } // Transaction already in the ledger? - dup, err := cow.isDup(txn.Txn.First(), txn.Txn.Last(), txid, txlease{sender: txn.Txn.Sender, lease: txn.Txn.Lease}) + err := cow.checkDup(txn.Txn.First(), txn.Txn.Last(), txid, txlease{sender: txn.Txn.Sender, lease: txn.Txn.Lease}) if err != nil { return err } - if dup { - return TransactionInLedgerError{txid} - } // Does the address that authorized the transaction actually match whatever address the sender has rekeyed to? // i.e., the sig/lsig/msig was checked against the txn.Authorizer() address, but does this match the sender's balrecord.AuthAddr? diff --git a/ledger/ledger.go b/ledger/ledger.go index 67450cc849..a461df75b3 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -470,10 +470,10 @@ func (l *Ledger) Totals(rnd basics.Round) (AccountTotals, error) { return l.accts.Totals(rnd) } -func (l *Ledger) isDup(currentProto config.ConsensusParams, current basics.Round, firstValid basics.Round, lastValid basics.Round, txid transactions.Txid, txl txlease) (bool, error) { +func (l *Ledger) checkDup(currentProto config.ConsensusParams, current basics.Round, firstValid basics.Round, lastValid basics.Round, txid transactions.Txid, txl txlease) error { l.trackerMu.RLock() defer l.trackerMu.RUnlock() - return l.txTail.isDup(currentProto, current, firstValid, lastValid, txid, txl) + return l.txTail.checkDup(currentProto, current, firstValid, lastValid, txid, txl) } // GetRoundTxIds returns a map of the transactions ids that we have for the given round diff --git a/ledger/txtail.go b/ledger/txtail.go index f51fb64f4b..e7ddff2a09 100644 --- a/ledger/txtail.go +++ b/ledger/txtail.go @@ -122,9 +122,21 @@ func (t *txTail) committedUpTo(rnd basics.Round) basics.Round { return (rnd + 1).SubSaturate(maxlife) } -func (t *txTail) isDup(proto config.ConsensusParams, current basics.Round, firstValid basics.Round, lastValid basics.Round, txid transactions.Txid, txl txlease) (bool, error) { +// txtailMissingRound is returned by checkDup when requested for a round number below the low watermark +type txtailMissingRound struct { + round basics.Round +} + +// Error satisfies builtin interface `error` +func (t txtailMissingRound) Error() string { + return fmt.Sprintf("txTail: tried to check for dup in missing round %d", t.round) +} + +// checkDup test to see if the given transaction id/lease already exists. It returns nil if neither exists, or +// TransactionInLedgerError / LeaseInLedgerError respectively. +func (t *txTail) checkDup(proto config.ConsensusParams, current basics.Round, firstValid basics.Round, lastValid basics.Round, txid transactions.Txid, txl txlease) error { if lastValid < t.lowWaterMark { - return true, fmt.Errorf("txTail: tried to check for dup in missing round %d", lastValid) + return &txtailMissingRound{round: lastValid} } if proto.SupportTransactionLeases && (txl.lease != [32]byte{}) { @@ -138,13 +150,15 @@ func (t *txTail) isDup(proto config.ConsensusParams, current basics.Round, first for rnd := firstChecked; rnd <= lastChecked; rnd++ { expires, ok := t.recent[rnd].txleases[txl] if ok && current <= expires { - return true, nil + return &LeaseInLedgerError{txid: txid, lease: txl} } } } - _, confirmed := t.lastValid[lastValid][txid] - return confirmed, nil + if _, confirmed := t.lastValid[lastValid][txid]; confirmed { + return &TransactionInLedgerError{Txid: txid} + } + return nil } func (t *txTail) getRoundTxIds(rnd basics.Round) (txMap map[transactions.Txid]bool) { diff --git a/ledger/txtail_test.go b/ledger/txtail_test.go new file mode 100644 index 0000000000..fd445927ff --- /dev/null +++ b/ledger/txtail_test.go @@ -0,0 +1,92 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package ledger + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/protocol" +) + +func TestTxTailCheckdup(t *testing.T) { + ledger := makeMockLedgerForTracker(t, true) + ledger.blocks = randomInitChain(protocol.ConsensusCurrentVersion, 1) + proto := config.Consensus[protocol.ConsensusCurrentVersion] + tail := txTail{} + require.NoError(t, tail.loadFromDisk(ledger)) + + lastRound := basics.Round(proto.MaxTxnLife) + lookback := basics.Round(100) + txvalidity := basics.Round(10) + leasevalidity := basics.Round(32) + + // push 1000 rounds into the txtail + for rnd := basics.Round(1); rnd < lastRound; rnd++ { + blk := bookkeeping.Block{ + BlockHeader: bookkeeping.BlockHeader{ + Round: rnd, + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: protocol.ConsensusCurrentVersion, + }, + }, + } + + txids := make(map[transactions.Txid]basics.Round) + txids[transactions.Txid(crypto.Hash([]byte{byte(rnd % 256), byte(rnd / 256), byte(1)}))] = rnd + txvalidity + txleases := make(map[txlease]basics.Round) + txleases[txlease{sender: basics.Address(crypto.Hash([]byte{byte(rnd % 256), byte(rnd / 256), byte(2)})), lease: crypto.Hash([]byte{byte(rnd % 256), byte(rnd / 256), byte(3)})}] = rnd + leasevalidity + + tail.newBlock(blk, StateDelta{accts: make(map[basics.Address]accountDelta), hdr: &blk.BlockHeader, Txids: txids, txleases: txleases}) + tail.committedUpTo(rnd.SubSaturate(lookback)) + } + + // test txid duplication testing. + for rnd := basics.Round(1); rnd < lastRound; rnd++ { + txid := transactions.Txid(crypto.Hash([]byte{byte(rnd % 256), byte(rnd / 256), byte(1)})) + err := tail.checkDup(proto, basics.Round(0), basics.Round(0), rnd+txvalidity, txid, txlease{}) + require.Errorf(t, err, "round %d", rnd) + if rnd < lastRound-lookback-txvalidity-1 { + var missingRoundErr *txtailMissingRound + require.Truef(t, errors.As(err, &missingRoundErr), "error a txtailMissingRound(%d) : %v ", rnd, err) + } else { + var txInLedgerErr *TransactionInLedgerError + require.Truef(t, errors.As(err, &txInLedgerErr), "error a TransactionInLedgerError(%d) : %v ", rnd, err) + } + } + + // test lease detection + for rnd := basics.Round(1); rnd < lastRound; rnd++ { + lease := txlease{sender: basics.Address(crypto.Hash([]byte{byte(rnd % 256), byte(rnd / 256), byte(2)})), lease: crypto.Hash([]byte{byte(rnd % 256), byte(rnd / 256), byte(3)})} + err := tail.checkDup(proto, rnd, basics.Round(0), rnd, transactions.Txid{}, lease) + require.Errorf(t, err, "round %d", rnd) + if rnd < lastRound-lookback-1 { + var missingRoundErr *txtailMissingRound + require.Truef(t, errors.As(err, &missingRoundErr), "error a txtailMissingRound(%d) : %v ", rnd, err) + } else { + var leaseInLedgerErr *LeaseInLedgerError + require.Truef(t, errors.As(err, &leaseInLedgerErr), "error a LeaseInLedgerError(%d) : %v ", rnd, err) + } + } +} diff --git a/test/scripts/e2e_subs/keyreg-teal-test.sh b/test/scripts/e2e_subs/keyreg-teal-test.sh index f5f0ee2719..acf8196c8e 100755 --- a/test/scripts/e2e_subs/keyreg-teal-test.sh +++ b/test/scripts/e2e_subs/keyreg-teal-test.sh @@ -100,7 +100,7 @@ ${gcmd} account changeonlinestatus -a ${ACCOUNTA} -x ${LEASE} --online --firstva dsign ${TEMPDIR}/delegate.keyregkey ${TEMPDIR}/kr.lsig < ${TEMPDIR}/keyreg.tx > ${TEMPDIR}/keyreg.stx RES=$(${gcmd} clerk rawsend -f ${TEMPDIR}/keyreg.stx || true) -EXPERROR='already in ledger' +EXPERROR='using an overlapping lease' if [[ $RES != *"${EXPERROR}"* ]]; then date '+keyreg-teal-test FAIL replayed txn should be rejected %Y%m%d_%H%M%S' false From a2d065a7cdfbebcc20a8ba0ed4bc96378e01b4fe Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 30 Oct 2020 20:59:33 -0400 Subject: [PATCH 04/34] Update goal error message for unauthorized access (#1678) Update error message to be more informative when the libgoal is failing to connect to a node with 401 error. --- cmd/goal/messages.go | 2 +- daemon/algod/api/client/restClient.go | 36 +++++++++++- .../cli/algoh/expect/algohTimeoutTest.exp | 2 +- .../goal/expect/goalNodeConnectionTest.exp | 58 +++++++++++++++++++ 4 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 test/e2e-go/cli/goal/expect/goalNodeConnectionTest.exp diff --git a/cmd/goal/messages.go b/cmd/goal/messages.go index 439b24014b..08cbad199a 100644 --- a/cmd/goal/messages.go +++ b/cmd/goal/messages.go @@ -69,7 +69,7 @@ const ( nodeLastCatchpoint = "Last Catchpoint: %s" errorNodeCreationIPFailure = "Parsing passed IP %v failed: need a valid IPv4 or IPv6 address with a specified port number" errorNodeNotDetected = "Algorand node does not appear to be running: %s" - errorNodeStatus = "Cannot contact Algorand node: %s." + errorNodeStatus = "Cannot contact Algorand node: %s" errorNodeFailedToStart = "Algorand node failed to start: %s" errorNodeRunning = "Node must be stopped before writing APIToken" errorNodeFailGenToken = "Cannot generate API token: %s" diff --git a/daemon/algod/api/client/restClient.go b/daemon/algod/api/client/restClient.go index f7d8449d8e..cb0fbebbf7 100644 --- a/daemon/algod/api/client/restClient.go +++ b/daemon/algod/api/client/restClient.go @@ -64,6 +64,19 @@ var rawRequestPaths = map[string]bool{ "/v2/teal/compile": true, } +// unauthorizedRequestError is generated when we receive 401 error from the server. This error includes the inner error +// as well as the likely parameters that caused the issue. +type unauthorizedRequestError struct { + errorString string + apiToken string + url string +} + +// Error format an error string for the unauthorizedRequestError error. +func (e unauthorizedRequestError) Error() string { + return fmt.Sprintf("Unauthorized request to `%s` when using token `%s` : %s", e.url, e.apiToken, e.errorString) +} + // RestClient manages the REST interface for a calling user. type RestClient struct { serverURL url.URL @@ -87,6 +100,20 @@ func (client *RestClient) SetAPIVersionAffinity(affinity APIVersion) (previousAf return } +// filterASCII filter out the non-ascii printable characters out of the given input string. +// It's used as a security qualifier before adding network provided data into an error message. +// The function allows only characters in the range of [32..126], which excludes all the +// control character, new lines, deletion, etc. All the alpha numeric and punctuation characters +// are included in this range. +func filterASCII(unfilteredString string) (filteredString string) { + for i, r := range unfilteredString { + if int(r) >= 0x20 && int(r) <= 0x7e { + filteredString += string(unfilteredString[i]) + } + } + return +} + // extractError checks if the response signifies an error (for now, StatusCode != 200 or StatusCode != 201). // If so, it returns the error. // Otherwise, it returns nil. @@ -96,7 +123,14 @@ func extractError(resp *http.Response) error { } errorBuf, _ := ioutil.ReadAll(resp.Body) // ignore returned error - return fmt.Errorf("HTTP %v: %s", resp.Status, errorBuf) + errorString := filterASCII(string(errorBuf)) + + if resp.StatusCode == http.StatusUnauthorized { + apiToken := resp.Request.Header.Get(authHeader) + return unauthorizedRequestError{errorString, apiToken, resp.Request.URL.String()} + } + + return fmt.Errorf("HTTP %s: %s", resp.Status, errorString) } // stripTransaction gets a transaction of the form "tx-XXXXXXXX" and truncates the "tx-" part, if it starts with "tx-" diff --git a/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp b/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp index 20716bef85..e0b18e5cc1 100644 --- a/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp +++ b/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp @@ -69,7 +69,7 @@ if { [catch { set timeout 201 spawn $env(GOPATH)/bin/goal node wait -d $TEST_NODE_DIR -w 200 expect { - "^Cannot contact Algorand node: open $TEST_NODE_DIR/algod.net: no such file or directory." {puts "ERROR: node shutdown"; close; exit 1} + "^Cannot contact Algorand node: open $TEST_NODE_DIR/algod.net: no such file or directory" {puts "ERROR: node shutdown"; close; exit 1} "^Timed out waiting for node to make progress" {puts "node correctly continued running despite relay shutdown"; close} timeout {puts "should not reached this case", close; exit 1} } diff --git a/test/e2e-go/cli/goal/expect/goalNodeConnectionTest.exp b/test/e2e-go/cli/goal/expect/goalNodeConnectionTest.exp new file mode 100644 index 0000000000..d8ae5cd002 --- /dev/null +++ b/test/e2e-go/cli/goal/expect/goalNodeConnectionTest.exp @@ -0,0 +1,58 @@ +#!/usr/bin/expect -f +#exp_internal 1 +set err 0 +log_user 1 + +if { [catch { + + source goalExpectCommon.exp + set TEST_ALGO_DIR [lindex $argv 0] + set TEST_DATA_DIR [lindex $argv 1] + + puts "TEST_ALGO_DIR: $TEST_ALGO_DIR" + puts "TEST_DATA_DIR: $TEST_DATA_DIR" + + set TIME_STAMP [clock seconds] + + set TEST_ROOT_DIR $TEST_ALGO_DIR/root + set TEST_PRIMARY_NODE_DIR $TEST_ROOT_DIR/Primary/ + set NETWORK_NAME test_net_expect_$TIME_STAMP + set NETWORK_TEMPLATE "$TEST_DATA_DIR/nettemplates/TwoNodes50Each.json" + + # Create network + ::AlgorandGoal::CreateNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + # Start node + ::AlgorandGoal::StartNode $TEST_PRIMARY_NODE_DIR + + exec cp $TEST_PRIMARY_NODE_DIR/algod.admin.token $TEST_PRIMARY_NODE_DIR/algod.admin.token.backup + exec echo "XXX" >> $TEST_PRIMARY_NODE_DIR/algod.admin.token + + set UNABLE_TO_CONNECT_MESSAGE 0 + spawn goal node status -d $TEST_PRIMARY_NODE_DIR + expect { + timeout { close; ::AlgorandGoal::Abort "failed to test node status" } + "Cannot contact Algorand node: Unauthorized request to * when using token *Invalid API Token*" { + set UNABLE_TO_CONNECT_MESSAGE 1 + exp_continue + } + eof { + if {$UNABLE_TO_CONNECT_MESSAGE == 0} { + puts "eof received before the expected output " + exit 1 + } + } + } + + exec cp $TEST_PRIMARY_NODE_DIR/algod.admin.token.backup $TEST_PRIMARY_NODE_DIR/algod.admin.token + + # Stop node + ::AlgorandGoal::StopNode $TEST_PRIMARY_NODE_DIR + + puts "Goal Node Connection Test Successful" + + exit 0 +} EXCEPTION] } { + puts "ERROR in goalNodeConnectionTest: $EXCEPTION" + exit 1 +} From 2b0510b05fefe2106862d443712a78b1f7b7cf09 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 4 Nov 2020 11:08:50 -0500 Subject: [PATCH 05/34] Remove buildnumber.dat and genesistimestamp.dat on merge back to master. --- buildnumber.dat | 1 - genesistimestamp.dat | 1 - 2 files changed, 2 deletions(-) delete mode 100644 buildnumber.dat delete mode 100644 genesistimestamp.dat diff --git a/buildnumber.dat b/buildnumber.dat deleted file mode 100644 index 573541ac97..0000000000 --- a/buildnumber.dat +++ /dev/null @@ -1 +0,0 @@ -0 diff --git a/genesistimestamp.dat b/genesistimestamp.dat deleted file mode 100644 index c72c6a7795..0000000000 --- a/genesistimestamp.dat +++ /dev/null @@ -1 +0,0 @@ -1558657885 From 40753dba547a47678d099e6c8105e96463851376 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Thu, 5 Nov 2020 09:26:06 -0500 Subject: [PATCH 06/34] Clarify which algod endpoints require developer API (#1684) Clarify in the algod API spec that /v2/teal/compile and /v2/teal/dryrun require EnableDeveloperAPI to be true. --- daemon/algod/api/algod.oas2.json | 10 +- daemon/algod/api/algod.oas3.yml | 12 +- .../algod/api/server/v2/generated/routes.go | 317 +++++++++--------- 3 files changed, 177 insertions(+), 162 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 9612ca71c7..02550a5803 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -895,7 +895,7 @@ }, "/v2/teal/compile": { "post": { - "description": "Given TEAL source code in plain text, return base64 encoded program bytes and base32 SHA512_256 hash of program bytes (Address style).", + "description": "Given TEAL source code in plain text, return base64 encoded program bytes and base32 SHA512_256 hash of program bytes (Address style). This endpoint is only enabled when a node's configureation file sets EnableDeveloperAPI to true.", "consumes": [ "text/plain" ], @@ -936,6 +936,9 @@ "$ref": "#/definitions/ErrorResponse" } }, + "404": { + "description": "Developer API not enabled" + }, "500": { "description": "Internal Error", "schema": { @@ -1055,7 +1058,7 @@ }, "/v2/teal/dryrun": { "post": { - "description": "Executes TEAL program(s) in context and returns debugging information about the execution.", + "description": "Executes TEAL program(s) in context and returns debugging information about the execution. This endpoint is only enabled when a node's configureation file sets EnableDeveloperAPI to true.", "consumes": [ "application/json", "application/msgpack" @@ -1095,6 +1098,9 @@ "$ref": "#/definitions/ErrorResponse" } }, + "404": { + "description": "Developer API not enabled" + }, "500": { "description": "Internal Error", "schema": { diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index 2f286218c4..a06170f41a 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -2593,7 +2593,7 @@ }, "/v2/teal/compile": { "post": { - "description": "Given TEAL source code in plain text, return base64 encoded program bytes and base32 SHA512_256 hash of program bytes (Address style).", + "description": "Given TEAL source code in plain text, return base64 encoded program bytes and base32 SHA512_256 hash of program bytes (Address style). This endpoint is only enabled when a node's configureation file sets EnableDeveloperAPI to true.", "operationId": "TealCompile", "requestBody": { "content": { @@ -2652,6 +2652,10 @@ }, "description": "Invalid API Token" }, + "404": { + "content": {}, + "description": "Developer API not enabled" + }, "500": { "content": { "application/json": { @@ -2673,7 +2677,7 @@ }, "/v2/teal/dryrun": { "post": { - "description": "Executes TEAL program(s) in context and returns debugging information about the execution.", + "description": "Executes TEAL program(s) in context and returns debugging information about the execution. This endpoint is only enabled when a node's configureation file sets EnableDeveloperAPI to true.", "operationId": "TealDryrun", "requestBody": { "content": { @@ -2742,6 +2746,10 @@ }, "description": "Invalid API Token" }, + "404": { + "content": {}, + "description": "Developer API not enabled" + }, "500": { "content": { "application/json": { diff --git a/daemon/algod/api/server/v2/generated/routes.go b/daemon/algod/api/server/v2/generated/routes.go index 8adb9de4a6..2f28bc41d9 100644 --- a/daemon/algod/api/server/v2/generated/routes.go +++ b/daemon/algod/api/server/v2/generated/routes.go @@ -561,164 +561,165 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9/XfbNrLov4Kre89p0itazld343N67nOTtPXbJM2J3b37bpzXhciRhDUJcAHQkprn", - "//0dDAASJEFJ/khad/VTYhEYDAbzhcFg8GmUiqIUHLhWo6NPo5JKWoAGiX/RNBUV1wnLzF8ZqFSyUjPB", - "R0f+G1FaMj4fjUfM/FpSvRiNR5wW0LQx/ccjCf+smIRsdKRlBeORShdQUANYr0vTuoa0SuYicSCOLYiT", - "l6OrDR9olklQqo/lTzxfE8bTvMqAaEm5oqn5pMiS6QXRC6aI60wYJ4IDETOiF63GZMYgz9SBn+Q/K5Dr", - "YJZu8OEpXTUoJlLk0MfzhSimjIPHCmqk6gUhWpAMZthoQTUxIxhcfUMtiAIq0wWZCbkFVYtEiC/wqhgd", - "fRgp4BlIXK0U2CX+dyYBfoVEUzkHPfo4jk1upkEmmhWRqZ046ktQVa4VwbY4xzm7BE5MrwPyplKaTIFQ", - "Tt5//4I8efLkuZlIQbWGzDHZ4Kya0cM52e6jo1FGNfjPfV6j+VxIyrOkbv/++xc4/qmb4K6tqFIQF5Zj", - "84WcvByagO8YYSHGNcxxHVrcb3pEhKL5eQozIWHHNbGN73RRwvF/01VJqU4XpWBcR9aF4FdiP0d1WNB9", - "kw6rEWi1Lw2lpAH64TB5/vHTo/Gjw6t//3Cc/I/789mTqx2n/6KGu4UC0YZpJSXwdJ3MJVCUlgXlfXq8", - "d/ygFqLKM7Kgl7j4tEBV7/oS09eqzkuaV4ZPWCrFcT4XilDHRhnMaJVr4gcmFc+NmjLQHLcTpkgpxSXL", - "IBsb7btcsHRBUqosCGxHlizPDQ9WCrIhXovPboMwXYUkMXjdiB44od8vMZp5baEErFAbJGkuFCRabDFP", - "3uJQnpHQoDS2Sl3PWJGzBRAc3HywxhZpxw1P5/maaFzXjFBFKPGmaUzYjKxFRZa4ODm7wP5uNoZqBTFE", - "w8Vp2VEjvEPk6xEjQrypEDlQjsTzctcnGZ+xeSVBkeUC9MLZPAmqFFwBEdN/QKrNsv/v05/eEiHJG1CK", - "zuEdTS8I8FRkw2vsBo1Z8H8oYRa8UPOSphdxc52zgkVQfkNXrKgKwqtiCtKsl7cPWhAJupJ8CCELcQuf", - "FXTVH/RMVjzFxW2GbTlqhpWYKnO6PiAnM1LQ1beHY4eOIjTPSQk8Y3xO9IoPOmlm7O3oJVJUPNvBh9Fm", - "wQKrqUpI2YxBRmooGzBxw2zDh/Hr4dN4VgE6HsggOvUoW9DhsIrwjBFd84WUdA4ByxyQn53mwq9aXACv", - "FRyZrvFTKeGSiUrVnQZwxKE3u9dcaEhKCTMW4bFTRw6jPWwbp14L5+CkgmvKOGRG8yLSQoPVRIM4BQNu", - "3sz0TfSUKvjm6ZABb77uuPoz0V31jSu+02pjo8SKZMQumq9OYONuU6v/Dpu/cGzF5on9ubeQbH5mTMmM", - "5Whm/mHWz5OhUqgEWoTwhkexOae6knB0zr82f5GEnGrKMyoz80thf3pT5Zqdsrn5Kbc/vRZzlp6y+QAx", - "a1yjuynsVth/DLy4Otar6KbhtRAXVRlOKG3tSqdrcvJyaJEtzOsy5nG9lQ13FWcrv9O4bg+9qhdyAMlB", - "2pXUNLyAtQSDLU1n+M9qhvxEZ/JX809Z5jGaGgZ2hhaDAi5Y8N79Zn4yIg92T2CgsJQaok7QfB59ChD6", - "Dwmz0dHo3ydNpGRiv6qJg2tGvBqPjhs4dz9S09POr7ORaT4Txu3qYNOx3RPePT4GahQTdFQ7OHyXi/Ti", - "RjiUUpQgNbPrODVw+pKC4MkCaAaSZFTTg2ZTZf2sAX7Hjj9iP9wlgYyYuJ/wPzQn5rORQqq9+2ZcV6aM", - "EyeCQFNmPD5rR+xIpgF6ooIU1skjxjm7FpYvmsGtgq416gdHlo9daJHVeWX9SoI9/CTM1Jtd4/FUyJvx", - "S4cROGn2woQaqLX3a2beXllsWpWJo0/En7YNOoCa8GNfrYYU6oKP0apFhVNNPwMVlIF6F1RoA7prKoii", - "ZDncgbwuqFr0J2EcnCePyemPx88ePf7l8bNvjIUupZhLWpDpWoMiD5xdIUqvc3jYnxkq+CrXcejfPPU7", - "qDbcrRRChGvYu0jUGRjNYClGbLzAYPdSrmXF74CEIKWQEZ8XWUeLVOTJJUjFRCR88c61IK6F0UPW7+78", - "brElS6qIGRu3YxXPQB7EKG/2WWjSNRRqm6GwoM9WvKGNA0ilpOveCtj5Rmbnxt1lTdrE9969IiXIRK84", - "yWBazUMbRWZSFISSDDuiQnwrMjjVVFfqDrRAA6xBxixEiAKdikoTSrjIjECbxnH9MBDLxCAKxn50qHL0", - "wtqfKRjvOKXVfKGJcStFbGmbjglN7aIkaCvUwNav3rPbVnY4GyfLJdBsTaYAnIip21+5nR9OkmJYRvsT", - "F6edGrTqPUELr1KKFJSCLHHHS1tR80dVuMh6A5kQb8S3HoQoQWZU3hBXLTTNt+CJbfrYqsabcHvSPta7", - "Db9p/bqDh6tIpdliWiYwrosR7hw0DJFwK02qcuA4wlm1M1YYkSCccqEgFTxTUWA5VTrZJgqmUcv0mmUN", - "uC/G/Qh4YNP9miptt72MZ+iGWRHGcbAPDjGM8KCWNpD/6hV0H3ZqdA9Xlaq1tarKUkgNWWwOHFYbxnoL", - "q3osMQtg1yZBC1Ip2AZ5iEoBfEcsOxNLIKpd3KWOC/UnhyFuo1vXUVK2kGgIsQmRU98qoG4Ykh1AxPjs", - "dU9kHKY6nFPHgccjpUVZGp2kk4rX/YbIdGpbH+ufm7Z95qK60ZWZADO69jg5zJeWsjYYv6DGX0LIpKAX", - "Rt+j92P3532cjTAmivEUkk2cb8Ty1LQKRWCLkA44nu64LxitIxwd/o0y3SATbFmFoQkPeMHvbFT5rIm4", - "3IEj8BI0ZbmqjX0dum5GwSh3NwPBeGYSUuA6XxtenTFZ2IMitBHK/2ZdicyNYo9EGvHjGZGwpDLzLfo7", - "kGAyCeMZrOLalbbiDRmsCIsjPatHZpqk/hiHhwAOooLuDsY2oOACDTcZ3HSND2uPfSyVVOxAED8YAShY", - "KgW153xmMtZI6vooS0JBDXZ44uSM+vCYjM8Te6wYMY/2uz929OHekGficD2fDEp2zRrLBeBJhlHXHSKG", - "3Ga2aaBgaCLzXExpnhjnFZIMcr01jGScYniJLY2dFGm/exvl8/MPeXZ+/pG8Nm3RTwZyAesJnr6SdEH5", - "HJqQeMin1gOGFaRVqNI7ZNxpU+Pifm3s29ua8agUIk/q7Vs3hN9T8126X7D0AjJi9AQ6nc76fNVeITMI", - "eWBYXNWHHMvF2vuzZQkcsocHhBxzAkWp1y5W0PE0OoPzr/Sm8Vc4albheSvlBCd5cM7j23R7WntLmfJg", - "NkuSTV+65VAWyOaB9IoPiBNd4mGDAReVz42RvlPsGZicniUNmMpisct++AfM6aGtVWYZbjYaq6KqacEw", - "sSdoNjaa05+19nerTB8Qcoa6w+wWFFyCpDlmLSgfBGWKFMxsOlWVpgDZ0TlPWpikonADP2j+a9XSeXV4", - "+ATI4cNuH6WNm+g2RlYGun2/JYdj+wnJRb4l56PzUQ+ShEJcQmY3hyFf215bwf5bDfec/9RTzKSga7ut", - "9LJIVDWbsZRZoufC6PW56Hh7XOAXkAY9MJszRZgeoylDiqKXbNelEcBR1Gu5i/hFBKrxj40pNdrOn7C1", - "eUcRWNHUzJKiklmTpWGUms/6zocWZRICiIZTN4zoAtqqpcdvKHd9fW5305vxO+vsp1vkCNj1YLvP3CNG", - "FINdxP+YlMKsOnO5ND7hImdK95B0G3s8zagZMmJ0Dsj/ERVJKcpvWWmo91RC4kYFN7BmBLSxfkznqTUU", - "ghwKsOEO/PL1192Jf/21W3OmyAyWPgHNNOyS4+uvrRAIpW8tAR3WXJ1EHCgMMhtrGkkaXlC1ONgacEa4", - "O8WZA9AnL/2AKExKoYm5Go/MVjdf34HAW0BEgvP3VCvoo+xXMQuT3dz6qbXSUPQjl7brLwOe6Hu/Q+tZ", - "WsFzxiEpBId1NL+bcXiDH6N2GllkoDMK61Df7g62hX8HrfY4u6zmbemLqx2wxLs69e4OFr8LtxO0DtP8", - "0MuEvCSUpDnDgKDgSssq1eecYoCi4wZ12MKHXYZDVi98k3iMLBLCcqDOOVWGhnXYInqYMYNIQPJ7AB+5", - "UtV8DqrjFpEZwDl3rRgnFWcax0KvMrELVoLEU6cD29J4AjOaY4TtV5CCTCvdVr2YjWQ9GxtBN8MQMTvn", - "VJMcqNLkDeNnKwTndzieZzjopZAXNRUGdmjAQTGVxA/mfrBff6Rq4advGnpl4zrbILGB36QsrTW00p3/", - "74P/OvpwnPwPTX49TJ7/5+Tjp6dXD7/u/fj46ttv/1/7pydX3z78r/+IrZTHPZYr4zA/eenckpOXaHua", - "4HkP9y8W/C0YT6JMZrYLBeOYctnhLfLAWFDPQA+bMLxb9XOuV9ww0iXNWWa2wDdhh66K68milY4O17QW", - "ohPL83P9GNvuzEVS0vQCz7VHc6YX1fQgFcXEu2OTuahds0lGoRAcv2UTWrKJ2d5OLh9tMY230Fckoq6u", - "xiOnddSdZ9A4wLEJdceso+j+by3IVz+8OiMTt1LqK5s4Z0EHGU8RD9odhrU2c2by9uKHzRw0m5mXMGOc", - "me9H5zyjmk6mVLFUTSoF8juaU57CwVyQI+JAvqSaYgygE1EcupuF8RmHTVlNc5aSi9AUN6I5FBg7P/9g", - "GOT8/GPvIKtvON1Q8WAjDpAsmV6ISicuKjscR2hiLQjZxuU2jTomDrblSBf1dfAHAqBlqZIgIhafflnm", - "ZvoBGyqCnTAPiigtpFeCRjO6mIZZ37fCHeVJuvTZ6JXZt/+9oOUHxvVHkrj993FZYrgN411/d7rG8OS6", - "hN1jZg2KDbDYPgsnbh2qa+fGIdBT28sHkVWccuYTkg7bGK3QxARvSicD6keRm8W9MZkCGFHqVHqRGJmK", - "zkoZ1kJ5CO4Q0rnRhf7szWybDfO5Oy1TIOkC0gvI8OABY4TjVnd/5O0sixdZpuw1FJsCh7nSuB2cAqnK", - "jDrbS/m6m7SqQGufqfseLmB9JppU6+tkqV6NRy6onxieGRKQ0tAjMAJi1hYXfzDQWXx3toKB97IkNrZt", - "sws9WxzVfOH7DAuQtUx3IDwxpqjJsIHfSyojhLDMP0CCG0zUwLsV60cj6VRqlrLSzn+32Py7Vh8DZJtS", - "j6pxMetq654yjWpv2ziZUhVX3GC+mPUwMtTNLvEj2ciKPSQjeJXZMe40h+BUSTnJphKdHT9tezdzCLU4", - "l4DkjTX1aLQpEprthTuWZJfNYSQeO+9i4LYeShku8vkCrB1+ZmbcHC7p4EnA4B2CkyAJILiaVt8Q8Iqt", - "Kwzj+raIvSXubxL46wP+zsBofK38//HI5XrFlkNwtO4Z5DCnLvCNWWSOURxqX6lggQweP81mOeNAklg+", - "AVVKpMyehTa63I0Bxvn7mhAbWCE7Q4ixcYA2RgwRMHkrQtnk8+sgyYFhiJF62BhrDP6G7RG35rq+cyu3", - "un993dEI0bi5TmOXsR/9GY+iKmnIM2+1IrbJFHpbmRiLGtXUj4f0oy4KckBznLQ0a3IRi5IZrwKQDU99", - "t8BdJw/YzBj5h0HgWMLc7L2b/aqRVh+A+bIxg0uhIZkxqXSCW+Xo9Eyj7xU6g9+bpnH10yIVsfd9WRbX", - "PjjsBayTjOVVfLXduH95aYZ9W+9bVDW9gDUaGaDpgkzxfrqxQq3hTZsNQ9ucmo0Tfm0n/Jre2Xx34yXT", - "1AwshdCdMe4JV3X0ySZhijBgjDn6qzZI0g3qJchG6OuWIA/C5kxgfsXBpt16T5iundExqHktpOhcAkd3", - "4yxs4o/N7Qmud/dzpgdkgJYly1advbOFGudxHOI6jrr1+HtUwNV1wLZQINgnx1IIJfi9vl3SwGbai/q9", - "NKvtlOkmdwUKIRyKKV9mpk8ow9qYjbONVmdA87/A+q+mLU5ndDUe3W7LH6O1g7iF1u/q5Y3SGWPIdgvY", - "ipxdk+S0LKW4pHnirqUMsaYUl441sbm/xfKFVV18+3326vj1O4c+Zq8BlS5pa9OssF15b2ZldsSxzK2z", - "IDKC3qrfO1tHLFj8+m5gGEzxiXYtX85oMcdcVrxqAxeKoguuzOJHWVtDJWFy3o0ks5Xdd9vIXJjqd6ci", - "35OwOIc2K7xFL4RjbSgsUNjaGYoI3k1wMG4c7jKRXQq6NqtoA7N9BcGrIjEikKicpfHQAZ8qI0W8KvDG", - "xloDwcYDDqGBWLGB8DmvWADLNFM7nBR1kAzGiBITwzobaDcVruhZxdk/KyAsA67NJ+kSnlrCYmTD5/D2", - "TVo8X9gBdinDNfjb2HkDasjCIxKbjXwY5Y1kiftNn59oHZ42PwTBuWsc0oQj9szShgMWxx+Om+1J96Id", - "rQ1rlPV1kGEMW89ie4E0HzpYWEQHxogWPBvU2MfD2hrzwHfX041aRnRDhWxz82iuRARMxZeU2/pFpp+l", - "oeutwO7bTa+lkHhpSUH0hJqpZCbFrxDfTc7MQkVysBwp0WXD3geRyyBdJVpHRprKdJ6+IR6DrD3kTQUf", - "SfsQbUDCkcuD8DUmlfogE+WWrW2tpdbRbVw4wnSLiYXfCIfDuZeiktPllMbKDhinxuB03ByUtMJhWhDf", - "2a+CqnOpHe8FZy51W2Zv+pQgm0TJ/k3NGzoo94vlM0hZQfN4dDRD6rfvemZszmzBqkpBUBHJAbKV/iwX", - "uapS9iiqIc3JjByOg5prbjUydskUm+aALR7ZFlOq0GrVIc+6i5kecL1Q2PzxDs0XFc8kZHqhLGGVILUT", - "aS83+PjzFPQSgJNDbPfoOXmAkXfFLuGhoaLzRUZHj55jSob94zBm7Fxluk16JUPF8t9OscT5GI8eLAxj", - "pBzUg+itM1tOdFiFbZAm23UXWcKWTuttl6WCcjqH+IlqsQUn2xdXEwN3HbrwzNbCU1qKNWE6Pj5oavTT", - "QFqWUX8WDZcrXxgB0oIoURh+asod2UE9OFtYz5Ug8Xj5j3jMUfo7D51N65cN0lpbHps1Hka9pQW0yTom", - "1F7OxGsb7lKvU4gHA7UiQF7GB5EDC+ztputLHnDBk8LITvawSfgL+C9aKkFomkeH1V53dTNXNoPe1dUy", - "UJJBwlYtwtJAJ92YxJWMz5NWZqif3792hqEQMlb3oNGGzkhI0JLBZVRiu4lrtWdSmwtP+ZiD8l3F8uyv", - "Tbppp8SQpDxdROOfU9Pxl6YmWk12S/XoFbwF5RzyKDgry794mY9opX+IXccpGN+xbbd0kJ1uZ3IN4m00", - "PVJ+QENepnMzQEjVdv5dnTiSz0VGcJzmknXDCP07UkEZlX9WoHTsPhd+sLlOuMc2/oqt4kGAZ2jtD4i9", - "/2Rwad1gQSvLiiq3tyEgm4N0AZiqzAXNxsTAOXt1/JrYUZW7Q4v3brCKyNzepWvNorO3Cqo/XOdy4VBq", - "1O5wNueMmFkrjVeylaZFGct6NS3OfANMrb2kLPfpB2h+QuockJfW8itvV+wgzR1SUg/ndA3yhPmP1jRd", - "oEltGaBhlt+9/I3nShWUgawr6tVFFey1SC18BRxbAGdMhPF7lkzZUrZwCe1E2zrr3Ll0PvG2PT1ZcW45", - "JW6fNtyKuAnZPXL2YM+HpKKYdQh/TTOjRCVTuG41oFPsFb1j1S0t1Kv/yCE7W/G6/povUZ5SLjhL8YZT", - "UDy3RtmVxd0lZrrDZbDudtmLuJPQiHBFCxrVqQOOioMljrwidITrB4yCr2ZRLXfYPzXWXzUbwTlo5TQb", - "ZGNftMrt4xhX4IpkYIXkQE+a7Xj3/DB6tNFc178mG2H634C78r35hq4Kcyk7F4zj5VVHNpcdZHdaWLVT", - "m+0d02QuQLn5tC9nqQ+mz8HZip8YjD8e+CqfCMOGkM207ZlFH9SxP8FwJwam7QvTlmC4uPm5lWpoBz0u", - "SzdoTBOoeoVjZbcGCRyJgic+DBkQt4YfQtvAbhuPHtGeGkaDSzy4gBLtcI8xBq7AvzKbWstR9iatPfKP", - "Xs1gPILGa8ahqUEbMRBp1CTgwqC8DvRTqaTauoA76bQzoDmelMQUmtIudHRbUJ0FRpLgHP0Yw8vYFF8b", - "UBx1g8Zxo3xdl7413B04Ey+w5rYjZL+UGnpVzonKMKmrU1wtpjiM4vZlCdsGoC8GfZ/IdteSWsm5jiUa", - "SkLPmDLbkWKaR9JYXtYfgwKDmC83XeO/sQvIwzNwB2s3LpiBHa/tX24uXpGbtU8Um99wVZr+d7gsHRkI", - "1yjG/a+MWgnv7fTuklvFU1+rwSN84cu94qaiTkxv8ywquuimrancuXnTOlyDc4yqcSCR531zY5Ra7Wtj", - "g0PpPOlg9hnVLrVUU7KpiowtnBmDYM8hbcFO+/hFNDAwdPZojx7N517v3fyGnheGsDcS1B9q9xH6i89a", - "ISVlLvDdiEifsi6/rZ9xuEvmS7PA3Um4rDEEEpvJDZO8dpK9PpUigh2mBmxhz4sWSe1tkI4nKSTcMWkD", - "E3pN0vaTHnadHs4DOaZS0J/nzgvQou0A7XchfKMX+sQdFmc93UWc40n1pjvqE0sQf+2jr02+mDZo1ft1", - "48ZW/a9D0QO7Qx4IVHVoWrE827a4rbBjc50ZA2u/TL952orefckL1b/YA/m+uLm7pdcx/N1FQMJE5toa", - "PBgqCCjuEEt03SKRQ6wFlVaS6TXm7nhPk/0SzUv+AbireuyKyNcnoO4Azr5f4kLT87p18+TED8KWgS6M", - "+4uuoMYiKa9WtChzcHLx7VfTP8GTPz/NDp88+tP0z4fPDlN4+uz54SF9/pQ+ev7kETz+87Onh/Bo9s3z", - "6ePs8dPH06ePn37z7Hn65Omj6dNvnv/pK//eg0W0eUvhb1h1IDl+d5KcGWQbmtCS/QXW9p6xYWN/g5mm", - "KIlQUJaPjvxP/8tL2EEqiuCJOvfryEX6RwutS3U0mSyXy4Owy2SOZfsSLap0MfHj9GvSvDupA7T2wB9X", - "1MbeDCvgojpWOMZv71+dnpHjdycHDcOMjkaHB4cHj7BQSAmclmx0NHqCP6H0LHDdJ47ZRkefrsajyQJo", - "rhfujwK0ZKn/pJZ0Pgd54K5ym58uH098fGfyyR1yXxmo81hWky+1VccX+zecxzZgYfYsdWmt4DKPcnd8", - "xmRq83eIq+7GM4wA2twMo9pqYp1kwYOYwcsL49Z7nh/u0RNVsbpPsavisUdH6yzz4Udngnf5/Ft8z/58", - "FTlo+th5SOTx4eFneDxk3ILi6XLDV0ie3iGK7R3UrRHtgutphTc0N3wD9cNyI5zQo3s7oROO9zmM2iJW", - "LV+NR8/u8QqdcCM4NCfYMkgh6avCn/kFF0vuWxqTXBUFlWs0uMFF8tC1uhpUue3kLXcjb1gPQ1CBLLjE", - "2wpsT9eez8ZE1YWeS8mEcRzwGcYMUgkUzbyQeB7U1DJzVxXBVrZ+c/w3jB6/Of6bLRIYfaIuGN4WzGwr", - "8R9AR2rtfbdunlnaqNF/KzU5/t2+6nd/bN5tTc2+YuO9rdi4g9Ler+6+Hue9rcd5v13SVZ14SwkXPOFY", - "1OASSBDW2vuov2sf9dnhk3s7m1OQlywFcgZFKSSVLF+Tn3mdEXQ7F7zWORUPcrQ26p9e9fnGiw7c96DA", - "0uRT61GJbHvwpHUjO2vVIKfxhy6D2jMuG3TcXDOlPLOZHP6sVo39dUuM1tl7zXY9xr3LmAcxJz04avlu", - "ffJyF7+8NafgBlrMN2/R63rP537WiMWNHyH9nBagh8d3NCM+ZfQz6+bdlOnTw6dfDoNwFd4KTb7HJLPP", - "rNI/a5wgzlaBssEiZpNP/rLaDgrGXQRtq5buy7UxpWIkdOxy9l2Z5vrxDaNPrCK0d3H7WsOMsKu+6N9V", - "jWmK5n7e70VHXOth4L1e2OuFG+uFLkM1GsE+Yzj5hAm2oTroiSS+rfwHOigJqudJUfjyLYLMQKcL9+xz", - "5yx76PX+jTpl07XCW+uX/aPft3n0e4cNyZ7AX+ZV9fsc+AisJUnIW3SHUMB9zvUfMezxOS3y557QW8GB", - "wIoprKppeXF/3Fi7C3gBH4ni30oIK97XroN7qnTyqXk7+KrJBLGXBCfW89/kV9gHXkZ3eqazf5TnHjzK", - "89vvKm4lIZ3ZSggfQAZ3SbaRFl+Us1+psp0s5ZqrRaUzsQxSq5rix4OS5J/Cv0NJ2r/Hv3+Pf/8e//49", - "/v17/Pv3+Pfv8d+P9/h/e8fquhupbrDuM+5u2q5q4LI0rpr9e7KkTCczIa0ZSrBCWiRQ2h79vynTri6f", - "20NpYZQCGEuMNdasQnFwgiopKsxZcY9X+AfPWRE5XDVDfS/kTnHZJtipBTETIxXXzGfV4yNH3m/7/QU5", - "9x7p3iPde6R7j3Tvke490r1Hej890t8meYEkiVfIPjM1lpdKRvfSa75HqZ9fMlezcaRrNxodb+P2Gjne", - "eKihgeYTV+8Lz3+FGsyOCmuHpWY4xkmZUyxovNL+jg7WMv7mqU9uqKvg2PIBRteYBk8ek9Mfj589evzL", - "42ff1G+Jt9s+8LVXlV7ntoBxe0dwBjR/4XC3SgOU/k5k6866GvQmiGl7RZtL9YxTGSkwFXlRuksDLbDI", - "nKuY1ts0XN1pwkO8CnCfnttIOVAJN8p9m5ZzawFWVxvUwd7lqMWsqScnccWpflONShAjx2aN9viXV583", - "UleejFExQiEcGw7LqhTw9S7HP6vENJoDT5yQJ1ORrf1TD65yXUul2ZJiwxrt1QrSykgGYuKY+oF66B5K", - "xNKIYawiWtI1qFAMCM/lTfW1lC1etVFJ3Xzx2qVwb3303gW36VV98kBIMpeiKh/amv98jZvQoqR87cMs", - "xm/CWrr4bCWmC92tWqzrCPaU2u6lYEPfHe9+dX+3ZCFLqnwd2MwWgo0Xw+mWK91O8aYY37biKXa+0cKh", - "A2VC+4voV9klKtahpRJkolc8Ur6vU6zvXz5H9z7q33dSXDKzJYyqMxvG1VHxPtiqhmWggFAPd+5fekXc", - "1o7v6TK8zbmrhlwlzme7tUO3APtSlndwIpdVjXGSgmYpVZhU6Oolf2ZnT69OIjtqRBOLDsx6l66Mtdxe", - "FB/h7uSKBaCbB5jwVrCy1ZV+W8esqQly7HI4W9TYa4k/yib3Oy98ilAi6bIrnEEN8x3UFF3qFY9qqUnz", - "wls05ygQiPpJqDs86emBbx/4BG8v2RMHyEtCSZozDKYLrrSsUn3OKQb3wjev+odBPmQ57Bi98E3i8eVI", - "+NeBOucUXympQ35RB2kGsYreAN7/UtV8Dkp3NPEM4Jy7Vow3L6IULJUisZl3JUjU6Ae2ZUHXZEZzjE7/", - "ClKQqXHZw4usGCpTmuW5O30ywxAxO+dUkxyM0n/DjHtmwPloSn2i6mrnh2+q90PPrszYwDM+P9ivP1K1", - "8NP3EREM3NjP9oDlyz/C43GPvc7nMD956YpMnLzEe8PNwVMP9y92cFIwnkSZzFh8d37b5S3ywD0JhQz0", - "sDnCcqt+zo1rrIV98bx5j/V67NANcPdk0UpHh2taC9GJg/u5foxdcJiLxGwAsfbwaM70oppiOT5/8WEy", - "F/UliElGoRAcv2UTWrKJKiGdXD7a4h/cQl+RiLraW+4/Tni6+2ZgvfDGie2t/YBdvoOaXr/vQl5bE1r2", - "ZbP2ZbP2hZX2ZbP2q7svm7UvKrUvKvWvWlTqYKOHOPmkV7uUeQmhssw+dSohtSPXCjxs1ioI0z8DZPqA", - "kDN8x5QaGwCXIGmOz1crfz2dKVKw+UITVaUpQHZ0zpMWJvZBTTPwg+a/dpt7Xh0ePgFy+JC0u9iwRaB4", - "+13RU8VP9lGbb8n56HzUBSShEJfgikNg66zCY1nbaSvUf3Ngz/lPsrdwBV3b0MqCliUYo6aq2YylzBI8", - "F2YrMBedvDUu8AtIgxwYfaoI07YOF1IT8/1c1gl1b/bEXO6+db9GLfTjDrPEU8YN212zYu5/7lIu91/F", - "vX4JmrJc1Znskd0U7mu6nLWkqhHcWqeMfQK08r+5w2c3Ss4uIMwtxYP+JZWZbxF9i6ypvObf2ou8r98q", - "SZXByjsEXaRn9ciseYi//zpMP67lCjttQMFVv7nJ4ANvSl+NR2kuFCSWSir2ygx+MJoIY7EUQ7HUvRLt", - "Hx81MIwwU4OdxKsiNmN9eEzG54l92CASorbf3cMHdSyuE/mOwPV8Mpi2WrOGfZoatU2XiCG3zYi7GT4Q", - "/rXvuNlkiBu/5tbp3nsoJ8/Ozz+S17ZoIb7ydAHriX1fJF1QPgdV0yjkU3u9w2awBPnKHTLe3Qtyxmok", - "A28/nvRzmLt0v2DpBWTE6An/MPmAC08e1BXY8HHf5WLtL2tYM/TwgJBjTqAo9dq/89uONHcG51/pTeOv", - "QsPZtkiRfLsU2CXIW8qUB7NZkhQYgbvlUBbI5oH0ig+IE11GNrS7luSJ7F87u8mAqSwWdxEW2FulvVXa", - "W6W9Vdpbpb1V+mxWqReCuf9Bim6fm0cpupDuLkzxmwcq/kBlAPcV/35nEwpTN1slfW8Ru60fLox5wS4q", - "2zwMGj60iVG1+onNDx+vPppv8tIH3Jp3I48mE/QqFkLpyehq/KnzpmT40ahSOrcQXECrlOwSC3Z+vPr/", - "AQAA//9ABOdBwOQAAA==", + "H4sIAAAAAAAC/+x9/ZPbNpLov4LTXVXsnDgaf2XXU5W6N7GdZN7ajsue7O07j18WIlsSdiiAS4AzUvzm", + "f3/VDYAESVDSfNiJs/rJHhFoAI3+RqPxcZSqZaEkSKNHRx9HBS/5EgyU9BdPU1VJk4gM/8pAp6UojFBy", + "dOS/MW1KIeej8UjgrwU3i9F4JPkSmjbYfzwq4Z+VKCEbHZmygvFIpwtYcgRs1gW2riGtkrlKHIhjC+Lk", + "+ehqwweeZSVo3Z/lTzJfMyHTvMqAmZJLzVP8pNmlMAtmFkIz15kJyZQEpmbMLFqN2UxAnukDv8h/VlCu", + "g1W6wYeXdNVMMSlVDv15PlPLqZDgZwX1pOoNYUaxDGbUaMENwxFwrr6hUUwDL9MFm6lyy1TtJML5gqyW", + "o6P3Iw0yg5J2KwVxQf+dlQC/QmJ4OQcz+jCOLW5moEyMWEaWduKwX4KucqMZtaU1zsUFSIa9DtirShs2", + "BcYle/v9M/bo0aOnuJAlNwYyR2SDq2pGD9dku4+ORhk34D/3aY3nc1VymSV1+7ffP6Px37kF7tqKaw1x", + "ZjnGL+zk+dACfMcICQlpYE770KJ+7BFhiubnKcxUCTvuiW18p5sSjv+b7krKTboolJAmsi+MvjL7OSrD", + "gu6bZFg9gVb7AjFVItD3h8nTDx8fjB8cXv37++Pkf9yfTx5d7bj8ZzXcLRiINkyrsgSZrpN5CZy4ZcFl", + "Hx9vHT3oharyjC34BW0+X5Kod30Z9rWi84LnFdKJSEt1nM+VZtyRUQYzXuWG+YFZJXMUUwjNUTsTmhWl", + "uhAZZGOUvpcLkS5YyrUFQe3YpchzpMFKQzZEa/HVbWCmqxAlOK8b4YMW9PtFRrOuLZiAFUmDJM2VhsSo", + "LerJaxwuMxYqlEZX6espK3a6AEaD4werbAl3Emk6z9fM0L5mjGvGmVdNYyZmbK0qdkmbk4tz6u9Wg1hb", + "MkQabU5LjyLzDqGvh4wI8qZK5cAlIc/zXR9lcibmVQmaXS7ALJzOK0EXSmpgavoPSA1u+/9+99Nrpkr2", + "CrTmc3jD03MGMlXZ8B67QWMa/B9a4YYv9bzg6XlcXediKSJTfsVXYlktmayWUyhxv7x+MIqVYKpSDk3I", + "QtxCZ0u+6g96WlYypc1thm0ZakhKQhc5Xx+wkxlb8tW3h2M3Hc14nrMCZCbknJmVHDTScOzt00tKVcls", + "BxvG4IYFWlMXkIqZgIzVUDbMxA2zbT5CXm8+jWUVTMcDGZxOPcqW6UhYRWgGWRe/sILPISCZA/azk1z0", + "1ahzkLWAY9M1fSpKuBCq0nWngTnS0JvNa6kMJEUJMxGhsXcOHSg9bBsnXpfOwEmVNFxIyFDy0qSVASuJ", + "BucUDLjZmemr6CnX8M3jIQXefN1x92equ+sbd3yn3aZGiWXJiF7Er45h42ZTq/8Ozl84thbzxP7c20gx", + "P0VVMhM5qZl/4P55NFSahEALEV7xaDGX3FQlHJ3Jr/EvlrB3hsuMlxn+srQ/vapyI96JOf6U259eqrlI", + "34n5ADLruUa9Keq2tP8gvLg4Nquo0/BSqfOqCBeUtrzS6ZqdPB/aZAvzuoR5XLuyoVdxuvKexnV7mFW9", + "kQOTHMRdwbHhOaxLwNnydEb/rGZET3xW/or/FEUewykSsFO0FBRwwYK37jf8CVkerE+AUETKEakTUp9H", + "H4MJ/UcJs9HR6N8nTaRkYr/qiYOLI16NR8cNnLsfqelp19dxZJrPTEi7O9R0bH3Cu58PQo3OhAzVzhy+", + "y1V6fqM5FKUqoDTC7uMU4fQ5hcCzBfAMSpZxww8ap8raWQP0Th1/pH7kJUEZUXE/0X94zvAzciE33nxD", + "01VoNOJUEGjK0OKzesSOhA3IElVsaY08hsbZtWb5rBncCuhaor53aPnQhRbZnRfWrmTUwy8Cl954jcdT", + "Vd6MXjqEIFnjCzOOUGvrF1fe3llqWhWJw0/EnrYNOoCa8GNfrIYY6oKP4aqFhXeGfwIsaIR6F1hoA7pr", + "LKhlIXK4A35dcL3oLwINnEcP2bsfj588ePjLwyffoIYuSjUv+ZJN1wY0u+f0CtNmncP9/spIwFe5iUP/", + "5rH3oNpwt2KIJlzD3oWjTgElg8UYs/ECnN3zcl1W8g5QCGWpyojNS6RjVKry5AJKLVQkfPHGtWCuBcoh", + "a3d3frezZZdcMxyb3LFKZlAexDCPfhapdANLvU1RWNCnK9ngxgHkZcnXvR2w642szo27y560ke+te80K", + "KBOzkiyDaTUPdRSblWrJOMuoIwnE1yqDd4abSt+BFGiANZPBjQinwKeqMowzqTJkaGwclw8DsUwKolDs", + "x4Qixyys/pkCWscpr+YLw9CsVLGtbTomPLWbkpCu0AOuX+2z21Z2OBsny0vg2ZpNASRTU+dfOc+PFskp", + "LGP8iYuTTs20ap+gNa+iVCloDVnijpe2Ts0fVdEmmw1oonnTfOtBmFZsxssbztUow/Mt86Q2/dnqxppw", + "Pml/1rsNv2n/uoOHu8hLdDEtEaDpgsydg4EhFG7FSVUMHEc4rXYqlsgSTHKpNKRKZjoKLOfaJNtYARu1", + "VC9ua0B9MeonwANO90uujXV7hczIDLMsTONQHxpieMKDUhoh/9UL6D7sFGWP1JWupbWuikKVBrLYGiSs", + "Noz1Glb1WGoWwK5VglGs0rAN8hCWAvgOWXYlFkHcuLhLHRfqL45C3Chb11FUtibRIGLTRN75VgF2w5Ds", + "wETQZq97EuEI3aGcOg48HmmjigJlkkkqWfcbQtM72/rY/Ny07RMXN42szBTg6MbPyc380mLWBuMXHO0l", + "gsyW/BzlPVk/1j/vzxmZMdFCppBsonxky3fYKmSBLUw6YHi6475gtA5zdOg3SnSDRLBlF4YWPGAFv7FR", + "5dMm4nIHhsBzMFzkulb2dei6GYWi3N0MBLTMSkhBmnyNtDoT5dIeFJGO0P43a0pkbhR7JNKwn8xYCZe8", + "zHyLvgcSLCYRMoNVXLryVrwhgxUT8UnP6pGFYak/xpEhgIMoo7uDsQ1TcIGGmwyOXePD2mMfiyUdOxCk", + "D8gAS5GWittzPlyMVZKmPsoqYclxdnTi5JT68JhCzhN7rBhRj/a7P3b04d6QZuJwPZ0McnZNGpcLoJMM", + "FNcdJIbUhm4aaBhayDxXU54naLxCkkFutoaR0CiG59QS9aRK+93bUz47e59nZ2cf2EtsS3YysHNYT+j0", + "laULLufQhMRDOrUWMKwgrUKR3kHjTk6Ni/u1Z992a8ajQqk8qd23bgi/J+a7eD8X6TlkDOUEGZ1O+3zV", + "3iEchN1DEtf1IcflYu3t2aIACdn9A8aOJYNlYdYuVtCxNDqDy6/MpvFXNGpW0Xkrl4wWeXAm4266Pa29", + "JU95MJs5yaYv3XIoC2TzQGYlB9iJX9JhA4KL8ufGSN876hmonJ4mDYjKzmIXf/gHyunhrV0WGTkbjVbR", + "1XQpKLEnaDZGyenPWvveqjAHjJ2S7EBvQcMFlDynrAXtg6BCs6VAp1NXaQqQHZ3JpDWTVC3dwPea/1qx", + "dFYdHj4Cdni/20cbNBOdY2R5oNv3W3Y4tp8IXexbdjY6G/UglbBUF5BZ5zCka9trK9h/q+GeyZ96gpkt", + "+dq6lZ4Xma5mM5EKi/RcoVyfq461JxV9gRKnB+icaSbMmFQZYZSsZLsvDQOOolbLXcQvIlDRPkZVitLO", + "n7C1aUczWPEUV8lJyKzZJRJKTWd948OoIgkBRMOpG0Z0AW3dkuM35Lu+PLfe9Ob5nXb86RY6AnI92G4z", + "95ARncEu7H/MCoW7LlwujU+4yIU2vUk6x55OM2qCjCidA/Z/VMVSTvxbVAZqn0qV5KiQA4sjkI71YzpL", + "rcEQ5LAEG+6gL19/3V3411+7PReazeDSJ6Bhwy46vv7aMoHS5tYc0CHN1UnEgKIgM2rTSNLwguvFwdaA", + "M8HdKc4cgD557gckZtKaVMzVeISubr6+A4a3gFgJzt7TraCPtl/VLEx2c/un19rAsh+5tF1/GbBE33oP", + "radplcyFhGSpJKyj+d1Cwiv6GNXTRCIDnYlZh/p2PdjW/DvTao+zy27eFr+02wFJvKlT7+5g87twO0Hr", + "MM2PrEzIC8ZZmgsKCCqpTVml5kxyClB0zKAOWfiwy3DI6plvEo+RRUJYDtSZ5BpxWIctoocZM4gEJL8H", + "8JErXc3noDtmEZsBnEnXSkhWSWFoLLIqE7thBZR06nRgW6IlMOM5Rdh+hVKxaWXaopeykaxlYyPoOAxT", + "szPJDcuBa8NeCXm6InDew/E0I8FcqvK8xsKAhwYStNBJ/GDuB/v1R64XfvnY0Asb19kGiRF+k7K0NtBK", + "d/6/9/7r6P1x8j88+fUwefqfkw8fH1/d/7r348Orb7/9f+2fHl19e/+//iO2U37usVwZN/OT584sOXlO", + "uqcJnvfm/tmCv0shkyiRobuwFJJSLju0xe6hBvUEdL8Jw7tdP5NmJZGQLnguMnSBb0IOXRHX40XLHR2q", + "aW1EJ5bn1/oh5u7MVVLw9JzOtUdzYRbV9CBVy4k3xyZzVZtmk4zDUkn6lk14ISbo3k4uHmxRjbeQVywi", + "rq7GIyd19J1n0DjAsQV1x6yj6P5vo9hXP7w4ZRO3U/ormzhnQQcZTxEL2h2GtZw5XLy9+GEzB9GZeQ4z", + "IQV+PzqTGTd8MuVapHpSaSi/4zmXKRzMFTtiDuRzbjjFADoRxaG7WRSfcbMpqmkuUnYequKGNYcCY2dn", + "75FAzs4+9A6y+orTDRUPNtIAyaUwC1WZxEVlh+MITayFINu43KZRx8zBthTpor4O/kAAtCh0EkTE4ssv", + "ihyXH5ChZtSJ8qCYNqr0QhAlo4tp4P6+Vu4or+SXPhu9Qr/970tevBfSfGCJ87+Pi4LCbRTv+ruTNUiT", + "6wJ2j5k1U2yAxfwsWrg1qK6dG0dA39lePois45jDT4Q6aoNSoYkJ3hRPCOpHlePm3hhNAYwodiqzSJCn", + "oqvSSFrED8EdQj5HWejP3tBtRuJzd1qmwNIFpOeQ0cEDxQjHre7+yNtpFs+yQttrKDYFjnKlyR2cAquK", + "jDvdy+W6m7SqwRifqfsWzmF9qppU6+tkqV6NRy6onyDNDDFIgfgIlICatdnFHwx0Nt+drVDgvSiYjW3b", + "7EJPFkc1Xfg+wwxkNdMdME+MKGo0bKD3gpcRRFjiH0DBDRaK8G5F+tFIOi+NSEVh179bbP5Nqw8C2SbU", + "o2JczbrSuidMo9LbNk6mXMcFN+AX3A/koW52iR/JRlbsIRmjq8yOcKc5BKdK2nE2L8nY8cu2dzOHphan", + "Eihlo039NNoYCdX2wh1LiovmMJKOnXdRcFsPpZCKfL6AaIefBY6bwwUfPAkYvENwEiQBBFfT6hsCXrB1", + "mWFc3xaxt8T9TQJ/fcDfGRiNr5X/Px65XK/YdihJ2j2DHObcBb4pi8wRipvaVzrYIJzHT7NZLiSwJJZP", + "wLVWqbBnoY0sd2MAGn9fM2YDK2xnCDEyDqZNEUMCzF6rkDfl/DqTlCAoxMg9bIo1Bn/D9ohbc13fmZVb", + "zb++7GiYaNxcp7Hb2I/+jEdRkTRkmbdaMdtkCj1XJkaiKJr68ZB+1EVDDqSOk5ZkTc5jUTK0KoDI8J3v", + "Fpjr7J6YoZK/HwSOS5ij7934q8itPgDzeWMGF8pAMhOlNgm5ytHlYaPvNRmD32PTuPhpoYrZ+74ii0sf", + "GvYc1kkm8iq+227cvzzHYV/XfouupuewJiUDPF2wKd1PRy3UGh7bbBja5tRsXPBLu+CX/M7WuxstYVMc", + "uFTKdMb4QqiqI082MVOEAGPE0d+1QZRuEC9BNkJftgR5EDZngvIrDjZ56z1munZGx6DktZCiawkM3Y2r", + "sIk/NrcnuN7dz5ke4AFeFCJbdXxnCzVO4zTEdQx1a/H3sEC764BtwUDgJ8dSCEvwvr7d0kBn2ov6vTSr", + "7ZjpJncFAiEcSmhfZqaPKCRtysbZhqtT4PlfYP1XbEvLGV2NR7dz+WO4dhC34PpNvb1RPFMM2bqArcjZ", + "NVHOi6JUFzxP3LWUIdIs1YUjTWrub7F8ZlEXd79PXxy/fOOmT9lrwEuXtLVpVdSu+GJWhR5xLHPrNIiM", + "kLXqfWdriAWbX98NDIMpPtGuZcuhFHPEZdmrVnAhK7rgyix+lLU1VBIm592IM1vZfbeNzIWpfnfK8j0O", + "i1Nos8Nb5EI41obCAktbO0MzJbsJDmjGkZdJ5LLka9xFG5jtCwhZLRNkgUTnIo2HDuRUIxfJakk3NtYG", + "GDUeMAgRYiUGwueyEgEsbKZ3OCnqTDIYI4pMCutswN1UuaJnlRT/rICJDKTBT6VLeGoxC/KGz+Htq7R4", + "vrAD7FKGa/C30fMIakjD0yQ2K/kwyhvJEvdOn19oHZ7GH4Lg3DUOacIRe2ppwwGLow9Hzfake9GO1oY1", + "yvoyCAnD1rPYXiDNhw4WdqIDY0QLng1K7ONhaU154LvL6UYs03RDgWxz83iuVQRMJS+5tPWLsJ/Foeut", + "wfrt2OtSlXRpSUP0hFroZFaqXyHuTc5woyI5WA6VZLJR74PIZZCuEK0jI01lOo/fcB6DpD1kTQUfWfsQ", + "bYDDicqD8DUllfogE5eWrG2tpdbRbZw5wnSLiYXfMIebcy9FJeeXUx4rO4BGDc7puDkoaYXDjGK+s98F", + "XedSO9oLzlzqtsLe9CmgbBIl+zc1b2igfFkkn0EqljyPR0czwn77rmcm5sIWrKo0BBWRHCBb6c9Skasq", + "ZY+iGtSczNjhOKi55nYjExdCi2kO1OKBbTHlmrRWHfKsu+DyQJqFpuYPd2i+qGRWQmYW2iJWK1YbkfZy", + "g48/T8FcAkh2SO0ePGX3KPKuxQXcRyw6W2R09OAppWTYPw5jys5VptskVzISLP/tBEucjunowcJAJeWg", + "HkRvndlyosMibAM32a678BK1dFJvOy8tueRziJ+oLrfMyfal3aTAXQcvMrO18LQp1ZoJEx8fDEf5NJCW", + "heLPTsPlyi+RgYxiWi2RnppyR3ZQD84W1nMlSPy8/Ec65ij8nYeO0/p5g7RWl8dWTYdRr/kS2mgdM24v", + "Z9K1DXep1wnEg4FaEVBexAcpBzbY603Xl92TSiZL5J3sfpPwF9BftFSCMjyPDmu87OpmrmwGvauphVCS", + "QcRWLcTyQCbdGMVVGV8nr3Con9++dIphqcpY3YNGGjolUYIpBVxEObabuFZbJrW68JiPGSjfVSLP/tqk", + "m3ZKDJVcpoto/HOKHX9paqLVaLdYj17BW3ApIY+Cs7z8i+f5iFT6h9p1nKWQO7btlg6yy+0srpl4e5p+", + "Un5ARK8wOQ4QYrWdf1cnjuRzlTEap7lk3RBC/45UUEblnxVoE7vPRR9srhP52Giv2CoeDGRG2v6A2ftP", + "OJfWDRbSsmJZ5fY2BGRzKF0ApipyxbMxQzinL45fMjuqdndo6d4NVRGZ27t0rVV0fKug+sN1LhcOpUbt", + "DmdzzgiuWhu6kq0NXxaxrFdsceobUGrtBRe5Tz8g9RNi54A9t5pfe71iB2nukLJ6OCdriCbwP8bwdEEq", + "taWAhkl+9/I3nip1UAayrqhXF1Ww1yKN8hVwbAGcMVNo91wKbUvZwgW0E23rrHNn0vnE2/byykpKSylx", + "/bThVsRN0O4nZw/2fEgqOrMO4q+pZrSqyhSuWw3oHfWK3rHqlhbq1X+UkJ2uZF1/zZcoT7lUUqR0wyko", + "nltP2ZXF3SVmusNlsK677FnccWiEuaIFjerUAYfFwRJHXhA6xPUDRsFX3FRLHfZPQ/VX0RGcg9FOskE2", + "9kWrnB8npAZXJIMqJAdyEt3x7vlh9Gijua5/TTKi9L8Bc+V7/EaminApO+dC0uVVhzaXHWQ9LaraadC9", + "E4bNFWi3nvblLP0e+xycruQJzvjDga/ySTBsCBmXbc8s+qCO/QmGOzHAts+wLaNwcfNzK9XQDnpcFG7Q", + "mCTQ9Q7Hym4NIjgSBU98GDJAbg0/hLaB3DYePZI+RUKDCzq4gIL0cI8wBq7Av0Cn1lKUvUlrj/yjVzOE", + "jEzjpZDQ1KCNKIg0qhJoY4hfB/rptOTGmoA7ybRT4DmdlMQEmjYudHRbUJ0NJpTQGv0Yw9vYFF8bEBx1", + "g8Zw43Jdl75F6g6MiWdUc9shsl9KjawqZ0RllNTVKa4WExwouH1ZwrYC6LNB3yay3U3JLedcRxMNJaFn", + "QqM7spzmkTSW5/XHoMAg5ctN1/Rv7ALy8ArcwdqNC2ZQx2vbl5uLV+S494kW8xvuStP/DrelwwPhHsWo", + "/wWKlfDeTu8uuRU89bUaOsJXvtwrORV1YnqbZknQRZ22pnLnZqd1uAbnmETjQCLP2+bGKLfS18YGh9J5", + "0sHsM25caqnhbFMVGVs4MwbBnkPagp328YtoYGDo7NEePeLnXu/d7IaeFUawNyLUH2r3J/QXn7XCCi5c", + "4LthkT5mXX5bP+Nwl8yXZoO7i3BZYwQktpIbJnntxHt9LEUYO0wN2EKe5y2U2tsgHUtSlXDHqA1U6DVR", + "20962HV5tA6imEpDf507b0ALtwO43wXxjVzoI3eYnc10F3aOJ9Vjd5InFiH+2kdfmnw2adCq9+vGje36", + "X4eiB9ZDHghUdXBaiTzbtrmtsGNznZkCa79Mv3ncit59zgvVv9gD+T67ubul11H83U0gxETW2ho8GCoI", + "KO4QS3TdIpFDqgWVVqUwa8rd8Zam+CWal/wDSFf12BWRr09A3QGcfb/EhabndevmyYkflC0DvUTzl0xB", + "Q0VSXqz4ssjB8cW3X03/BI/+/Dg7fPTgT9M/Hz45TOHxk6eHh/zpY/7g6aMH8PDPTx4fwoPZN0+nD7OH", + "jx9OHz98/M2Tp+mjxw+mj795+qev/HsPdqLNWwp/o6oDyfGbk+QUJ9vghBfiL7C294yRjP0NZp4SJ8KS", + "i3x05H/6X57DDlK1DJ6oc7+OXKR/tDCm0EeTyeXl5UHYZTKnsn2JUVW6mPhx+jVp3pzUAVp74E87amNv", + "SAq0qY4Ujunb2xfvTtnxm5ODhmBGR6PDg8ODB1QopADJCzE6Gj2in4h7FrTvE0dso6OPV+PRZAE8Nwv3", + "xxJMKVL/SV/y+RzKA3eVG3+6eDjx8Z3JR3fIfYVQ57GsJl9qq44v9m84j23AAn2WurRWcJlHuzs+Yza1", + "+TvMVXeTGUUAbW4GirYaWSdZ8CBm8PLCuPWe5/sv6ImqWN2n2FXx2KOjdZb58KMzwbt8/i2+J3++ihw0", + "feg8JPLw8PATPB4ybkHxeLnhKySP73CKbQ/q1hPtgutJhVc8R7qB+mG5ES3owRe7oBNJ9zlQbDErlq/G", + "oydf8A6dSGQcnjNqGaSQ9EXhz/JcqkvpW6JKrpZLXq5J4QYXyUPT6mpQ5LaTt9yNvGE5DEEFsuASbyuw", + "PV17OhszXRd6Lkqh0HCgZxgzSEvgpOZVSedBTS0zd1URbGXrV8d/o+jxq+O/2SKB0SfqguFtwcy2EP8B", + "TKTW3nfr5pmljRL9txKT49/tq35fjs67rarZV2z8Yis27iC097u7r8f5xdbj/LJN0lWdeMuZVDKRVNTg", + "AlgQ1trbqL9rG/XJ4aMvdjXvoLwQKbBTWBaq5KXI1+xnWWcE3c4Er2VOJYMcrY3yp1d9vrGiA/M9KLA0", + "+dh6VCLbHjxp3cjOWjXIefyhy6D2jMsGHTfXTLnMbCaHP6vVY3/dkqJ19l6z3Y9x7zLmQcxID45avluf", + "PN/FLm+tKbiBFrPNW/i63vO5nzRiceNHSD+lBujN4zueMZ8y+oll827C9PHh4883g3AXXivDvqcks08s", + "0j9pnCBOVoGwoSJmk4/+stoOAsZdBG2Llu7LtTGhghw6djn7rkxz/fgGyhMrCO1d3L7UwBF2lRf9u6ox", + "SdHcz/u9yIhrPQy8lwt7uXBjudAlqEYi2GcMJx8pwTYUBz2WpLeV/0AHJUH1vFItffkWxWZg0oV79rlz", + "lj30ev9GmbLpWuGt5cv+0e/bPPq9g0OyR/DneVX9Sw58BNqSJew1mUPE4D7n+o8Y9viUGvlTL+i1ksBg", + "JTRV1bS0uD9urM0FuoBPSPFvJYQV72vTwT1VOvnYvB181WSC2EuCE2v5b7Ir7AMvozs909k/yvMFPMrz", + "23sVt+KQzmpLCB9ABndJtuEWX5SzX6mynSzlmutFZTJ1GaRWNcWPBznJP4V/h5y0f49//x7//j3+/Xv8", + "+/f49+/x79/j/zLe4//tDavrOlLdYN0n9G7apmpgsjSmmv17csmFSWaqtGoooQppkUBpe/T/5sK4unzO", + "hzIKhQKgJqYaa1agODhBlRQd5qy4xyv8g+diGTlcxaG+V+VOcdkm2GkUw4WxShrhs+rpkSNvt/3+gpx7", + "i3Rvke4t0r1FurdI9xbp3iL9Mi3S3yZ5gSWJF8g+MzWWl8pGX6TV/AWlfn7OXM3GkK7NaDK80exFPt54", + "qGGA5xNX74vOf5UezI4Ka4elOJyQrMg5FTReGX9Hh2oZf/PYJzfUVXBs+QCUNdjg0UP27sfjJw8e/vLw", + "yTf1W+Lttvd87VVt1jncd4e/9d1gfwoMklO5HToE5t6jSH1mhrWQZyIHRi9avqDmz+ECcjSPkSKprFkV", + "cTlOgefPHHKsVAJtvlPZukM4uP4JoaJNMs2tfSF5GalgFXmyuotko6iKnSvJ1vNKru40oyJeZri/Ydv2", + "aqDUbpS8N9HL1gqvrviog73LWQ7uqUcnc9WvflORzWhGjswa8fS7SULrvkDhGIfaolXh+O9LTRjziI8y", + "HrHtGGkyq1KgB8Ucxa0SbDQHmTixkExVtvavT7hiei0pa6ucDQvZFytIK+Qlmoljg3v6vnu7kao1huGT", + "aJXZoGgyELzmvaHPLThtwa6NcvPm1NEu/3vrdIMuuL7UCO423FMlm5eqKu7bdw7kmhzvZcHl2oeW0Fak", + "+sH0VCelSN2tpK5rJ/bk7O7lb0N/he67dX+3aGGXXPvat5ktfhsvANQt0bod400Bwm0FY+x6o8VSB0qj", + "9jfR77JLzqzDaQWUiVnJSMnCToHCfV7yv4RKeFOqC4GOc1TC2mC3iQqEg62aoQxEFqmGzi1Vrxva8vQt", + "vwzvvO4qU1eJMzxvbZUuwL4n5q20yJVe1Jel4lnKNaVeuqrSn9hiNauTSNyBpkmlGWa9q2mowLc/HUBw", + "d7InA9DNM1V0d1rbGlS/rXXZVE45dpmuLWzsQwF/lFDAd575NOOs5Jdd5gwqve8gpvilWcmolJo07+BF", + "M7MChqgfzrrD87Ae+PaxWPBClT2XgbxgnKW5oCMHJbUpq9ScSU4h0PBlsP6RmQ/sDptSz3yTeBQ+EiR3", + "oM4kp7dc6sBo1KSaQazuOYC32HQ1n4M2HUk8AziTrpWQzbsxS5GWKrH5iaiuUaIf2JZLvmYznlMM/1co", + "FZuiFxFe96WAojYiz90ZHQ7D1OxMcsNyQKH/SqBBh+B8zKk+d3YvDIQvz/cD9K4Y28BjRz/Yrz9yvfDL", + "93EjCm/Zz/YY6vM/VeTnHnvD0M385LkrxXHynG5XN8dzvbl/tuOlpZBJlMhQ47tT7i5tsXvu4SwioPvN", + "QZ/b9TOJxrRR9l345tXa65FD9xigx4uWOzpU09qIzmmBX+uH2DWQuUrQZaQKzaO5MItqSkUL/fWQyVzV", + "V0UmGYelkvQtm/BCTHQB6eTiwRb74BbyikXE1V5z/3GC+N2XFeuNRyO2t/cDevkOKp/9vsudbU372RcX", + "2xcX25ef2hcX2+/uvrjYvvTWvvTWv2rprYONFuLko1ntUgwnhCoy+yBsCakduRbgYbNW2Zz+saQwB4yd", + "0muvHHUAXEDJc3rkW/tL/EKzpZgvDNNVmgJkR2cyac3EPjuKA99r/mvd3LPq8PARsMP73T42bhFI3n5f", + "MlXpk33751t2Njob9SCVsFQX4IpoUPOsorNi22sr2H+r4f5U9rZuydc2uLLgRQGo1nQ1m4lUWJTnCp2B", + "uerk90lFX6DEyQFKVM2EsfXKCJ+UF+myc7h72yhmdPf1+zVqxh93yCWeWo+Ed83Kwv+5S1nhfxUD+zkY", + "LnJdZ/xH/CnybLqUdcl1w7q1VBn7RHHtf3MH1m6UXJxDmINL2QeXvMx8i+ibbU2FOv8mYT+01C7dlcHK", + "mwTdSc/qkYWxxbbQ4ey9otOPbLkCWBum4KoE3WTwgbe3r8ajNFcaEoslHXuNhz6gKKJoLKdgLHevaftH", + "WhEGMjPH2ZV0pcZm9g+PKeQ8sQ9ARILU9rt7IKKOxnVi3xG4nk4G03tr0rBPeJO06SIxpLYZczfoBwLA", + "9r07m0Bx41fvOt17Dwrl2dnZB/bSFnek17DOYT2x77CkCy7noGschXRqr8HYtJogr7uDxrt7aQ+1RjLw", + "RuZJP9e7i/dzkZ5DxlBO+AfcB4x4dq+uVEePIF8u1v5Si1VD9w8YO5YMloVZ+/eQ27HmzuDyK7Np/FWo", + "ONsaKZI2mIK4gPKWPOXBbOYkDchwtxzKAtk8kFnJAXbilxGXdtfSRREPtuNPBkRlZ3EXgYG9Vtprpb1W", + "2mulvVbaa6VPppV6QZh9mOJzhCl+80DFH6hc4r4y4u9sQWHyZqv08S2it/UDjzEr2MVlmwdUwwdJKapW", + "P0X6/sPVB/xWXviAW/O+5tFkQlbFQmkzGV2NP3be3gw/oijlcwvBBbSKUlxQYdMPV/8/AAD//3Jx7Gro", + "5QAA", } // GetSwagger returns the Swagger specification corresponding to the generated code From 3300ce38944d7876c8e636b84124ef4ca509c025 Mon Sep 17 00:00:00 2001 From: btoll Date: Thu, 5 Nov 2020 09:55:14 -0500 Subject: [PATCH 07/34] Mount .docker dir to allow container to push to docker hub (#1682) The container launched by the mule task creates and pushes a docker image to docker hub. The fix mounts the .docker directory that contains the docker credentials. --- package-deploy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/package-deploy.yaml b/package-deploy.yaml index 853c809e72..bf2484095c 100644 --- a/package-deploy.yaml +++ b/package-deploy.yaml @@ -12,6 +12,7 @@ agents: - VERSION=$VERSION volumes: - /var/run/docker.sock:/var/run/docker.sock + - $HOME/.docker:/root/.docker workDir: $HOME/projects/go-algorand - name: releases-page From ea849096ab6164971370b25937d37ef606c59a0b Mon Sep 17 00:00:00 2001 From: btoll Date: Thu, 5 Nov 2020 09:55:53 -0500 Subject: [PATCH 08/34] Added package test to verify gpg signature (#1680) --- scripts/release/mule/test/tests/pre/verify_signature.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 scripts/release/mule/test/tests/pre/verify_signature.sh diff --git a/scripts/release/mule/test/tests/pre/verify_signature.sh b/scripts/release/mule/test/tests/pre/verify_signature.sh new file mode 100755 index 0000000000..e2962a4a4a --- /dev/null +++ b/scripts/release/mule/test/tests/pre/verify_signature.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -ex + +echo "[$0] Verifying gpg signatures" + +find tmp/node_pkgs -type f -name "*.sig" -exec gpg --verify {} \; + From 558d069b58972bcc2a15cd9f0db4bb5bb165193e Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Mon, 9 Nov 2020 17:04:02 -0500 Subject: [PATCH 09/34] Add e2e test for LegderForLogic (#1688) Add unit tests to coin existing teal behavior --- test/scripts/e2e_subs/e2e-app-cross-round.sh | 35 ++++ .../e2e_subs/tealprogs/cross-round.teal | 170 ++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100755 test/scripts/e2e_subs/e2e-app-cross-round.sh create mode 100644 test/scripts/e2e_subs/tealprogs/cross-round.teal diff --git a/test/scripts/e2e_subs/e2e-app-cross-round.sh b/test/scripts/e2e_subs/e2e-app-cross-round.sh new file mode 100755 index 0000000000..bdaedd8f7d --- /dev/null +++ b/test/scripts/e2e_subs/e2e-app-cross-round.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +date '+app-cross-round-test start %Y%m%d_%H%M%S' + +set -e +set -x +set -o pipefail +export SHELLOPTS + +WALLET=$1 + +# Directory of this bash program +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +gcmd="goal -w ${WALLET}" + +ACCOUNT=$(${gcmd} account list|awk '{ print $3 }') + +printf '#pragma version 2\nint 1' > "${TEMPDIR}/int1.teal" +APPID=$(${gcmd} app create --creator ${ACCOUNT} --approval-prog ${DIR}/tealprogs/cross-round.teal --global-byteslices 1 --global-ints 1 --local-byteslices 1 --local-ints 1 --clear-prog "${TEMPDIR}/int1.teal" | grep Created | awk '{ print $6 }') + +# Should succeed to opt in with first arg hello +${gcmd} app optin --app-id $APPID --from $ACCOUNT --app-arg "str:first" + +# Write should now succeed +${gcmd} app call --app-id $APPID --from $ACCOUNT --app-arg "str:second" + +# Write should now succeed +${gcmd} app call --app-id $APPID --from $ACCOUNT --app-arg "str:third" + +# Delete application should still succeed +${gcmd} app delete --app-id $APPID --from $ACCOUNT --app-arg "str:any" + +# Clear should still succeed with arbitrary args +${gcmd} app clear --app-id $APPID --from $ACCOUNT --app-arg "str:any" diff --git a/test/scripts/e2e_subs/tealprogs/cross-round.teal b/test/scripts/e2e_subs/tealprogs/cross-round.teal new file mode 100644 index 0000000000..7bbfed8aff --- /dev/null +++ b/test/scripts/e2e_subs/tealprogs/cross-round.teal @@ -0,0 +1,170 @@ +#pragma version 2 + +// This program writes some data on optin and then +// validates stored values on subsequent invocations. +// Invocation parameter is controlled by app arg ("first", "second", "third"). +// The program also establishes behavior for reading and deleting non-existing keys. + +// allow app creation and deletion +txn ApplicationID +int 0 +== +bnz success + +txn OnCompletion +int DeleteApplication +== +bnz success + +txn OnCompletion +int OptIn +== +txna ApplicationArgs 0 +byte "first" +== +&& +bnz first +txna ApplicationArgs 0 +byte "second" +== +bnz second +txna ApplicationArgs 0 +byte "third" +== +bnz third +b fail + +// check non-existing keys deletion and reading +// write some initial values +// check same-txn opt-in +first: +int 0 // account idx +int 0 // app idx +app_opted_in +bz fail // must be opted in +// check global non-existing +int 0 +byte "foo" +app_global_get_ex +bnz fail // value may not exist +// keep one zero value (non-initialized value) on the stack from app_global_get_ex +byte "foo" +app_global_get +// check local non-existing +// re-use 0 in the top of the stack as account idx = 0 => sender +// re-use 0 from the stack as app idx +// this both are done by purpose to coin possible side effects of non-initialized stack values +byte "bar" +app_local_get_ex +bnz fail // value may not exist +// re-use 0 in the top of the stack as account idx = 0 => sender +byte "bar" +app_local_get +// keep one zero value (non-initialized value) on the stack from app_local_get +byte "foobar" +app_global_del +// re-use 0 in the top of the stack as account idx = 0 => sender +byte "barfoo" +app_local_del + +// now set some keys +byte "gki" +int 100 +app_global_put +byte "gkb" +byte "test" +app_global_put +int 0 +byte "lki" +int 200 +app_local_put +int 0 +byte "lkb" +byte "anothertest" +app_local_put +b success + +// check keys written during previous invocation +// delete one key and update another +second: +int 0 // app idx +byte "gki" +app_global_get_ex +bz fail +int 100 +!= +bnz fail +byte "gkb" +app_global_get +byte "test" +!= +bnz fail +// check opted-in +int 0 // account idx +int 0 // app idx +app_opted_in +bz fail // must be opted in +int 0 // account idx +int 0 // app idx +byte "lki" +app_local_get_ex +bz fail +int 200 +!= +bnz fail +int 0 +byte "lkb" +app_local_get +byte "anothertest" +!= +bnz fail +// delete and update keys +byte "gki" +app_global_del +int 0 +byte "lkb" +app_local_del +byte "gkb" +byte "data" +app_global_put +int 0 +byte "lki" +int 201 +app_local_put +b success + +// check keys updated +// check deleted keys do not exist +third: +int 0 // app idx +byte "gki" +app_global_get_ex +bnz fail +// re-use 0 in the top of the stack as app idx +byte "gkb" +app_global_get_ex +bz fail +byte "data" +!= +bnz fail +int 0 // acc idx +int 0 // app idx +byte "lki" +app_local_get_ex +bz fail +int 201 +!= +bnz fail +int 0 +byte "lkb" +app_local_get +int 0 // no such value +== +bnz success + +// program tail +fail: +int 0 +return +success: +int 1 From 75d289a0c5ea2acb6820023e7159e250c09d6e69 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Tue, 10 Nov 2020 12:19:17 -0500 Subject: [PATCH 10/34] Better 'goal asset' help documentation. (#1687) Better 'goal asset' help documentation --- cmd/goal/asset.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/goal/asset.go b/cmd/goal/asset.go index 074e00bfbb..f8d9fb68cb 100644 --- a/cmd/goal/asset.go +++ b/cmd/goal/asset.go @@ -67,7 +67,7 @@ func init() { createAssetCmd.MarkFlagRequired("creator") destroyAssetCmd.Flags().StringVar(&assetManager, "manager", "", "Manager account to issue the destroy transaction (defaults to creator)") - destroyAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Account address for asset to destroy") + destroyAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Creator account address for asset to destroy") destroyAssetCmd.Flags().Uint64Var(&assetID, "assetid", 0, "Asset ID to destroy") destroyAssetCmd.Flags().StringVar(&assetUnitName, "asset", "", "Unit name of asset to destroy") @@ -256,7 +256,7 @@ var createAssetCmd = &cobra.Command{ var destroyAssetCmd = &cobra.Command{ Use: "destroy", Short: "Destroy an asset", - Long: `Issue a transaction deleting an asset from the network. This transaction must be issued by the asset owner, who must hold all outstanding asset tokens.`, + Long: `Issue a transaction deleting an asset from the network. This transaction must be issued by the asset manager while the creator holds all of the asset's tokens.`, Args: validateNoPosArgsFn, Run: func(cmd *cobra.Command, _ []string) { checkTxValidityPeriodCmdFlags(cmd) From 6d67e88dcf2a5365951572122dcec46cee3efbeb Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 10 Nov 2020 14:46:06 -0500 Subject: [PATCH 11/34] Dynamically adjust catchpoint file serving timeouts based on file size (#1693) Dynamically adjust catchpoint file serving timeouts based on file size --- ledger/acctupdates.go | 34 ++++++++++++++++++++++++++++------ ledger/acctupdates_test.go | 5 ++++- ledger/ledger.go | 5 ++--- rpcs/ledgerService.go | 28 +++++++++++++++++----------- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 24c51e6d86..21519e4200 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -739,13 +739,35 @@ func (au *accountUpdates) Totals(rnd basics.Round) (totals AccountTotals, err er return au.totalsImpl(rnd) } -// GetCatchpointStream returns an io.Reader to the catchpoint file associated with the provided round -func (au *accountUpdates) GetCatchpointStream(round basics.Round) (io.ReadCloser, error) { +// ReadCloseSizer interface implements the standard io.Reader and io.Closer as well +// as supporting the Size() function that let the caller know what the size of the stream would be (in bytes). +type ReadCloseSizer interface { + io.ReadCloser + Size() (int64, error) +} + +// readCloseSizer is an instance of the ReadCloseSizer interface +type readCloseSizer struct { + io.ReadCloser + size int64 +} + +// Size returns the length of the assiciated stream. +func (r *readCloseSizer) Size() (int64, error) { + if r.size < 0 { + return 0, fmt.Errorf("unknown stream size") + } + return r.size, nil +} + +// GetCatchpointStream returns a ReadCloseSizer to the catchpoint file associated with the provided round +func (au *accountUpdates) GetCatchpointStream(round basics.Round) (ReadCloseSizer, error) { dbFileName := "" + fileSize := int64(0) start := time.Now() ledgerGetcatchpointCount.Inc(nil) err := au.dbs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { - dbFileName, _, _, err = getCatchpoint(tx, round) + dbFileName, _, fileSize, err = getCatchpoint(tx, round) return }) ledgerGetcatchpointMicros.AddMicrosecondsSince(start, nil) @@ -757,7 +779,7 @@ func (au *accountUpdates) GetCatchpointStream(round basics.Round) (io.ReadCloser catchpointPath := filepath.Join(au.dbDirectory, dbFileName) file, err := os.OpenFile(catchpointPath, os.O_RDONLY, 0666) if err == nil && file != nil { - return file, nil + return &readCloseSizer{ReadCloser: file, size: fileSize}, nil } // else, see if this is a file-not-found error if os.IsNotExist(err) { @@ -784,14 +806,14 @@ func (au *accountUpdates) GetCatchpointStream(round basics.Round) (io.ReadCloser fileInfo, err := file.Stat() if err != nil { // we couldn't get the stat, so just return with the file. - return file, nil + return &readCloseSizer{ReadCloser: file, size: -1}, nil } err = au.saveCatchpointFile(round, fileName, fileInfo.Size(), "") if err != nil { au.log.Warnf("accountUpdates: getCatchpointStream: unable to save missing catchpoint entry: %v", err) } - return file, nil + return &readCloseSizer{ReadCloser: file, size: fileInfo.Size()}, nil } return nil, ErrNoEntry{} } diff --git a/ledger/acctupdates_test.go b/ledger/acctupdates_test.go index efba00a1a0..f40b6545fd 100644 --- a/ledger/acctupdates_test.go +++ b/ledger/acctupdates_test.go @@ -1094,7 +1094,7 @@ func TestGetCatchpointStream(t *testing.T) { require.NoError(t, err) // Store the catchpoint into the database - err := au.accountsq.storeCatchpoint(context.Background(), basics.Round(i), fileName, "", 0) + err := au.accountsq.storeCatchpoint(context.Background(), basics.Round(i), fileName, "", int64(len(data))) require.NoError(t, err) } @@ -1108,6 +1108,9 @@ func TestGetCatchpointStream(t *testing.T) { require.Equal(t, 3, n) outData := []byte{1, 2, 3} require.Equal(t, outData, dataRead) + len, err := reader.Size() + require.NoError(t, err) + require.Equal(t, int64(3), len) // File deleted, but record in the database err = os.Remove(filepath.Join(temporaryDirectroy, "catchpoints", "2.catchpoint")) diff --git a/ledger/ledger.go b/ledger/ledger.go index a461df75b3..4c2c9dcb45 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -20,7 +20,6 @@ import ( "context" "database/sql" "fmt" - "io" "os" "time" @@ -604,12 +603,12 @@ func (l *Ledger) GetCatchpointCatchupState(ctx context.Context) (state Catchpoin return MakeCatchpointCatchupAccessor(l, l.log).GetState(ctx) } -// GetCatchpointStream returns an io.ReadCloser file stream from which the catchpoint file +// GetCatchpointStream returns a ReadCloseSizer file stream from which the catchpoint file // for the provided round could be retrieved. If no such stream can be generated, a non-nil // error is returned. The io.ReadCloser and the error are mutually exclusive - // if error is returned, the file stream is guaranteed to be nil, and vice versa, // if the file stream is not nil, the error is guaranteed to be nil. -func (l *Ledger) GetCatchpointStream(round basics.Round) (io.ReadCloser, error) { +func (l *Ledger) GetCatchpointStream(round basics.Round) (ReadCloseSizer, error) { l.trackerMu.RLock() defer l.trackerMu.RUnlock() return l.accts.GetCatchpointStream(round) diff --git a/rpcs/ledgerService.go b/rpcs/ledgerService.go index 5093eb470d..12188ce034 100644 --- a/rpcs/ledgerService.go +++ b/rpcs/ledgerService.go @@ -47,14 +47,11 @@ const ( // e.g. .Handle(LedgerServiceLedgerPath, &ls) LedgerServiceLedgerPath = "/v{version:[0-9.]+}/{genesisID}/ledger/{round:[0-9a-z]+}" - // maxCatchpointFileSize is a rough estimate for the worst-case scenario we're going to have of all the accounts data per a single catchpoint file chunk. + // maxCatchpointFileSize is the default catchpoint file size, if we can't get a concreate number from the ledger. maxCatchpointFileSize = 512 * 1024 * 1024 // 512MB - // expectedWorstDownloadSpeedBytesPerSecond defines the worst-case scenario upload speed we expect to get while uploading a catchpoint file - expectedWorstDownloadSpeedBytesPerSecond = 200 * 1024 - - // maxCatchpointFileChunkDownloadDuration is the maximum amount of time we would wait to download a single chunk off a catchpoint file - maxCatchpointFileWritingDuration = 2*time.Minute + maxCatchpointFileSize*time.Second/expectedWorstDownloadSpeedBytesPerSecond + // expectedWorstUploadSpeedBytesPerSecond defines the worst-case scenario upload speed we expect to get while uploading a catchpoint file + expectedWorstUploadSpeedBytesPerSecond = 20 * 1024 ) // LedgerService represents the Ledger RPC API @@ -177,11 +174,6 @@ func (ls *LedgerService) ServeHTTP(response http.ResponseWriter, request *http.R response.Write([]byte(fmt.Sprintf("specified round number could not be parsed using base 36 : %v", err))) return } - if conn := ls.net.GetHTTPRequestConnection(request); conn != nil { - conn.SetWriteDeadline(time.Now().Add(maxCatchpointFileWritingDuration)) - } else { - logging.Base().Warnf("LedgerService.ServeHTTP unable to set connection timeout") - } cs, err := ls.ledger.GetCatchpointStream(basics.Round(round)) if err != nil { switch err.(type) { @@ -199,6 +191,20 @@ func (ls *LedgerService) ServeHTTP(response http.ResponseWriter, request *http.R } } defer cs.Close() + if conn := ls.net.GetHTTPRequestConnection(request); conn != nil { + maxCatchpointFileWritingDuration := 2 * time.Minute + + catchpointFileSize, err := cs.Size() + if err != nil || catchpointFileSize <= 0 { + maxCatchpointFileWritingDuration += maxCatchpointFileSize * time.Second / expectedWorstUploadSpeedBytesPerSecond + } else { + maxCatchpointFileWritingDuration += time.Duration(catchpointFileSize) * time.Second / expectedWorstUploadSpeedBytesPerSecond + } + conn.SetWriteDeadline(time.Now().Add(maxCatchpointFileWritingDuration)) + } else { + logging.Base().Warnf("LedgerService.ServeHTTP unable to set connection timeout") + } + response.Header().Set("Content-Type", LedgerResponseContentType) requestedCompressedResponse := strings.Contains(request.Header.Get("Accept-Encoding"), "gzip") if requestedCompressedResponse { From e7a7c804b9ad8ae8b4f070433b6074a265211f67 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Wed, 11 Nov 2020 07:59:36 -0500 Subject: [PATCH 12/34] Extend catchpoint file download time (#1692) Extend catchpoint file download time --- catchup/catchpointService.go | 2 +- catchup/ledgerFetcher.go | 25 +++-- catchup/ledgerFetcher_test.go | 7 +- config/config.go | 12 +- config/local_defaults.go | 156 +++++++++++++------------- installer/config.json.example | 4 +- test/testdata/configs/config-v13.json | 81 +++++++++++++ 7 files changed, 196 insertions(+), 91 deletions(-) create mode 100644 test/testdata/configs/config-v13.json diff --git a/catchup/catchpointService.go b/catchup/catchpointService.go index 73a2db4436..1059d35276 100644 --- a/catchup/catchpointService.go +++ b/catchup/catchpointService.go @@ -262,7 +262,7 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) { } // download balances file. - ledgerFetcher := makeLedgerFetcher(cs.net, cs.ledgerAccessor, cs.log, cs) + ledgerFetcher := makeLedgerFetcher(cs.net, cs.ledgerAccessor, cs.log, cs, cs.config) attemptsCount := 0 for { diff --git a/catchup/ledgerFetcher.go b/catchup/ledgerFetcher.go index 1d6a8550a9..12d6a48208 100644 --- a/catchup/ledgerFetcher.go +++ b/catchup/ledgerFetcher.go @@ -27,6 +27,7 @@ import ( "strconv" "time" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" @@ -39,12 +40,8 @@ var errNoLedgerForRound = errors.New("No ledger available for given round") const ( // maxCatchpointFileChunkSize is a rough estimate for the worst-case scenario we're going to have of all the accounts data per a single catchpoint file chunk. maxCatchpointFileChunkSize = ledger.BalancesPerCatchpointFileChunk * basics.MaxEncodedAccountDataSize - // maxCatchpointFileDownloadDuration is the maximum time we would wait for download an entire catchpoint file - maxCatchpointFileDownloadDuration = 45 * time.Minute - // expectedWorstDownloadSpeedBytesPerSecond defines the worst case-scenario download speed we expect to get while downloading a catchpoint file - expectedWorstDownloadSpeedBytesPerSecond = 20 * 1024 - // maxCatchpointFileChunkDownloadDuration is the maximum amount of time we would wait to download a single chunk off a catchpoint file - maxCatchpointFileChunkDownloadDuration = 2*time.Minute + maxCatchpointFileChunkSize*time.Second/expectedWorstDownloadSpeedBytesPerSecond + // defaultMinCatchpointFileDownloadBytesPerSecond defines the worst-case scenario download speed we expect to get while downloading a catchpoint file + defaultMinCatchpointFileDownloadBytesPerSecond = 20 * 1024 // catchpointFileStreamReadSize defines the number of bytes we would attempt to read at each itration from the incoming http data stream catchpointFileStreamReadSize = 4096 ) @@ -62,14 +59,16 @@ type ledgerFetcher struct { log logging.Logger peers []network.Peer reporter ledgerFetcherReporter + config config.Local } -func makeLedgerFetcher(net network.GossipNode, accessor ledger.CatchpointCatchupAccessor, log logging.Logger, reporter ledgerFetcherReporter) *ledgerFetcher { +func makeLedgerFetcher(net network.GossipNode, accessor ledger.CatchpointCatchupAccessor, log logging.Logger, reporter ledgerFetcherReporter, cfg config.Local) *ledgerFetcher { return &ledgerFetcher{ net: net, accessor: accessor, log: log, reporter: reporter, + config: cfg, } } @@ -101,7 +100,8 @@ func (lf *ledgerFetcher) getPeerLedger(ctx context.Context, peer network.HTTPPee if err != nil { return err } - timeoutContext, timeoutContextCancel := context.WithTimeout(ctx, maxCatchpointFileDownloadDuration) + + timeoutContext, timeoutContextCancel := context.WithTimeout(ctx, lf.config.MaxCatchpointDownloadDuration) defer timeoutContextCancel() request = request.WithContext(timeoutContext) network.SetUserAgentHeader(request.Header) @@ -133,6 +133,15 @@ func (lf *ledgerFetcher) getPeerLedger(ctx context.Context, peer network.HTTPPee err = fmt.Errorf("getPeerLedger : http ledger fetcher response has an invalid content type : %s", contentTypes[0]) return err } + + // maxCatchpointFileChunkDownloadDuration is the maximum amount of time we would wait to download a single chunk off a catchpoint file + maxCatchpointFileChunkDownloadDuration := 2 * time.Minute + if lf.config.MinCatchpointFileDownloadBytesPerSecond > 0 { + maxCatchpointFileChunkDownloadDuration += maxCatchpointFileChunkSize * time.Second / time.Duration(lf.config.MinCatchpointFileDownloadBytesPerSecond) + } else { + maxCatchpointFileChunkDownloadDuration += maxCatchpointFileChunkSize * time.Second / defaultMinCatchpointFileDownloadBytesPerSecond + } + watchdogReader := makeWatchdogStreamReader(response.Body, catchpointFileStreamReadSize, 2*maxCatchpointFileChunkSize, maxCatchpointFileChunkDownloadDuration) defer watchdogReader.Close() tarReader := tar.NewReader(watchdogReader) diff --git a/catchup/ledgerFetcher_test.go b/catchup/ledgerFetcher_test.go index cccedc3f2b..9bc7846477 100644 --- a/catchup/ledgerFetcher_test.go +++ b/catchup/ledgerFetcher_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/require" "github.com/algorand/go-algorand/components/mocks" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" @@ -38,7 +39,7 @@ func (lf *dummyLedgerFetcherReporter) updateLedgerFetcherProgress(*ledger.Catchp } func TestNoPeersAvailable(t *testing.T) { - lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}) + lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal()) err := lf.downloadLedger(context.Background(), basics.Round(0)) require.Equal(t, errNoPeersAvailable, err) lf.peers = append(lf.peers, &lf) // The peer is an opaque interface.. we can add anything as a Peer. @@ -47,7 +48,7 @@ func TestNoPeersAvailable(t *testing.T) { } func TestNonParsableAddress(t *testing.T) { - lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}) + lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal()) peer := testHTTPPeer(":def") err := lf.getPeerLedger(context.Background(), &peer, basics.Round(0)) require.Error(t, err) @@ -75,7 +76,7 @@ func TestLedgerFetcherErrorResponseHandling(t *testing.T) { w.WriteHeader(httpServerResponse) }) - lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}) + lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal()) peer := testHTTPPeer(listener.Addr().String()) err = lf.getPeerLedger(context.Background(), &peer, basics.Round(0)) require.Equal(t, errNoLedgerForRound, err) diff --git a/config/config.go b/config/config.go index 9c3fae01bb..97afc5a20f 100644 --- a/config/config.go +++ b/config/config.go @@ -63,7 +63,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13"` // environmental (may be overridden) // When enabled, stores blocks indefinitally, otherwise, only the most recents blocks @@ -349,6 +349,16 @@ type Local struct { // and is expected to happen only on either startup ( after enabling the catchpoint interval, or on certain database upgrades ) or during fast catchup. The values specified here // and their meanings are identical to the ones in LedgerSynchronousMode. AccountsRebuildSynchronousMode int `version[12]:"1"` + + // MaxCatchpointDownloadDuration defines the maximum duration a client will be keeping the outgoing connection of a catchpoint download request open for processing before + // shutting it down. Networks that have large catchpoint files, slow connection or slow storage could be a good reason to increase this value. Note that this is a client-side only + // configuration value, and it's independent of the actual catchpoint file size. + MaxCatchpointDownloadDuration time.Duration `version[13]:"7200000000000"` + + // MinCatchpointFileDownloadBytesPerSecond defines the minimal download speed that would be considered to be "acceptable" by the catchpoint file fetcher, measured in bytes per seconds. If the + // provided stream speed drops below this threshold, the connection would be recycled. Note that this field is evaluated per catchpoint "chunk" and not on it's own. If this field is zero, + // the default of 20480 would be used. + MinCatchpointFileDownloadBytesPerSecond uint64 `version[13]:"20480"` } // Filenames of config files within the configdir (e.g. ~/.algorand) diff --git a/config/local_defaults.go b/config/local_defaults.go index 33f6fd1c2c..4b8f2cd1cf 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,81 +20,83 @@ package config var defaultLocal = Local{ - Version: 12, - AccountsRebuildSynchronousMode: 1, - AnnounceParticipationKey: true, - Archival: false, - BaseLoggerDebugLevel: 4, - BroadcastConnectionsLimit: -1, - CadaverSizeTarget: 1073741824, - CatchpointFileHistoryLength: 365, - CatchpointInterval: 10000, - CatchpointTracking: 0, - CatchupBlockDownloadRetryAttempts: 1000, - CatchupFailurePeerRefreshRate: 10, - CatchupGossipBlockFetchTimeoutSec: 4, - CatchupHTTPBlockFetchTimeoutSec: 4, - CatchupLedgerDownloadRetryAttempts: 50, - CatchupParallelBlocks: 16, - ConnectionsRateLimitingCount: 60, - ConnectionsRateLimitingWindowSeconds: 1, - DNSBootstrapID: ".algorand.network", - DNSSecurityFlags: 1, - DeadlockDetection: 0, - DisableOutgoingConnectionThrottling: false, - EnableAgreementReporting: false, - EnableAgreementTimeMetrics: false, - EnableAssembleStats: false, - EnableBlockService: false, - EnableDeveloperAPI: false, - EnableGossipBlockService: true, - EnableIncomingMessageFilter: false, - EnableLedgerService: false, - EnableMetricReporting: false, - EnableOutgoingNetworkMessageFiltering: true, - EnablePingHandler: true, - EnableProcessBlockStats: false, - EnableProfiler: false, - EnableRequestLogger: false, - EnableTopAccountsReporting: false, - EndpointAddress: "127.0.0.1:0", - FallbackDNSResolverAddress: "", - ForceRelayMessages: false, - GossipFanout: 4, - IncomingConnectionsLimit: 10000, - IncomingMessageFilterBucketCount: 5, - IncomingMessageFilterBucketSize: 512, - IsIndexerActive: false, - LedgerSynchronousMode: 2, - LogArchiveMaxAge: "", - LogArchiveName: "node.archive.log", - LogSizeLimit: 1073741824, - MaxConnectionsPerIP: 30, - NetAddress: "", - NetworkProtocolVersion: "", - NodeExporterListenAddress: ":9100", - NodeExporterPath: "./node_exporter", - OptimizeAccountsDatabaseOnStartup: false, - OutgoingMessageFilterBucketCount: 3, - OutgoingMessageFilterBucketSize: 128, - PeerConnectionsUpdateInterval: 3600, - PeerPingPeriodSeconds: 0, - PriorityPeers: map[string]bool{}, - PublicAddress: "", - ReconnectTime: 60000000000, - ReservedFDs: 256, - RestReadTimeoutSeconds: 15, - RestWriteTimeoutSeconds: 120, - RunHosted: false, - SuggestedFeeBlockHistory: 3, - SuggestedFeeSlidingWindowSize: 50, - TLSCertFile: "", - TLSKeyFile: "", - TelemetryToLog: true, - TxPoolExponentialIncreaseFactor: 2, - TxPoolSize: 15000, - TxSyncIntervalSeconds: 60, - TxSyncServeResponseSize: 1000000, - TxSyncTimeoutSeconds: 30, - UseXForwardedForAddressField: "", + Version: 13, + AccountsRebuildSynchronousMode: 1, + AnnounceParticipationKey: true, + Archival: false, + BaseLoggerDebugLevel: 4, + BroadcastConnectionsLimit: -1, + CadaverSizeTarget: 1073741824, + CatchpointFileHistoryLength: 365, + CatchpointInterval: 10000, + CatchpointTracking: 0, + CatchupBlockDownloadRetryAttempts: 1000, + CatchupFailurePeerRefreshRate: 10, + CatchupGossipBlockFetchTimeoutSec: 4, + CatchupHTTPBlockFetchTimeoutSec: 4, + CatchupLedgerDownloadRetryAttempts: 50, + CatchupParallelBlocks: 16, + ConnectionsRateLimitingCount: 60, + ConnectionsRateLimitingWindowSeconds: 1, + DNSBootstrapID: ".algorand.network", + DNSSecurityFlags: 1, + DeadlockDetection: 0, + DisableOutgoingConnectionThrottling: false, + EnableAgreementReporting: false, + EnableAgreementTimeMetrics: false, + EnableAssembleStats: false, + EnableBlockService: false, + EnableDeveloperAPI: false, + EnableGossipBlockService: true, + EnableIncomingMessageFilter: false, + EnableLedgerService: false, + EnableMetricReporting: false, + EnableOutgoingNetworkMessageFiltering: true, + EnablePingHandler: true, + EnableProcessBlockStats: false, + EnableProfiler: false, + EnableRequestLogger: false, + EnableTopAccountsReporting: false, + EndpointAddress: "127.0.0.1:0", + FallbackDNSResolverAddress: "", + ForceRelayMessages: false, + GossipFanout: 4, + IncomingConnectionsLimit: 10000, + IncomingMessageFilterBucketCount: 5, + IncomingMessageFilterBucketSize: 512, + IsIndexerActive: false, + LedgerSynchronousMode: 2, + LogArchiveMaxAge: "", + LogArchiveName: "node.archive.log", + LogSizeLimit: 1073741824, + MaxCatchpointDownloadDuration: 7200000000000, + MaxConnectionsPerIP: 30, + MinCatchpointFileDownloadBytesPerSecond: 20480, + NetAddress: "", + NetworkProtocolVersion: "", + NodeExporterListenAddress: ":9100", + NodeExporterPath: "./node_exporter", + OptimizeAccountsDatabaseOnStartup: false, + OutgoingMessageFilterBucketCount: 3, + OutgoingMessageFilterBucketSize: 128, + PeerConnectionsUpdateInterval: 3600, + PeerPingPeriodSeconds: 0, + PriorityPeers: map[string]bool{}, + PublicAddress: "", + ReconnectTime: 60000000000, + ReservedFDs: 256, + RestReadTimeoutSeconds: 15, + RestWriteTimeoutSeconds: 120, + RunHosted: false, + SuggestedFeeBlockHistory: 3, + SuggestedFeeSlidingWindowSize: 50, + TLSCertFile: "", + TLSKeyFile: "", + TelemetryToLog: true, + TxPoolExponentialIncreaseFactor: 2, + TxPoolSize: 15000, + TxSyncIntervalSeconds: 60, + TxSyncServeResponseSize: 1000000, + TxSyncTimeoutSeconds: 30, + UseXForwardedForAddressField: "", } diff --git a/installer/config.json.example b/installer/config.json.example index 9df7ffc657..63eefd7c7c 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,5 @@ { - "Version": 12, + "Version": 13, "AccountsRebuildSynchronousMode": 1, "AnnounceParticipationKey": true, "Archival": false, @@ -48,7 +48,9 @@ "LogArchiveMaxAge": "", "LogArchiveName": "node.archive.log", "LogSizeLimit": 1073741824, + "MaxCatchpointDownloadDuration": 7200000000000, "MaxConnectionsPerIP": 30, + "MinCatchpointFileDownloadBytesPerSecond": 20480, "NetAddress": "", "NetworkProtocolVersion": "", "NodeExporterListenAddress": ":9100", diff --git a/test/testdata/configs/config-v13.json b/test/testdata/configs/config-v13.json new file mode 100644 index 0000000000..63eefd7c7c --- /dev/null +++ b/test/testdata/configs/config-v13.json @@ -0,0 +1,81 @@ +{ + "Version": 13, + "AccountsRebuildSynchronousMode": 1, + "AnnounceParticipationKey": true, + "Archival": false, + "BaseLoggerDebugLevel": 4, + "BroadcastConnectionsLimit": -1, + "CadaverSizeTarget": 1073741824, + "CatchpointFileHistoryLength": 365, + "CatchpointInterval": 10000, + "CatchpointTracking": 0, + "CatchupBlockDownloadRetryAttempts": 1000, + "CatchupFailurePeerRefreshRate": 10, + "CatchupGossipBlockFetchTimeoutSec": 4, + "CatchupHTTPBlockFetchTimeoutSec": 4, + "CatchupLedgerDownloadRetryAttempts": 50, + "CatchupParallelBlocks": 16, + "ConnectionsRateLimitingCount": 60, + "ConnectionsRateLimitingWindowSeconds": 1, + "DNSBootstrapID": ".algorand.network", + "DNSSecurityFlags": 1, + "DeadlockDetection": 0, + "DisableOutgoingConnectionThrottling": false, + "EnableAgreementReporting": false, + "EnableAgreementTimeMetrics": false, + "EnableAssembleStats": false, + "EnableBlockService": false, + "EnableDeveloperAPI": false, + "EnableGossipBlockService": true, + "EnableIncomingMessageFilter": false, + "EnableLedgerService": false, + "EnableMetricReporting": false, + "EnableOutgoingNetworkMessageFiltering": true, + "EnablePingHandler": true, + "EnableProcessBlockStats": false, + "EnableProfiler": false, + "EnableRequestLogger": false, + "EnableTopAccountsReporting": false, + "EndpointAddress": "127.0.0.1:0", + "FallbackDNSResolverAddress": "", + "ForceRelayMessages": false, + "GossipFanout": 4, + "IncomingConnectionsLimit": 10000, + "IncomingMessageFilterBucketCount": 5, + "IncomingMessageFilterBucketSize": 512, + "IsIndexerActive": false, + "LedgerSynchronousMode": 2, + "LogArchiveMaxAge": "", + "LogArchiveName": "node.archive.log", + "LogSizeLimit": 1073741824, + "MaxCatchpointDownloadDuration": 7200000000000, + "MaxConnectionsPerIP": 30, + "MinCatchpointFileDownloadBytesPerSecond": 20480, + "NetAddress": "", + "NetworkProtocolVersion": "", + "NodeExporterListenAddress": ":9100", + "NodeExporterPath": "./node_exporter", + "OptimizeAccountsDatabaseOnStartup": false, + "OutgoingMessageFilterBucketCount": 3, + "OutgoingMessageFilterBucketSize": 128, + "PeerConnectionsUpdateInterval": 3600, + "PeerPingPeriodSeconds": 0, + "PriorityPeers": {}, + "PublicAddress": "", + "ReconnectTime": 60000000000, + "ReservedFDs": 256, + "RestReadTimeoutSeconds": 15, + "RestWriteTimeoutSeconds": 120, + "RunHosted": false, + "SuggestedFeeBlockHistory": 3, + "SuggestedFeeSlidingWindowSize": 50, + "TLSCertFile": "", + "TLSKeyFile": "", + "TelemetryToLog": true, + "TxPoolExponentialIncreaseFactor": 2, + "TxPoolSize": 15000, + "TxSyncIntervalSeconds": 60, + "TxSyncServeResponseSize": 1000000, + "TxSyncTimeoutSeconds": 30, + "UseXForwardedForAddressField": "" +} From e8a24ee9447541fcbd46154fdc76b63f0bdf092e Mon Sep 17 00:00:00 2001 From: btoll Date: Thu, 12 Nov 2020 12:43:04 -0500 Subject: [PATCH 13/34] Improved task that updates debian repository (#1689) The operation that uses the aptly tool to create a new snapshot with the new package(s) and upload to the mirror on S3 is now a mule task. --- .../deploy/deb/.aptly.conf => .aptly.conf | 0 docker/build/aptly.Dockerfile | 26 +++++++ package-deploy.yaml | 27 ++++++++ scripts/release/mule/Makefile.mule | 5 +- scripts/release/mule/README.md | 12 +--- scripts/release/mule/deploy/deb/Dockerfile | 27 -------- .../release/mule/deploy/deb/create_and_push | 68 ------------------ scripts/release/mule/deploy/deb/deploy.sh | 69 +++++++++++++++++++ 8 files changed, 128 insertions(+), 106 deletions(-) rename scripts/release/mule/deploy/deb/.aptly.conf => .aptly.conf (100%) create mode 100644 docker/build/aptly.Dockerfile delete mode 100644 scripts/release/mule/deploy/deb/Dockerfile delete mode 100755 scripts/release/mule/deploy/deb/create_and_push create mode 100755 scripts/release/mule/deploy/deb/deploy.sh diff --git a/scripts/release/mule/deploy/deb/.aptly.conf b/.aptly.conf similarity index 100% rename from scripts/release/mule/deploy/deb/.aptly.conf rename to .aptly.conf diff --git a/docker/build/aptly.Dockerfile b/docker/build/aptly.Dockerfile new file mode 100644 index 0000000000..9d317861b3 --- /dev/null +++ b/docker/build/aptly.Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:18.04 + +ARG ARCH=amd64 +ARG GOLANG_VERSION +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get install aptly awscli binutils build-essential curl gnupg2 -y +RUN curl https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz | tar -xzf - && mv go /usr/local +ENV GOROOT=/usr/local/go \ + GOPATH=/root/go \ + PATH=$GOPATH/bin:$GOROOT/bin:$PATH + +WORKDIR /root +COPY .aptly.conf . +RUN curl https://releases.algorand.com/key.pub | gpg --no-default-keyring --keyring trustedkeys.gpg --import - && \ + aptly mirror create stable https://releases.algorand.com/deb/ stable main && \ + aptly mirror create beta https://releases.algorand.com/deb/ beta main && \ + aptly repo create -distribution=stable -architectures=amd64 -component=main -comment=mainnet stable && \ + aptly repo create -distribution=beta -architectures=amd64 -component=main -comment=betanet beta && \ + aptly mirror update stable && \ + aptly mirror update beta && \ + aptly repo import stable stable algorand algorand-devtools + aptly repo import beta beta algorand-beta algorand-devtools-beta + +CMD ["/bin/bash"] + diff --git a/package-deploy.yaml b/package-deploy.yaml index bf2484095c..746d647542 100644 --- a/package-deploy.yaml +++ b/package-deploy.yaml @@ -24,6 +24,24 @@ agents: - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY workDir: $HOME/projects/go-algorand + - name: deb + dockerFilePath: docker/build/aptly.Dockerfile + image: algorand/aptly + version: scripts/configure_dev-deps.sh + buildArgs: + - GOLANG_VERSION=`./scripts/get_golang_version.sh` + env: + - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + - CHANNEL=$CHANNEL + - SNAPSHOT=$SNAPSHOT + - STAGING=$STAGING + - VERSION=$VERSION + volumes: + - $XDG_RUNTIME_DIR/gnupg/S.gpg-agent:/root/.gnupg/S.gpg-agent + - $HOME/.gnupg/pubring.kbx:/root/.gnupg/pubring.kbx + workDir: $HOME/projects/go-algorand + - name: rpm dockerFilePath: docker/build/cicd.centos.Dockerfile image: algorand/cicd-centos @@ -54,12 +72,21 @@ tasks: agent: releases-page target: mule-releases-page + - task: docker.Make + name: deb + agent: deb + target: mule-deploy-deb + - task: docker.Make name: rpm agent: rpm target: mule-deploy-rpm jobs: + package-deploy-deb: + tasks: + - docker.Make.deb + package-deploy-rpm: tasks: - docker.Make.rpm diff --git a/scripts/release/mule/Makefile.mule b/scripts/release/mule/Makefile.mule index 69f35d7dfd..65294063b5 100644 --- a/scripts/release/mule/Makefile.mule +++ b/scripts/release/mule/Makefile.mule @@ -42,8 +42,9 @@ ci-build: buildsrc gen ci-setup # https://scene-si.org/2019/12/04/make-dynamic-makefile-targets/ mule = $(shell ls -d scripts/release/mule/*/ | awk 'BEGIN { FS="/" ; OFS="-" } { print $$3, $$4 }') -mule-deploy-rpm: - scripts/release/mule/deploy/rpm/deploy.sh +mule-deploy-%: PKG_TYPE=$* +mule-deploy-%: + scripts/release/mule/deploy/$(PKG_TYPE)/deploy.sh mule-docker: scripts/release/mule/deploy/docker/docker.sh diff --git a/scripts/release/mule/README.md b/scripts/release/mule/README.md index 0c2caf026f..8439767725 100644 --- a/scripts/release/mule/README.md +++ b/scripts/release/mule/README.md @@ -4,7 +4,6 @@ - [Build Stages](#build-stages) - [Custom Builds](#custom-builds) - [Examples](#examples) -- [Manual Deploy](#manual-deploy) # Environment Variables @@ -219,26 +218,21 @@ Let's look at some examples. 1. The new rpm packages will be downloaded from staging if the `S3_SOURCE` variable is set. Each package will then be pushed to `s3:algorand-releases:`. + S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-deploy.yaml package-deploy-deb S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-deploy.yaml package-deploy-rpm 1. Packages are not downloaded from staging but rather are copied from the location on the local filesystem specified by `PACKAGES_DIR` in the `mule` yaml file. Each package will then be pushed to `s3:algorand-releases:`. + PACKAGES_DIR=/packages_location/foo VERSION=2.1.86615 mule -f package-deploy.yaml package-deploy-deb PACKAGES_DIR=/packages_location/foo VERSION=2.1.86615 mule -f package-deploy.yaml package-deploy-rpm 1. `NO_DEPLOY` is set to `true`. Instead of automatically pushing to `s3:algorand-releases:`, this will copy the `rpmrepo` directory that was created in the container to the `WORKDIR` in the host environment (the `WORKDIR` is set in the `mule` yaml file). This is handy when testing a deployment and not yet ready to deploy. + NO_DEPLOY=true S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-deploy.yaml package-deploy-deb NO_DEPLOY=true S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-deploy.yaml package-deploy-rpm -# Manual Deploy - -> Before any processes are run, make sure that the signing keys have been added to the `gpg-agent`. The `gpg_preset_passphrase.sh` helper script is provided just for this purpose. - -Currently, it is still necessary to run two stages manually: sign and deploy. This is for several reasons, though principally because GPG signing of the build assets occurs in both stages. - -The processes that make up both stages have been `mule-ified` as much as possible, and all but one can be run as a `mule` task (deploying deb packages, which are done in its own separate docker container). - ### Signing Usually, the packages are pulled down from S3 where the eks pipeline or the `mule` `package-upload` task had placed them. Issue the following command to download and sign them: diff --git a/scripts/release/mule/deploy/deb/Dockerfile b/scripts/release/mule/deploy/deb/Dockerfile deleted file mode 100644 index 5492ba5a93..0000000000 --- a/scripts/release/mule/deploy/deb/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM ubuntu:18.04 - -ARG AWS_ACCESS_KEY_ID -ARG AWS_SECRET_ACCESS_KEY - -ENV DEBIAN_FRONTEND noninteractive \ - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - -RUN apt-get update && apt-get install aptly awscli binutils curl gnupg2 silversearcher-ag tree -y - -WORKDIR /root - -COPY . . - -RUN aws s3 sync s3://algorand-internal/packages packages && \ - aptly repo create -distribution=stable -architectures=amd64 -component=main -comment=mainnet algorand && \ - aptly repo create -distribution=beta -architectures=amd64 -component=main -comment=betanet algorand-beta && \ - aptly repo create -distribution=indexer -architectures=amd64 -component=main -comment=indexer algorand-indexer && \ - aptly repo create -distribution=indexer-beta -architectures=amd64 -component=main -comment=indexer-beta algorand-indexer-beta && \ - aptly repo add algorand packages/deb/stable/*.deb && \ - aptly repo add algorand-beta packages/deb/beta/*.deb && \ - aptly repo add algorand-indexer packages/deb/indexer/stable/*.deb && \ - aptly repo add algorand-indexer-beta packages/deb/indexer/beta/*.deb - -CMD ["/bin/bash"] - diff --git a/scripts/release/mule/deploy/deb/create_and_push b/scripts/release/mule/deploy/deb/create_and_push deleted file mode 100755 index ef7df6f74c..0000000000 --- a/scripts/release/mule/deploy/deb/create_and_push +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# This script is doing the following: -# 1. Copying the new packages from staging to the packages cache. -# 2. Syncing from the packages cache to the local container. -# 3. Aptly operations: -# i. Add all the packages to the correct $REPO. This includes the newly-synced packages. -# ii. Create the new snapshot that contains the new packages and the previous ones for -# the channel. -# iii. Switch out the old snapshot for the new one. Note that in the "publish" command that -# the $CHANNEL is the distro. -# 4. Copy the new packages from staging to `algorand-dev-deb-repo` so that the releases page can -# be generated. - -CHANNEL=${CHANNEL:-stable} -REPO=${REPO:-algorand} - -usage() { - echo "Usage: -e CHANNEL=\$CHANNEL -e VERSION=\$VERSION -e REPO=\$REPO -e SNAPSHOT=\$SNAPSHOT $0 " - exit "$1" -} - -if [ "$HELP" = true ] -then - echo ------------------------------ - echo "default values:" - echo + CHANNEL=stable - echo + REPO=algorand - echo + SNAPSHOT="\$CHANNEL-\$VERSION" - echo - echo "aptly repos (values for REPO):" - echo "+ algorand" - echo "+ algorand-beta" - echo "+ algorand-indexer" - echo "+ algorand-indexer-beta" - echo ------------------------------ - exit 0 -fi - -if [ -z "$VERSION" ] -then - echo "Version is a required parameter." - usage 1 -fi - -PKG_DIR="/root/packages/deb/$CHANNEL/" - -if [ -z "$SNAPSHOT" ] -then - SNAPSHOT="$CHANNEL-$VERSION" -fi - -echo -e "REPO $REPO\nSNAPSHOT $SNAPSHOT\nPKG_DIR $PKG_DIR" - -# It's necessary to copy packages from staging to packages/. -SOURCE_PREFIX="algorand-staging/releases/$CHANNEL/$VERSION" -DEST="s3://algorand-internal/packages/deb/$CHANNEL/" -aws s3 cp "s3://$SOURCE_PREFIX/algorand_${CHANNEL}_linux-amd64_${VERSION}.deb" "$DEST" -aws s3 cp "s3://$SOURCE_PREFIX/algorand-devtools_${CHANNEL}_linux-amd64_${VERSION}.deb" "$DEST" - -aws s3 sync s3://algorand-internal/packages packages - -aptly repo add "$REPO" "$PKG_DIR"/*.deb -aptly snapshot create "$SNAPSHOT" from repo "$REPO" -aptly publish switch "$CHANNEL" s3:algorand-releases: "$SNAPSHOT" - -aws s3 sync "s3://algorand-staging/releases/$CHANNEL/$VERSION" "s3://algorand-dev-deb-repo/releases/$CHANNEL/$("/root/reverse_hex_timestamp")_$VERSION" - diff --git a/scripts/release/mule/deploy/deb/deploy.sh b/scripts/release/mule/deploy/deb/deploy.sh new file mode 100755 index 0000000000..bebdc0db13 --- /dev/null +++ b/scripts/release/mule/deploy/deb/deploy.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -e + +if [ -z "$STAGING" ] +then + echo "[$0] Staging is a required parameter." + exit 1 +fi + +if [ -z "$CHANNEL" ] +then + echo "[$0] Channel is a required parameter." + exit 1 +fi + +if [[ ! "$CHANNEL" =~ ^beta$|^stable$ ]] +then + echo "[$0] Repository values must be either \`beta\` or \`stable\`." + exit 1 +fi + +if [ -z "$VERSION" ] +then + echo "[$0] Version is a required parameter." + exit 1 +fi + +if [ -z "$SNAPSHOT" ] +then + SNAPSHOT="$CHANNEL-$VERSION" +fi + +PACKAGES_DIR=/root/packages +mkdir -p /root/packages + +aptly mirror update stable +aptly mirror update beta + +# aptly repo import ... +aptly repo import stable stable algorand algorand-devtools +aptly repo import beta beta algorand-beta algorand-devtools-beta + +KEY_PREFIX="releases/$CHANNEL/$VERSION" +FILENAME_SUFFIX="${CHANNEL}_linux-amd64_${VERSION}.deb" +ALGORAND_KEY="$KEY_PREFIX/algorand_${FILENAME_SUFFIX}" +DEVTOOLS_KEY="$KEY_PREFIX/algorand-devtools_${FILENAME_SUFFIX}" + +for key in {"$ALGORAND_KEY","$DEVTOOLS_KEY"} +do + if aws s3api head-object --bucket "$STAGING" --key "$key" + then + aws s3 cp "s3://$STAGING/$key" "$PACKAGES_DIR" + else + echo "[$0] The package \`$key\` failed to download." + fi +done + +if ls -A $PACKAGES_DIR +then + aptly repo add "$CHANNEL" "$PACKAGES_DIR"/*.deb + aptly repo show -with-packages "$CHANNEL" + aptly snapshot create "$SNAPSHOT" from repo "$CHANNEL" + aptly publish snapshot -gpg-key=dev@algorand.com -origin=Algorand -label=Algorand "$SNAPSHOT" s3:algorand-releases: +else + echo "[$0] The packages directory is empty, so there is nothing to add the \`$CHANNEL\` repo." + exit 1 +fi + From 5f2c041a9978e6cca367789da836c859eda76102 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Sun, 15 Nov 2020 14:23:14 -0300 Subject: [PATCH 14/34] Env variables while make execution list is created (#1695) Pass the GOPATH to makefile go calls --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d2eea50bcf..52a93461c7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ UNAME := $(shell uname) -ifneq (, $(findstring MINGW,$(UNAME))) +ifneq (,$(findstring MINGW,$(UNAME))) #Gopath is not saved across sessions, probably existing Windows env vars, override them -export GOPATH := ${HOME}/go +export GOPATH := $(HOME)/go GOPATH1 := $(GOPATH) export PATH := $(PATH):$(GOPATH)/bin else @@ -51,7 +51,7 @@ export GOBUILDMODE := -buildmode=exe endif GOTAGS := --tags "$(GOTAGSLIST)" -GOTRIMPATH := $(shell go help build | grep -q .-trimpath && echo -trimpath) +GOTRIMPATH := $(shell GOPATH=$(GOPATH) && go help build | grep -q .-trimpath && echo -trimpath) GOLDFLAGS_BASE := -X github.com/algorand/go-algorand/config.BuildNumber=$(BUILDNUMBER) \ -X github.com/algorand/go-algorand/config.CommitHash=$(COMMITHASH) \ @@ -62,8 +62,8 @@ GOLDFLAGS_BASE := -X github.com/algorand/go-algorand/config.BuildNumber=$(BUILD GOLDFLAGS := $(GOLDFLAGS_BASE) \ -X github.com/algorand/go-algorand/config.Channel=$(CHANNEL) -UNIT_TEST_SOURCES := $(sort $(shell GO111MODULE=off go list ./... | grep -v /go-algorand/test/ )) -ALGOD_API_PACKAGES := $(sort $(shell GO111MODULE=off cd daemon/algod/api; go list ./... )) +UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ )) +ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... )) MSGP_GENERATE := ./protocol ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger From b42042ce08e0dddc48425a51e67f49bd1441e774 Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 16 Nov 2020 11:53:22 -0500 Subject: [PATCH 15/34] added swagger with version v0.25.0 to go.mod file to fix build. --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 4bb9aedb2c..f2eeacfc92 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/algorand/go-algorand go 1.14 require ( + github.com/go-swagger/go-swagger/cmd/swagger v0.25.0 github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d github.com/algorand/go-deadlock v0.2.1 github.com/algorand/msgp v1.1.45 From 6e54040ab473d478e1959d01a1120c2b11cd079e Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 16 Nov 2020 12:03:19 -0500 Subject: [PATCH 16/34] Replace spaces with tab. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f2eeacfc92..b015d82f3f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/algorand/go-algorand go 1.14 require ( - github.com/go-swagger/go-swagger/cmd/swagger v0.25.0 + github.com/go-swagger/go-swagger/cmd/swagger v0.25.0 github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d github.com/algorand/go-deadlock v0.2.1 github.com/algorand/msgp v1.1.45 From d1711cb62e6215589774bcea4b7660b00695cba8 Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 16 Nov 2020 13:00:42 -0500 Subject: [PATCH 17/34] Update configure_dev-deps.sh to accept optional version for go module. --- go.mod | 1 - scripts/configure_dev-deps.sh | 19 ++++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index b015d82f3f..4bb9aedb2c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/algorand/go-algorand go 1.14 require ( - github.com/go-swagger/go-swagger/cmd/swagger v0.25.0 github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d github.com/algorand/go-deadlock v0.2.1 github.com/algorand/msgp v1.1.45 diff --git a/scripts/configure_dev-deps.sh b/scripts/configure_dev-deps.sh index 59083ed004..52f286b28c 100755 --- a/scripts/configure_dev-deps.sh +++ b/scripts/configure_dev-deps.sh @@ -3,18 +3,23 @@ set -ex function get_go_version { - ( - cd $(dirname "$0") - VERSION=$(cat ../go.mod | grep "$1" 2>/dev/null | awk -F " " '{print $2}') - echo $VERSION - ) + if [[ -z "$2" ]] + then + ( + cd $(dirname "$0") + VERSION=$(cat ../go.mod | grep "$1" 2>/dev/null | awk -F " " '{print $2}') + echo $VERSION + ) + else + (echo $2) + fi return } function install_go_module { local OUTPUT # Check for version to go.mod version - VERSION=$(get_go_version "$1") + VERSION=$(get_go_version "$1" "$2") if [ -z "$VERSION" ]; then OUTPUT=$(GO111MODULE=off go get -u "$1" 2>&1) else @@ -28,5 +33,5 @@ function install_go_module { install_go_module golang.org/x/lint/golint install_go_module golang.org/x/tools/cmd/stringer -install_go_module github.com/go-swagger/go-swagger/cmd/swagger +install_go_module github.com/go-swagger/go-swagger/cmd/swagger v0.25.0 install_go_module github.com/algorand/msgp From d7e53bf450487b8ae4b525f7c4d9aca9fcdd38c1 Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 16 Nov 2020 13:35:26 -0500 Subject: [PATCH 18/34] Adjustments for shellcheck. --- scripts/configure_dev-deps.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/configure_dev-deps.sh b/scripts/configure_dev-deps.sh index 52f286b28c..8a24fa1958 100755 --- a/scripts/configure_dev-deps.sh +++ b/scripts/configure_dev-deps.sh @@ -6,12 +6,12 @@ function get_go_version { if [[ -z "$2" ]] then ( - cd $(dirname "$0") - VERSION=$(cat ../go.mod | grep "$1" 2>/dev/null | awk -F " " '{print $2}') - echo $VERSION + cd "$(dirname "$0")" + VERSION=$( grep "$1" 2>/dev/null < ../go.mod | awk -F " " '{print $2}') + echo "$VERSION" ) else - (echo $2) + (echo "$2") fi return } From dcd2082acfb58a2337bd168ac8cbf7fbcb334741 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 17 Nov 2020 11:38:15 -0300 Subject: [PATCH 19/34] Static linking in Windows executables (#1701) Make windows executable statically linked to avoid DLL dependency with MINGW --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52a93461c7..6092470fb9 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ endif endif ifneq (, $(findstring MINGW,$(UNAME))) -EXTLDFLAGS := -static-libstdc++ -static-libgcc +EXTLDFLAGS := -static -static-libstdc++ -static-libgcc export GOBUILDMODE := -buildmode=exe endif From 5a05207727087d18a7391c52059cb244ca265221 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 17 Nov 2020 12:11:06 -0500 Subject: [PATCH 20/34] Ordered account insertion into merkle trie (I) (#1697) Refactor merkle trie rebuild to use ordered accounts insertions --- ledger/accountdb.go | 233 ++++++++++++++++++++++++++++++++ ledger/acctupdates.go | 80 +++++------ ledger/acctupdates_test.go | 62 ++++++++- ledger/catchpointwriter_test.go | 4 +- 4 files changed, 337 insertions(+), 42 deletions(-) diff --git a/ledger/accountdb.go b/ledger/accountdb.go index 8ea1c4b626..bd6b524e33 100644 --- a/ledger/accountdb.go +++ b/ledger/accountdb.go @@ -1131,3 +1131,236 @@ func (iterator *encodedAccountsBatchIter) Close() { iterator.rows = nil } } + +// orderedAccountsIterStep is used by orderedAccountsIter to define the current step +//msgp:ignore orderedAccountsIterStep +type orderedAccountsIterStep int + +const ( + // startup step + oaiStepStartup = orderedAccountsIterStep(0) + // delete old ordering table if we have any leftover from previous invocation + oaiStepDeleteOldOrderingTable = orderedAccountsIterStep(0) + // create new ordering table + oaiStepCreateOrderingTable = orderedAccountsIterStep(1) + // query the existing accounts + oaiStepQueryAccounts = orderedAccountsIterStep(2) + // iterate over the existing accounts and insert their hash & address into the staging ordering table + oaiStepInsertAccountData = orderedAccountsIterStep(3) + // create an index on the ordering table so that we can efficiently scan it. + oaiStepCreateOrderingAccountIndex = orderedAccountsIterStep(4) + // query the ordering table + oaiStepSelectFromOrderedTable = orderedAccountsIterStep(5) + // iterate over the ordering table + oaiStepIterateOverOrderedTable = orderedAccountsIterStep(6) + // cleanup and delete ordering table + oaiStepShutdown = orderedAccountsIterStep(7) + // do nothing as we're done. + oaiStepDone = orderedAccountsIterStep(8) +) + +// orderedAccountsIter allows us to iterate over the accounts addresses in the order of the account hashes. +type orderedAccountsIter struct { + step orderedAccountsIterStep + rows *sql.Rows + tx *sql.Tx + accountCount int + fetchAccountData bool + insertStmt *sql.Stmt +} + +// makeOrderedAccountsIter creates an ordered account iterator. Note that due to implementation reasons, +// only a single iterator can be active at a time. +func makeOrderedAccountsIter(tx *sql.Tx, accountCount int, fetchAccountData bool) *orderedAccountsIter { + return &orderedAccountsIter{ + tx: tx, + accountCount: accountCount, + fetchAccountData: fetchAccountData, + step: oaiStepStartup, + } +} + +// accountAddressHashData is used by Next to return a single account address, hash and account data. +type accountAddressHashData struct { + address basics.Address + digest []byte + encodedAccountData []byte +} + +// Next returns an array containing the account address, hash and potentially data +// the Next function works in multiple processing stages, where it first processs the current accounts and order them +// followed by returning the ordered accounts. In the first phase, it would return empty accountAddressHashData array +// and sets the processedRecords to the number of accounts that were processed. On the second phase, the acct +// would contain valid data ( and optionally the account data as well, if was asked in makeOrderedAccountsIter) and +// the processedRecords would be zero. If err is sql.ErrNoRows it means that the iterator have completed it's work and no further +// accounts exists. Otherwise, the caller is expected to keep calling "Next" to retrieve the next set of accounts +// ( or let the Next function make some progress toward that goal ) +func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAddressHashData, processedRecords int, err error) { + if iterator.step == oaiStepDeleteOldOrderingTable { + // although we're going to delete this table anyway when completing the iterator execution, we'll try to + // clean up any intermediate table. + _, err = iterator.tx.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes") + if err != nil { + return + } + iterator.step = oaiStepCreateOrderingTable + return + } + if iterator.step == oaiStepCreateOrderingTable { + // create the temporary table + _, err = iterator.tx.ExecContext(ctx, "CREATE TABLE accountsiteratorhashes(address blob, hash blob)") + if err != nil { + return + } + iterator.step = oaiStepQueryAccounts + return + } + if iterator.step == oaiStepQueryAccounts { + // iterate over the existing accounts + iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT address, data FROM accountbase") + if err != nil { + return + } + // prepare the insert statement into the temporary table + iterator.insertStmt, err = iterator.tx.PrepareContext(ctx, "INSERT INTO accountsiteratorhashes(address, hash) VALUES(?, ?)") + if err != nil { + return + } + iterator.step = oaiStepInsertAccountData + return + } + if iterator.step == oaiStepInsertAccountData { + var addr basics.Address + count := 0 + for iterator.rows.Next() { + var addrbuf []byte + var buf []byte + err = iterator.rows.Scan(&addrbuf, &buf) + if err != nil { + iterator.Close(ctx) + return + } + + if len(addrbuf) != len(addr) { + err = fmt.Errorf("Account DB address length mismatch: %d != %d", len(addrbuf), len(addr)) + iterator.Close(ctx) + return + } + + copy(addr[:], addrbuf) + + var accountData basics.AccountData + err = protocol.Decode(buf, &accountData) + if err != nil { + iterator.Close(ctx) + return + } + hash := accountHashBuilder(addr, accountData, buf) + _, err = iterator.insertStmt.ExecContext(ctx, addrbuf, hash) + if err != nil { + iterator.Close(ctx) + return + } + + count++ + if count == iterator.accountCount { + // we're done with this iteration. + processedRecords = count + return + } + } + processedRecords = count + iterator.rows.Close() + iterator.rows = nil + iterator.insertStmt.Close() + iterator.insertStmt = nil + iterator.step = oaiStepCreateOrderingAccountIndex + return + } + if iterator.step == oaiStepCreateOrderingAccountIndex { + // create an index. It shown that even when we're making a single select statement in step 5, it would be better to have this index vs. not having it at all. + // note that this index is using the rowid of the accountsiteratorhashes table. + _, err = iterator.tx.ExecContext(ctx, "CREATE INDEX accountsiteratorhashesidx ON accountsiteratorhashes(hash)") + if err != nil { + iterator.Close(ctx) + return + } + iterator.step = oaiStepSelectFromOrderedTable + return + } + if iterator.step == oaiStepSelectFromOrderedTable { + // select the data from the ordered table + if iterator.fetchAccountData { + iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT accountsiteratorhashes.address, accountsiteratorhashes.hash, accountbase.data FROM accountsiteratorhashes JOIN accountbase ON accountbase.address=accountsiteratorhashes.address ORDER BY accountsiteratorhashes.hash") + } else { + iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT address, hash FROM accountsiteratorhashes ORDER BY hash") + } + + if err != nil { + iterator.Close(ctx) + return + } + iterator.step = oaiStepIterateOverOrderedTable + return + } + + if iterator.step == oaiStepIterateOverOrderedTable { + acct = make([]accountAddressHashData, 0, iterator.accountCount) + var addr basics.Address + for iterator.rows.Next() { + var addrbuf []byte + var acctdata []byte + var hash []byte + if iterator.fetchAccountData { + err = iterator.rows.Scan(&addrbuf, &hash, &acctdata) + } else { + err = iterator.rows.Scan(&addrbuf, &hash) + } + if err != nil { + iterator.Close(ctx) + return + } + + if len(addrbuf) != len(addr) { + err = fmt.Errorf("Account DB address length mismatch: %d != %d", len(addrbuf), len(addr)) + iterator.Close(ctx) + return + } + + copy(addr[:], addrbuf) + + acct = append(acct, accountAddressHashData{address: addr, digest: hash, encodedAccountData: acctdata}) + if len(acct) == iterator.accountCount { + // we're done with this iteration. + return + } + } + iterator.step = oaiStepShutdown + iterator.rows.Close() + iterator.rows = nil + return + } + if iterator.step == oaiStepShutdown { + err = iterator.Close(ctx) + if err != nil { + return + } + iterator.step = oaiStepDone + // fallthrough + } + return nil, 0, sql.ErrNoRows +} + +// Close shuts down the orderedAccountsBuilderIter, releasing database resources. +func (iterator *orderedAccountsIter) Close(ctx context.Context) (err error) { + if iterator.rows != nil { + iterator.rows.Close() + iterator.rows = nil + } + if iterator.insertStmt != nil { + iterator.insertStmt.Close() + iterator.insertStmt = nil + } + _, err = iterator.tx.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes") + return +} diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 21519e4200..2a252351bf 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -1119,56 +1119,58 @@ func (au *accountUpdates) accountsInitialize(ctx context.Context, tx *sql.Tx) (b if rootHash.IsZero() { au.log.Infof("accountsInitialize rebuilding merkle trie for round %d", rnd) - var accountsIterator encodedAccountsBatchIter - defer accountsIterator.Close() + accountBuilderIt := makeOrderedAccountsIter(tx, trieRebuildAccountChunkSize, false) + defer accountBuilderIt.Close(ctx) startTrieBuildTime := time.Now() accountsCount := 0 lastRebuildTime := startTrieBuildTime pendingAccounts := 0 + totalOrderedAccounts := 0 for { - bal, err := accountsIterator.Next(ctx, tx, trieRebuildAccountChunkSize) - if err != nil { - return rnd, err - } - if len(bal) == 0 { + accts, processedRows, err := accountBuilderIt.Next(ctx) + if err == sql.ErrNoRows { + // the account builder would return sql.ErrNoRows when no more data is available. break - } - accountsCount += len(bal) - pendingAccounts += len(bal) - for _, balance := range bal { - var accountData basics.AccountData - err = protocol.Decode(balance.AccountData, &accountData) - if err != nil { - return rnd, err - } - hash := accountHashBuilder(balance.Address, accountData, balance.AccountData) - added, err := trie.Add(hash) - if err != nil { - return rnd, fmt.Errorf("accountsInitialize was unable to add changes to trie: %v", err) - } - if !added { - au.log.Warnf("accountsInitialize attempted to add duplicate hash '%s' to merkle trie for account %v", hex.EncodeToString(hash), balance.Address) - } + } else if err != nil { + return rnd, err } - if pendingAccounts >= trieRebuildCommitFrequency { - // this trie Evict will commit using the current transaction. - // if anything goes wrong, it will still get rolled back. - _, err = trie.Evict(true) - if err != nil { - return 0, fmt.Errorf("accountsInitialize was unable to commit changes to trie: %v", err) + if len(accts) > 0 { + accountsCount += len(accts) + pendingAccounts += len(accts) + for _, acct := range accts { + added, err := trie.Add(acct.digest) + if err != nil { + return rnd, fmt.Errorf("accountsInitialize was unable to add changes to trie: %v", err) + } + if !added { + au.log.Warnf("accountsInitialize attempted to add duplicate hash '%s' to merkle trie for account %v", hex.EncodeToString(acct.digest), acct.address) + } } - pendingAccounts = 0 - } - if len(bal) < trieRebuildAccountChunkSize { - break - } + if pendingAccounts >= trieRebuildCommitFrequency { + // this trie Evict will commit using the current transaction. + // if anything goes wrong, it will still get rolled back. + _, err = trie.Evict(true) + if err != nil { + return 0, fmt.Errorf("accountsInitialize was unable to commit changes to trie: %v", err) + } + pendingAccounts = 0 + } - if time.Now().Sub(lastRebuildTime) > 5*time.Second { - // let the user know that the trie is still being rebuilt. - au.log.Infof("accountsInitialize still building the trie, and processed so far %d accounts", accountsCount) - lastRebuildTime = time.Now() + if time.Now().Sub(lastRebuildTime) > 5*time.Second { + // let the user know that the trie is still being rebuilt. + au.log.Infof("accountsInitialize still building the trie, and processed so far %d accounts", accountsCount) + lastRebuildTime = time.Now() + } + } else if processedRows > 0 { + totalOrderedAccounts += processedRows + // if it's not ordered, we can ignore it for now; we'll just increase the counters and emit logs periodically. + if time.Now().Sub(lastRebuildTime) > 5*time.Second { + // let the user know that the trie is still being rebuilt. + au.log.Infof("accountsInitialize still building the trie, and hashed so far %d accounts", totalOrderedAccounts) + lastRebuildTime = time.Now() + } } } diff --git a/ledger/acctupdates_test.go b/ledger/acctupdates_test.go index f40b6545fd..55a6aae902 100644 --- a/ledger/acctupdates_test.go +++ b/ledger/acctupdates_test.go @@ -554,7 +554,7 @@ func TestLargeAccountCountCatchpointGeneration(t *testing.T) { if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" { t.Skip("This test is too slow on ARM and causes travis builds to time out") } - // create new protocol version, which has lower back balance. + // create new protocol version, which has lower lookback testProtocolVersion := protocol.ConsensusVersion("test-protocol-TestLargeAccountCountCatchpointGeneration") protoParams := config.Consensus[protocol.ConsensusCurrentVersion] protoParams.MaxBalLookback = 32 @@ -1130,3 +1130,63 @@ func TestGetCatchpointStream(t *testing.T) { err = au.deleteStoredCatchpoints(context.Background(), au.accountsq) require.NoError(t, err) } + +func BenchmarkLargeMerkleTrieRebuild(b *testing.B) { + proto := config.Consensus[protocol.ConsensusCurrentVersion] + + ml := makeMockLedgerForTracker(b, true) + defer ml.close() + ml.blocks = randomInitChain(protocol.ConsensusCurrentVersion, 10) + + accts := []map[basics.Address]basics.AccountData{randomAccounts(5, true)} + + pooldata := basics.AccountData{} + pooldata.MicroAlgos.Raw = 1000 * 1000 * 1000 * 1000 + pooldata.Status = basics.NotParticipating + accts[0][testPoolAddr] = pooldata + + sinkdata := basics.AccountData{} + sinkdata.MicroAlgos.Raw = 1000 * 1000 * 1000 * 1000 + sinkdata.Status = basics.NotParticipating + accts[0][testSinkAddr] = sinkdata + + au := &accountUpdates{} + cfg := config.GetDefaultLocal() + cfg.Archival = true + au.initialize(cfg, ".", proto, accts[0]) + defer au.close() + + err := au.loadFromDisk(ml) + require.NoError(b, err) + + // at this point, the database was created. We want to fill the accounts data + accountsNumber := 6000000 * b.N + for i := 0; i < accountsNumber; { + updates := make(map[basics.Address]accountDelta, 0) + for k := 0; i < accountsNumber && k < 1024; k++ { + addr := randomAddress() + acctData := basics.AccountData{} + acctData.MicroAlgos.Raw = 1 + updates[addr] = accountDelta{new: acctData} + i++ + } + + err := ml.dbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + return accountsNewRound(tx, updates, nil, proto) + }) + require.NoError(b, err) + } + + err = ml.dbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + return updateAccountsRound(tx, 0, 1) + }) + require.NoError(b, err) + + au.close() + + b.ResetTimer() + err = au.loadFromDisk(ml) + require.NoError(b, err) + b.StopTimer() + b.ReportMetric(float64(accountsNumber), "entries/trie") +} diff --git a/ledger/catchpointwriter_test.go b/ledger/catchpointwriter_test.go index d39f3a1615..37fd63f98e 100644 --- a/ledger/catchpointwriter_test.go +++ b/ledger/catchpointwriter_test.go @@ -124,7 +124,7 @@ func TestCatchpointFileBalancesChunkEncoding(t *testing.T) { } func TestBasicCatchpointWriter(t *testing.T) { - // create new protocol version, which has lower back balance. + // create new protocol version, which has lower lookback testProtocolVersion := protocol.ConsensusVersion("test-protocol-TestBasicCatchpointWriter") protoParams := config.Consensus[protocol.ConsensusCurrentVersion] protoParams.MaxBalLookback = 32 @@ -223,7 +223,7 @@ func TestBasicCatchpointWriter(t *testing.T) { } func TestFullCatchpointWriter(t *testing.T) { - // create new protocol version, which has lower back balance. + // create new protocol version, which has lower lookback testProtocolVersion := protocol.ConsensusVersion("test-protocol-TestFullCatchpointWriter") protoParams := config.Consensus[protocol.ConsensusCurrentVersion] protoParams.MaxBalLookback = 32 From f5355dec01b414939527c2a0a500bb595b6d27d3 Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Tue, 17 Nov 2020 12:58:50 -0500 Subject: [PATCH 21/34] Add app state read-write into pingpong (#1700) Add app state read-write into pingpong --- cmd/pingpong/runCmd.go | 31 +- shared/pingpong/accounts.go | 357 ++++++++++++++---- shared/pingpong/config.go | 7 +- shared/pingpong/pingpong.go | 58 ++- .../cc_agent/component/pingPongComponent.go | 8 +- 5 files changed, 349 insertions(+), 112 deletions(-) diff --git a/cmd/pingpong/runCmd.go b/cmd/pingpong/runCmd.go index 5b33ccd591..3d316efb40 100644 --- a/cmd/pingpong/runCmd.go +++ b/cmd/pingpong/runCmd.go @@ -58,9 +58,12 @@ var teal string var groupSize uint32 var numAsset uint32 var numApp uint32 +var numAppOptIn uint32 var appProgOps uint32 -var appProgHashs uint32 +var appProgHashes uint32 var appProgHashSize string +var appProgGlobKeys uint32 +var appProgLocalKeys uint32 var duration uint32 var rekey bool @@ -92,10 +95,13 @@ func init() { runCmd.Flags().StringVar(&teal, "teal", "", "teal test scenario, can be light, normal, or heavy, this overrides --program") runCmd.Flags().Uint32Var(&groupSize, "groupsize", 1, "The number of transactions in each group") runCmd.Flags().Uint32Var(&numAsset, "numasset", 0, "The number of assets each account holds") - runCmd.Flags().Uint32Var(&numApp, "numapp", 0, "The number of apps each account opts in to") + runCmd.Flags().Uint32Var(&numApp, "numapp", 0, "The total number of apps to create") + runCmd.Flags().Uint32Var(&numAppOptIn, "numappoptin", 0, "The number of apps each account opts in to") runCmd.Flags().Uint32Var(&appProgOps, "appprogops", 0, "The approximate number of TEAL operations to perform in each ApplicationCall transaction") - runCmd.Flags().Uint32Var(&appProgHashs, "appproghashes", 0, "The number of hashes to include in the Application") + runCmd.Flags().Uint32Var(&appProgHashes, "appproghashes", 0, "The number of hashes to include in the Application") runCmd.Flags().StringVar(&appProgHashSize, "appproghashsize", "sha256", "The size of hashes to include in the Application") + runCmd.Flags().Uint32Var(&appProgGlobKeys, "appproggk", 0, "Number of global state writes in the Application") + runCmd.Flags().Uint32Var(&appProgLocalKeys, "appproglk", 0, "Number of local state writes in the Application. Number or local keys per account will be appproglk / proto.MaxAppTxnAccounts") runCmd.Flags().BoolVar(&randomLease, "randomlease", false, "set the lease to contain a random value") runCmd.Flags().BoolVar(&rekey, "rekey", false, "Create RekeyTo transactions. Requires groupsize=2 and any of random flags exc random dst") runCmd.Flags().Uint32Var(&duration, "duration", 0, "The number of seconds to run the pingpong test, forever if 0") @@ -258,7 +264,7 @@ var runCmd = &cobra.Command{ } cfg.AppProgOps = appProgOps - cfg.AppProgHashs = appProgHashs + cfg.AppProgHashes = appProgHashes cfg.AppProgHashSize = appProgHashSize if numApp <= 1000 { @@ -267,6 +273,19 @@ var runCmd = &cobra.Command{ reportErrorf("Invalid number of apps: %d, (valid number: 0 - 1000)\n", numApp) } + if numAppOptIn > cfg.NumApp { + reportErrorf("Cannot opt in %d times of %d total apps\n", numAppOptIn, numApp) + } + + cfg.NumAppOptIn = numAppOptIn + + if appProgGlobKeys > 0 { + cfg.AppGlobKeys = appProgGlobKeys + } + if appProgLocalKeys > 0 { + cfg.AppLocalKeys = appProgLocalKeys + } + if numAsset != 0 && numApp != 0 { reportErrorf("only one of numapp and numasset may be specified\n") } @@ -285,7 +304,7 @@ var runCmd = &cobra.Command{ cfg.Dump(os.Stdout) // Initialize accounts if necessary - accounts, assetParams, appParams, cfg, err := pingpong.PrepareAccounts(ac, cfg) + accounts, cinfo, cfg, err := pingpong.PrepareAccounts(ac, cfg) if err != nil { reportErrorf("Error preparing accounts for transfers: %v\n", err) } @@ -298,7 +317,7 @@ var runCmd = &cobra.Command{ cfg.Dump(os.Stdout) // Kick off the real processing - pingpong.RunPingPong(context.Background(), ac, accounts, assetParams, appParams, cfg) + pingpong.RunPingPong(context.Background(), ac, accounts, cinfo, cfg) }, } diff --git a/shared/pingpong/accounts.go b/shared/pingpong/accounts.go index ce1083e62b..b9bb2896c6 100644 --- a/shared/pingpong/accounts.go +++ b/shared/pingpong/accounts.go @@ -19,17 +19,20 @@ package pingpong import ( "fmt" "math" + "math/rand" "os" "sort" "strings" "time" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" v1 "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/libgoal" + "github.com/algorand/go-algorand/protocol" ) func ensureAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]uint64, cfg PpConfig, err error) { @@ -290,7 +293,7 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf fmt.Printf("Distributing assets from %v to %v \n", addr, addr2) } - tx, sendErr := constructTxn(addr, addr2, cfg.MaxFee, assetAmt, k, client, cfg) + tx, sendErr := constructTxn(addr, addr2, cfg.MaxFee, assetAmt, k, CreatablesInfo{}, client, cfg) if sendErr != nil { fmt.Printf("Cannot transfer asset %v from account %v\n", k, addr) err = sendErr @@ -357,47 +360,89 @@ func genBigNoOpAndBigHashes(numOps uint32, numHashes uint32, hashSize string) [] return progBytes } -func genMaxClone(numKeys int) []byte { +func genAppProgram(numOps uint32, numHashes uint32, hashSize string, numGlobalKeys uint32, numLocalKeys uint32) ([]byte, string) { + prologueSize := uint32(2 + 3 + 2 + 1 + 1 + 3) + prologue := `#pragma version 2 + txn ApplicationID + bz ok + txn OnCompletion + int OptIn + == + bnz ok + ` + // goto flip if first key exists + flipBranchSize := uint32(1 + 1 + 1 + 1 + 3 + 1) flipBranch := ` - int 0 // current app id - int 1 // key - itob - app_global_get_ex - bnz flip + int 0 // current app id [0] + int 1 // key [1, 0] + itob // ["\x01", 0] + app_global_get_ex // [0|1, x] + bnz flip // [x] + pop // [] ` writePrefix := ` write: - int 0 + int 0 // [0] ` + writeBlockSize := uint32(6) writeBlock := ` - int 1 - + - dup - itob - dup - app_global_put + int 1 // [1, 0] + + // [1] + dup // [1, 1] + itob // ["\x01", 1] + dup // ["\x01", "\x01", 1] + app_global_put // [1] ` writeSuffix := ` - int 1 + pop // [] + int 1 // [1] return ` + writeLocBlockPrefix := ` + // handle a rare case when there is no opt ins for this app + // the caller adds opted in accounts to txn.Accounts + txn NumAccounts // [x, n] + int 1 // [1, x, n] + == // [0/1, n] + bnz ok // [n] + ` + + writeLocBlockSize := uint32(15 + 2) + writeLocBlock := ` + int 1 // [1, n] + + // [1+n] + dup // [1+n, 1+n] + dup // [1+n, 1+n, 1+n] + store 0 // [1+n, 1+n] + txn NumAccounts // [N, 1+n, 1+n] + int 1 // [1, N, 1+n, 1+n] + - // [N-1, 1+n, 1+n], exclude sender + % // [A, 1+n], A = 1+n mod N-1 + int 1 // [1, A, 1+n], + + // [A+1, 1+n] + load 0 // [1+n, A+1, 1+n] + itob // ["\x n+1", A+1, 1+n] + dup // ["\x n+1", "\x n+1", A, 1+n] + app_local_put // [1+n] + ` + // flip stored value's low bit flipPrefix := ` - flip: - btoi - int 1 - ^ - itob - store 0 - int 1 - itob - load 0 - app_global_put + flip: // [x] + btoi // [n] + int 1 // [1, n] + ^ // [n^1] + itob // ["\x n^1"] + store 0 // [] + int 1 // [1] + itob // ["x01"] + load 0 // ["\x n^1", "x01"] + app_global_put // [] ` flipSuffix := ` @@ -405,16 +450,41 @@ func genMaxClone(numKeys int) []byte { return ` + epilogue := ` + ok: + int 1 + return + ` + // generate assembly - progParts := []string{"#pragma version 2"} + progParts := append([]string{}, prologue) progParts = append(progParts, flipBranch) progParts = append(progParts, writePrefix) - for i := 0; i < numKeys; i++ { + for i := uint32(0); i < numGlobalKeys; i++ { progParts = append(progParts, writeBlock) } + progParts = append(progParts, writeLocBlockPrefix) + for i := uint32(0); i < numLocalKeys; i++ { + progParts = append(progParts, writeLocBlock) + } + if numHashes > 0 { + progParts = append(progParts, `byte base64 AA==`) + for i := uint32(0); i < numHashes; i++ { + progParts = append(progParts, hashSize) + } + } + written := prologueSize + flipBranchSize + numHashes + numGlobalKeys*writeBlockSize + numLocalKeys*writeLocBlockSize + if written < numOps { + left := numOps - written - 20 // allow some space + for i := uint32(0); i < left/2; i++ { + progParts = append(progParts, `int 1`) + progParts = append(progParts, `pop`) + } + } progParts = append(progParts, writeSuffix) progParts = append(progParts, flipPrefix) progParts = append(progParts, flipSuffix) + progParts = append(progParts, epilogue) progAsm := strings.Join(progParts, "\n") // assemble @@ -422,25 +492,93 @@ func genMaxClone(numKeys int) []byte { if err != nil { panic(err) } - return progBytes + return progBytes, progAsm } -func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) (appParams map[uint64]v1.AppParams, err error) { +func sendAsGroup(txgroup []transactions.Transaction, client libgoal.Client, h []byte) (err error) { + if len(txgroup) == 0 { + err = fmt.Errorf("sendAsGroup: empty group") + return + } + gid, gidErr := client.GroupID(txgroup) + if gidErr != nil { + err = gidErr + return + } + var stxgroup []transactions.SignedTxn + for _, txn := range txgroup { + txn.Group = gid + signedTxn, signErr := client.SignTransactionWithWallet(h, nil, txn) + if signErr != nil { + fmt.Printf("Cannot sign app creation txn\n") + err = signErr + return + } + stxgroup = append(stxgroup, signedTxn) + } + + broadcastErr := client.BroadcastTransactionGroup(stxgroup) + if broadcastErr != nil { + fmt.Printf("Cannot broadcast app creation txn group\n") + err = broadcastErr + return + } + return +} + +func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) (appParams map[uint64]v1.AppParams, optIns map[uint64][]string, err error) { + status, err := client.Status() + if err != nil { + return + } + proto, err := client.ConsensusParams(status.LastRound) + if err != nil { + return + } + + toCreate := int(cfg.NumApp) + appsPerAcct := proto.MaxAppsCreated + // create min(groupSize, maxAppsPerAcct) per account to optimize sending in batches + groupSize := proto.MaxTxGroupSize + if appsPerAcct > groupSize { + appsPerAcct = groupSize + } + + acctNeeded := toCreate / appsPerAcct + if toCreate%appsPerAcct != 0 { + acctNeeded++ + } + if acctNeeded >= len(accounts) { // >= because cfg.SrcAccount is skipped + err = fmt.Errorf("Need %d accts to create %d apps but got only %d accts", acctNeeded, toCreate, len(accounts)) + return + } + maxOptIn := uint32(config.Consensus[protocol.ConsensusCurrentVersion].MaxAppsOptedIn) + if cfg.NumAppOptIn > maxOptIn { + err = fmt.Errorf("Each acct can only opt in to %d but %d requested", maxOptIn, cfg.NumAppOptIn) + return + } - var appAccount v1.Account + var appAccounts []v1.Account for tempAccount := range accounts { if tempAccount != cfg.SrcAccount { + var appAccount v1.Account appAccount, err = client.AccountInformation(tempAccount) if err != nil { fmt.Printf("Warning, cannot lookup tempAccount account %s", tempAccount) return } - break + appAccounts = append(appAccounts, appAccount) + if len(appAccounts) == acctNeeded { + break + } } } if !cfg.Quiet { - fmt.Printf("Selected temp account: %s\n", appAccount.Address) + fmt.Printf("Selected temp account:\n") + for _, acct := range appAccounts { + fmt.Printf("%s\n", acct.Address) + } } // Get wallet handle token @@ -450,73 +588,132 @@ func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig return } - toCreate := int(cfg.NumApp) - - // create apps in srcAccount - for i := 0; i < toCreate; i++ { - var tx transactions.Transaction - - // generate app program with roughly some number of operations - prog := genBigNoOpAndBigHashes(cfg.AppProgOps, cfg.AppProgHashs, cfg.AppProgHashSize) - if !cfg.Quiet { - fmt.Printf("generated program: \n%s\n", prog) + // create apps + for idx, appAccount := range appAccounts { + begin := idx * appsPerAcct + end := (idx + 1) * appsPerAcct + if end > toCreate { + end = toCreate } + var txgroup []transactions.Transaction + for i := begin; i < end; i++ { + var tx transactions.Transaction - globSchema := basics.StateSchema{NumByteSlice: 64} - locSchema := basics.StateSchema{} - tx, err = client.MakeUnsignedAppCreateTx(transactions.NoOpOC, prog, prog, globSchema, locSchema, nil, nil, nil, nil) - if err != nil { - fmt.Printf("Cannot create app txn\n") - panic(err) - } + // generate app program with roughly some number of operations + prog, asm := genAppProgram(cfg.AppProgOps, cfg.AppProgHashes, cfg.AppProgHashSize, cfg.AppGlobKeys, cfg.AppLocalKeys) + if !cfg.Quiet { + fmt.Printf("generated program: \n%s\n", asm) + } - tx, err = client.FillUnsignedTxTemplate(appAccount.Address, 0, 0, cfg.MaxFee, tx) - if err != nil { - fmt.Printf("Cannot fill app creation txn\n") - panic(err) - } + globSchema := basics.StateSchema{NumByteSlice: proto.MaxGlobalSchemaEntries} + locSchema := basics.StateSchema{NumByteSlice: proto.MaxLocalSchemaEntries} + tx, err = client.MakeUnsignedAppCreateTx(transactions.NoOpOC, prog, prog, globSchema, locSchema, nil, nil, nil, nil) + if err != nil { + fmt.Printf("Cannot create app txn\n") + panic(err) + } - // Ensure different txids - var note [8]byte - crypto.RandBytes(note[:]) - tx.Note = note[:] + tx, err = client.FillUnsignedTxTemplate(appAccount.Address, 0, 0, cfg.MaxFee, tx) + if err != nil { + fmt.Printf("Cannot fill app creation txn\n") + panic(err) + } - signedTxn, signErr := client.SignTransactionWithWallet(h, nil, tx) - if signErr != nil { - fmt.Printf("Cannot sign app creation txn\n") - err = signErr - return + // Ensure different txids + var note [8]byte + crypto.RandBytes(note[:]) + tx.Note = note[:] + + txgroup = append(txgroup, tx) + accounts[appAccount.Address] -= tx.Fee.Raw } - txid, broadcastErr := client.BroadcastTransaction(signedTxn) - if broadcastErr != nil { - fmt.Printf("Cannot broadcast app creation txn\n") - err = broadcastErr + err = sendAsGroup(txgroup, client, h) + if err != nil { return } if !cfg.Quiet { - fmt.Printf("Create a new app: txid=%s\n", txid) + fmt.Printf("Created new %d apps\n", len(txgroup)) } - - accounts[appAccount.Address] -= tx.Fee.Raw } - var account v1.Account // get these apps - for { - account, err = client.AccountInformation(appAccount.Address) - if err != nil { - fmt.Printf("Warning, cannot lookup source account") - return + var aidxs []uint64 + appParams = make(map[uint64]v1.AppParams) + for _, appAccount := range appAccounts { + var account v1.Account + for { + account, err = client.AccountInformation(appAccount.Address) + if err != nil { + fmt.Printf("Warning, cannot lookup source account") + return + } + if len(account.AppParams) >= appsPerAcct || len(aidxs) >= int(cfg.NumApp) { + break + } + time.Sleep(time.Second) } - if len(account.AppParams) >= int(cfg.NumApp) { - break + for idx := range account.AppParams { + aidxs = append(aidxs, idx) + } + for k, v := range account.AppParams { + appParams[k] = v + } + } + if len(aidxs) != len(appParams) { + err = fmt.Errorf("duplicates in aidxs, %d != %d", len(aidxs), len(appParams)) + return + } + + // time to opt in to these apps + if cfg.NumAppOptIn > 0 { + optIns = make(map[uint64][]string) + for addr := range accounts { + var txgroup []transactions.Transaction + permAppIndices := rand.Perm(len(aidxs)) + for i := uint32(0); i < cfg.NumAppOptIn; i++ { + j := permAppIndices[i] + aidx := aidxs[j] + var tx transactions.Transaction + tx, err = client.MakeUnsignedAppOptInTx(aidx, nil, nil, nil, nil) + if err != nil { + fmt.Printf("Cannot create app txn\n") + panic(err) + } + + tx, err = client.FillUnsignedTxTemplate(addr, 0, 0, cfg.MaxFee, tx) + if err != nil { + fmt.Printf("Cannot fill app creation txn\n") + panic(err) + } + + // Ensure different txids + var note [8]byte + crypto.RandBytes(note[:]) + tx.Note = note[:] + + optIns[aidx] = append(optIns[aidx], addr) + + txgroup = append(txgroup, tx) + if len(txgroup) == groupSize { + err = sendAsGroup(txgroup, client, h) + if err != nil { + return + } + txgroup = txgroup[:0] + } + } + // broadcast leftovers + if len(txgroup) > 0 { + err = sendAsGroup(txgroup, client, h) + if err != nil { + return + } + } } - time.Sleep(time.Second) } - appParams = account.AppParams return } diff --git a/shared/pingpong/config.go b/shared/pingpong/config.go index 40b61a0776..f29606bf3a 100644 --- a/shared/pingpong/config.go +++ b/shared/pingpong/config.go @@ -53,9 +53,12 @@ type PpConfig struct { NumAsset uint32 MinAccountAsset uint64 NumApp uint32 + NumAppOptIn uint32 AppProgOps uint32 - AppProgHashs uint32 + AppProgHashes uint32 AppProgHashSize string + AppGlobKeys uint32 + AppLocalKeys uint32 Rekey bool MaxRuntime time.Duration } @@ -81,7 +84,7 @@ var DefaultConfig = PpConfig{ MinAccountAsset: 10000000, NumApp: 0, AppProgOps: 0, - AppProgHashs: 0, + AppProgHashes: 0, AppProgHashSize: "sha256", Rekey: false, MaxRuntime: 0, diff --git a/shared/pingpong/pingpong.go b/shared/pingpong/pingpong.go index 266d87a4e6..f715d5a9cb 100644 --- a/shared/pingpong/pingpong.go +++ b/shared/pingpong/pingpong.go @@ -25,14 +25,21 @@ import ( "time" "github.com/algorand/go-algorand/crypto" - "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" + v1 "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/libgoal" ) +// CreatablesInfo has information about created assets, apps and opting in +type CreatablesInfo struct { + AssetParams map[uint64]v1.AssetParams + AppParams map[uint64]v1.AppParams + OptIns map[uint64][]string +} + // PrepareAccounts to set up accounts and asset accounts required for Ping Pong run -func PrepareAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]uint64, assetParams map[uint64]v1.AssetParams, appParams map[uint64]v1.AppParams, cfg PpConfig, err error) { +func PrepareAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]uint64, cinfo CreatablesInfo, cfg PpConfig, err error) { cfg = initCfg accounts, cfg, err = ensureAccounts(ac, cfg) if err != nil { @@ -57,7 +64,7 @@ func PrepareAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]u return } - assetParams, err = prepareAssets(assetAccounts, ac, cfg) + cinfo.AssetParams, err = prepareAssets(assetAccounts, ac, cfg) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "prepare assets failed %v\n", err) return @@ -76,7 +83,7 @@ func PrepareAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]u _, _ = fmt.Fprintf(os.Stderr, "prepare new accounts failed: %v\n", err) return } - appParams, err = prepareApps(appAccounts, ac, cfg) + cinfo.AppParams, cinfo.OptIns, err = prepareApps(appAccounts, ac, cfg) if err != nil { return } @@ -239,7 +246,7 @@ func listSufficientAccounts(accounts map[string]uint64, minimumAmount uint64, ex } // RunPingPong starts ping pong process -func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uint64, assetParam map[uint64]v1.AssetParams, appParam map[uint64]v1.AppParams, cfg PpConfig) { +func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uint64, cinfo CreatablesInfo, cfg PpConfig) { // Infinite loop given: // - accounts -> map of accounts to include in transfers (including src account, which we don't want to use) // - cfg -> configuration for how to proceed @@ -292,7 +299,7 @@ func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uin } toList := listSufficientAccounts(accounts, minimumAmount, cfg.SrcAccount) - sent, succeeded, err := sendFromTo(fromList, toList, accounts, assetParam, appParam, ac, cfg) + sent, succeeded, err := sendFromTo(fromList, toList, accounts, cinfo, ac, cfg) totalSent += sent totalSucceeded += succeeded if err != nil { @@ -320,11 +327,11 @@ func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uin } } -func getCreatableID(cfg PpConfig, assetParams map[uint64]v1.AssetParams, appParams map[uint64]v1.AppParams) (aidx uint64) { +func getCreatableID(cfg PpConfig, cinfo CreatablesInfo) (aidx uint64) { if cfg.NumAsset > 0 { - rindex := rand.Intn(len(assetParams)) + rindex := rand.Intn(len(cinfo.AssetParams)) i := 0 - for k := range assetParams { + for k := range cinfo.AssetParams { if i == rindex { aidx = k break @@ -332,9 +339,9 @@ func getCreatableID(cfg PpConfig, assetParams map[uint64]v1.AssetParams, appPara i++ } } else if cfg.NumApp > 0 { - rindex := rand.Intn(len(appParams)) + rindex := rand.Intn(len(cinfo.AppParams)) i := 0 - for k := range appParams { + for k := range cinfo.AppParams { if i == rindex { aidx = k break @@ -347,7 +354,7 @@ func getCreatableID(cfg PpConfig, assetParams map[uint64]v1.AssetParams, appPara func sendFromTo( fromList, toList []string, accounts map[string]uint64, - assetParams map[uint64]v1.AssetParams, appParams map[uint64]v1.AppParams, + cinfo CreatablesInfo, client libgoal.Client, cfg PpConfig, ) (sentCount, successCount uint64, err error) { @@ -380,9 +387,9 @@ func sendFromTo( if cfg.GroupSize == 1 { // generate random assetID or appId if we send asset/app txns - aidx := getCreatableID(cfg, assetParams, appParams) + aidx := getCreatableID(cfg, cinfo) // Construct single txn - txn, consErr := constructTxn(from, to, fee, amt, aidx, client, cfg) + txn, consErr := constructTxn(from, to, fee, amt, aidx, cinfo, client, cfg) if consErr != nil { err = consErr _, _ = fmt.Fprintf(os.Stderr, "constructTxn failed: %v\n", err) @@ -425,17 +432,17 @@ func sendFromTo( var txn transactions.Transaction var signer string if j%2 == 0 { - txn, err = constructTxn(from, to, fee, amt, 0, client, cfg) + txn, err = constructTxn(from, to, fee, amt, 0, cinfo, client, cfg) fromBalanceChange -= int64(txn.Fee.Raw + amt) toBalanceChange += int64(amt) signer = from } else if cfg.GroupSize == 2 && cfg.Rekey { - txn, err = constructTxn(from, to, fee, amt, 0, client, cfg) + txn, err = constructTxn(from, to, fee, amt, 0, cinfo, client, cfg) fromBalanceChange -= int64(txn.Fee.Raw + amt) toBalanceChange += int64(amt) signer = to } else { - txn, err = constructTxn(to, from, fee, amt, 0, client, cfg) + txn, err = constructTxn(to, from, fee, amt, 0, cinfo, client, cfg) toBalanceChange -= int64(txn.Fee.Raw + amt) fromBalanceChange += int64(amt) signer = to @@ -518,7 +525,7 @@ func sendFromTo( return } -func constructTxn(from, to string, fee, amt, aidx uint64, client libgoal.Client, cfg PpConfig) (txn transactions.Transaction, err error) { +func constructTxn(from, to string, fee, amt, aidx uint64, cinfo CreatablesInfo, client libgoal.Client, cfg PpConfig) (txn transactions.Transaction, err error) { var noteField []byte const pingpongTag = "pingpong" const tagLen = uint32(len(pingpongTag)) @@ -540,7 +547,20 @@ func constructTxn(from, to string, fee, amt, aidx uint64, client libgoal.Client, } if cfg.NumApp > 0 { // Construct app transaction - txn, err = client.MakeUnsignedAppNoOpTx(aidx, nil, nil, nil, nil) + // generate random assetID or appId if we send asset/app txns + var accounts []string + if len(cinfo.OptIns[aidx]) > 0 { + indices := rand.Perm(len(cinfo.OptIns[aidx])) + limit := 4 + if len(indices) < limit { + limit = len(indices) + } + for i := 0; i < limit; i++ { + idx := indices[i] + accounts = append(accounts, cinfo.OptIns[aidx][idx]) + } + } + txn, err = client.MakeUnsignedAppNoOpTx(aidx, nil, accounts, nil, nil) if err != nil { return } diff --git a/test/commandandcontrol/cc_agent/component/pingPongComponent.go b/test/commandandcontrol/cc_agent/component/pingPongComponent.go index c195bb5036..757ddebc58 100644 --- a/test/commandandcontrol/cc_agent/component/pingPongComponent.go +++ b/test/commandandcontrol/cc_agent/component/pingPongComponent.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "fmt" - v1 "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" "io/ioutil" "time" @@ -122,13 +121,12 @@ func (componentInstance *PingPongComponentInstance) startPingPong(cfg *pingpong. log.Infof("Preparing to initialize PingPong with config: %+v\n", cfg) var accounts map[string]uint64 - var assetParams map[uint64]v1.AssetParams - var appParams map[uint64]v1.AppParams + var cinfo pingpong.CreatablesInfo var resultCfg pingpong.PpConfig // Initialize accounts if necessary, this may take several attempts while previous transactions to settle for i := 0; i < 10; i++ { - accounts, assetParams, appParams, resultCfg, err = pingpong.PrepareAccounts(ac, *cfg) + accounts, cinfo, resultCfg, err = pingpong.PrepareAccounts(ac, *cfg) if err == nil { break } else { @@ -147,7 +145,7 @@ func (componentInstance *PingPongComponentInstance) startPingPong(cfg *pingpong. componentInstance.ctx, componentInstance.cancelFunc = context.WithCancel(context.Background()) // Kick off the real processing - go pingpong.RunPingPong(componentInstance.ctx, ac, accounts, assetParams, appParams, resultCfg) + go pingpong.RunPingPong(componentInstance.ctx, ac, accounts, cinfo, resultCfg) return } From f6239ced9620c56fe107eded52108303892b5998 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Wed, 18 Nov 2020 16:16:08 -0500 Subject: [PATCH 22/34] optional network propagation tracing (#1618) Add network propagation tracing to `algod` Compiled out by default, it can be enabled by building with: `make GOTAGSCUSTOM=msgtrace` And setting NetworkMessageTraceServer in config.json `algocfg set -p NetworkMessageTraceServer -v server.addr:6225` Traces when a block proposal message is handled. --- Makefile | 4 ++ agreement/gossip/network.go | 12 +++++ config/config.go | 3 ++ config/local_defaults.go | 1 + go.mod | 1 + go.sum | 4 ++ installer/config.json.example | 1 + network/messagetracer/graphtrace.go | 69 +++++++++++++++++++++++++++ network/messagetracer/interface.go | 58 ++++++++++++++++++++++ node/node.go | 8 +++- test/testdata/configs/config-v13.json | 1 + 11 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 network/messagetracer/graphtrace.go create mode 100644 network/messagetracer/interface.go diff --git a/Makefile b/Makefile index 6092470fb9..0966d019eb 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,9 @@ DEFAULT_DEADLOCK ?= $(shell ./scripts/compute_branch_deadlock_default.sh $(BUILD GOTAGSLIST := sqlite_unlock_notify sqlite_omit_load_extension +# e.g. make GOTAGSCUSTOM=msgtrace +GOTAGSLIST += ${GOTAGSCUSTOM} + ifeq ($(UNAME), Linux) EXTLDFLAGS := -static-libstdc++ -static-libgcc ifeq ($(ARCH), amd64) @@ -304,3 +307,4 @@ include ./scripts/release/mule/Makefile.mule archive: CHANNEL=$(CHANNEL) \ aws s3 cp tmp/node_pkgs s3://algorand-internal/channel/${CHANNEL}/$(FULLBUILDNUMBER) --recursive --exclude "*" --include "*${CHANNEL}*$(FULLBUILDNUMBER)*" + diff --git a/agreement/gossip/network.go b/agreement/gossip/network.go index f59a7460b5..e6953f6fe2 100644 --- a/agreement/gossip/network.go +++ b/agreement/gossip/network.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/messagetracer" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/metrics" ) @@ -50,6 +51,8 @@ type networkImpl struct { net network.GossipNode log logging.Logger + + trace messagetracer.MessageTracer } // WrapNetwork adapts a network.GossipNode into an agreement.Network. @@ -66,6 +69,12 @@ func WrapNetwork(net network.GossipNode, log logging.Logger) agreement.Network { return i } +// SetTrace modifies the result of WrapNetwork to add network propagation tracing +func SetTrace(net agreement.Network, trace messagetracer.MessageTracer) { + i := net.(*networkImpl) + i.trace = trace +} + func (i *networkImpl) Start() { handlers := []network.TaggedMessageHandler{ {Tag: protocol.AgreementVoteTag, MessageHandler: network.HandlerFunc(i.processVoteMessage)}, @@ -87,6 +96,9 @@ func (i *networkImpl) processVoteMessage(raw network.IncomingMessage) network.Ou } func (i *networkImpl) processProposalMessage(raw network.IncomingMessage) network.OutgoingMessage { + if i.trace != nil { + i.trace.HashTrace(messagetracer.Proposal, raw.Data) + } return i.processMessage(raw, i.proposalCh) } diff --git a/config/config.go b/config/config.go index 97afc5a20f..4c0cdddbe5 100644 --- a/config/config.go +++ b/config/config.go @@ -359,6 +359,9 @@ type Local struct { // provided stream speed drops below this threshold, the connection would be recycled. Note that this field is evaluated per catchpoint "chunk" and not on it's own. If this field is zero, // the default of 20480 would be used. MinCatchpointFileDownloadBytesPerSecond uint64 `version[13]:"20480"` + + // TraceServer is a host:port to report graph propagation trace info to. + NetworkMessageTraceServer string `version[13]:""` } // Filenames of config files within the configdir (e.g. ~/.algorand) diff --git a/config/local_defaults.go b/config/local_defaults.go index 4b8f2cd1cf..d1d7655ec8 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -73,6 +73,7 @@ var defaultLocal = Local{ MaxConnectionsPerIP: 30, MinCatchpointFileDownloadBytesPerSecond: 20480, NetAddress: "", + NetworkMessageTraceServer: "", NetworkProtocolVersion: "", NodeExporterListenAddress: ":9100", NodeExporterPath: "./node_exporter", diff --git a/go.mod b/go.mod index 4bb9aedb2c..c8e58e0d63 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d github.com/algorand/go-deadlock v0.2.1 + github.com/algorand/graphtrace v0.0.0-20201117160756-e524ed1a6f64 github.com/algorand/msgp v1.1.45 github.com/algorand/oapi-codegen v1.3.5-algorand5 github.com/algorand/websocket v1.4.1 diff --git a/go.sum b/go.sum index 6ffd738e6d..7447e5910d 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d h1:W9MgGUo github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d/go.mod h1:qm6LyXvDa1+uZJxaVg8X+OEjBqt/zDinDa2EohtTDxU= github.com/algorand/go-deadlock v0.2.1 h1:TQPQwWAB133bS5uwHpmrgH5hCMyZK5hnUW26aqWMvq4= github.com/algorand/go-deadlock v0.2.1/go.mod h1:HgdF2cwtBIBCL7qmUaozuG/UIZFR6PLpSMR58pvWiXE= +github.com/algorand/graphtrace v0.0.0-20201023193244-96dde3c58a7a h1:0vQcZhPB7rCI7w0CXMiAmxg1R92JROMxTG6RjvVz1oQ= +github.com/algorand/graphtrace v0.0.0-20201023193244-96dde3c58a7a/go.mod h1:qFtQmC+kmsfnLfS9j3xgKtzsWyozemL5ek1R4dWZa5c= +github.com/algorand/graphtrace v0.0.0-20201117160756-e524ed1a6f64 h1:yvKeJdS/mvLRiyIxu8j5BQDXIzs1XbC9/22KycJnt3A= +github.com/algorand/graphtrace v0.0.0-20201117160756-e524ed1a6f64/go.mod h1:qFtQmC+kmsfnLfS9j3xgKtzsWyozemL5ek1R4dWZa5c= github.com/algorand/msgp v1.1.45 h1:uLEPvg0BfTrb2JjBcXexON8il4vv0EsD7HYFaA7e5jg= github.com/algorand/msgp v1.1.45/go.mod h1:LtOntbYiCHj/Sl/Sqxtf8CZOrDt2a8Dv3tLaS6mcnUE= github.com/algorand/oapi-codegen v1.3.5-algorand5 h1:y576Ca2/guQddQrQA7dtL5KcOx5xQgPeIupiuFMGyCI= diff --git a/installer/config.json.example b/installer/config.json.example index 63eefd7c7c..61d37b3457 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -52,6 +52,7 @@ "MaxConnectionsPerIP": 30, "MinCatchpointFileDownloadBytesPerSecond": 20480, "NetAddress": "", + "NetworkMessageTraceServer": "", "NetworkProtocolVersion": "", "NodeExporterListenAddress": ":9100", "NodeExporterPath": "./node_exporter", diff --git a/network/messagetracer/graphtrace.go b/network/messagetracer/graphtrace.go new file mode 100644 index 0000000000..5b189dde27 --- /dev/null +++ b/network/messagetracer/graphtrace.go @@ -0,0 +1,69 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . +// +// +build msgtrace + +package messagetracer + +import ( + "hash/fnv" + + "github.com/algorand/graphtrace/graphtrace" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/logging" +) + +type graphtraceMessageTracer struct { + tracer graphtrace.Client + log logging.Logger +} + +func (gmt *graphtraceMessageTracer) Init(cfg config.Local) MessageTracer { + if cfg.NetworkMessageTraceServer == "" { + gmt.log.Info("NetworkMessageTraceServer empty graphtrace disabled") + return nil + } + var err error + gmt.tracer, err = graphtrace.NewTCPClient(cfg.NetworkMessageTraceServer, gmt.log) + if err != nil { + gmt.log.Errorf("unable to create trace client: %v", err) + return nil + } + gmt.log.Infof("tracing network messages to %s", cfg.NetworkMessageTraceServer) + return gmt +} +func (gmt *graphtraceMessageTracer) HashTrace(prefix string, data []byte) { + hasher := fnv.New64a() + hasher.Write(data) + pb := []byte(prefix) + msg := make([]byte, len(pb)+8) + copy(msg, pb) + hash := hasher.Sum(msg[0:len(pb)]) + gmt.tracer.Trace(hash) +} + +// NewGraphtraceMessageTracer returns a new MessageTracer that sends data to a graphtrace collector +func NewGraphtraceMessageTracer(log logging.Logger) MessageTracer { + return &graphtraceMessageTracer{log: log} +} + +func init() { + if implFactory != nil { + panic("at most one MessageTracer impl should be compiled in, dup found at graphtrace.go init()") + } + implFactory = NewGraphtraceMessageTracer +} diff --git a/network/messagetracer/interface.go b/network/messagetracer/interface.go new file mode 100644 index 0000000000..aacfa009f8 --- /dev/null +++ b/network/messagetracer/interface.go @@ -0,0 +1,58 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package messagetracer + +import ( + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/logging" +) + +// MessageTracer interface for configuring trace client and sending trace messages +type MessageTracer interface { + // Init configures trace client or returns nil. + // Caller is expected to check for nil, e.g. `if t != nil {t.HashTrace(...)}` + Init(cfg config.Local) MessageTracer + + // HashTrace submits a trace message to the statistics server. + HashTrace(prefix string, data []byte) +} + +var implFactory func(logging.Logger) MessageTracer + +type nopMessageTracer struct { +} + +func (gmt *nopMessageTracer) Init(cfg config.Local) MessageTracer { + return nil +} +func (gmt *nopMessageTracer) HashTrace(prefix string, data []byte) { +} + +var singletonNopMessageTracer nopMessageTracer + +// NewTracer constructs a new MessageTracer if that has been compiled in with the build tag `msgtrace` +func NewTracer(log logging.Logger) MessageTracer { + if implFactory != nil { + log.Info("graphtrace factory enabled") + return implFactory(log) + } + log.Info("graphtrace factory DISabled") + return &singletonNopMessageTracer +} + +// Proposal is a prefix for HashTrace() +const Proposal = "prop" diff --git a/node/node.go b/node/node.go index db198df34a..580f13c682 100644 --- a/node/node.go +++ b/node/node.go @@ -42,6 +42,7 @@ import ( "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/messagetracer" "github.com/algorand/go-algorand/node/indexer" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" @@ -117,7 +118,7 @@ type AlgorandFullNode struct { // syncStatusMu used for locking lastRoundTimestamp and hasSyncedSinceStartup // syncStatusMu added so OnNewBlock wouldn't be blocked by oldKeyDeletionThread during catchup - syncStatusMu deadlock.Mutex + syncStatusMu deadlock.Mutex lastRoundTimestamp time.Time hasSyncedSinceStartup bool @@ -128,6 +129,8 @@ type AlgorandFullNode struct { oldKeyDeletionNotify chan struct{} monitoringRoutinesWaitGroup sync.WaitGroup + + tracer messagetracer.MessageTracer } // TxnWithStatus represents information about a single transaction, @@ -271,6 +274,9 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd } } + node.tracer = messagetracer.NewTracer(log).Init(cfg) + gossip.SetTrace(agreementParameters.Network, node.tracer) + return node, err } diff --git a/test/testdata/configs/config-v13.json b/test/testdata/configs/config-v13.json index 63eefd7c7c..61d37b3457 100644 --- a/test/testdata/configs/config-v13.json +++ b/test/testdata/configs/config-v13.json @@ -52,6 +52,7 @@ "MaxConnectionsPerIP": 30, "MinCatchpointFileDownloadBytesPerSecond": 20480, "NetAddress": "", + "NetworkMessageTraceServer": "", "NetworkProtocolVersion": "", "NodeExporterListenAddress": ":9100", "NodeExporterPath": "./node_exporter", From 96895b26b440fc7b70df9e0aab13642850308eac Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Thu, 19 Nov 2020 22:34:42 -0500 Subject: [PATCH 23/34] Write the catchpoint file in parallel of reading from the database. (#1712) Optimize the catchpoint file writer performance. --- ledger/acctupdates_test.go | 72 ++++++++++++++++++++- ledger/catchpointwriter.go | 111 ++++++++++++++++++++++++-------- ledger/catchpointwriter_test.go | 2 +- 3 files changed, 156 insertions(+), 29 deletions(-) diff --git a/ledger/acctupdates_test.go b/ledger/acctupdates_test.go index 55a6aae902..74d7ba2c4d 100644 --- a/ledger/acctupdates_test.go +++ b/ledger/acctupdates_test.go @@ -1161,9 +1161,9 @@ func BenchmarkLargeMerkleTrieRebuild(b *testing.B) { // at this point, the database was created. We want to fill the accounts data accountsNumber := 6000000 * b.N - for i := 0; i < accountsNumber; { + for i := 0; i < accountsNumber-5-2; { // subtract the account we've already created above, plus the sink/reward updates := make(map[basics.Address]accountDelta, 0) - for k := 0; i < accountsNumber && k < 1024; k++ { + for k := 0; i < accountsNumber-5-2 && k < 1024; k++ { addr := randomAddress() acctData := basics.AccountData{} acctData.MicroAlgos.Raw = 1 @@ -1190,3 +1190,71 @@ func BenchmarkLargeMerkleTrieRebuild(b *testing.B) { b.StopTimer() b.ReportMetric(float64(accountsNumber), "entries/trie") } + +func BenchmarkLargeCatchpointWriting(b *testing.B) { + proto := config.Consensus[protocol.ConsensusCurrentVersion] + + ml := makeMockLedgerForTracker(b, true) + defer ml.close() + ml.blocks = randomInitChain(protocol.ConsensusCurrentVersion, 10) + + accts := []map[basics.Address]basics.AccountData{randomAccounts(5, true)} + + pooldata := basics.AccountData{} + pooldata.MicroAlgos.Raw = 1000 * 1000 * 1000 * 1000 + pooldata.Status = basics.NotParticipating + accts[0][testPoolAddr] = pooldata + + sinkdata := basics.AccountData{} + sinkdata.MicroAlgos.Raw = 1000 * 1000 * 1000 * 1000 + sinkdata.Status = basics.NotParticipating + accts[0][testSinkAddr] = sinkdata + + au := &accountUpdates{} + cfg := config.GetDefaultLocal() + cfg.Archival = true + au.initialize(cfg, ".", proto, accts[0]) + defer au.close() + + temporaryDirectroy, err := ioutil.TempDir(os.TempDir(), "catchpoints") + require.NoError(b, err) + defer func() { + os.RemoveAll(temporaryDirectroy) + }() + catchpointsDirectory := filepath.Join(temporaryDirectroy, "catchpoints") + err = os.Mkdir(catchpointsDirectory, 0777) + require.NoError(b, err) + + au.dbDirectory = temporaryDirectroy + + err = au.loadFromDisk(ml) + require.NoError(b, err) + + // at this point, the database was created. We want to fill the accounts data + accountsNumber := 6000000 * b.N + err = ml.dbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + for i := 0; i < accountsNumber-5-2; { // subtract the account we've already created above, plus the sink/reward + updates := make(map[basics.Address]accountDelta, 0) + for k := 0; i < accountsNumber-5-2 && k < 1024; k++ { + addr := randomAddress() + acctData := basics.AccountData{} + acctData.MicroAlgos.Raw = 1 + updates[addr] = accountDelta{new: acctData} + i++ + } + + err = accountsNewRound(tx, updates, nil, proto) + if err != nil { + return + } + } + + return updateAccountsRound(tx, 0, 1) + }) + require.NoError(b, err) + + b.ResetTimer() + au.generateCatchpoint(basics.Round(0), "0#ABCD", crypto.Digest{}, time.Second) + b.StopTimer() + b.ReportMetric(float64(accountsNumber), "accounts") +} diff --git a/ledger/catchpointwriter.go b/ledger/catchpointwriter.go index 5d8313fdc1..4884c26dff 100644 --- a/ledger/catchpointwriter.go +++ b/ledger/catchpointwriter.go @@ -170,6 +170,24 @@ func (cw *catchpointWriter) WriteStep(stepCtx context.Context) (more bool, err e cw.headerWritten = true } + writerRequest := make(chan catchpointFileBalancesChunk, 1) + writerResponse := make(chan error, 2) + go cw.asyncWriter(writerRequest, writerResponse, cw.balancesChunkNum) + defer func() { + close(writerRequest) + // wait for the writerResponse to close. + for { + select { + case writerError, open := <-writerResponse: + if open { + err = writerError + } else { + return + } + } + } + }() + for { // have we timed-out / canceled by that point ? if more, err = hasContextDeadlineExceeded(stepCtx); more == true || err != nil { @@ -188,43 +206,84 @@ func (cw *catchpointWriter) WriteStep(stepCtx context.Context) (more bool, err e return } + // check if we had any error on the writer from previous iterations. + select { + case err := <-writerResponse: + // we ran into an error. wait for the channel to close before returning with the error. + select { + case <-writerResponse: + } + return false, err + default: + } + // write to disk. if len(cw.balancesChunk.Balances) > 0 { cw.balancesChunkNum++ - encodedChunk := protocol.Encode(&cw.balancesChunk) - err = cw.tar.WriteHeader(&tar.Header{ - Name: fmt.Sprintf("balances.%d.%d.msgpack", cw.balancesChunkNum, cw.fileHeader.TotalChunks), - Mode: 0600, - Size: int64(len(encodedChunk)), - }) - - if err != nil { - return - } - _, err = cw.tar.Write(encodedChunk) - if err != nil { - return - } - + writerRequest <- cw.balancesChunk if len(cw.balancesChunk.Balances) < BalancesPerCatchpointFileChunk || cw.balancesChunkNum == cw.fileHeader.TotalChunks { - cw.tar.Close() - cw.gzip.Close() - cw.file.Close() - cw.balancesChunk.Balances = nil - cw.file = nil - var fileInfo os.FileInfo - fileInfo, err = os.Stat(cw.filePath) - if err != nil { - return false, err + cw.accountsIterator.Close() + // if we're done, wait for the writer to complete it's writing. + select { + case err, opened := <-writerResponse: + if opened { + // we ran into an error. wait for the channel to close before returning with the error. + select { + case <-writerResponse: + } + return false, err + } + // channel is closed. we're done writing and no issues detected. + return false, nil } - cw.writtenBytes = fileInfo.Size() - return false, nil } cw.balancesChunk.Balances = nil } } } +func (cw *catchpointWriter) asyncWriter(balances chan catchpointFileBalancesChunk, response chan error, initialBalancesChunkNum uint64) { + defer close(response) + balancesChunkNum := initialBalancesChunkNum + for bc := range balances { + balancesChunkNum++ + if len(bc.Balances) == 0 { + break + } + + encodedChunk := protocol.Encode(&bc) + err := cw.tar.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("balances.%d.%d.msgpack", balancesChunkNum, cw.fileHeader.TotalChunks), + Mode: 0600, + Size: int64(len(encodedChunk)), + }) + if err != nil { + response <- err + break + } + _, err = cw.tar.Write(encodedChunk) + if err != nil { + response <- err + break + } + + if len(bc.Balances) < BalancesPerCatchpointFileChunk || balancesChunkNum == cw.fileHeader.TotalChunks { + cw.tar.Close() + cw.gzip.Close() + cw.file.Close() + cw.file = nil + var fileInfo os.FileInfo + fileInfo, err = os.Stat(cw.filePath) + if err != nil { + response <- err + break + } + cw.writtenBytes = fileInfo.Size() + break + } + } +} + func (cw *catchpointWriter) readDatabaseStep(ctx context.Context, tx *sql.Tx) (err error) { cw.balancesChunk.Balances, err = cw.accountsIterator.Next(ctx, tx, BalancesPerCatchpointFileChunk) if err == nil { diff --git a/ledger/catchpointwriter_test.go b/ledger/catchpointwriter_test.go index 37fd63f98e..05d3f07a60 100644 --- a/ledger/catchpointwriter_test.go +++ b/ledger/catchpointwriter_test.go @@ -217,7 +217,7 @@ func TestBasicCatchpointWriter(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(len(accts)), uint64(len(balances.Balances))) } else { - require.Failf(t, "unexpected tar chunk name %s", header.Name) + require.Failf(t, "unexpected tar chunk name", "tar chunk name %s", header.Name) } } } From 93fd1c1a4b3c9706343d6a301f7ab1f3e2ec3d5e Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Fri, 20 Nov 2020 09:30:42 -0500 Subject: [PATCH 24/34] Add applications to min balance requirements calculation (#1715) Improve pingpong testing : add applications to min balance requirements calculation --- shared/pingpong/accounts.go | 25 ++++++++++++++++++++----- shared/pingpong/pingpong.go | 29 ++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/shared/pingpong/accounts.go b/shared/pingpong/accounts.go index b9bb2896c6..ba1fd56bf9 100644 --- a/shared/pingpong/accounts.go +++ b/shared/pingpong/accounts.go @@ -526,12 +526,27 @@ func sendAsGroup(txgroup []transactions.Transaction, client libgoal.Client, h [] return } -func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) (appParams map[uint64]v1.AppParams, optIns map[uint64][]string, err error) { - status, err := client.Status() - if err != nil { - return +var proto *config.ConsensusParams + +func getProto(client libgoal.Client) (config.ConsensusParams, error) { + if proto == nil { + var err error + status, err := client.Status() + if err != nil { + return config.ConsensusParams{}, err + } + currentProto, err := client.ConsensusParams(status.LastRound) + if err != nil { + return config.ConsensusParams{}, err + } + proto = ¤tProto } - proto, err := client.ConsensusParams(status.LastRound) + + return *proto, nil +} + +func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) (appParams map[uint64]v1.AppParams, optIns map[uint64][]string, err error) { + proto, err := getProto(client) if err != nil { return } diff --git a/shared/pingpong/pingpong.go b/shared/pingpong/pingpong.go index f715d5a9cb..3249ada4a6 100644 --- a/shared/pingpong/pingpong.go +++ b/shared/pingpong/pingpong.go @@ -127,8 +127,13 @@ func prepareNewAccounts(client libgoal.Client, cfg PpConfig, wallet []byte, acco } // determine the min balance per participant account -func computeAccountMinBalance(cfg PpConfig) (requiredBalance uint64) { - const minActiveAccountBalance uint64 = 100000 // min balance for any active account +func computeAccountMinBalance(client libgoal.Client, cfg PpConfig) (requiredBalance uint64, err error) { + proto, err := getProto(client) + if err != nil { + return + } + + minActiveAccountBalance := proto.MinBalance if cfg.NumApp > 0 { requiredBalance = (cfg.MinAccountFunds + (cfg.MaxAmt+cfg.MaxFee)*10) * 2 @@ -141,6 +146,12 @@ func computeAccountMinBalance(cfg PpConfig) (requiredBalance uint64) { } if cfg.MaxFee != 0 { fee = cfg.MaxFee + } else { + // follow the same logic as constructTxn + fee, err = client.SuggestedFee() + if err != nil { + return + } } requiredBalance = minActiveAccountBalance @@ -152,6 +163,15 @@ func computeAccountMinBalance(cfg PpConfig) (requiredBalance uint64) { (fee)*uint64(cfg.NumAsset)*uint64(cfg.NumPartAccounts) // asset distributions requiredBalance += assetCost } + if cfg.NumApp > 0 { + creationCost := uint64(cfg.NumApp) * proto.AppFlatParamsMinBalance * uint64(proto.MaxAppsCreated) + optInCost := uint64(cfg.NumApp) * proto.AppFlatOptInMinBalance * uint64(proto.MaxAppsOptedIn) + maxGlobalSchema := basics.StateSchema{NumUint: proto.MaxGlobalSchemaEntries, NumByteSlice: proto.MaxGlobalSchemaEntries} + maxLocalSchema := basics.StateSchema{NumUint: proto.MaxLocalSchemaEntries, NumByteSlice: proto.MaxLocalSchemaEntries} + schemaCost := uint64(cfg.NumApp) * (maxGlobalSchema.MinBalance(&proto).Raw*uint64(proto.MaxAppsCreated) + + maxLocalSchema.MinBalance(&proto).Raw*uint64(proto.MaxAppsOptedIn)) + requiredBalance += creationCost + optInCost + schemaCost + } // add cost of transactions requiredBalance += (cfg.MaxAmt + fee) * 2 * cfg.TxnPerSec * uint64(math.Ceil(cfg.RefreshTime.Seconds())) @@ -172,7 +192,10 @@ func fundAccounts(accounts map[string]uint64, client libgoal.Client, cfg PpConfi // Fee of 0 will make cause the function to use the suggested one by network fee := uint64(0) - minFund := computeAccountMinBalance(cfg) + minFund, err := computeAccountMinBalance(client, cfg) + if err != nil { + return err + } fmt.Printf("adjusting account balance to %d\n", minFund) for addr, balance := range accounts { From 97673c3fc0eb8ab18f5b9f3b8f6214a05bc812ad Mon Sep 17 00:00:00 2001 From: egieseke Date: Fri, 20 Nov 2020 21:53:37 -0500 Subject: [PATCH 25/34] Fix intermittent failure of algohTimeoutTest.exp (#1706) Fix error running algohTimeoutTest.exp on arm64 by providing sufficient time for network to start --- test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp b/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp index e0b18e5cc1..4067776ca1 100644 --- a/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp +++ b/test/e2e-go/cli/algoh/expect/algohTimeoutTest.exp @@ -13,7 +13,15 @@ if { [catch { exec mkdir -p $TEST_ALGO_DIR set TIME_STAMP [clock seconds] - set timeout 5 + + set archType [exec $TEST_DATA_DIR/../../scripts/archtype.sh] + puts "arch type: $archType" + if {[regexp "arm64" $archType]} { + set timeout 60 + } else { + set timeout 5 + } + puts "set timeout to $timeout" set TEST_ROOT_DIR $TEST_ALGO_DIR/root set TEST_PRIMARY_NODE_DIR $TEST_ROOT_DIR/Primary @@ -22,11 +30,14 @@ if { [catch { set NETWORK_TEMPLATE "$TEST_DATA_DIR/nettemplates/TwoNodes50Each.json" puts "TEST_ALGO_DIR: $TEST_ALGO_DIR" - ::Algoh::CreateNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ROOT_DIR ::Algoh::StartNode $TEST_PRIMARY_NODE_DIR + puts "Successfully started node" + + set timeout 5 + exec cat ./disabled_profiler_config.json > $TEST_NODE_DIR/config.json exec cat ./host-config.json > $TEST_NODE_DIR/host-config.json From 6e7776e61580ef59dc0b5558131eab3bc7302303 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Mon, 23 Nov 2020 16:35:08 -0500 Subject: [PATCH 26/34] Derive the swagger version from the go.mod file (#1709) Derive the swagger version from the go.mod file --- go.mod | 7 +- go.sum | 482 ++++++++++++++++++++++++++++++++++ logging/telemetryCommon.go | 19 +- logging/telemetryConfig.go | 15 +- scripts/configure_dev-deps.sh | 29 +- 5 files changed, 523 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index c8e58e0d63..ff8d7fe5b9 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/gen2brain/beeep v0.0.0-20180718162406-4e430518395f github.com/getkin/kin-openapi v0.22.0 github.com/go-chi/chi v4.1.2+incompatible // indirect + github.com/go-swagger/go-swagger v0.25.0 github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f // indirect github.com/gofrs/flock v0.7.0 github.com/google/go-querystring v1.0.0 @@ -26,12 +27,10 @@ require ( github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.6.2 github.com/gorilla/schema v1.0.2 - github.com/gorilla/websocket v1.4.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmoiron/sqlx v1.2.0 github.com/karalabe/hid v0.0.0-20181128192157-d815e0c1a2e2 github.com/labstack/echo/v4 v4.1.17 - github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect github.com/matryer/moq v0.1.3 // indirect github.com/mattn/go-sqlite3 v1.10.0 github.com/miekg/dns v1.1.27 @@ -42,16 +41,14 @@ require ( github.com/onsi/gomega v1.5.0 // indirect github.com/russross/blackfriday v1.5.2 // indirect github.com/satori/go.uuid v1.2.0 - github.com/sirupsen/logrus v1.0.5 + github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.3 - github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.6.1 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/net v0.0.0-20200904194848-62affa334b73 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f golang.org/x/text v0.3.3 // indirect golang.org/x/tools v0.0.0-20200904185747-39188db58858 // indirect - google.golang.org/appengine v1.6.1 // indirect gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 7447e5910d..0b2d0e7956 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,26 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/algorand/go-codec v1.1.2 h1:QWS9YC3EEWBpJq5AqFPELcCJ2QPpTIg9aqR2K/sRDq4= github.com/algorand/go-codec v1.1.2/go.mod h1:A3YI4V24jUUnU1eNekNmx2fLi60FvlNssqOiUsyfNM8= github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d h1:W9MgGUodEl4Y4+CxeEr+T3fZ26kOcWA4yfqhjbFxxmI= @@ -14,10 +37,32 @@ github.com/algorand/oapi-codegen v1.3.5-algorand5 h1:y576Ca2/guQddQrQA7dtL5KcOx5 github.com/algorand/oapi-codegen v1.3.5-algorand5/go.mod h1:/k0Ywn0lnt92uBMyE+yiRf/Wo3/chxHHsAfenD09EbY= github.com/algorand/websocket v1.4.1 h1:FPoNHI8i2VZWZzhCscY8JTzsAE7Vv73753cMbzb3udk= github.com/algorand/websocket v1.4.1/go.mod h1:0nFSn+xppw/GZS9hgWPS3b8/4FcA3Pj7XQxm+wqHGx8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/aws/aws-sdk-go v1.16.5 h1:NVxzZXIuwX828VcJrpNxxWjur1tlOBISdMdDdHIKHcc= github.com/aws/aws-sdk-go v1.16.5/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M= github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20191230040416-20a325f050d4 h1:Fphwr1XDjkTR/KFbrrkLfY6D2CEOlHqFGomQQrxcHFs= github.com/cyberdelia/templates v0.0.0-20191230040416-20a325f050d4/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,12 +74,19 @@ github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4= github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gen2brain/beeep v0.0.0-20180718162406-4e430518395f h1:eyHMPp7tXlBMF8PZHdsL89G0ehuRNflu7zKUeoQjcJ0= github.com/gen2brain/beeep v0.0.0-20180718162406-4e430518395f/go.mod h1:GprdPCZglWh5OMcIDpeKBxuUJI+fEDOTVUfxZeda4zo= github.com/getkin/kin-openapi v0.3.1/go.mod h1:W8dhxZgpE84ciM+VIItFqkmZ4eHtuomrdIHtASQIqi0= @@ -44,49 +96,234 @@ github.com/getkin/kin-openapi v0.22.0 h1:J5IFyKd/5yuB6AZAgwK0CMBKnabWcmkowtsl6bR github.com/getkin/kin-openapi v0.22.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4= github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/analysis v0.19.10 h1:5BHISBAXOc/aJK25irLZnx2D3s6WyYaY9D4gmuz9fdE= +github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.6 h1:xZMThgv5SQ7SMbWtKFkCf9bBdvR2iEyw9k3zGZONuys= +github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4= +github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.4 h1:3Vw+rh13uq2JFNxgnMTGE1rnoieU9FmyE1gvnyylsYg= +github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= +github.com/go-openapi/loads v0.19.5 h1:jZVYWawIQiA1NBnHla28ktg6hrcfTHsCE+3QLVRBIls= +github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= +github.com/go-openapi/runtime v0.19.20 h1:J/t+QIjbcoq8WJvjGxRKiFBhqUE8slS9SbmD0Oi/raQ= +github.com/go-openapi/runtime v0.19.20/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8 h1:qAdZLh1r6QF/hI/gTq+TJTvsQUodZsM7KLqkAJdiJNg= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.5 h1:0utjKrw+BAh8s57XE9Xz8DUBsVvPmRUB6styvl9wWIM= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9 h1:1IxuqvBUU3S2Bi4YC7tlP9SJF1gVpCvqN0T2Qof4azE= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/go-openapi/validate v0.19.10 h1:tG3SZ5DC5KF4cyt7nqLVcQXGj5A7mpaYkAcNPlDK+Yk= +github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-swagger/go-swagger v0.25.0 h1:FxhyrWWV8V/A9P6GtI5szWordAdbb6Y0nqdY/y9So2w= +github.com/go-swagger/go-swagger v0.25.0/go.mod h1:9639ioXrPX9E6BbnbaDklGXjNz7upAXoNBwL4Ok11Vk= +github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013/go.mod h1:b65mBPzqzZWxOZGxSWrqs4GInLIn+u99Q9q7p+GKni0= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f h1:zlOR3rOlPAVvtfuxGKoghCmop5B0TRyu/ZieziZuGiM= github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g= github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherwasm v1.0.1 h1:Gmj9RMDjh+P9EFzzQltoCtjAxR5mUkaJqaNPfeaNe2I= github.com/gopherjs/gopherwasm v1.0.1/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/hid v0.0.0-20181128192157-d815e0c1a2e2 h1:BkkpZxPVs3gIf+3Tejt8lWzuo2P29N1ChGUMEpuSJ8U= github.com/karalabe/hid v0.0.0-20181128192157-d815e0c1a2e2/go.mod h1:YvbcH+3Wo6XPs9nkgTY3u19KXLauXW+J5nB7hEHuX0A= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= github.com/labstack/echo/v4 v4.1.17 h1:PQIBaRplyRy3OjwILGkPg89JRtH2x5bssi59G2EL3fo= @@ -95,17 +332,28 @@ github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/moq v0.0.0-20200310130814-7721994d1b54 h1:p8zN0Xu28xyEkPpqLbFXAnjdgBVvTJCpfOtoDf/+/RQ= github.com/matryer/moq v0.0.0-20200310130814-7721994d1b54/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/matryer/moq v0.1.3 h1:+fW3u2jmlPw59a3V6spZKOLCcvrDKzPjMsRvUhnZ/c0= github.com/matryer/moq v0.1.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -113,12 +361,28 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olivere/elastic v6.2.14+incompatible h1:k+KadwNP/dkXE0/eu+T6otk1+5fe0tEpPyQJ4XVm5i8= github.com/olivere/elastic v6.2.14+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -126,30 +390,88 @@ github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.3.2 h1:GDarE4TJQI52kYSbSAmLiId1Elfj+xgSDqrUZxFhxlU= +github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ= +github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -159,72 +481,183 @@ github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.5 h1:S0ZOruh4YGHjD7JoN7mIsTrNjnQbOjrmgrx6l6pZN7I= +go.mongodb.org/mongo-driver v1.3.5/go.mod h1:Ual6Gkco7ZGQw8wE1t4tLnvBsf6yVSM60qW6TgOeJ5c= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200423205358-59e73619c742 h1:9OGWpORUXvk8AsaBJlpzzDx7Srv/rSK6rvjcsJq4rJo= golang.org/x/tools v0.0.0-20200423205358-59e73619c742/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858 h1:xLt+iB5ksWcZVxqc+g9K41ZHy+6MKWfXCDsjSThnsPA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= @@ -234,29 +667,78 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 h1:q/fZgS8MMadqFFGa8WL4Oyz+TmjiZfi8UrzWhTl8d5w= gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009/go.mod h1:O0bY1e/dSoxMYZYTHP0SWKxG5EWLEvKR9/cOjWPPMKU= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/toast.v1 v1.0.0-20180812000517-0a84660828b2 h1:MZF6J7CV6s/h0HBkfqebrYfKCVEo5iN+wzE4QhV3Evo= gopkg.in/toast.v1 v1.0.0-20180812000517-0a84660828b2/go.mod h1:s1Sn2yZos05Qfs7NKt867Xe18emOmtsO3eAKbDaon0o= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/logging/telemetryCommon.go b/logging/telemetryCommon.go index af24d732fe..c7202fd000 100644 --- a/logging/telemetryCommon.go +++ b/logging/telemetryCommon.go @@ -59,15 +59,24 @@ type TelemetryConfig struct { URI string Name string GUID string - MinLogLevel logrus.Level - ReportHistoryLevel logrus.Level - FilePath string // Path to file on disk, if any - ChainID string `json:"-"` - SessionGUID string `json:"-"` + MinLogLevel logrus.Level `json:"-"` // these are the logrus.Level, but we can't use it directly since on logrus version 1.4.2 they added + ReportHistoryLevel logrus.Level `json:"-"` // text marshalers which breaks our backward compatibility. + FilePath string // Path to file on disk, if any + ChainID string `json:"-"` + SessionGUID string `json:"-"` UserName string Password string } +// MarshalingTelemetryConfig is used for json serialization of the TelemetryConfig +// so that we could replace the MinLogLevel/ReportHistoryLevel with our own types. +type MarshalingTelemetryConfig struct { + TelemetryConfig + + MinLogLevel uint32 + ReportHistoryLevel uint32 +} + type asyncTelemetryHook struct { deadlock.Mutex wrappedHook logrus.Hook diff --git a/logging/telemetryConfig.go b/logging/telemetryConfig.go index d8fd544698..49da8e1378 100644 --- a/logging/telemetryConfig.go +++ b/logging/telemetryConfig.go @@ -83,11 +83,14 @@ func (cfg TelemetryConfig) Save(configPath string) error { } defer f.Close() - sanitizedCfg := cfg - sanitizedCfg.FilePath = "" + var marshaledConfig MarshalingTelemetryConfig + marshaledConfig.TelemetryConfig = cfg + marshaledConfig.TelemetryConfig.FilePath = "" + marshaledConfig.MinLogLevel = uint32(cfg.MinLogLevel) + marshaledConfig.ReportHistoryLevel = uint32(cfg.ReportHistoryLevel) enc := json.NewEncoder(f) - err = enc.Encode(sanitizedCfg) + err = enc.Encode(marshaledConfig) return err } @@ -131,8 +134,12 @@ func loadTelemetryConfig(path string) (TelemetryConfig, error) { } defer f.Close() cfg := createTelemetryConfig() + var marshaledConfig MarshalingTelemetryConfig dec := json.NewDecoder(f) - err = dec.Decode(&cfg) + err = dec.Decode(&marshaledConfig) + cfg = marshaledConfig.TelemetryConfig + cfg.MinLogLevel = logrus.Level(marshaledConfig.MinLogLevel) + cfg.ReportHistoryLevel = logrus.Level(marshaledConfig.ReportHistoryLevel) cfg.FilePath = path // Sanitize user-defined name. diff --git a/scripts/configure_dev-deps.sh b/scripts/configure_dev-deps.sh index 8a24fa1958..ed74f64d22 100755 --- a/scripts/configure_dev-deps.sh +++ b/scripts/configure_dev-deps.sh @@ -3,35 +3,34 @@ set -ex function get_go_version { - if [[ -z "$2" ]] - then - ( - cd "$(dirname "$0")" - VERSION=$( grep "$1" 2>/dev/null < ../go.mod | awk -F " " '{print $2}') - echo "$VERSION" - ) - else - (echo "$2") - fi + cd "$(dirname "$0")" + VERSION=$( grep "$1" 2>/dev/null < ../go.mod | awk -F " " '{print $2}') + echo "$VERSION" return } function install_go_module { local OUTPUT + local MODULE + if [[ "$2" != "" ]]; then + MODULE=$2 + else + MODULE=$1 + fi # Check for version to go.mod version - VERSION=$(get_go_version "$1" "$2") + VERSION=$(get_go_version "$1") if [ -z "$VERSION" ]; then - OUTPUT=$(GO111MODULE=off go get -u "$1" 2>&1) + OUTPUT=$(GO111MODULE=off go get -u "${MODULE}" 2>&1) else - OUTPUT=$(cd && GO111MODULE=on go get "$1@${VERSION}" 2>&1) + OUTPUT=$(cd && GO111MODULE=on go get "${MODULE}@${VERSION}" 2>&1) fi if [ $? != 0 ]; then - echo "error: executing \"go get $1\" failed : ${OUTPUT}" + echo "error: executing \"go get ${MODULE}\" failed : ${OUTPUT}" exit 1 fi } install_go_module golang.org/x/lint/golint install_go_module golang.org/x/tools/cmd/stringer -install_go_module github.com/go-swagger/go-swagger/cmd/swagger v0.25.0 +install_go_module github.com/go-swagger/go-swagger github.com/go-swagger/go-swagger/cmd/swagger install_go_module github.com/algorand/msgp From 35b83c9460428893abb78a3af21df74611086e37 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 24 Nov 2020 08:51:29 -0500 Subject: [PATCH 27/34] Optimize fast catchup (#1717) Restore the merkle tree during the catchpoint catchup in it's hash order. --- catchup/catchpointService.go | 18 +- cmd/goal/messages.go | 2 +- cmd/goal/node.go | 2 +- .../mocks/mockCatchpointCatchupAccessor.go | 5 + daemon/algod/api/Makefile | 10 +- daemon/algod/api/algod.oas2.json | 6 +- daemon/algod/api/algod.oas3.yml | 18 +- .../api/server/v2/generated/private/routes.go | 252 +++++++------- .../api/server/v2/generated/private/types.go | 5 +- .../algod/api/server/v2/generated/routes.go | 323 +++++++++--------- daemon/algod/api/server/v2/generated/types.go | 5 +- daemon/algod/api/server/v2/handlers.go | 1 + .../algod/api/server/v2/test/handlers_test.go | 1 + daemon/algod/api/server/v2/test/helpers.go | 1 + ledger/accountdb.go | 216 +++++++++--- ledger/accountdb_test.go | 4 +- ledger/acctupdates.go | 2 +- ledger/catchpointwriter.go | 1 - ledger/catchupaccessor.go | 308 ++++++++++++----- node/node.go | 2 + 20 files changed, 751 insertions(+), 431 deletions(-) diff --git a/catchup/catchpointService.go b/catchup/catchpointService.go index 1059d35276..efbef2b542 100644 --- a/catchup/catchpointService.go +++ b/catchup/catchpointService.go @@ -43,6 +43,7 @@ type CatchpointCatchupStats struct { CatchpointLabel string TotalAccounts uint64 ProcessedAccounts uint64 + VerifiedAccounts uint64 TotalBlocks uint64 AcquiredBlocks uint64 VerifiedBlocks uint64 @@ -277,11 +278,15 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) { } err = ledgerFetcher.downloadLedger(cs.ctx, round) if err == nil { - break + err = cs.ledgerAccessor.BuildMerkleTrie(cs.ctx, cs.updateVerifiedAccounts) + if err == nil { + break + } + // failed to build the merkle trie for the above catchpoint file. } // instead of testing for err == cs.ctx.Err() , we'll check on the context itself. // this is more robust, as the http client library sometimes wrap the context canceled - // error with other erros. + // error with other errors. if cs.ctx.Err() != nil { return cs.stopOrAbort() } @@ -300,6 +305,13 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) { return nil } +// updateVerifiedAccounts update the user's statistics for the given verified accounts +func (cs *CatchpointCatchupService) updateVerifiedAccounts(verifiedAccounts uint64) { + cs.statsMu.Lock() + defer cs.statsMu.Unlock() + cs.stats.VerifiedAccounts = verifiedAccounts +} + // processStageLastestBlockDownload is the third catchpoint catchup stage. It downloads the latest block and verify that against the previously downloaded ledger. func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err error) { blockRound, err := cs.ledgerAccessor.GetCatchupBlockRound(cs.ctx) @@ -328,6 +340,7 @@ func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err erro if attemptsCount <= cs.config.CatchupBlockDownloadRetryAttempts { // try again. blk = nil + cs.log.Infof("processStageLastestBlockDownload: block %d download failed, another attempt will be made; err = %v", blockRound, err) continue } return cs.abort(fmt.Errorf("processStageLastestBlockDownload failed to get block %d : %v", blockRound, err)) @@ -369,6 +382,7 @@ func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err erro if attemptsCount <= cs.config.CatchupBlockDownloadRetryAttempts { // try again. blk = nil + cs.log.Infof("processStageLastestBlockDownload: block %d verification against catchpoint failed, another attempt will be made; err = %v", blockRound, err) continue } return cs.abort(fmt.Errorf("processStageLastestBlockDownload failed when calling VerifyCatchpoint : %v", err)) diff --git a/cmd/goal/messages.go b/cmd/goal/messages.go index 08cbad199a..c2caea95c6 100644 --- a/cmd/goal/messages.go +++ b/cmd/goal/messages.go @@ -64,7 +64,7 @@ const ( infoNodeStatus = "Last committed block: %d\nTime since last block: %s\nSync Time: %s\nLast consensus protocol: %s\nNext consensus protocol: %s\nRound for next consensus protocol: %d\nNext consensus protocol supported: %v" catchupStoppedOnUnsupported = "Last supported block (%d) is committed. The next block consensus protocol is not supported. Catchup service is stopped." infoNodeCatchpointCatchupStatus = "Last committed block: %d\nSync Time: %s\nCatchpoint: %s" - infoNodeCatchpointCatchupAccounts = "Catchpoint total accounts: %d\nCatchpoint accounts processed: %d" + infoNodeCatchpointCatchupAccounts = "Catchpoint total accounts: %d\nCatchpoint accounts processed: %d\nCatchpoint accounts verified: %d" infoNodeCatchpointCatchupBlocks = "Catchpoint total blocks: %d\nCatchpoint downloaded blocks: %d" nodeLastCatchpoint = "Last Catchpoint: %s" errorNodeCreationIPFailure = "Parsing passed IP %v failed: need a valid IPv4 or IPv6 address with a specified port number" diff --git a/cmd/goal/node.go b/cmd/goal/node.go index 74b8daaefa..ed6ba63749 100644 --- a/cmd/goal/node.go +++ b/cmd/goal/node.go @@ -423,7 +423,7 @@ func makeStatusString(stat generatedV2.NodeStatusResponse) string { if stat.CatchpointTotalAccounts != nil && (*stat.CatchpointTotalAccounts > 0) && stat.CatchpointProcessedAccounts != nil { statusString = statusString + "\n" + fmt.Sprintf(infoNodeCatchpointCatchupAccounts, *stat.CatchpointTotalAccounts, - *stat.CatchpointProcessedAccounts) + *stat.CatchpointProcessedAccounts, *stat.CatchpointVerifiedAccounts) } if stat.CatchpointAcquiredBlocks != nil && stat.CatchpointTotalBlocks != nil && (*stat.CatchpointAcquiredBlocks+*stat.CatchpointTotalBlocks > 0) { statusString = statusString + "\n" + fmt.Sprintf(infoNodeCatchpointCatchupBlocks, *stat.CatchpointTotalBlocks, diff --git a/components/mocks/mockCatchpointCatchupAccessor.go b/components/mocks/mockCatchpointCatchupAccessor.go index 53363e7bc8..b5c9af34f2 100644 --- a/components/mocks/mockCatchpointCatchupAccessor.go +++ b/components/mocks/mockCatchpointCatchupAccessor.go @@ -57,6 +57,11 @@ func (m *MockCatchpointCatchupAccessor) ProgressStagingBalances(ctx context.Cont return nil } +// BuildMerkleTrie inserts the account hashes into the merkle trie +func (m *MockCatchpointCatchupAccessor) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64)) (err error) { + return nil +} + // GetCatchupBlockRound returns the latest block round matching the current catchpoint func (m *MockCatchpointCatchupAccessor) GetCatchupBlockRound(ctx context.Context) (round basics.Round, err error) { return basics.Round(0), nil diff --git a/daemon/algod/api/Makefile b/daemon/algod/api/Makefile index 0512d68827..c2818fce5c 100644 --- a/daemon/algod/api/Makefile +++ b/daemon/algod/api/Makefile @@ -16,13 +16,15 @@ server/v2/generated/private/types.go: algod.oas3.yml server/v2/generated/private/routes.go: algod.oas3.yml oapi-codegen -package private -type-mappings integer=uint64 -generate server,spec -include-tags=private -o ./server/v2/generated/private/routes.go algod.oas3.yml -algod.oas3.yml: .3tmp.json - python3 jsoncanon.py < .3tmp.json > algod.oas3.yml - -.3tmp.json: algod.oas2.json +algod.oas3.yml: algod.oas2.json curl -s -X POST "https://converter.swagger.io/api/convert" -H "accept: application/json" -H "Content-Type: application/json" -d @./algod.oas2.json -o .3tmp.json + python3 jsoncanon.py < .3tmp.json > algod.oas3.yml + rm -f .3tmp.json oapi-codegen: .PHONY GO111MODULE=on go get -u "github.com/algorand/oapi-codegen/...@v1.3.5-algorand5" +clean: + rm -rf server/v2/generated/types.go server/v2/generated/routes.go server/v2/generated/private/types.go server/v2/generated/private/routes.go algod.oas3.yml + .PHONY: diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 02550a5803..87a98306ae 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -2104,7 +2104,11 @@ "type": "integer" }, "catchpoint-processed-accounts": { - "description": "The number of account from the current catchpoint that have been processed so far as part of the catchup", + "description": "The number of accounts from the current catchpoint that have been processed so far as part of the catchup", + "type": "integer" + }, + "catchpoint-verified-accounts": { + "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup", "type": "integer" }, "catchpoint-total-blocks": { diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index a06170f41a..b38d27985f 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -386,7 +386,7 @@ "type": "integer" }, "catchpoint-processed-accounts": { - "description": "The number of account from the current catchpoint that have been processed so far as part of the catchup", + "description": "The number of accounts from the current catchpoint that have been processed so far as part of the catchup", "type": "integer" }, "catchpoint-total-accounts": { @@ -397,6 +397,10 @@ "description": "The total number of blocks that are required to complete the current catchpoint catchup", "type": "integer" }, + "catchpoint-verified-accounts": { + "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup", + "type": "integer" + }, "catchup-time": { "description": "CatchupTime in nanoseconds", "type": "integer" @@ -2357,7 +2361,7 @@ "type": "integer" }, "catchpoint-processed-accounts": { - "description": "The number of account from the current catchpoint that have been processed so far as part of the catchup", + "description": "The number of accounts from the current catchpoint that have been processed so far as part of the catchup", "type": "integer" }, "catchpoint-total-accounts": { @@ -2368,6 +2372,10 @@ "description": "The total number of blocks that are required to complete the current catchpoint catchup", "type": "integer" }, + "catchpoint-verified-accounts": { + "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup", + "type": "integer" + }, "catchup-time": { "description": "CatchupTime in nanoseconds", "type": "integer" @@ -2480,7 +2488,7 @@ "type": "integer" }, "catchpoint-processed-accounts": { - "description": "The number of account from the current catchpoint that have been processed so far as part of the catchup", + "description": "The number of accounts from the current catchpoint that have been processed so far as part of the catchup", "type": "integer" }, "catchpoint-total-accounts": { @@ -2491,6 +2499,10 @@ "description": "The total number of blocks that are required to complete the current catchpoint catchup", "type": "integer" }, + "catchpoint-verified-accounts": { + "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup", + "type": "integer" + }, "catchup-time": { "description": "CatchupTime in nanoseconds", "type": "integer" diff --git a/daemon/algod/api/server/v2/generated/private/routes.go b/daemon/algod/api/server/v2/generated/private/routes.go index 14c53e1f99..8c3a30f411 100644 --- a/daemon/algod/api/server/v2/generated/private/routes.go +++ b/daemon/algod/api/server/v2/generated/private/routes.go @@ -235,132 +235,132 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9e3PcNpL4V8Fvdqv8+A1n5Fd2rarUnmI7iS6O47KU3N1avgRD9swgIgGGACVNfPru", - "V90ASJAEZ0ayznep3b9sDYBGo9EvNBrNj5NUFaWSII2eHH6clLziBRio6C+epqqWJhEZ/pWBTitRGqHk", - "5NC3MW0qIVeT6UTgryU368l0InkBbR8cP51U8FstKsgmh6aqYTrR6RoKjoDNpsTeDaSrZKUSB+LIgjh+", - "Obne0sCzrAKth1j+IPMNEzLN6wyYqbjUPMUmzS6FWTOzFpq5wUxIpiQwtWRm3enMlgLyTM/8In+rodoE", - "q3STjy/pukUxqVQOQzxfqGIhJHisoEGq2RBmFMtgSZ3W3DCcAXH1HY1iGniVrtlSVTtQtUiE+IKsi8nh", - "+4kGmUFFu5WCuKD/LiuA3yExvFqBmXyYxha3NFAlRhSRpR076leg69xoRn1pjStxAZLhqBn7vtaGLYBx", - "yd59/YI9efLkOS6k4MZA5phsdFXt7OGa7PDJ4STjBnzzkNd4vlIVl1nS9H/39Qua/8QtcN9eXGuIC8sR", - "trDjl2ML8AMjLCSkgRXtQ4f7cUREKNqfF7BUFey5J7bznW5KOP//6q6k3KTrUglpIvvCqJXZ5qgOC4Zv", - "02ENAp3+JVKqQqDvD5LnHz4+mj46uP7T+6Pk7+7PZ0+u91z+iwbuDgpEO6Z1VYFMN8mqAk7SsuZySI93", - "jh/0WtV5xtb8gjafF6Tq3ViGY63qvOB5jXwi0kod5SulGXdslMGS17lhfmJWyxzVFEJz3M6EZmWlLkQG", - "2RS17+VapGuWcm1BUD92KfIcebDWkI3xWnx1W4TpOiQJ4nUretCC/u8So13XDkrAFWmDJM2VhsSoHebJ", - "WxwuMxYalNZW6ZsZK3a6BkaTY4M1tkQ7iTyd5xtmaF8zxjXjzJumKRNLtlE1u6TNycU5jXerQaoVDIlG", - "m9Oxoyi8Y+QbECNCvIVSOXBJxPNyNySZXIpVXYFml2swa2fzKtClkhqYWvwKqcFt/9eTH94wVbHvQWu+", - "grc8PWcgU5WN77GbNGbBf9UKN7zQq5Kn53FznYtCRFD+nl+Joi6YrIsFVLhf3j4YxSowdSXHELIQd/BZ", - "wa+Gk55WtUxpc9tpO44aspLQZc43M3a8ZAW/+vJg6tDRjOc5K0FmQq6YuZKjThrOvRu9pFK1zPbwYQxu", - "WGA1dQmpWArIWANlCyZuml34CHkzfFrPKkDHAxlFp5llBzoSriI8g6KLLazkKwhYZsZ+dJqLWo06B9ko", - "OLbYUFNZwYVQtW4GjeBIU293r6UykJQVLEWEx04cOVB72D5OvRbOwUmVNFxIyFDzEtLKgNVEozgFE24/", - "zAxN9IJr+OLpmAFvW/fc/aXq7/rWHd9rt6lTYkUyYhex1Qls3G3qjN/j8BfOrcUqsT8PNlKsTtGULEVO", - "ZuZX3D9PhlqTEugQwhseLVaSm7qCwzP5EP9iCTsxXGa8yvCXwv70fZ0bcSJW+FNuf3qtViI9EasRYja4", - "Rk9TNKyw/yC8uDo2V9FDw2ulzusyXFDaOZUuNuz45dgmW5g3Zcyj5igbnipOr/xJ46YjzFWzkSNIjtKu", - "5NjxHDYVILY8XdI/V0viJ76sfsd/yjKP0RQZ2BlaCgq4YME79xv+hCIP9kyAUETKkahzMp+HHwOE/lzB", - "cnI4+dO8jZTMbaueO7g44/V0ctTCufuZ2pF2fb2DTNvMhLS7Q12n9kx49/gg1Cgm5Kj2cPgqV+n5rXAo", - "K1VCZYTdxwXCGUoKgWdr4BlULOOGz9pDlfWzRvidBn5L4+iUBFXExP1A/+E5w2aUQm68+4auq9DoxKkg", - "0JShx2ftiJ0JO5AnqlhhnTyGztmNsHzRTm4VdKNR3zuyfOhDi+zOK+tXMhrhF4FLb0+NRwtV3Y5feowg", - "WXsWZhyhNt4vrry7s9S1LhNHn4g/bTv0ALXhx6FaDSnUBx+jVYcKJ4b/D1BBI9S7oEIX0F1TQRWlyOEO", - "5HXN9Xq4CHRwnjxmJ98ePXv0+OfHz75AC11WalXxgi02BjS77+wK02aTw4PhykjB17mJQ//iqT9BdeHu", - "pBAh3MDeR6JOATWDpRiz8QLE7mW1qWp5BySEqlJVxOcl1jEqVXlyAZUWKhK+eOt6MNcD9ZD1u3u/W2zZ", - "JdcM56bjWC0zqGYxyuM5i0y6gULvMhQW9OmVbGnjAPKq4pvBDtj1Rlbn5t1nT7rE9969ZiVUibmSLINF", - "vQptFFtWqmCcZTSQFOIblcGJ4abWd6AFWmAtMrgRIQp8oWrDOJMqQ4HGznH9MBLLpCAKxX5MqHLM2tqf", - "BaB3nPJ6tTYM3UoV29p2YMJTuykJ2Qo9cvRrzuy2l53OxsnyCni2YQsAydTCna/cyY8WySksY/yNi9NO", - "LVrNmaCDV1mpFLSGLHHXSztR81dVtMlmC5kIb8K3mYRpxZa8uiWuRhme78CT+gyx1a034c6kQ6z3m37b", - "/vUnD3eRV3jEtEyArgsKdw4Gxki4kyZ1OXId4azaqShQJJjkUmlIlcx0FFjOtUl2iQJ26phe3NaA+2Lc", - "T4BHDt2vuTb22CtkRm6YFWGah8bQFOMIj2pphPyTV9BD2CnqHqlr3WhrXZelqgxksTVIuNoy1xu4auZS", - "ywB2YxKMYrWGXZDHqBTAd8SyK7EE4sbFXZq40HBxFOJG3bqJkrKDREuIbYic+F4BdcOQ7Agi6LM3I4lx", - "hO5xThMHnk60UWWJOskktWzGjZHpxPY+Mj+2fYfMxU2rKzMFOLvxODnMLy1lbTB+zdFfIsis4Oeo78n7", - "sefzIc4ojIkWMoVkG+ejWJ5gr1AEdgjpiOPprvuC2XrC0ePfKNONMsGOXRhb8IgX/NZGlU/biMsdOAIv", - "wXCR68bYN6HrdhaKcvczENAzqyAFafIN8upSVIW9KCIbof1v1pXI3Cz2SqQVP5mxCi55lfkewxNIsJhE", - "yAyu4tqVd+INGVwxEUd62cwsDEv9NY4MAcyigu4uxrag4AINt5kch8antdc+lko6diFIDSgAhUgrxe09", - "Hy7GGknTXGVVUHDEjm6cnFEfn1PIVWKvFSPm0bb7a0cf7g15Jg7X88moZDescbkGuslAdd0jYshteEwD", - "DWMLWeVqwfMEnVdIMsjNzjASOsXwknqinVTpcHgX5bOz93l2dvaBvca+5CcDO4fNnG5fWbrmcgVtSDzk", - "U+sBwxWkdajSe2Tc61Dj4n5d7LvHmumkVCpPmuNbP4Q/UPN9up+L9BwyhnqCnE5nfe51dwgnYfeRxXVz", - "yXG53nh/tixBQvZgxtiRZFCUZuNiBT1Poze5vGe2zX9Fs2Y13bdyyWiRszMZP6bb29pPlCkPZrsk2fSl", - "T5zKAtk+kbmSI+LEL+myAcFF5XNrpO+ERgYmZ2BJA6ayWOxzHv6Gcnp4Z5dFRoeN1qroelEISuwJuk1R", - "c/q71uFpVZgZY6ekO/C0oOECKp5T1oL2QVChWSHw0KnrNAXIDs9k0sEkVYWb+H77X6uWzuqDgyfADh70", - "x2iDbqI7GFkZ6I/9kh1MbRORi33JziZnkwGkCgp1AZk9HIZ8bUftBPv/Grhn8oeBYmYF39hjpZdFpuvl", - "UqTCEj1XqNdXquftSUUtUCF6gIczzYSZkikjipKXbPelFcBJ1Gu5i/hFBCr6x2hKUdv5G7Yu72gGVzzF", - "VXJSMht2iYzS8NnQ+TCqTEIA0XDqlhldQFt39Pgt5W6oz+1pejt+p73zdIccAbvOdvvMA2JEMdhH/I9Y", - "qXDXhcul8QkXudBmgKQ72NNtRsOQEaMzY/+hapZykt+yNtCcqVRFBxU6wOIMZGP9nM5TaykEORRgwx3U", - "8vBhf+EPH7o9F5ot4dInoGHHPjkePrRCoLT5ZAnosebVccSBoiAzWtNI0vCa6/VsZ8CZ4O4VZw5AH7/0", - "E5IwaU0m5no6waNuvrkDgbeAWAXO39OdoI+2rWoZJru5/dMbbaAYRi7t0J9HPNF3/oQ2sLRK5kJCUigJ", - "m2h+t5DwPTVG7TSxyMhgEtaxsf0TbAf/HlrdefbZzU+lL+12wBJvm9S7O9j8Ptxe0DpM8yMvE/KScZbm", - "ggKCSmpT1ak5k5wCFD03qMcWPuwyHrJ64bvEY2SREJYDdSa5Rho2YYvoZcYSIgHJrwF85ErXqxXonlvE", - "lgBn0vUSktVSGJqLvMrEblgJFd06zWxP9ASWPKcI2+9QKbaoTVf1UjaS9WxsBB2nYWp5JrlhOXBt2PdC", - "nl4ROH/C8TwjwVyq6ryhwsgJDSRooZP4xdw3tvVbrtd++djRKxs32AaJEX6bsrQx0El3/s/7fzt8f5T8", - "nSe/HyTP///8w8en1w8eDn58fP3ll//V/enJ9ZcP/vbn2E553GO5Mg7z45fOLTl+SbanDZ4PcP9swd9C", - "yCTKZHhcKISklMseb7H7aEE9Az1ow/Bu18+kuZLISBc8FxkegW/DDn0VN5BFKx09rulsRC+W59f6IXbc", - "Wamk5Ok53WtPVsKs68UsVcXcu2PzlWpcs3nGoVCS2rI5L8Ucj7fzi0c7TOMn6CsWUVfX04nTOvrOM2gc", - "4NiC+nM2UXT/t1Hs3jevTtnc7ZS+ZxPnLOgg4yniQbvLsM5hDhdvH37YzEE8zLyEpZAC2w/PZMYNny+4", - "Fqme1xqqr3jOZQqzlWKHzIF8yQ2nGEAvojj2NoviMw6bsl7kImXnoSluRXMsMHZ29h4Z5Ozsw+Aia2g4", - "3VTxYCNNkFwKs1a1SVxUdjyO0MZaCLKNy22bdcocbMuRLurr4I8EQMtSJ0FELL78ssxx+QEbakaDKA+K", - "aaMqrwRRM7qYBu7vG+Wu8ip+6bPRazy3/1Lw8r2Q5gNL3Pn7qCwp3Ebxrl+crkGe3JSwf8ysRbEFFjtn", - "0cKtQ3Xj3DgCemJH+SCyjlMOm4h01Ae1QhsTvC2dENS3KsfNvTWZAhhR6tRmnaBMRVelkbVIHoI3hHyF", - "utDfveGxGZnPvWlZAEvXkJ5DRhcPFCOcdob7K29nWbzICm2fodgUOMqVpuPgAlhdZtzZXi43/aRVDcb4", - "TN13cA6bU9WmWt8kS/V6OnFB/QR5ZkxASqRHYATUsisu/mKgt/nuboUC72XJbGzbZhd6tjhs+MKPGRcg", - "a5nuQHhiTNGQYQu/l7yKEMIy/wgJbrFQhPdJrB+NpPPKiFSUdv37xebfdsYgkF1KParG1bKvrQfKNKq9", - "bedkwXVccQO24H6gDPWzS/xMNrJiL8kYPWV2jLvIIbhV0k6yeUXOjl+2fZs5hlqcS6CSrTX1aHQpEprt", - "tbuWFBftZSRdO+9j4HZeSiEX+XwB0Q0/C5w3hws+ehMw+obgOEgCCJ6mNS8EvGLrC8O0eS1iX4n7lwT+", - "+YB/MzCZ3ij/fzpxuV6x7VCSrHsGOay4C3xTFpljFIfaPR1sEOLxw3KZCwksieUTcK1VKuxdaKvL3RyA", - "zt9Dxmxghe0NIcbGAdoUMSTA7I0KZVOuboKkBEEhRu5hU6wx+Bt2R9za5/rOrdzp/g11RytE0/Y5jd3G", - "YfRnOomqpDHPvNOL2S4LGBxlYiyKqmkYDxlGXTTkQOY46WjW5DwWJUOvAogNT/ywwF1n98USjfyDIHBc", - "wQrP3u15FaXVB2A+b8zgQhlIlqLSJqGjcnR52OlrTc7g19g1rn46pGL2va/I4tqHpj2HTZKJvI7vtpv3", - "u5c47Zvm3KLrxTlsyMgAT9dsQe/T0Qp1psc+W6a2OTVbF/zaLvg1v7P17sdL2BUnrpQyvTn+IFzV0yfb", - "hCnCgDHmGO7aKEm3qJcgG2GoW4I8CJszQfkVs22n9YEw3TijY1TzWkjRtQSO7tZV2MQfm9sTPO8e5kyP", - "yAAvS5Fd9c7OFmqcx2mKmzjq1uMfUIF21wHbQYHgnBxLIazAn/XtlgY20z7UH6RZ7aZMP7krUAjhVEL7", - "MjNDQiFrUzbOLlqdAs+/g81P2JeWM7meTj7tyB+jtYO4g9Zvm+2N0pliyPYI2Imc3ZDkvCwrdcHzxD1L", - "GWPNSl041qTu/hXLZ1Z18eP36auj128d+pS9BrxySVvbVkX9yj/MqvBEHMvcOg0iI+St+rOzdcSCzW/e", - "BobBFJ9o1/HlUIs55rLi1Ri4UBRdcGUZv8raGSoJk/NuJZmd7L5PjcyFqX53KvIDCYtzaLvDO/RCONeW", - "wgKFrZ2hmZL9BAd04+iUSexS8A3uog3MDhWErIsERSDRuUjjoQO50ChFsi7oxcbGAKPOIw4hQqzFSPhc", - "1iKAhd30HjdFPSSDOaLEpLDOFtotlCt6VkvxWw1MZCANNlUu4akjLCgbPod3aNLi+cIOsEsZbsB/ip1H", - "UGMWnpDYbuTDKG8kS9wf+vxCm/A0/hAE525wSRPOODBLWy5YHH84brY33etutDasUTbUQcgYtp7F7gJp", - "PnSwtoiOzBEteDaqsY/GtTXlge+vp1u1TOiGCtnm5vFcqwiYWl5yaesX4ThLQzdagz2346hLVdGjJQ3R", - "G2qhk2Wlfof4aXKJGxXJwXKkJJeNRs8ij0H6SrSJjLSV6Tx9QzxGWXvMmwoaWfcSbUTCicuD8DUllfog", - "E5eWrW2tpc7VbVw4wnSLuYXfCofDeZCikvPLBY+VHUCnBnE6ai9KOuEwo5gf7HdBN7nUjveCO5emr7Av", - "fUqo2kTJ4UvNWzoofyyWzyAVBc/j0dGMqN9965mJlbAFq2oNQUUkB8hW+rNc5KpK2auoljTHS3YwDWqu", - "ud3IxIXQYpED9Xhkeyy4JqvVhDybIbg8kGatqfvjPbqva5lVkJm1toTVijVOpH3c4OPPCzCXAJIdUL9H", - "z9l9irxrcQEPkIrOF5kcPnpOKRn2j4OYsXOV6bbplYwUy785xRLnY7p6sDDQSDmos+irM1tOdFyFbZEm", - "O3QfWaKeTuvtlqWCS76C+I1qsQMnO5Z2kwJ3PbrIzNbC06ZSGyZMfH4wHPXTSFoWqj+LhsuVL1CAjGJa", - "FchPbbkjO6kHZwvruRIkHi/fSNccpX/z0Du0ft4grbXlsVXTZdQbXkCXrFPG7eNMerbhHvU6hTgbqRUB", - "1UV8kmpkg73ddGPZfalkUqDsZA/ahL+A/6KlEpTheXRa43VXP3NlO+h9XS2EkowStu4Qlgc66dYkrqv4", - "OnmNU/347rUzDIWqYnUPWm3ojEQFphJwEZXYfuJa45k05sJTPuagfFWLPPupTTftlRiquEzX0fjnAgf+", - "3NZEa8huqR59grfmUkIeBWdl+Wcv8xGt9Kvad55CyD379ksH2eX2Ftci3kXTI+UnRPIKk+MEIVW7+XdN", - "4ki+UhmjedpH1i0jDN9IBWVUfqtBm9h7LmqwuU50xkZ/xVbxYCAzsvYzZt8/IS6dFyxkZUVR5/Y1BGQr", - "qFwApi5zxbMpQzinr45eMzurdm9o6d0NVRFZ2bd0nVX0zlZB9YebPC4cS43aH872nBFctTb0JFsbXpSx", - "rFfsceo7UGrtBRe5Tz8g8xNSZ8ZeWsuvvV2xk7RvSFkzndM1xBP4H2N4uiaT2jFA4yy/f/kbz5U6KAPZ", - "VNRriirYZ5FG+Qo4tgDOlCn0ey6FtqVs4QK6ibZN1rlz6XzibXd5VS2l5ZS4fdryKuI2ZPfI2Ys9H5KK", - "YtYj/A3NjFZ1lcJNqwGd0KjoG6t+aaFB/UcJ2emVbOqv+RLlKZdKipReOAXFcxuUXVncfWKmezwG6x+X", - "vYg7CY0IV7SgUZM64Kg4WuLIK0JHuGHAKGjFTbXcYf80VH8VD4IrMNppNsimvmiVO8cJqcEVyaAKyYGe", - "xON4//4werXRPte/IRtR+t+Iu/I1tpGrIlzKzrmQ9HjVkc1lB9mTFlXtNHi8E4atFGi3nu7jLP0ex8xO", - "r+QxYvxh5qt8EgwbQsZl2zuLIagjf4Phbgyw7wvsyyhc3P7cSTW0kx6VpZs0pgl0s8OxslujBI5EwRMf", - "hgyI28APoW1ht61Xj2RPkdHggi4uoCQ7PGCMkSfwr/BQaznKvqS1V/7RpxlCRtB4LSS0NWgjBiKNmgTa", - "GJLXkXE6rbixLuBeOu0UeE43JTGFpo0LHX0qqN4GE0lojX6O8W1si6+NKI6mQ+u4cblpSt8idwfOxAuq", - "ue0IOSylRl6Vc6IySurqFVeLKQ5U3L4sYdcADMVg6BPZ4abiVnJuYonGktAzofE4UizySBrLy6YxKDBI", - "+XKLDf0be4A8vgJ3sXbrghk08Mb+5fbiFTnufaLF6pa70o6/w23pyUC4RzHuf4VqJXy3M3hLbhVP86yG", - "rvCVL/dKh4omMb3Ls6Toooe2tnLn9kPreA3OKanGkUSed+2LUW61r40NjqXzpKPZZ9y41FLD2bYqMrZw", - "ZgyCvYe0BTvtxy+igYGxu0d79YjNg9H7+Q0DL4xgbyWov9QeIvSdz1phJRcu8N2KyJCyLr9tmHG4T+ZL", - "u8H9RbisMQISW8ktk7z2kr0hlSKCHaYG7GDP8w5J7WuQniepKrhj0gYm9IakHSY97Ls8WgdxTK1huM69", - "N6BD2xHa70P4Vi8MiTsuzmaxjzjHk+pxOOkTSxD/7GOoTT6bNujU+3Xzxnb9p7HogT0hjwSqejStRZ7t", - "2txO2LF9zkyBtZ8XXzztRO8+54Pqn+2F/FDc3NvSmxj+/iYQYSJr7UweTBUEFPeIJbphkcgh1YJK60qY", - "DeXueE9T/BzNS/4GpKt67IrINzeg7gLOfr/EhaZXTe/2kxPfKFsGukD3l1xBQ0VSXl3xoszBycWX9xZ/", - "gSd/fZodPHn0l8VfD54dpPD02fODA/78KX/0/MkjePzXZ08P4NHyi+eLx9njp48XTx8//eLZ8/TJ00eL", - "p188/8s9/70Hi2j7LYV/p6oDydHb4+QUkW1pwkvxHWzsO2NkY/+CmackiVBwkU8O/U//4iVslqoi+ESd", - "+3XiIv2TtTGlPpzPLy8vZ+GQ+YrK9iVG1el67ucZ1qR5e9wEaO2FP+2ojb0hK9CmOlY4orZ3r05O2dHb", - "41nLMJPDycHsYPaICoWUIHkpJoeTJ/QTSc+a9n3umG1y+PF6Opmvgedm7f4owFQi9U36kq9WUM3cU278", - "6eLx3Md35h/dJff1trZuloF7OhIMCN4ezj926i1mIVx6mTf/6DMwgiZbpHf+kcJHwe+uyub8Y1v29try", - "eg6xc7wvC9Z2p3JfVGFf21+Rvf0todDd0sPNXh1nuEc46kVTAjj8kOj7f9DP7n3ofYXk8cHBP9j3FJ7e", - "cMVbvdPOaS5SNeErnjF/U0RzP/p8cx9LereB6olZ9Xs9nTz7nKs/lsjyPGfUM8jtGG79j/Jcqkvpe6Kt", - "rIuCVxsvxrqjFHxhb9LIfKWpDGElLvCI/oHqXMau5kaUC3244sbKhb7G8U/l8rmUyx/jMyWPbyjgf/wV", - "/1Od/tHU6YlVd/urU+fK2WSEuS0E13p4/g3k8GFg1zcd08nu4MLuU9RTwuUDl9BgwUYemTaXxyqzEQ5f", - "KMinSblZZwOd/c4B7bxn/g42epcCP10D+6X9dPwvlMxJVwlTpir2C8/z4Df6Aqh3wmcj36FvHh7u+xH6", - "6+tpDK0lgE8tpRRSV8sUDdk5+Ceqlgad68bhDX1bVm4Jo9+itdW3Qg3mWPDRwcFBLLWnj7OLxliMKZX3", - "UiU5XEA+3OoxJHovVbd9uXH0OxzDB8bhKTrCdf5Dx82b49EPWXZfzd4Eu5dK3jPskgtXUj2oUmO/dlII", - "47/xalN+XDpgYyPi3wVNEOT2zwZ/qvH+49Umvd6i7PS6Npm6lOOKi94K8dwl21L6axM8MIp5AI2mmjH/", - "0b584786yzglH6nadD8G7YtP9EowN+WRVkLSBCTlNIvNKudBzqb7EMZQCZ44zN7Y74b09F70m5gWx7jc", - "x4T+U3lp6Ghs3StfrKTz9xxZHt1V+7GhhCg0DFAY4PncpZ30frWXw8GP3TLLkV/nzUOtaGM/7BJrnX80", - "V0FkpY13hvFD2qkmcvj+AxKcMoLdJrbhsMP5nC5k10qb+QQVTjdUFjZ+aGj80e+8p/X1h+v/DgAA//8U", - "2bjJl4UAAA==", + "H4sIAAAAAAAC/+x9/XPcNpLov4I3u1X+eMMZ+Su7VlVqn2I7iV4cx2Upubu1fAmG7JlBRAIMAUqa+PS/", + "X3UDIEESnBnJWt+ldn+yNQAajUZ/odFofpykqiiVBGn05PDjpOQVL8BARX/xNFW1NInI8K8MdFqJ0ggl", + "J4e+jWlTCbmaTCcCfy25WU+mE8kLaPvg+Omkgt9qUUE2OTRVDdOJTtdQcARsNiX2biBdJSuVOBBHFsTx", + "y8n1lgaeZRVoPcTyB5lvmJBpXmfATMWl5ik2aXYpzJqZtdDMDWZCMiWBqSUz605nthSQZ3rmF/lbDdUm", + "WKWbfHxJ1y2KSaVyGOL5QhULIcFjBQ1SzYYwo1gGS+q05obhDIir72gU08CrdM2WqtqBqkUixBdkXUwO", + "3080yAwq2q0UxAX9d1kB/A6J4dUKzOTDNLa4pYEqMaKILO3YUb8CXedGM+pLa1yJC5AMR83Y97U2bAGM", + "S/bu6xfsyZMnz3EhBTcGMsdko6tqZw/XZIdPDicZN+Cbh7zG85WquMySpv+7r1/Q/Cdugfv24lpDXFiO", + "sIUdvxxbgB8YYSEhDaxoHzrcjyMiQtH+vIClqmDPPbGd73RTwvn/R3cl5SZdl0pIE9kXRq3MNkd1WDB8", + "mw5rEOj0L5FSFQJ9f5A8//Dx0fTRwfWf3h8lf3d/PntyvefyXzRwd1Ag2jGtqwpkuklWFXCSljWXQ3q8", + "c/yg16rOM7bmF7T5vCBV78YyHGtV5wXPa+QTkVbqKF8pzbhjowyWvM4N8xOzWuaophCa43YmNCsrdSEy", + "yKaofS/XIl2zlGsLgvqxS5HnyIO1hmyM1+Kr2yJM1yFJEK9b0YMW9L+XGO26dlACrkgbJGmuNCRG7TBP", + "3uJwmbHQoLS2St/MWLHTNTCaHBussSXaSeTpPN8wQ/uaMa4ZZ940TZlYso2q2SVtTi7OabxbDVKtYEg0", + "2pyOHUXhHSPfgBgR4i2UyoFLIp6XuyHJ5FKs6go0u1yDWTubV4EuldTA1OJXSA1u+/8/+eENUxX7HrTm", + "K3jL03MGMlXZ+B67SWMW/FetcMMLvSp5eh4317koRATl7/mVKOqCybpYQIX75e2DUawCU1dyDCELcQef", + "FfxqOOlpVcuUNredtuOoISsJXeZ8M2PHS1bwqy8Ppg4dzXiesxJkJuSKmSs56qTh3LvRSypVy2wPH8bg", + "hgVWU5eQiqWAjDVQtmDiptmFj5A3w6f1rAJ0PJBRdJpZdqAj4SrCMyi62MJKvoKAZWbsR6e5qNWoc5CN", + "gmOLDTWVFVwIVetm0AiONPV291oqA0lZwVJEeOzEkQO1h+3j1GvhHJxUScOFhAw1LyGtDFhNNIpTMOH2", + "w8zQRC+4hi+ejhnwtnXP3V+q/q5v3fG9dps6JVYkI3YRW53Axt2mzvg9Dn/h3FqsEvvzYCPF6hRNyVLk", + "ZGZ+xf3zZKg1KYEOIbzh0WIluakrODyTD/EvlrATw2XGqwx/KexP39e5ESdihT/l9qfXaiXSE7EaIWaD", + "a/Q0RcMK+w/Ci6tjcxU9NLxW6rwuwwWlnVPpYsOOX45tsoV5U8Y8ao6y4ani9MqfNG46wlw1GzmC5Cjt", + "So4dz2FTAWLL0yX9c7UkfuLL6nf8pyzzGE2RgZ2hpaCACxa8c7/hTyjyYM8ECEWkHIk6J/N5+DFA6M8V", + "LCeHkz/N20jJ3LbquYOLM15PJ0ctnLufqR1p19c7yLTNTEi7O9R1as+Ed48PQo1iQo5qD4evcpWe3wqH", + "slIlVEbYfVwgnKGkEHi2Bp5BxTJu+Kw9VFk/a4TfaeC3NI5OSVBFTNwP9B+eM2xGKeTGu2/ougqNTpwK", + "Ak0ZenzWjtiZsAN5oooV1slj6JzdCMsX7eRWQTca9b0jy4c+tMjuvLJ+JaMRfhG49PbUeLRQ1e34pccI", + "krVnYcYRauP94sq7O0td6zJx9In407ZDD1Abfhyq1ZBCffAxWnWocGL4P4AKGqHeBRW6gO6aCqooRQ53", + "IK9rrtfDRaCD8+QxO/n26Nmjxz8/fvYFWuiyUquKF2yxMaDZfWdXmDabHB4MV0YKvs5NHPoXT/0Jqgt3", + "J4UI4Qb2PhJ1CqgZLMWYjRcgdi+rTVXLOyAhVJWqIj4vsY5RqcqTC6i0UJHwxVvXg7keqIes39373WLL", + "LrlmODcdx2qZQTWLUR7PWWTSDRR6l6GwoE+vZEsbB5BXFd8MdsCuN7I6N+8+e9IlvvfuNSuhSsyVZBks", + "6lVoo9iyUgXjLKOBpBDfqAxODDe1vgMt0AJrkcGNCFHgC1UbxplUGQo0do7rh5FYJgVRKPZjQpVj1tb+", + "LAC945TXq7Vh6Faq2Na2AxOe2k1JyFbokaNfc2a3vex0Nk6WV8CzDVsASKYW7nzlTn60SE5hGeNvXJx2", + "atFqzgQdvMpKpaA1ZIm7XtqJmu9nd9lsoRMhTgg3szCt2JJXt0TWKMPzHYhSnxi6jTvhDqVDrPebftsG", + "9icPt5FXeMa0XIC+C0p3DgbGSLgnTS6gosPZP3T//CS33b66HLk6cRb4VBQovkxyqTSkSmY6Cizn2iS7", + "xBY7ddwEXEEgKTFJJcAjAYLXXBt7RBcyI5fRqhuah8bQFOMIj1oUhPyTNyZD2CnqSalr3VgWXZelqgxk", + "sTVIuNoy1xu4auZSywB2Y76MYrWGXZDHqBTAd8SyK7EE4sbFiJoY1nBxFI5HO7CJkrKDREuIbYic+F4B", + "dcPw8QgieL5oRhLjCN3jnCZmPZ1oo8oS5c8ktWzGjZHpxPY+Mj+2fYfMxU2r1zMFOLvxODnMLy1l7cXB", + "mqNvR5BZwc/RNpGnZmMJQ5xRGBMtZArJNs5HsTzBXqEI7BDSESfZXU0Gs/WEo8e/UaYbZYIduzC24BGP", + "/a2NgJ+20aE7cFpeguEi141j0oTZ21koIt/PlkAvsoIUpMk3yKtLURX2UovMmfa/Wbcnc7PY65tW/GTG", + "KrjkVeZ7DE9LwWISITO4imtX3omNZHDFRBzpZTOzMCz1V04yBDCLCrq7xNuCgguK3GZyHBqf1l5RWSrp", + "2OUlNaAAFCKtFLd3krgYa89Nc+1WQcERO7odc+Z2fE4hV4m9Ao2YR9vur0h9aDrkmThczyejkt2wxuUa", + "6NYF1XWPiCG34ZESNIwtZJWrBc8TdLQhySA3O0Ne6MDDS+qJdlKlw+FdlM/O3ufZ2dkH9hr7kk8P7Bw2", + "c7opZumayxW04fuQT623DleQ1qFK75FxrwOYi1F2se8ewaaTUqk8aY6a/euGgZrv0/1cpOeQMdQT5GA5", + "63Ovu0M4CbuPLK6bC5nL9ca7bmUJErIHM8aOJIOiNBsX1+h5Gr3J5T2zbf4rmjWr6W6YS0aLnJ3JeEjB", + "3ix/okx5MNslyaZafeJUFsj2icyVHBEnfkkXIwguKp9bo5InNDIwOQNLGjCVxWKfs/s3lH/EO7ssMjoG", + "tFZF14tCUBJS0G2KmtPfCw9P1sLMGDsl3YEHGw0XUPGcMiy0D9gKzQqBB2RdpylAdngmkw4mqSrcxPfb", + "/1q1dFYfHDwBdvCgP0YbdBPdGc7KQH/sl+xgapuIXOxLdjY5mwwgVVCoC8jsOSjkaztqJ9j/08A9kz8M", + "FDMr+MaeoLwsMl0vlyIVlui5Qr2+Uj1vTypqgQrRAzyzaSbMlEwZUZS8ZLsvrQBOol7LXcRaIlDRP0ZT", + "itrO3wZ2eUczuOIprpKTktmwS2SUhs+GzodRZRICiIZ+t8zogu+6o8dvKXdDfW4P/tvxO+0d/TvkCNh1", + "tttnHhAjisE+4n/ESoW7Llzej08OyYU2AyRdGIBuXhqGjBidGfsPVbOUk/yWtYHmTKUqOqjQARZnIBvr", + "53SeWkshyKEAG5mhlocP+wt/+NDtudBsCZc+WQ479snx8KEVAqXNJ0tAjzWvjiMOFAXE0ZpGEpzXXK9n", + "O4PjBHevmHgA+viln5CESWsyMdfTCR51880dCLwFxCpw/p7uhIi0bVXLMDHP7Z/eaAPFMMpqh/484om+", + "8ye0gaVVMhcSkkJJ2ERz0YWE76kxaqeJRUYGk7COje2fYDv499DqzrPPbn4qfWm3A5Z426QJ3sHm9+H2", + "AuxhSiJ5mZCXjLM0FxQ+VFKbqk7NmeQUoOi5QT228GGX8ZDVC98lHiOLhLAcqDPJNdKwCVtEL16WEAlI", + "fg3gI1e6Xq1A99witgQ4k66XkKyWwtBc5FUmdsNKqOiGbGZ7oiew5DlF2H6HSrFFbbqqlzKnrGdjo/04", + "DVPLM8kNy4Frw74X8vSKwPkTjucZCeZSVecNFUZOaCBBC53ELxG/sa3fcr32y8eOXtm4wTaejfDb9KqN", + "gU5q9n/e/9vh+6Pk7zz5/SB5/n/nHz4+vX7wcPDj4+svv/yv7k9Prr988Lc/x3bK4x7L63GYH790bsnx", + "S7I9bZx/gPtnC/4WQiZRJsPjQiEkpYf2eIvdRwvqGehBe2Pgdv1MmiuJjHTBc5HhEfg27NBXcQNZtNLR", + "45rORvRieX6tH2LHnZVKSp6e0x38ZCXMul7MUlXMvTs2X6nGNZtnHAolqS2b81LM8Xg7v3i0wzR+gr5i", + "EXV1PZ04raPvPNvHAY4tqD9nE0X3fxvF7n3z6pTN3U7pezbJz4IOsrMiHrR7Y9Y5zOHi7SMVm+WIh5mX", + "sBRSYPvhmcy44fMF1yLV81pD9RXPuUxhtlLskDmQL7nhFAPoRRTH3pFRfMZhU9aLXKTsPDTFrWiOBcbO", + "zt4jg5ydfRjcuQ0Np5sqHmykCZJLYdaqNomLyo7HEdpYC0G2cblts06Zg2050kV9HfyRAGhZ6iSIiMWX", + "X5Y5Lj9gQ81oEOVsMW1U5ZUgakYX08D9faPcrWPFL33mfI3n9l8KXr4X0nxgiTt/H5Ulhdso3vWL0zXI", + "k5sS9o+ZtSi2wGLnLFq4dahunMdHQE/sKB9E1nHKYRORjvqgVmhjgrelE4L6VuW4ubcmUwAjSp3arBOU", + "qeiqNLIWyUPw3pGvUBf6uzc8NiPzufc3C2DpGtJzyOjigWKE085wf73rLIsXWaHtkxmbrkd53XQcXACr", + "y4w728vlpp9gq8EYn1X8Ds5hc6ratPCbZNReTycuqJ8gz4wJSIn0CIyAWnbFxV8M9Dbf3a1Q4L0smY1t", + "20xIzxaHDV/4MeMCZC3THQhPjCkaMmzh95JXEUJY5h8hwS0WivA+ifWjkXReGZGK0q5/v9j8284YBLJL", + "qUfVuFr2tfVAmUa1t+2cLLiOK27AFtwPlKF+JoWfyUZW7CUZo2fXjnEXOQS3StpJNq/I2fHLtu9Ix1CL", + "cwlUsrWmHo0uRUKzvXbXkuKivYyka+d9DNzOSynkIp8vILrhZ4Hz5nDBR28CRt87HAdJAMEzuuY1g1ds", + "fWGYNi9b7It2/+rBP3Xw7xsm0xu9VZhOXF5abDuUJOueQQ4r7gLflPHmGMWhdk8HG4R4/LBc5kICS2L5", + "BFxrlQp7F9rqcjcHoPP3kDEbWGF7Q4ixcYA2RQwJMHujQtmUq5sgKUFQiJF72BRrDP6G3RG3trSAcyt3", + "un9D3dEK0bR9+mO3cRj9mU6iKmnMM+/0YrbLAgZHmRiLomoaxkOGURcNOZA5TjqaNTmPRcnQqwBiwxM/", + "LHDX2X2xRCP/IAgcV7DCs3d7XkVp9QGYzxszuFAGkqWotEnoqBxdHnb6WpMz+DV2jaufDqmYfZsssrj2", + "oWnPYZNkIq/ju+3m/e4lTvumObfoenEOGzIywNM1W9BberRCnemxz5apbU7N1gW/tgt+ze9svfvxEnbF", + "iSulTG+OPwhX9fTJNmGKMGCMOYa7NkrSLeolyEYY6pYgD8LmTFB+xWzbaX0gTDfO6BjVvBZSdC2Bo7t1", + "FTbxx+b2BE/Rh/ndIzLAy1JkV72zs4Ua53Ga4iaOuvX4B1Sg3XXAdlAgOCfHUggr8Gd9u6WBzbRFBQZp", + "Vrsp00/uChRCOJXQviTOkFDI2pSNs4tWp8Dz72DzE/al5Uyup5NPO/LHaO0g7qD122Z7o3SmGLI9AnYi", + "ZzckOS/LSl3wPHFPaMZYs1IXjjWpu39x85lVXfz4ffrq6PVbhz5lrwGvXNLWtlVRv/IPsyo8Eccyt06D", + "yAh5q/7sbB2xYPObd4xhMMUn2nV8OdRijrmseDUGLhRFF1xZxq+ydoZKwuS8W0lmJ7vvUyNzYarfnYr8", + "QMLiHNru8A69EM61pQhCYet8aKZkP8EB3Tg6ZRK7FHyDu2gDs0MFIesiQRFIdC7SeOhALjRKkawLelyy", + "McCo84hDiBBrMRI+l7UIYGE3vcdNUQ/JYI4oMSmss4V2C+UKtNVS/FYDExlIg02VS3jqCAvKhs/hHZq0", + "eL6wA+xShhvwn2LnEdSYhSckthv5MMobyRL3hz6/0CY8jT8EwbkbXNKEMw7M0pYLFscfjpvtTfe6G60N", + "66kNdRAyhq29sbuYmw8drC2iI3NEi7ONauyjcW1NeeD76+lWLRO6oUK2uXk81yoCppaXXNpaSzjO0tCN", + "1mDP7TjqUlX0aElD9IZa6GRZqd8hfppc4kZFcrAcKcllo9GzyGOQvhJtIiNtFT1P3xCPUdYe86aCRta9", + "RBuRcOLyIHxNSaU+yMSlZWtbF6pzdRsXjjDdYm7ht8LhcB6kqOT8csFjJRLQqUGcjtqLkk44zCjmB/td", + "0E0uteO94M6l6SvsS58SqjZRcviq9JYOyh+L5TNIRcHzeHQ0I+p33zVmYiVsca1aQ1C9yQGyVQktF7kK", + "WPYqqiXN8ZIdTIP6cG43MnEhtFjkQD0e2R4LrslqNSHPZgguD6RZa+r+eI/u61pmFWRmrS1htWKNE2kf", + "N/j48wLMJYBkB9Tv0XN2nyLvWlzAA6Si80Umh4+eU0qG/eMgZuxcFb1teiUjxfJvTrHE+ZiuHiwMNFIO", + "6iz66syWPh1XYVukyQ7dR5aop9N6u2Wp4JKvIH6jWuzAyY6l3aTAXY8uMrN1+7Sp1IYJE58fDEf9NJKW", + "herPouFy5QsUIKOYVgXyU1uayU7qwdkigK5cisfLN9I1R+nfPPQOrZ83SGtteWzVdBn1hhfQJeuUcfs4", + "k55tuEe9TiHORupaQHURn6Qa2WBvN91Ydl8qmRQoO9mDNuEv4L9oWQdleB6d1njd1c9c2Q56X1cLoSSj", + "hK07hOWBTro1iesqvk5e41Q/vnvtDEOhqliNhlYbOiNRgakEXEQltp+41ngmjbnwlI85KF/VIs9+atNN", + "e+WQKi7TdTT+ucCBP7f12xqyW6pHn+CtuZSQR8FZWf7Zy3xEK/2q9p2nEHLPvv0yR3a5vcW1iHfR9Ej5", + "CZG8wuQ4QUjVbv5dkziSr1TGaJ72kXXLCMM3UkHJl99q0Cb2nosabK4TnbHRX7EVRxjIjKz9jNn3T4hL", + "5wULWVlR1Ll9DQHZCioXgKnLXPFsyhDO6auj18zOqt0bWnp3QxVPVvYtXWcVvbNVUJHhJo8Lx1Kj9oez", + "PWcEV60NPcnWhhdlLOsVe5z6DpRae8FF7tMPyPyE1Jmxl9bya29X7CTtG1LWTOd0DfEE/scYnq7JpHYM", + "0DjL71+qx3OlDkpWNtX/mqIK9lmkUb5ajy3WM2UK/Z5LoW3ZXbiAbqJtk3XuXDqfeNtdXlVLaTklbp+2", + "vIq4Ddk9cvZiz4ekopj1CH9DM6NVXaVw08pFJzQq+saqXwZpUKtSQnZ6JZtacb6cesqlkiKlF05Bod8G", + "ZVfCd5+Y6R6PwfrHZS/iTkIjwhUtvtSkDjgqjpZj8orQEW4YMApacVMtd9g/DdWKxYPgCox2mg2yqS+w", + "5c5xQmpwRTKomnOgJ/E43r8/jF5ttM/1b8hGlP434q58jW3kqgiXsnMuJD1edWRz2UH2pEUVRg0e74Rh", + "KwXaraf7OEu/xzGz0yt5jBh/mPmKpATDhpBx2fbOYgjqyN9guBsD7PsC+zIKF7c/d1IN7aRHZekmjWkC", + "3exwrETYKIEjUfDEhyED4jbwQ2hb2G3r1SPZU2Q0uKCLCyjJDg8YY+QJ/Cs81FqOsi9p7ZV/9GmGkBE0", + "XgsJbb3ciIFIoyaBNobkdWScTiturAu4l047BZ7TTUlMoWnjQkefCqq3wUQSWqOfY3wb20JxI4qj6dA6", + "blxumjK9yN2BM/GC6oM7Qg7LvpFX5ZyojJK6eoXgYooDFbcvodg1AEMxGPpEdripuJWcm1iisST0TGg8", + "jhSLPJLG8rJpDIohUr7cYkP/xh4gj6/AXazdumAGDbyxf7m9eEWOe59osbrlrrTj73BbejIQ7lGM+1+h", + "Wgnf7QzeklvF0zyroSt85UvT0qGiSUzv8iwpuuihra0yuv3QOl4vdEqqcSSR5137YpRb7Wtjg2PpPOlo", + "9hk3LrXUcLatiowt8hmDYO8hbXFR+6GOaGBg7O7RXj1i82D0fn7DwAsj2FsJ6i+1hwh957NWWMmFC3y3", + "IjKkrMtvG2Yc7pP50m5wfxEua4yAxFZyyySvvWRvSKWIYIepATvY87xDUvsapOdJqgrumLSBCb0haYdJ", + "D/suj9ZBHFNrGK5z7w3o0HaE9vsQvtULQ+KOi7NZ7CPO8aR6HE76xBLEP/sYapPPpg06tYndvLFd/2ks", + "emBPyCOBqh5Na5Fnuza3E3ZsnzNTYO3nxRdPO9G7z/mg+md7IT8UN/e29CaGv78JRJjIWjuTB1MFAcU9", + "YoluWCRySLWg0roSZkO5O97TFD9H85K/AekqNLuC980NqLuAs99acaHpVdO7/TzGN8qWrC7Q/SVX0FCR", + "lFdXvChzcHLx5b3FX+DJX59mB08e/WXx14NnByk8ffb84IA/f8ofPX/yCB7/9dnTA3i0/OL54nH2+Onj", + "xdPHT7949jx98vTR4ukXz/9yz3+bwiLafvfh36nqQHL09jg5RWRbmvBSfAcb+84Y2di/YOYpSSIUXOST", + "Q//T//MSNktVEXxOz/06cZH+ydqYUh/O55eXl7NwyHxFZfsSo+p0PffzDGvSvD1uArT2wp921MbekBVo", + "Ux0rHFHbu1cnp+zo7fGsZZjJ4eRgdjB7RIVCSpC8FJPDyRP6iaRnTfs+d8w2Ofx4PZ3M18Bzs3Z/FGAq", + "kfomfclXK6hm7ik3/nTxeO7jO/OP7pL7eltbN8vAPR0JBgRvD+cfO/UWsxAuvcybf/QZGEGTrSc8/0jh", + "o+B3V2Vz/rEte3tteT2H2DnelwVru1O5L/oagLa/Inv7W0Khu1WSm706znCPcNSLpgRw+NHT9/+knwj8", + "0PtiyuODg3+ybz88veGKt3qnndNcpGrCVzxj/qaI5n70+eY+lvRuA9UTs+r3ejp59jlXfyyR5XnOqGeQ", + "2zHc+h/luVSX0vdEW1kXBa82Xox1Ryn4wt6kkflKUxnCSlzgEf0D1bmMXc2NKBf6yMaNlQt9OeRfyuVz", + "KZc/xidVHt9QwP/4K/6XOv2jqdMTq+72V6fOlbPJCHNbCK718PwbyOHDwK5vOqaT3cGF3aeop4TLBy6h", + "wYKNPDJtLo9VZiMcvlCQT5Nys84GOvudA9p5z/wdbPQuBX66BvZL+5n7XyiZk64SpkxV7Bee58Fv9LVS", + "74TPRr6Z3zw83PeD+dfX0xhaSwCfWkoppK6WKRqyc/BPVC0NOteNwxv6tqzcEka/m2urb4UazLHgo4OD", + "g1hqTx9nF42xGFMq76VKcriAfLjVY0j0Xqpu+8rk6Hc4hg+Mw1N0hOv8R5mbN8ejH93svpq9CXYvlbxn", + "2CUXrqR6UKXGfpilEMZ/j9am/Lh0wMZGxL9hmiDI7Z84/lTj/cerTXq9RdnpdW0ydSnHFRe9FeK5S7al", + "9NcmeGAU8wAaTTVj/gOD+cZ/IZdxSj5Stel+uNoXn+iVYG7KI62EpAlIymkWm1XOg5xN9yGMoRI8cZi9", + "sd8N6em96Pc7LY5xuY8J/afy0tDR2LpXvlhJ5+85sjy6q/a7SAlRaBigMMDzuUs76f1qL4eDH7tlliO/", + "zpuHWtHGftgl1jr/aK6CyEob7wzjh7RTTeTw/QckOGUEu01sw2GH8zldyK6VNvMJKpxuqCxs/NDQ+KPf", + "eU/r6w/X/x0AAP//DI8H3kOGAAA=", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/daemon/algod/api/server/v2/generated/private/types.go b/daemon/algod/api/server/v2/generated/private/types.go index 3a44a1a51f..e8a30a9289 100644 --- a/daemon/algod/api/server/v2/generated/private/types.go +++ b/daemon/algod/api/server/v2/generated/private/types.go @@ -483,7 +483,7 @@ type NodeStatusResponse struct { // The number of blocks that have already been obtained by the node as part of the catchup CatchpointAcquiredBlocks *uint64 `json:"catchpoint-acquired-blocks,omitempty"` - // The number of account from the current catchpoint that have been processed so far as part of the catchup + // The number of accounts from the current catchpoint that have been processed so far as part of the catchup CatchpointProcessedAccounts *uint64 `json:"catchpoint-processed-accounts,omitempty"` // The total number of accounts included in the current catchpoint @@ -492,6 +492,9 @@ type NodeStatusResponse struct { // The total number of blocks that are required to complete the current catchpoint catchup CatchpointTotalBlocks *uint64 `json:"catchpoint-total-blocks,omitempty"` + // The number of accounts from the current catchpoint that have been verified so far as part of the catchup + CatchpointVerifiedAccounts *uint64 `json:"catchpoint-verified-accounts,omitempty"` + // CatchupTime in nanoseconds CatchupTime uint64 `json:"catchup-time"` diff --git a/daemon/algod/api/server/v2/generated/routes.go b/daemon/algod/api/server/v2/generated/routes.go index 2f28bc41d9..a84f3ce68e 100644 --- a/daemon/algod/api/server/v2/generated/routes.go +++ b/daemon/algod/api/server/v2/generated/routes.go @@ -561,165 +561,170 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9/ZPbNpLov4LTXVXsnDgaf2XXU5W6N7GdZN7ajsue7O07j18WIlsSdiiAS4AzUvzm", - "f3/VDYAESVDSfNiJs/rJHhFoAI3+RqPxcZSqZaEkSKNHRx9HBS/5EgyU9BdPU1VJk4gM/8pAp6UojFBy", - "dOS/MW1KIeej8UjgrwU3i9F4JPkSmjbYfzwq4Z+VKCEbHZmygvFIpwtYcgRs1gW2riGtkrlKHIhjC+Lk", - "+ehqwweeZSVo3Z/lTzJfMyHTvMqAmZJLzVP8pNmlMAtmFkIz15kJyZQEpmbMLFqN2UxAnukDv8h/VlCu", - "g1W6wYeXdNVMMSlVDv15PlPLqZDgZwX1pOoNYUaxDGbUaMENwxFwrr6hUUwDL9MFm6lyy1TtJML5gqyW", - "o6P3Iw0yg5J2KwVxQf+dlQC/QmJ4OQcz+jCOLW5moEyMWEaWduKwX4KucqMZtaU1zsUFSIa9DtirShs2", - "BcYle/v9M/bo0aOnuJAlNwYyR2SDq2pGD9dku4+ORhk34D/3aY3nc1VymSV1+7ffP6Px37kF7tqKaw1x", - "ZjnGL+zk+dACfMcICQlpYE770KJ+7BFhiubnKcxUCTvuiW18p5sSjv+b7krKTboolJAmsi+MvjL7OSrD", - "gu6bZFg9gVb7AjFVItD3h8nTDx8fjB8cXv37++Pkf9yfTx5d7bj8ZzXcLRiINkyrsgSZrpN5CZy4ZcFl", - "Hx9vHT3oharyjC34BW0+X5Kod30Z9rWi84LnFdKJSEt1nM+VZtyRUQYzXuWG+YFZJXMUUwjNUTsTmhWl", - "uhAZZGOUvpcLkS5YyrUFQe3YpchzpMFKQzZEa/HVbWCmqxAlOK8b4YMW9PtFRrOuLZiAFUmDJM2VhsSo", - "LerJaxwuMxYqlEZX6espK3a6AEaD4werbAl3Emk6z9fM0L5mjGvGmVdNYyZmbK0qdkmbk4tz6u9Wg1hb", - "MkQabU5LjyLzDqGvh4wI8qZK5cAlIc/zXR9lcibmVQmaXS7ALJzOK0EXSmpgavoPSA1u+/9+99Nrpkr2", - "CrTmc3jD03MGMlXZ8B67QWMa/B9a4YYv9bzg6XlcXediKSJTfsVXYlktmayWUyhxv7x+MIqVYKpSDk3I", - "QtxCZ0u+6g96WlYypc1thm0ZakhKQhc5Xx+wkxlb8tW3h2M3Hc14nrMCZCbknJmVHDTScOzt00tKVcls", - "BxvG4IYFWlMXkIqZgIzVUDbMxA2zbT5CXm8+jWUVTMcDGZxOPcqW6UhYRWgGWRe/sILPISCZA/azk1z0", - "1ahzkLWAY9M1fSpKuBCq0nWngTnS0JvNa6kMJEUJMxGhsXcOHSg9bBsnXpfOwEmVNFxIyFDy0qSVASuJ", - "BucUDLjZmemr6CnX8M3jIQXefN1x92equ+sbd3yn3aZGiWXJiF7Er45h42ZTq/8Ozl84thbzxP7c20gx", - "P0VVMhM5qZl/4P55NFSahEALEV7xaDGX3FQlHJ3Jr/EvlrB3hsuMlxn+srQ/vapyI96JOf6U259eqrlI", - "34n5ADLruUa9Keq2tP8gvLg4Nquo0/BSqfOqCBeUtrzS6ZqdPB/aZAvzuoR5XLuyoVdxuvKexnV7mFW9", - "kQOTHMRdwbHhOaxLwNnydEb/rGZET3xW/or/FEUewykSsFO0FBRwwYK37jf8CVkerE+AUETKEakTUp9H", - "H4MJ/UcJs9HR6N8nTaRkYr/qiYOLI16NR8cNnLsfqelp19dxZJrPTEi7O9R0bH3Cu58PQo3OhAzVzhy+", - "y1V6fqM5FKUqoDTC7uMU4fQ5hcCzBfAMSpZxww8ap8raWQP0Th1/pH7kJUEZUXE/0X94zvAzciE33nxD", - "01VoNOJUEGjK0OKzesSOhA3IElVsaY08hsbZtWb5rBncCuhaor53aPnQhRbZnRfWrmTUwy8Cl954jcdT", - "Vd6MXjqEIFnjCzOOUGvrF1fe3llqWhWJw0/EnrYNOoCa8GNfrIYY6oKP4aqFhXeGfwIsaIR6F1hoA7pr", - "LKhlIXK4A35dcL3oLwINnEcP2bsfj588ePjLwyffoIYuSjUv+ZJN1wY0u+f0CtNmncP9/spIwFe5iUP/", - "5rH3oNpwt2KIJlzD3oWjTgElg8UYs/ECnN3zcl1W8g5QCGWpyojNS6RjVKry5AJKLVQkfPHGtWCuBcoh", - "a3d3frezZZdcMxyb3LFKZlAexDCPfhapdANLvU1RWNCnK9ngxgHkZcnXvR2w642szo27y560ke+te80K", - "KBOzkiyDaTUPdRSblWrJOMuoIwnE1yqDd4abSt+BFGiANZPBjQinwKeqMowzqTJkaGwclw8DsUwKolDs", - "x4Qixyys/pkCWscpr+YLw9CsVLGtbTomPLWbkpCu0AOuX+2z21Z2OBsny0vg2ZpNASRTU+dfOc+PFskp", - "LGP8iYuTTs20ap+gNa+iVCloDVnijpe2Ts0fVdEmmw1oonnTfOtBmFZsxssbztUow/Mt86Q2/dnqxppw", - "Pml/1rsNv2n/uoOHu8hLdDEtEaDpgsydg4EhFG7FSVUMHEc4rXYqlsgSTHKpNKRKZjoKLOfaJNtYARu1", - "VC9ua0B9MeonwANO90uujXV7hczIDLMsTONQHxpieMKDUhoh/9UL6D7sFGWP1JWupbWuikKVBrLYGiSs", - "Noz1Glb1WGoWwK5VglGs0rAN8hCWAvgOWXYlFkHcuLhLHRfqL45C3Chb11FUtibRIGLTRN75VgF2w5Ds", - "wETQZq97EuEI3aGcOg48HmmjigJlkkkqWfcbQtM72/rY/Ny07RMXN42szBTg6MbPyc380mLWBuMXHO0l", - "gsyW/BzlPVk/1j/vzxmZMdFCppBsonxky3fYKmSBLUw6YHi6475gtA5zdOg3SnSDRLBlF4YWPGAFv7FR", - "5dMm4nIHhsBzMFzkulb2dei6GYWi3N0MBLTMSkhBmnyNtDoT5dIeFJGO0P43a0pkbhR7JNKwn8xYCZe8", - "zHyLvgcSLCYRMoNVXLryVrwhgxUT8UnP6pGFYak/xpEhgIMoo7uDsQ1TcIGGmwyOXePD2mMfiyUdOxCk", - "D8gAS5GWittzPlyMVZKmPsoqYclxdnTi5JT68JhCzhN7rBhRj/a7P3b04d6QZuJwPZ0McnZNGpcLoJMM", - "FNcdJIbUhm4aaBhayDxXU54naLxCkkFutoaR0CiG59QS9aRK+93bUz47e59nZ2cf2EtsS3YysHNYT+j0", - "laULLufQhMRDOrUWMKwgrUKR3kHjTk6Ni/u1Z992a8ajQqk8qd23bgi/J+a7eD8X6TlkDOUEGZ1O+3zV", - "3iEchN1DEtf1IcflYu3t2aIACdn9A8aOJYNlYdYuVtCxNDqDy6/MpvFXNGpW0Xkrl4wWeXAm4266Pa29", - "JU95MJs5yaYv3XIoC2TzQGYlB9iJX9JhA4KL8ufGSN876hmonJ4mDYjKzmIXf/gHyunhrV0WGTkbjVbR", - "1XQpKLEnaDZGyenPWvveqjAHjJ2S7EBvQcMFlDynrAXtg6BCs6VAp1NXaQqQHZ3JpDWTVC3dwPea/1qx", - "dFYdHj4Cdni/20cbNBOdY2R5oNv3W3Y4tp8IXexbdjY6G/UglbBUF5BZ5zCka9trK9h/q+GeyZ96gpkt", - "+dq6lZ4Xma5mM5EKi/RcoVyfq461JxV9gRKnB+icaSbMmFQZYZSsZLsvDQOOolbLXcQvIlDRPkZVitLO", - "n7C1aUczWPEUV8lJyKzZJRJKTWd948OoIgkBRMOpG0Z0AW3dkuM35Lu+PLfe9Ob5nXb86RY6AnI92G4z", - "95ARncEu7H/MCoW7LlwujU+4yIU2vUk6x55OM2qCjCidA/Z/VMVSTvxbVAZqn0qV5KiQA4sjkI71YzpL", - "rcEQ5LAEG+6gL19/3V3411+7PReazeDSJ6Bhwy46vv7aMoHS5tYc0CHN1UnEgKIgM2rTSNLwguvFwdaA", - "M8HdKc4cgD557gckZtKaVMzVeISubr6+A4a3gFgJzt7TraCPtl/VLEx2c/un19rAsh+5tF1/GbBE33oP", - "radplcyFhGSpJKyj+d1Cwiv6GNXTRCIDnYlZh/p2PdjW/DvTao+zy27eFr+02wFJvKlT7+5g87twO0Hr", - "MM2PrEzIC8ZZmgsKCCqpTVml5kxyClB0zKAOWfiwy3DI6plvEo+RRUJYDtSZ5BpxWIctoocZM4gEJL8H", - "8JErXc3noDtmEZsBnEnXSkhWSWFoLLIqE7thBZR06nRgW6IlMOM5Rdh+hVKxaWXaopeykaxlYyPoOAxT", - "szPJDcuBa8NeCXm6InDew/E0I8FcqvK8xsKAhwYStNBJ/GDuB/v1R64XfvnY0Asb19kGiRF+k7K0NtBK", - "d/6/9/7r6P1x8j88+fUwefqfkw8fH1/d/7r348Orb7/9f+2fHl19e/+//iO2U37usVwZN/OT584sOXlO", - "uqcJnvfm/tmCv0shkyiRobuwFJJSLju0xe6hBvUEdL8Jw7tdP5NmJZGQLnguMnSBb0IOXRHX40XLHR2q", - "aW1EJ5bn1/oh5u7MVVLw9JzOtUdzYRbV9CBVy4k3xyZzVZtmk4zDUkn6lk14ISbo3k4uHmxRjbeQVywi", - "rq7GIyd19J1n0DjAsQV1x6yj6P5vo9hXP7w4ZRO3U/ormzhnQQcZTxEL2h2GtZw5XLy9+GEzB9GZeQ4z", - "IQV+PzqTGTd8MuVapHpSaSi/4zmXKRzMFTtiDuRzbjjFADoRxaG7WRSfcbMpqmkuUnYequKGNYcCY2dn", - "75FAzs4+9A6y+orTDRUPNtIAyaUwC1WZxEVlh+MITayFINu43KZRx8zBthTpor4O/kAAtCh0EkTE4ssv", - "ihyXH5ChZtSJ8qCYNqr0QhAlo4tp4P6+Vu4or+SXPhu9Qr/970tevBfSfGCJ87+Pi4LCbRTv+ruTNUiT", - "6wJ2j5k1U2yAxfwsWrg1qK6dG0dA39lePois45jDT4Q6aoNSoYkJ3hRPCOpHlePm3hhNAYwodiqzSJCn", - "oqvSSFrED8EdQj5HWejP3tBtRuJzd1qmwNIFpOeQ0cEDxQjHre7+yNtpFs+yQttrKDYFjnKlyR2cAquK", - "jDvdy+W6m7SqwRifqfsWzmF9qppU6+tkqV6NRy6onyDNDDFIgfgIlICatdnFHwx0Nt+drVDgvSiYjW3b", - "7EJPFkc1Xfg+wwxkNdMdME+MKGo0bKD3gpcRRFjiH0DBDRaK8G5F+tFIOi+NSEVh179bbP5Nqw8C2SbU", - "o2JczbrSuidMo9LbNk6mXMcFN+AX3A/koW52iR/JRlbsIRmjq8yOcKc5BKdK2nE2L8nY8cu2dzOHphan", - "Eihlo039NNoYCdX2wh1LiovmMJKOnXdRcFsPpZCKfL6AaIefBY6bwwUfPAkYvENwEiQBBFfT6hsCXrB1", - "mWFc3xaxt8T9TQJ/fcDfGRiNr5X/Px65XK/YdihJ2j2DHObcBb4pi8wRipvaVzrYIJzHT7NZLiSwJJZP", - "wLVWqbBnoY0sd2MAGn9fM2YDK2xnCDEyDqZNEUMCzF6rkDfl/DqTlCAoxMg9bIo1Bn/D9ohbc13fmZVb", - "zb++7GiYaNxcp7Hb2I/+jEdRkTRkmbdaMdtkCj1XJkaiKJr68ZB+1EVDDqSOk5ZkTc5jUTK0KoDI8J3v", - "Fpjr7J6YoZK/HwSOS5ij7934q8itPgDzeWMGF8pAMhOlNgm5ytHlYaPvNRmD32PTuPhpoYrZ+74ii0sf", - "GvYc1kkm8iq+227cvzzHYV/XfouupuewJiUDPF2wKd1PRy3UGh7bbBja5tRsXPBLu+CX/M7WuxstYVMc", - "uFTKdMb4QqiqI082MVOEAGPE0d+1QZRuEC9BNkJftgR5EDZngvIrDjZ56z1munZGx6DktZCiawkM3Y2r", - "sIk/NrcnuN7dz5ke4AFeFCJbdXxnCzVO4zTEdQx1a/H3sEC764BtwUDgJ8dSCEvwvr7d0kBn2ov6vTSr", - "7ZjpJncFAiEcSmhfZqaPKCRtysbZhqtT4PlfYP1XbEvLGV2NR7dz+WO4dhC34PpNvb1RPFMM2bqArcjZ", - "NVHOi6JUFzxP3LWUIdIs1YUjTWrub7F8ZlEXd79PXxy/fOOmT9lrwEuXtLVpVdSu+GJWhR5xLHPrNIiM", - "kLXqfWdriAWbX98NDIMpPtGuZcuhFHPEZdmrVnAhK7rgyix+lLU1VBIm592IM1vZfbeNzIWpfnfK8j0O", - "i1Nos8Nb5EI41obCAktbO0MzJbsJDmjGkZdJ5LLka9xFG5jtCwhZLRNkgUTnIo2HDuRUIxfJakk3NtYG", - "GDUeMAgRYiUGwueyEgEsbKZ3OCnqTDIYI4pMCutswN1UuaJnlRT/rICJDKTBT6VLeGoxC/KGz+Htq7R4", - "vrAD7FKGa/C30fMIakjD0yQ2K/kwyhvJEvdOn19oHZ7GH4Lg3DUOacIRe2ppwwGLow9Hzfake9GO1oY1", - "yvoyCAnD1rPYXiDNhw4WdqIDY0QLng1K7ONhaU154LvL6UYs03RDgWxz83iuVQRMJS+5tPWLsJ/Foeut", - "wfrt2OtSlXRpSUP0hFroZFaqXyHuTc5woyI5WA6VZLJR74PIZZCuEK0jI01lOo/fcB6DpD1kTQUfWfsQ", - "bYDDicqD8DUllfogE5eWrG2tpdbRbZw5wnSLiYXfMIebcy9FJeeXUx4rO4BGDc7puDkoaYXDjGK+s98F", - "XedSO9oLzlzqtsLe9CmgbBIl+zc1b2igfFkkn0EqljyPR0czwn77rmcm5sIWrKo0BBWRHCBb6c9Skasq", - "ZY+iGtSczNjhOKi55nYjExdCi2kO1OKBbTHlmrRWHfKsu+DyQJqFpuYPd2i+qGRWQmYW2iJWK1YbkfZy", - "g48/T8FcAkh2SO0ePGX3KPKuxQXcRyw6W2R09OAppWTYPw5jys5VptskVzISLP/tBEucjunowcJAJeWg", - "HkRvndlyosMibAM32a678BK1dFJvOy8tueRziJ+oLrfMyfal3aTAXQcvMrO18LQp1ZoJEx8fDEf5NJCW", - "heLPTsPlyi+RgYxiWi2RnppyR3ZQD84W1nMlSPy8/Ec65ij8nYeO0/p5g7RWl8dWTYdRr/kS2mgdM24v", - "Z9K1DXep1wnEg4FaEVBexAcpBzbY603Xl92TSiZL5J3sfpPwF9BftFSCMjyPDmu87OpmrmwGvauphVCS", - "QcRWLcTyQCbdGMVVGV8nr3Con9++dIphqcpY3YNGGjolUYIpBVxEObabuFZbJrW68JiPGSjfVSLP/tqk", - "m3ZKDJVcpoto/HOKHX9paqLVaLdYj17BW3ApIY+Cs7z8i+f5iFT6h9p1nKWQO7btlg6yy+0srpl4e5p+", - "Un5ARK8wOQ4QYrWdf1cnjuRzlTEap7lk3RBC/45UUEblnxVoE7vPRR9srhP52Giv2CoeDGRG2v6A2ftP", - "OJfWDRbSsmJZ5fY2BGRzKF0ApipyxbMxQzinL45fMjuqdndo6d4NVRGZ27t0rVV0fKug+sN1LhcOpUbt", - "DmdzzgiuWhu6kq0NXxaxrFdsceobUGrtBRe5Tz8g9RNi54A9t5pfe71iB2nukLJ6OCdriCbwP8bwdEEq", - "taWAhkl+9/I3nip1UAayrqhXF1Ww1yKN8hVwbAGcMVNo91wKbUvZwgW0E23rrHNn0vnE2/byykpKSylx", - "/bThVsRN0O4nZw/2fEgqOrMO4q+pZrSqyhSuWw3oHfWK3rHqlhbq1X+UkJ2uZF1/zZcoT7lUUqR0wyko", - "nltP2ZXF3SVmusNlsK677FnccWiEuaIFjerUAYfFwRJHXhA6xPUDRsFX3FRLHfZPQ/VX0RGcg9FOskE2", - "9kWrnB8npAZXJIMqJAdyEt3x7vlh9Gijua5/TTKi9L8Bc+V7/EaminApO+dC0uVVhzaXHWQ9LaraadC9", - "E4bNFWi3nvblLP0e+xycruQJzvjDga/ySTBsCBmXbc8s+qCO/QmGOzHAts+wLaNwcfNzK9XQDnpcFG7Q", - "mCTQ9Q7Hym4NIjgSBU98GDJAbg0/hLaB3DYePZI+RUKDCzq4gIL0cI8wBq7Av0Cn1lKUvUlrj/yjVzOE", - "jEzjpZDQ1KCNKIg0qhJoY4hfB/rptOTGmoA7ybRT4DmdlMQEmjYudHRbUJ0NJpTQGv0Yw9vYFF8bEBx1", - "g8Zw43Jdl75F6g6MiWdUc9shsl9KjawqZ0RllNTVKa4WExwouH1ZwrYC6LNB3yay3U3JLedcRxMNJaFn", - "QqM7spzmkTSW5/XHoMAg5ctN1/Rv7ALy8ArcwdqNC2ZQx2vbl5uLV+S494kW8xvuStP/DrelwwPhHsWo", - "/wWKlfDeTu8uuRU89bUaOsJXvtwrORV1YnqbZknQRZ22pnLnZqd1uAbnmETjQCLP2+bGKLfS18YGh9J5", - "0sHsM25caqnhbFMVGVs4MwbBnkPagp328YtoYGDo7NEePeLnXu/d7IaeFUawNyLUH2r3J/QXn7XCCi5c", - "4LthkT5mXX5bP+Nwl8yXZoO7i3BZYwQktpIbJnntxHt9LEUYO0wN2EKe5y2U2tsgHUtSlXDHqA1U6DVR", - "20962HV5tA6imEpDf507b0ALtwO43wXxjVzoI3eYnc10F3aOJ9Vjd5InFiH+2kdfmnw2adCq9+vGje36", - "X4eiB9ZDHghUdXBaiTzbtrmtsGNznZkCa79Mv3ncit59zgvVv9gD+T67ubul11H83U0gxETW2ho8GCoI", - "KO4QS3TdIpFDqgWVVqUwa8rd8Zam+CWal/wDSFf12BWRr09A3QGcfb/EhabndevmyYkflC0DvUTzl0xB", - "Q0VSXqz4ssjB8cW3X03/BI/+/Dg7fPTgT9M/Hz45TOHxk6eHh/zpY/7g6aMH8PDPTx4fwoPZN0+nD7OH", - "jx9OHz98/M2Tp+mjxw+mj795+qev/HsPdqLNWwp/o6oDyfGbk+QUJ9vghBfiL7C294yRjP0NZp4SJ8KS", - "i3x05H/6X57DDlK1DJ6oc7+OXKR/tDCm0EeTyeXl5UHYZTKnsn2JUVW6mPhx+jVp3pzUAVp74E87amNv", - "SAq0qY4Ujunb2xfvTtnxm5ODhmBGR6PDg8ODB1QopADJCzE6Gj2in4h7FrTvE0dso6OPV+PRZAE8Nwv3", - "xxJMKVL/SV/y+RzKA3eVG3+6eDjx8Z3JR3fIfYVQ57GsJl9qq44v9m84j23AAn2WurRWcJlHuzs+Yza1", - "+TvMVXeTGUUAbW4GirYaWSdZ8CBm8PLCuPWe5/sv6ImqWN2n2FXx2KOjdZb58KMzwbt8/i2+J3++ihw0", - "feg8JPLw8PATPB4ybkHxeLnhKySP73CKbQ/q1hPtgutJhVc8R7qB+mG5ES3owRe7oBNJ9zlQbDErlq/G", - "oydf8A6dSGQcnjNqGaSQ9EXhz/JcqkvpW6JKrpZLXq5J4QYXyUPT6mpQ5LaTt9yNvGE5DEEFsuASbyuw", - "PV17OhszXRd6Lkqh0HCgZxgzSEvgpOZVSedBTS0zd1URbGXrV8d/o+jxq+O/2SKB0SfqguFtwcy2EP8B", - "TKTW3nfr5pmljRL9txKT49/tq35fjs67rarZV2z8Yis27iC097u7r8f5xdbj/LJN0lWdeMuZVDKRVNTg", - "AlgQ1trbqL9rG/XJ4aMvdjXvoLwQKbBTWBaq5KXI1+xnWWcE3c4Er2VOJYMcrY3yp1d9vrGiA/M9KLA0", - "+dh6VCLbHjxp3cjOWjXIefyhy6D2jMsGHTfXTLnMbCaHP6vVY3/dkqJ19l6z3Y9x7zLmQcxID45avluf", - "PN/FLm+tKbiBFrPNW/i63vO5nzRiceNHSD+lBujN4zueMZ8y+oll827C9PHh4883g3AXXivDvqcks08s", - "0j9pnCBOVoGwoSJmk4/+stoOAsZdBG2Llu7LtTGhghw6djn7rkxz/fgGyhMrCO1d3L7UwBF2lRf9u6ox", - "SdHcz/u9yIhrPQy8lwt7uXBjudAlqEYi2GcMJx8pwTYUBz2WpLeV/0AHJUH1vFItffkWxWZg0oV79rlz", - "lj30ev9GmbLpWuGt5cv+0e/bPPq9g0OyR/DneVX9Sw58BNqSJew1mUPE4D7n+o8Y9viUGvlTL+i1ksBg", - "JTRV1bS0uD9urM0FuoBPSPFvJYQV72vTwT1VOvnYvB181WSC2EuCE2v5b7Ir7AMvozs909k/yvMFPMrz", - "23sVt+KQzmpLCB9ABndJtuEWX5SzX6mynSzlmutFZTJ1GaRWNcWPBznJP4V/h5y0f49//x7//j3+/Xv8", - "+/f49+/x79/j/zLe4//tDavrOlLdYN0n9G7apmpgsjSmmv17csmFSWaqtGoooQppkUBpe/T/5sK4unzO", - "hzIKhQKgJqYaa1agODhBlRQd5qy4xyv8g+diGTlcxaG+V+VOcdkm2GkUw4WxShrhs+rpkSNvt/3+gpx7", - "i3Rvke4t0r1FurdI9xbp3iL9Mi3S3yZ5gSWJF8g+MzWWl8pGX6TV/AWlfn7OXM3GkK7NaDK80exFPt54", - "qGGA5xNX74vOf5UezI4Ka4elOJyQrMg5FTReGX9Hh2oZf/PYJzfUVXBs+QCUNdjg0UP27sfjJw8e/vLw", - "yTf1W+Lttvd87VVt1jncd4e/9d1gfwoMklO5HToE5t6jSH1mhrWQZyIHRi9avqDmz+ECcjSPkSKprFkV", - "cTlOgefPHHKsVAJtvlPZukM4uP4JoaJNMs2tfSF5GalgFXmyuotko6iKnSvJ1vNKru40oyJeZri/Ydv2", - "aqDUbpS8N9HL1gqvrviog73LWQ7uqUcnc9WvflORzWhGjswa8fS7SULrvkDhGIfaolXh+O9LTRjziI8y", - "HrHtGGkyq1KgB8Ucxa0SbDQHmTixkExVtvavT7hiei0pa6ucDQvZFytIK+Qlmoljg3v6vnu7kao1huGT", - "aJXZoGgyELzmvaHPLThtwa6NcvPm1NEu/3vrdIMuuL7UCO423FMlm5eqKu7bdw7kmhzvZcHl2oeW0Fak", - "+sH0VCelSN2tpK5rJ/bk7O7lb0N/he67dX+3aGGXXPvat5ktfhsvANQt0bod400Bwm0FY+x6o8VSB0qj", - "9jfR77JLzqzDaQWUiVnJSMnCToHCfV7yv4RKeFOqC4GOc1TC2mC3iQqEg62aoQxEFqmGzi1Vrxva8vQt", - "vwzvvO4qU1eJMzxvbZUuwL4n5q20yJVe1Jel4lnKNaVeuqrSn9hiNauTSNyBpkmlGWa9q2mowLc/HUBw", - "d7InA9DNM1V0d1rbGlS/rXXZVE45dpmuLWzsQwF/lFDAd575NOOs5Jdd5gwqve8gpvilWcmolJo07+BF", - "M7MChqgfzrrD87Ae+PaxWPBClT2XgbxgnKW5oCMHJbUpq9ScSU4h0PBlsP6RmQ/sDptSz3yTeBQ+EiR3", - "oM4kp7dc6sBo1KSaQazuOYC32HQ1n4M2HUk8AziTrpWQzbsxS5GWKrH5iaiuUaIf2JZLvmYznlMM/1co", - "FZuiFxFe96WAojYiz90ZHQ7D1OxMcsNyQKH/SqBBh+B8zKk+d3YvDIQvz/cD9K4Y28BjRz/Yrz9yvfDL", - "93EjCm/Zz/YY6vM/VeTnHnvD0M385LkrxXHynG5XN8dzvbl/tuOlpZBJlMhQ47tT7i5tsXvu4SwioPvN", - "QZ/b9TOJxrRR9l345tXa65FD9xigx4uWOzpU09qIzmmBX+uH2DWQuUrQZaQKzaO5MItqSkUL/fWQyVzV", - "V0UmGYelkvQtm/BCTHQB6eTiwRb74BbyikXE1V5z/3GC+N2XFeuNRyO2t/cDevkOKp/9vsudbU372RcX", - "2xcX25ef2hcX2+/uvrjYvvTWvvTWv2rprYONFuLko1ntUgwnhCoy+yBsCakduRbgYbNW2Zz+saQwB4yd", - "0muvHHUAXEDJc3rkW/tL/EKzpZgvDNNVmgJkR2cyac3EPjuKA99r/mvd3LPq8PARsMP73T42bhFI3n5f", - "MlXpk33751t2Njob9SCVsFQX4IpoUPOsorNi22sr2H+r4f5U9rZuydc2uLLgRQGo1nQ1m4lUWJTnCp2B", - "uerk90lFX6DEyQFKVM2EsfXKCJ+UF+myc7h72yhmdPf1+zVqxh93yCWeWo+Ed83Kwv+5S1nhfxUD+zkY", - "LnJdZ/xH/CnybLqUdcl1w7q1VBn7RHHtf3MH1m6UXJxDmINL2QeXvMx8i+ibbU2FOv8mYT+01C7dlcHK", - "mwTdSc/qkYWxxbbQ4ey9otOPbLkCWBum4KoE3WTwgbe3r8ajNFcaEoslHXuNhz6gKKJoLKdgLHevaftH", - "WhEGMjPH2ZV0pcZm9g+PKeQ8sQ9ARILU9rt7IKKOxnVi3xG4nk4G03tr0rBPeJO06SIxpLYZczfoBwLA", - "9r07m0Bx41fvOt17Dwrl2dnZB/bSFnek17DOYT2x77CkCy7noGschXRqr8HYtJogr7uDxrt7aQ+1RjLw", - "RuZJP9e7i/dzkZ5DxlBO+AfcB4x4dq+uVEePIF8u1v5Si1VD9w8YO5YMloVZ+/eQ27HmzuDyK7Np/FWo", - "ONsaKZI2mIK4gPKWPOXBbOYkDchwtxzKAtk8kFnJAXbilxGXdtfSRREPtuNPBkRlZ3EXgYG9Vtprpb1W", - "2mulvVbaa6VPppV6QZh9mOJzhCl+80DFH6hc4r4y4u9sQWHyZqv08S2it/UDjzEr2MVlmwdUwwdJKapW", - "P0X6/sPVB/xWXviAW/O+5tFkQlbFQmkzGV2NP3be3gw/oijlcwvBBbSKUlxQYdMPV/8/AAD//3Jx7Gro", - "5QAA", + "H4sIAAAAAAAC/+x9a3fcNpLoX8H27jmxs021/MpOdE7OXiXOQ3ccxydSZueu5ZtFk9XdGJEAhwCl7vjq", + "v99TBYAESbC79bATz+iTrSYehUK9UChUvZ+kqiiVBGn05Oj9pOQVL8BARX/xNFW1NInI8K8MdFqJ0ggl", + "J0f+G9OmEnI5mU4E/lpys5pMJ5IX0LbB/tNJBX+vRQXZ5MhUNUwnOl1BwXFgsymxdTPSOlmqxA1xbIc4", + "eTm53vKBZ1kFWg+h/EnmGyZkmtcZMFNxqXmKnzS7EmbFzEpo5jozIZmSwNSCmVWnMVsIyDN94Bf59xqq", + "TbBKN/n4kq5bEJNK5TCE8xtVzIUEDxU0QDUbwoxiGSyo0YobhjMgrL6hUUwDr9IVW6hqB6gWiBBekHUx", + "OXo70SAzqGi3UhCX9N9FBfAbJIZXSzCTd9PY4hYGqsSIIrK0E4f9CnSdG82oLa1xKS5BMux1wH6stWFz", + "YFyyn7/7hj179uxLXEjBjYHMEdnoqtrZwzXZ7pOjScYN+M9DWuP5UlVcZknT/ufvvqH5T90C923FtYY4", + "sxzjF3bycmwBvmOEhIQ0sKR96FA/9ogwRfvzHBaqgj33xDa+100J5/9ddyXlJl2VSkgT2RdGX5n9HJVh", + "QfdtMqwBoNO+RExVOOjbw+TLd++fTJ8cXv/r2+Pkv92fL55d77n8b5pxd2Ag2jCtqwpkukmWFXDilhWX", + "Q3z87OhBr1SdZ2zFL2nzeUGi3vVl2NeKzkue10gnIq3Ucb5UmnFHRhkseJ0b5idmtcxRTOFojtqZ0Kys", + "1KXIIJui9L1aiXTFUq7tENSOXYk8RxqsNWRjtBZf3RZmug5RgnDdCh+0oD8uMtp17cAErEkaJGmuNCRG", + "7VBPXuNwmbFQobS6St9MWbGzFTCaHD9YZUu4k0jTeb5hhvY1Y1wzzrxqmjKxYBtVsyvanFxcUH+3GsRa", + "wRBptDkdPYrMO4a+ATIiyJsrlQOXhDzPd0OUyYVY1hVodrUCs3I6rwJdKqmBqfnfIDW47f/79KfXTFXs", + "R9CaL+ENTy8YyFRl43vsJo1p8L9phRte6GXJ04u4us5FISIg/8jXoqgLJutiDhXul9cPRrEKTF3JMYDs", + "iDvorODr4aRnVS1T2tx22o6hhqQkdJnzzQE7WbCCr786nDpwNON5zkqQmZBLZtZy1EjDuXeDl1Sqltke", + "NozBDQu0pi4hFQsBGWtG2QKJm2YXPELeDJ7WsgrA8YOMgtPMsgMcCesIzSDr4hdW8iUEJHPAfnGSi74a", + "dQGyEXBsvqFPZQWXQtW66TQCI0293byWykBSVrAQERo7dehA6WHbOPFaOAMnVdJwISFDyUtAKwNWEo3C", + "FEy4/TAzVNFzruGL52MKvP265+4vVH/Xt+74XrtNjRLLkhG9iF8dw8bNpk7/PQ5/4dxaLBP782AjxfIM", + "VclC5KRm/ob759FQaxICHUR4xaPFUnJTV3B0Lj/Hv1jCTg2XGa8y/KWwP/1Y50aciiX+lNufXqmlSE/F", + "cgSZDazR0xR1K+w/OF5cHJt19NDwSqmLugwXlHZOpfMNO3k5tsl2zJsS5nFzlA1PFWdrf9K4aQ+zbjZy", + "BMhR3JUcG17ApgKElqcL+me9IHrii+o3/Kcs8xhOkYCdoiWngHMW/Ox+w5+Q5cGeCXAUkXJE6ozU59H7", + "AKB/q2AxOZr866z1lMzsVz1z4+KM19PJcTvO/c/U9rTr6x1k2s9MSLs71HRqz4T3Dw+OGoWEDNUeDF/n", + "Kr24FQxlpUqojLD7OMdxhpxCw7MV8AwqlnHDD9pDlbWzRuidOv5A/eiUBFVExf1E/+E5w8/Ihdx48w1N", + "V6HRiFOBoylDi8/qETsTNiBLVLHCGnkMjbMbQflNO7kV0I1EfevQ8q4/WmR3vrV2JaMefhG49PbUeDxX", + "1e3opUcIkrVnYcZx1Mb6xZV3d5aa1mXi8BOxp22D3kCt+3EoVkMM9YeP4aqDhVPDPwAWNI56H1joDnTf", + "WFBFKXK4B35dcb0aLgINnGdP2ekPxy+ePP316YsvUEOXlVpWvGDzjQHNHjm9wrTZ5PB4uDIS8HVu4qN/", + "8dyfoLrj7sQQAdyMvQ9HnQFKBosxZv0FCN3LalPV8h5QCFWlqojNS6RjVKry5BIqLVTEffHGtWCuBcoh", + "a3f3frfQsiuuGc5Nx7FaZlAdxDCP5yxS6QYKvUtR2KHP1rLFjRuQVxXfDHbArjeyOjfvPnvSRb637jUr", + "oUrMWrIM5vUy1FFsUamCcZZRRxKIr1UGp4abWt+DFGgHa4HBjQhB4HNVG8aZVBkyNDaOy4cRXyY5Ucj3", + "Y0KRY1ZW/8wBreOU18uVYWhWqtjWth0TntpNSUhX6JGjX3Nmt63sdNZPllfAsw2bA0im5u585U5+tEhO", + "bhnjb1ycdGrBas4EHbjKSqWgNWSJu17aCZpvZ3fZbMETAU4AN7MwrdiCV7cE1ijD8x2AUpsYuI054Q6l", + "Q6j3m37bBvYnD7eRV3jGtFSAtgtydw4GxlC4J04uoaLD2QfdPz/JbbevLkeuTpwGPhMFsi+TXCoNqZKZ", + "jg6Wc22SXWyLjTpmAq4g4JQYp9LAIw6CV1wbe0QXMiOT0Yobmof60BTjAI9qFBz5L16ZDMdOUU5KXetG", + "s+i6LFVlIIutQcJ6y1yvYd3MpRbB2I36MorVGnaNPIalYHyHLLsSiyBunI+o8WENF0fueNQDmygqO0C0", + "iNgGyKlvFWA3dB+PAILni6YnEY7QPcppfNbTiTaqLJH/TFLLpt8Ymk5t62PzS9t2SFzctHI9U4CzGw+T", + "g/zKYtZeHKw42nY0Miv4BeomstSsL2EIMzJjooVMIdlG+ciWp9gqZIEdTDpiJLuryWC2HnP06DdKdKNE", + "sGMXxhY8YrG/sR7ws9Y7dA9Gy0swXOS6MUwaN3s7C3nk+9ESaEVWkII0+QZpdSGqwl5qkTrT/jdr9mRu", + "Fnt907KfzFgFV7zKfIvhaSlYTCJkBuu4dOUd30gGaybiQC+amYVhqb9ykuEAB1FGd5d4W0BwTpHbTI5d", + "49PaKyqLJR27vKQPyACFSCvF7Z0kLsbqc9Ncu1VQcISObsecuh2fU8hlYq9AI+rRfvdXpN41HdJMfFxP", + "J6Oc3ZDG1Qro1gXFdQ+JIbXhkRI0jC1kmas5zxM0tCHJIDc7XV5owMNLaol6UqXD7l2Qz8/f5tn5+Tv2", + "CtuSTQ/sAjYzuilm6YrLJbTu+5BOrbUOa0jrUKT30LjXAcz5KLvQd49g00mpVJ40R83+dcNAzPfxfiHS", + "C8gYygkysJz2+ay7QzgJe4QkrpsLmavVxptuZQkSsscHjB1LBkVpNs6v0bM0epPLz8y2+dc0a1bT3TCX", + "jBZ5cC7jLgV7s3xHnvLDbOckG2p1x6nsINsnMms5wk78ii5GcLgof271Sp5Sz0DlDDRpQFQWin3O7t9T", + "/BHv7LLI6BjQahVdzwtBQUhBsylKTn8vPDxZC3PA2BnJDjzYaLiEiucUYaG9w1ZoVgg8IOs6TQGyo3OZ", + "dCBJVeEmftT+14ql8/rw8Bmww8f9PtqgmejOcJYH+n2/YodT+4nQxb5i55PzyWCkCgp1CZk9B4V0bXvt", + "HPZfmnHP5U8DwcwKvrEnKM+LTNeLhUiFRXquUK4vVc/ak4q+QIXgAZ7ZNBNmSqqMMEpWst2XlgEnUavl", + "PnwtkVHRPkZVitLO3wZ2aUczWPMUV8lJyGzYFRJKQ2dD48OoMgkHiLp+t8zonO+6I8dvyXdDeW4P/tvh", + "O+sd/TvoCMj1YLfNPEBGFIJ92P+YlQp3Xbi4Hx8ckgttBkA6NwDdvDQEGVE6B+z/qJqlnPi3rA00ZypV", + "0UGFDrA4A+lYP6ez1FoMQQ4FWM8Mffn88/7CP//c7bnQbAFXPlgOG/bR8fnnlgmUNnfmgB5prk8iBhQ5", + "xFGbRgKcV1yvDnY6x2ncvXziwdAnL/2ExExak4q5nk7wqJtv7oHh7UCsAmfv6Y6LSNuvahEG5rn90xtt", + "oBh6WW3XX0cs0Z/9CW2gaZXMhYSkUBI20Vh0IeFH+hjV00QiI52JWcf69k+wHfh7YHXn2Wc374pf2u2A", + "JN40YYL3sPn9cXsO9jAkkaxMyEvGWZoLch8qqU1Vp+ZccnJQ9MygHll4t8u4y+ob3yTuI4u4sNxQ55Jr", + "xGHjtohevCwg4pD8DsB7rnS9XILumUVsAXAuXSshWS2FobnIqkzshpVQ0Q3ZgW2JlsCC5+Rh+w0qxea1", + "6Ypeipyylo319uM0TC3OJTcsB64N+1HIszUN5084nmYkmCtVXTRYGDmhgQQtdBK/RPzefv2B65VfPjb0", + "wsZ1tv5sHL8Nr9oY6IRm/99H/3n09jj5b578dph8+e+zd++fXz/+fPDj0+uvvvp/3Z+eXX/1+D//LbZT", + "HvZYXI+D/OSlM0tOXpLuaf38A9g/mvO3EDKJEhkeFwohKTy0R1vsEWpQT0CP2xsDt+vn0qwlEtIlz0WG", + "R+DbkENfxA140XJHj2o6G9Hz5fm1vosdd5YqKXl6QXfwk6Uwq3p+kKpi5s2x2VI1ptks41AoSd+yGS/F", + "DI+3s8snO1TjHeQVi4ir6+nESR1979E+buDYgvpzNl50/7dR7LPvvz1jM7dT+jMb5GeHDqKzIha0e2PW", + "Oczh4u0jFRvliIeZl7AQUuD3o3OZccNnc65Fqme1huprnnOZwsFSsSPmhnzJDScfQM+jOPaOjPwzDpqy", + "nuciZRehKm5Zc8wxdn7+Fgnk/Pzd4M5tqDjdVHFnI02QXAmzUrVJnFd23I/Q+lpoZOuX2zbrlLmxLUU6", + "r68bf8QBWpY6CTxi8eWXZY7LD8hQM+pEMVtMG1V5IYiS0fk0cH9fK3frWPErHzlf47n9fwpevhXSvGOJ", + "O38flyW528jf9T9O1iBNbkrY32fWgtgOFjtn0cKtQXXjOD4a9NT28k5kHcccfiLUURuUCq1P8LZ4wqF+", + "UDlu7q3RFIwRxU5tVgnyVHRVGkmL+CF478iXKAv93Rsem5H43PubObB0BekFZHTxQD7Caae7v951msWz", + "rND2yYwN16O4bjoOzoHVZcad7uVy0w+w1WCMjyr+GS5gc6basPCbRNReTyfOqZ8gzYwxSIn4CJSAWnTZ", + "xV8M9Dbf3a2Q470smfVt20hITxZHDV34PuMMZDXTPTBPjCgaNGyh95JXEURY4h9BwS0WiuPdifSjnnRe", + "GZGK0q5/P9/8m04fHGSXUI+KcbXoS+uBMI1Kb9s4mXMdF9yAX3A/kIf6kRR+JutZsZdkjJ5dO8Kd5xDc", + "KmnH2bwiY8cv274jHQMtTiVQyVabejC6GAnV9spdS4rL9jKSrp33UXA7L6WQiny8gOi6nwXOm8MlH70J", + "GH3vcBIEAQTP6JrXDF6w9Zlh2rxssS/a/asH/9TBv2+YTG/0VmE6cXFpse1QkrR7BjksuXN8U8SbIxQH", + "2mc62CCE46fFIhcSWBKLJ+Baq1TYu9BWlrs5AI2/zxmzjhW29wgxMg7AJo8hDcxeq5A35fImQEoQ5GLk", + "fmzyNQZ/w26PW5tawJmVO82/oexomWjaPv2x2zj0/kwnUZE0Zpl3WjHbZA6Do0yMRFE0Df0hQ6+LhhxI", + "HScdyZpcxLxkaFUAkeGp7xaY6+yRWKCSfxw4jitY4tm7Pa8it3oHzMf1GVwqA8lCVNokdFSOLg8bfafJ", + "GPwOm8bFTwdVzL5NFllc+tC0F7BJMpHX8d128/75JU77ujm36Hp+ARtSMsDTFZvTW3rUQp3psc2WqW1M", + "zdYFv7ILfsXvbb370RI2xYkrpUxvjk+EqnryZBszRQgwRhzDXRtF6RbxEkQjDGVLEAdhYyYovuJg22l9", + "wEw3jugYlbx2pOhaAkN36yps4I+N7Qmeog/ju0d4gJelyNa9s7MdNU7jNMVNDHVr8Q+wQLvrBtuBgeCc", + "HAshrMCf9e2WBjrTJhUYhFntxkw/uCsQCOFUQvuUOENEIWlTNM4uXJ0Bz/8Mm79gW1rO5Ho6uduRP4Zr", + "N+IOXL9ptjeKZ/Ih2yNgx3N2Q5TzsqzUJc8T94RmjDQrdelIk5r7FzcfWdTFj99n3x6/euPAp+g14JUL", + "2tq2KmpXfjKrwhNxLHLrLPCMkLXqz87WEAs2v3nHGDpTfKBdx5ZDKeaIy7JXo+BCVnTOlUX8KmunqyQM", + "zrsVZ3ai++7qmQtD/e6V5QccFqfQdod3yIVwri1JEAqb50MzJfsBDmjG0SmTyKXgG9xF65gdCghZFwmy", + "QKJzkcZdB3KukYtkXdDjko0BRo1HDEIcsRYj7nNZi2AsbKb3uCnqARnMEUUmuXW24G6uXIK2Woq/18BE", + "BtLgp8oFPHWYBXnDx/AOVVo8XtgN7EKGm+HvoudxqDENT0BsV/KhlzcSJe4PfX6hjXsafwiccze4pAln", + "HKilLRcsjj4cNdub7lXXWxvmUxvKICQMm3tjdzI37zpYWUBH5ogmZxuV2Mfj0priwPeX061YJnBDgWxj", + "83iuVWSYWl5xaXMtYT+LQ9dbgz23Y68rVdGjJQ3RG2qhk0WlfoP4aXKBGxWJwXKoJJONeh9EHoP0hWjj", + "GWmz6Hn8hnCMkvaYNRV8ZN1LtBEOJyoP3NcUVOqdTFxasrZ5oTpXt3HmCMMtZnb8ljkczIMQlZxfzXks", + "RQIaNQjTcXtR0nGHGcV8Z78LuomldrQX3Lk0bYV96VNC1QZKDl+V3tJA+bRIPoNUFDyPe0czwn73XWMm", + "lsIm16o1BNmb3EA2K6GlIpcBy15Ftag5WbDDaZAfzu1GJi6FFvMcqMUT22LONWmtxuXZdMHlgTQrTc2f", + "7tF8VcusgsystEWsVqwxIu3jBu9/noO5ApDskNo9+ZI9Is+7FpfwGLHobJHJ0ZMvKSTD/nEYU3Yui942", + "uZKRYPkvJ1jidExXD3YMVFJu1IPoqzOb+nRchG3hJtt1H16ilk7q7ealgku+hPiNarEDJtuXdpMcdz28", + "yMzm7dOmUhsmTHx+MBzl00hYFoo/C4aLlS+QgYxiWhVIT21qJjupH84mAXTpUjxc/iNdc5T+zUPv0Ppx", + "nbRWl8dWTZdRr3kBXbROGbePM+nZhnvU6wTiwUheC6gu45NUIxvs9abryx5JJZMCeSd73Ab8BfQXTeug", + "DM+j0xovu/qRK9uH3tfUwlGSUcTWHcTyQCbdGsV1FV8nr3GqX35+5RRDoapYjoZWGjolUYGpBFxGObYf", + "uNZYJo268JiPGShf1yLP/tKGm/bSIVVcpquo/3OOHX9t87c1aLdYjz7BW3EpIY8OZ3n5V8/zEan0N7Xv", + "PIWQe7btpzmyy+0trgW8C6YHyk+I6BUmxwlCrHbj75rAkXypMkbztI+sW0IYvpEKUr78vQZtYu+56ION", + "daIzNtorNuMIA5mRtj9g9v0TwtJ5wUJaVhR1bl9DQLaEyjlg6jJXPJsyHOfs2+NXzM6q3RtaendDGU+W", + "9i1dZxW9s1WQkeEmjwvHQqP2H2d7zAiuWht6kq0NL8pY1Cu2OPMNKLT2kovchx+Q+gmxc8BeWs2vvV6x", + "k7RvSFkznZM1RBP4H2N4uiKV2lFA4yS/f6oeT5U6SFnZZP9rkirYZ5FG+Ww9NlnPlCm0e66Etml34RK6", + "gbZN1Lkz6XzgbXd5VS2lpZS4ftryKuI2aPfA2Ys975KKQtZD/A3VjFZ1lcJNMxedUq/oG6t+GqRBrkoJ", + "2dlaNrnifDr1lEslRUovnIJEvw3ILoXvPj7TPR6D9Y/LnsUdh0aYK5p8qQkdcFgcTcfkBaFD3NBhFHzF", + "TbXUYf80lCsWD4JLMNpJNsimPsGWO8cJqcElyaBszoGcxON4//4werXRPte/IRlR+N+IufIdfiNTRbiQ", + "nQsh6fGqQ5uLDrInLcowavB4JwxbKtBuPd3HWfot9jk4W8sThPjdgc9ISmNYFzIu295ZDIc69jcY7sYA", + "236DbRm5i9ufO6GGdtLjsnSTxiSBbnY4liJsFMERL3ji3ZABcpvxw9G2kNvWq0fSp0hocEkXF1CSHh4Q", + "xsgT+G/xUGspyr6ktVf+0acZQkbAeCUktPlyIwoijaoE2hji15F+Oq24sSbgXjLtDHhONyUxgaaNcx3d", + "dajeBhNKaI1+jvFtbBPFjQiOpkFruHG5adL0InUHxsQ3lB/cIXKY9o2sKmdEZRTU1UsEFxMcKLh9CsWu", + "AhiywdAmst1NxS3n3EQTjQWhZ0LjcaSY55EwlpfNxyAZIsXLzTf0b+wB8vgK3MXarRNmUMcb25fbk1fk", + "uPeJFstb7krb/x63pccD4R7FqP9bFCvhu53BW3IreJpnNXSFr3xqWjpUNIHpXZolQRc9tLVZRrcfWsfz", + "hU5JNI4E8vzcvhjlVvpa3+BYOE86Gn3GjQstNZxtyyJjk3zGRrD3kDa5qC3UEXUMjN092qtH/DzovZ/d", + "MLDCaOytCPWX2kOA/uyjVljJhXN8tywyxKyLbxtGHO4T+dJucH8RLmqMBomt5JZBXnvx3hBLEcYOQwN2", + "kOdFB6X2NUjPklQV3DNqAxV6Q9QOgx72XR6tgyim1jBc594b0MHtCO73QXwrF4bIHWdnM9+HneNB9did", + "5IlFiH/2MZQmH00adHITu3lju/6XMe+BPSGPOKp6OK1Fnu3a3I7bsX3OTI61X+dfPO947z7mg+pf7YX8", + "kN3c29KbKP7+JhBiImvtTB5MFTgU9/Alum4RzyHlgkrrSpgNxe54S1P8Go1L/h6ky9DsEt43N6DuAs7W", + "WnGu6WXTui2P8b2yKasLNH/JFDSUJOXbNS/KHBxffPXZ/D/g2Z+eZ4fPnvzH/E+HLw5TeP7iy8ND/uVz", + "/uTLZ0/g6Z9ePD+EJ4svvpw/zZ4+fzp//vT5Fy++TJ89fzJ//sWX//GZr01hAW3rPvyVsg4kx29OkjME", + "tsUJL8WfYWPfGSMZ+xfMPCVOhIKLfHLkf/pfnsMOUlUE5fTcrxPn6Z+sjCn10Wx2dXV1EHaZLSltX2JU", + "na5mfp5hTpo3J42D1l74045a3xuSAm2qI4Vj+vbzt6dn7PjNyUFLMJOjyeHB4cETShRSguSlmBxNntFP", + "xD0r2veZI7bJ0fvr6WS2Ap6blfujAFOJ1H/SV3y5hOrAPeXGny6fzrx/Z/beXXJf46jLWFSTT7XV+BeH", + "L5yn1mGBZ5YmtVbwmEe7Nz5TNrfxO8xld5MZeQBtbAaKtgZZJ1lQvDOoEjHt1B59+wmV04rlfYo9FY8V", + "SG2izMcL5AQ1BH3dwBd/uo5cNL3rFT15enj4AQqdTDujeLzcsmLK83sEsXuCujOg/eEGUuFHniPdQFME", + "b0ILevLJLuhE0nsOFFvMiuXr6eTFJ7xDJxIZh+eMWgYhJENR+Iu8kOpK+paokuui4NWGFG7wkDw0ra5H", + "RW43eMu9yBuXwxBkIAse8XYc2/ONp7Mp002i57ISCg0HKhmZQVoBJzWvKroPanOZuaeKYDNb/3j8V/Ie", + "/3j8V5skMFpOL5jeJszsCvHvwURy7X29aUtCbZXov5eYnP5hKxB+OjrvrqrmIWPjJ5uxcQ+h/bC7D/k4", + "P9l8nJ+2SbpuAm85k0omkpIaXAIL3FoPNuof2kZ9cfjsk13NKVSXIgV2BkWpKl6JfMN+kU1E0N1M8Ebm", + "1DKI0doqfwbZ51srOjDfgwRLs/edohLZbudJ50V21slBzuNFOYPcMy4adNo+M+Uys5Ec/q5WT/1zS/LW", + "2XfNdj+mg8eYBzEjPbhq+Xpz8nIfu7yzpuAFWsw27+DrZqV+P6jH4tYFUz+kBhjA8TXPmA8Z/cCyeT9h", + "+vzw+ceDINyF18qw7yjI7AOL9A/qJ4iTVSBsKInZ7L1/rLaHgHEPQbuipV9lNyZUkEOnLmbfpWluim+g", + "PLGC0L7FHUoNnGFfeTF8qxqTFO37vD+KjLhREeMHufAgF24tF/oE1UoEW3Fx9p4CbENxMGBJqgP9D3RR", + "EmTPq1Th07cotgCTrlyJ6t5ddkSs+MDkcZmy7VnhneXLQ4HyuxQo3+NA8oDgj1MB/lN2fATakiXsNZlD", + "xOA+5vof0e3xITXyh17QayWBwVpoyqppafHhurExF+gBPiHF10oIM943poMrVTp739YOvm4jQewjwZm1", + "/LfZFbbAy+Re73QeivJ8AkV5fv9TxZ04pLfaCsICyOAeybbc4pNyDjNVdoOlXHO9qk2mroLQqjb58Sgn", + "+bL998hJr1UGdtxueOHwXTq3Bfm0B6LHQDuKkI/VVBfaVSNNeb1cGZuIIZrlpS3xzlNL+FtLz8eLzttq", + "QnkFPNvYUu5qjotu95UWebty/E1V/w9ae76Z5bbF58Pi/dsB7SdkaMBtDFHH9UOo95t+2wb2Jw+3kVfQ", + "VgIyii76cnDVKSIo3BMnvqz/B90/P8ltt88XDx+WB7Nfz0QBu8uSu3JFu9jW1t1v16LB5rXxnPIxy0d1", + "yqNHR47XRbNrGFS1D6ulb63wv7WkPm5bpPCazbq0a+QxLAXjNw/fg+LlpslFO1qyn6qicmd4RTIhxsvG", + "bwPk1LcKsBu6gEYAEbpFdJOkvUs5QUakrVXrY69vsfWx+aVtOySusJpspsBmQXLtHeRXFrM2p8WKa+bg", + "YAW/oCIOlVq6ELEhzPFS+kNuEgWcYquQBXYwad/IC9m/V2+swxw9+o0S3SgR7NiFsQXHzMo/hBF400Nf", + "37H4AU9iXbM6MK9as9L+PbviwiQLVVmNmVA2t4hTtzv7f3FhXA5Bd94zCoUCoNahfHBWoLhxgowuOoyv", + "cYU2fHF2UUQugnGq71S1lw+5dcwaxXBhrJZG+BcAVJDJ25h/PIfsg/X8YD0/WM8P1vOD9fxgPT9Yzw/W", + "84e2nn+foBCWJF5O+4jfWLwvm3ySFv4nFFL7MWNgW6O/MfnpkIAmOvLx1ssiAzyfuTxqdK+u9GjUWZiT", + "LcXphGRlzilR9Nr4t0+UI/qL5z5opMkuZNMyoKzBBs+estMfjl88efrr0xdfNDXau20f+Zy22mxyeOwu", + "1Zs31/52HSSnNEZ0uc796Sf1ES/Wml+IHBhVCv2Wmr+ES8jRlEeKpHRxdeR4dAY8/8Yhx0ol0OZrlW16", + "hIPrnxEquiTTZkMQkleRzGCRUuB9JBtF2QFdqrvBCer6XiNV4umbhxu2a69GUhhHyXsbvezMnOuSurqx", + "97kjwz316GQuq9jvKrIZQeTIrBVPf5jgvn5lD8c41BatCsd/n2ognkd8lPGIbadIk1mdAhVqcxS3TrDR", + "EmTixEIyV9nGV/VwSQo7UtZmjxsXst+uIa2RlwgSxwaP9GNXE5OyYIaunmj23iAZNdB4bR2njy04bSK0", + "rXLz9tTRTat85zCO/nBDqRG8GXmkKrasVF0+tvUj5IaOxEXJ5ca7wdBWpLzMVAKVQs/uV1I3OSkHcnb/", + "tMLheYXeEfZ/t2hhV1z7nMKZTSocT6zUT327G+NtYsddiXjseqNJaEdSzg430e+yC3ptXH8lVIlZy0gq", + "yF7ix4d4738KlfCmUpcCD85RCWsd8yYqEA52aoYqEFmkGnqvf71u6MrTn/lV+JZ4X5m6TpzheWerdAW2", + "Tpu30iJPpVFfVopnKdcU0uqydX9gi9WsTyJ+BwKTUl4sBk/+UIHvLslA4+5lTwZDt+W/6E26trm9fl/r", + "ss1Ic+wiiDvYeHAF/KO4Ar72zKcZZxW/6jNnkEF/DzHFr8xaRqXUrK0vGI14CxiiKUh2j3d3g+G7V3hB", + "5S97BQF5yThLc0EXFEpqU9WpOZecXKBhxbXh9Z537I6bUt/4JnEvfMRJ7oY6l5xq5DSO0ahJtYBYPnkA", + "b7HperkEbXqSeAFwLl0rIdt6PIVIK5XYuE9U1yjRD2zLgm/Ygufkw/8NKsXmeIoIn1GTQ1EbkefuPhGn", + "YWpxLrlhOaDQ/1GgQYfDeZ9Tc0fuKjeEFf2HDnqX5G6kiNT39usPXK/88r3fiNxb9rO9Mfv4JaA87LHa", + "kA7yk5cuxcnJS3q13t4kDmD/aNdLhZBJlMhQ47sb+T5tsUeuIBkR0OP2TtLt+rlEY9ooW2+/rQZ8M3Lo", + "XwMMeNFyR49qOhvRuy3wa30Xe16zVAkeGSnz9WQpzKqeUzJI/+xmtlTNE5xZxqFQkr5lM16KmS4hnV0+", + "2WEf3EFesYi4etDc/zhO/H7Fymbj0Ygd7P2IXr6HjHJ/7DRyO0OUHpK2PSRte0jr9ZC07WF3H5K2PaQ0", + "e0hp9s+a0uxgq4U4e2/W+yQZCkcVmS20W0FqZ24EeNisk45oeC0pzAFjZ1RFl6MOgEuoeE7F07VPjiA0", + "K8RyZZiu0xQgOzqXSQcSW84VJ37U/tcec8/rw8NnwA4f9/tYv0UgeYd9yVSlT7am0lfsfHI+GYxUQaEu", + "wSUnoeZZTXfFttfOYf+lGfenarB1Bd9Y58qKlyWgWtP1YiFSYVGeKzwMLFUvvk8q+gIVAgcoUTUTxuaB", + "I3xSXKSLzuGuZlTM6B7q9xvk4j/ukUv8GQAS3g0zNv/7Puma/1kM7JdguMh18zohcp6ik02fsq64blm3", + "kSpTH9Ou/W/uwtrNkosLCGNwKfrgileZbxGthddm/vO1HoeupW5KtAzW3iToA71oZhbGJjHDA+egOtHQ", + "s+USi20BwWVfus3kIzXNr6eTNFcaEoslHatyRB9QFJE3lpMzlrsq5b74LY6BzMwRuoqe/9iY+/E5hVwm", + "trBGxEltv7vCG403ruf7jozr6WQ0vLchDVsanaRNH4khtS2Yy0ww4gC2dQRtAMWtqwn2ug8KNeXZ+fk7", + "9somzaQqYxewmdn6NumKyyXoBkchndonOzasJojr7qHx/ioYotZIRmqPngxjvft4vxDpBWQM5YQvjD9i", + "xLNHTQZAKi59tdr49xtWDT0+YOxYMihKs/F1pru+5t7k8jOzbf51qDi7GikSNpiCuITqjjzlh9nOSRqQ", + "4e44lR1k+0RmLUfYiV9FjrT7poSKnGB758mAqCwU9+EYeNBKD1rpQSs9aKUHrfSglT6YVho4YR7cFB/D", + "TfG7Oyr+gdJQPmSc/IMtKAze7KSUvoP3timcGbOCnV+2LUwbFnolr1pT4vXtu+t3+K269A63tm7p0WxG", + "VsVKaTObXE/f92qahh9RlPKlHcE5tMpKXFLC2HfX/z8AAP//ufS+0+znAAA=", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/daemon/algod/api/server/v2/generated/types.go b/daemon/algod/api/server/v2/generated/types.go index 9b31e0f89e..ea289a1a74 100644 --- a/daemon/algod/api/server/v2/generated/types.go +++ b/daemon/algod/api/server/v2/generated/types.go @@ -483,7 +483,7 @@ type NodeStatusResponse struct { // The number of blocks that have already been obtained by the node as part of the catchup CatchpointAcquiredBlocks *uint64 `json:"catchpoint-acquired-blocks,omitempty"` - // The number of account from the current catchpoint that have been processed so far as part of the catchup + // The number of accounts from the current catchpoint that have been processed so far as part of the catchup CatchpointProcessedAccounts *uint64 `json:"catchpoint-processed-accounts,omitempty"` // The total number of accounts included in the current catchpoint @@ -492,6 +492,9 @@ type NodeStatusResponse struct { // The total number of blocks that are required to complete the current catchpoint catchup CatchpointTotalBlocks *uint64 `json:"catchpoint-total-blocks,omitempty"` + // The number of accounts from the current catchpoint that have been verified so far as part of the catchup + CatchpointVerifiedAccounts *uint64 `json:"catchpoint-verified-accounts,omitempty"` + // CatchupTime in nanoseconds CatchupTime uint64 `json:"catchup-time"` diff --git a/daemon/algod/api/server/v2/handlers.go b/daemon/algod/api/server/v2/handlers.go index 4c7131481d..c22e2fd3a2 100644 --- a/daemon/algod/api/server/v2/handlers.go +++ b/daemon/algod/api/server/v2/handlers.go @@ -222,6 +222,7 @@ func (v2 *Handlers) GetStatus(ctx echo.Context) error { Catchpoint: &stat.Catchpoint, CatchpointTotalAccounts: &stat.CatchpointCatchupTotalAccounts, CatchpointProcessedAccounts: &stat.CatchpointCatchupProcessedAccounts, + CatchpointVerifiedAccounts: &stat.CatchpointCatchupVerifiedAccounts, CatchpointTotalBlocks: &stat.CatchpointCatchupTotalBlocks, CatchpointAcquiredBlocks: &stat.CatchpointCatchupAcquiredBlocks, } diff --git a/daemon/algod/api/server/v2/test/handlers_test.go b/daemon/algod/api/server/v2/test/handlers_test.go index 0998d8449f..59e7e189e4 100644 --- a/daemon/algod/api/server/v2/test/handlers_test.go +++ b/daemon/algod/api/server/v2/test/handlers_test.go @@ -135,6 +135,7 @@ func TestGetStatus(t *testing.T) { Catchpoint: &stat.Catchpoint, CatchpointTotalAccounts: &stat.CatchpointCatchupTotalAccounts, CatchpointProcessedAccounts: &stat.CatchpointCatchupProcessedAccounts, + CatchpointVerifiedAccounts: &stat.CatchpointCatchupVerifiedAccounts, CatchpointTotalBlocks: &stat.CatchpointCatchupTotalBlocks, CatchpointAcquiredBlocks: &stat.CatchpointCatchupAcquiredBlocks, } diff --git a/daemon/algod/api/server/v2/test/helpers.go b/daemon/algod/api/server/v2/test/helpers.go index dd283dd921..3c1cf1e294 100644 --- a/daemon/algod/api/server/v2/test/helpers.go +++ b/daemon/algod/api/server/v2/test/helpers.go @@ -50,6 +50,7 @@ var cannedStatusReportGolden = node.StatusReport{ Catchpoint: "", CatchpointCatchupAcquiredBlocks: 0, CatchpointCatchupProcessedAccounts: 0, + CatchpointCatchupVerifiedAccounts: 0, CatchpointCatchupTotalAccounts: 0, CatchpointCatchupTotalBlocks: 0, LastCatchpoint: "", diff --git a/ledger/accountdb.go b/ledger/accountdb.go index bd6b524e33..ff1fff42fc 100644 --- a/ledger/accountdb.go +++ b/ledger/accountdb.go @@ -140,30 +140,64 @@ const ( catchpointStateCatchupBalancesRound = catchpointState("catchpointCatchupBalancesRound") ) -func writeCatchpointStagingCreatable(ctx context.Context, tx *sql.Tx, addr basics.Address, cidx basics.CreatableIndex, ctype basics.CreatableType) error { - _, err := tx.ExecContext(ctx, "INSERT INTO catchpointassetcreators(asset, creator, ctype) VALUES(?, ?, ?)", cidx, addr[:], ctype) - if err != nil { - return err +// normalizedAccountBalance is a staging area for a catchpoint file account information before it's being added to the catchpoint staging tables. +type normalizedAccountBalance struct { + address basics.Address + accountData basics.AccountData + encodedAccountData []byte + accountHash []byte + normalizedBalance uint64 +} + +// prepareNormalizedBalances converts an array of encodedBalanceRecord into an equal size array of normalizedAccountBalances. +func prepareNormalizedBalances(bals []encodedBalanceRecord, proto config.ConsensusParams) (normalizedAccountBalances []normalizedAccountBalance, err error) { + normalizedAccountBalances = make([]normalizedAccountBalance, len(bals), len(bals)) + for i, balance := range bals { + normalizedAccountBalances[i].address = balance.Address + err = protocol.Decode(balance.AccountData, &(normalizedAccountBalances[i].accountData)) + if err != nil { + return nil, err + } + normalizedAccountBalances[i].normalizedBalance = normalizedAccountBalances[i].accountData.NormalizedOnlineBalance(proto) + normalizedAccountBalances[i].encodedAccountData = balance.AccountData + normalizedAccountBalances[i].accountHash = accountHashBuilder(balance.Address, normalizedAccountBalances[i].accountData, balance.AccountData) } - return nil + return } -func writeCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, bals []encodedBalanceRecord, proto config.ConsensusParams) error { +// writeCatchpointStagingBalances inserts all the account balances in the provided array into the catchpoint balance staging table catchpointbalances. +func writeCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, bals []normalizedAccountBalance) error { insertStmt, err := tx.PrepareContext(ctx, "INSERT INTO catchpointbalances(address, normalizedonlinebalance, data) VALUES(?, ?, ?)") if err != nil { return err } for _, balance := range bals { - var data basics.AccountData - err = protocol.Decode(balance.AccountData, &data) + result, err := insertStmt.ExecContext(ctx, balance.address[:], balance.normalizedBalance, balance.encodedAccountData) if err != nil { return err } - normBalance := data.NormalizedOnlineBalance(proto) + aff, err := result.RowsAffected() + if err != nil { + return err + } + if aff != 1 { + return fmt.Errorf("number of affected record in insert was expected to be one, but was %d", aff) + } + } + return nil +} + +// writeCatchpointStagingBalances inserts all the account hashes in the provided array into the catchpoint pending hashes table catchpointpendinghashes. +func writeCatchpointStagingHashes(ctx context.Context, tx *sql.Tx, bals []normalizedAccountBalance) error { + insertStmt, err := tx.PrepareContext(ctx, "INSERT INTO catchpointpendinghashes(data) VALUES(?)") + if err != nil { + return err + } - result, err := insertStmt.ExecContext(ctx, balance.Address[:], normBalance, balance.AccountData) + for _, balance := range bals { + result, err := insertStmt.ExecContext(ctx, balance.accountHash[:]) if err != nil { return err } @@ -179,11 +213,51 @@ func writeCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, bals []enco return nil } +// createCatchpointStagingHashesIndex creates an index on catchpointpendinghashes to allow faster scanning according to the hash order +func createCatchpointStagingHashesIndex(ctx context.Context, tx *sql.Tx) (err error) { + _, err = tx.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS catchpointpendinghashesidx ON catchpointpendinghashes(data)") + if err != nil { + return + } + return +} + +// writeCatchpointStagingCreatable inserts all the creatables in the provided array into the catchpoint asset creator staging table catchpointassetcreators. +func writeCatchpointStagingCreatable(ctx context.Context, tx *sql.Tx, bals []normalizedAccountBalance) error { + insertStmt, err := tx.PrepareContext(ctx, "INSERT INTO catchpointassetcreators(asset, creator, ctype) VALUES(?, ?, ?)") + if err != nil { + return err + } + + for _, balance := range bals { + // if the account has any asset params, it means that it's the creator of an asset. + if len(balance.accountData.AssetParams) > 0 { + for aidx := range balance.accountData.AssetParams { + _, err := insertStmt.ExecContext(ctx, basics.CreatableIndex(aidx), balance.address[:], basics.AssetCreatable) + if err != nil { + return err + } + } + } + + if len(balance.accountData.AppParams) > 0 { + for aidx := range balance.accountData.AppParams { + _, err := insertStmt.ExecContext(ctx, basics.CreatableIndex(aidx), balance.address[:], basics.AppCreatable) + if err != nil { + return err + } + } + } + } + return nil +} + func resetCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, newCatchup bool) (err error) { s := []string{ "DROP TABLE IF EXISTS catchpointbalances", "DROP TABLE IF EXISTS catchpointassetcreators", "DROP TABLE IF EXISTS catchpointaccounthashes", + "DROP TABLE IF EXISTS catchpointpendinghashes", "DELETE FROM accounttotals where id='catchpointStaging'", } @@ -198,6 +272,7 @@ func resetCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, newCatchup s = append(s, "CREATE TABLE IF NOT EXISTS catchpointassetcreators (asset integer primary key, creator blob, ctype integer)", "CREATE TABLE IF NOT EXISTS catchpointbalances (address blob primary key, data blob, normalizedonlinebalance integer)", + "CREATE TABLE IF NOT EXISTS catchpointpendinghashes (data blob)", "CREATE TABLE IF NOT EXISTS catchpointaccounthashes (id integer primary key, data blob)", createNormalizedOnlineBalanceIndex(idxname, "catchpointbalances"), ) @@ -1069,20 +1144,14 @@ func (mc *merkleCommitter) LoadPage(page uint64) (content []byte, err error) { // encodedAccountsBatchIter allows us to iterate over the accounts data stored in the accountbase table. type encodedAccountsBatchIter struct { - rows *sql.Rows - orderByAddress bool + rows *sql.Rows } // Next returns an array containing the account data, in the same way it appear in the database // returning accountCount accounts data at a time. func (iterator *encodedAccountsBatchIter) Next(ctx context.Context, tx *sql.Tx, accountCount int) (bals []encodedBalanceRecord, err error) { if iterator.rows == nil { - if iterator.orderByAddress { - iterator.rows, err = tx.QueryContext(ctx, "SELECT address, data FROM accountbase ORDER BY address") - } else { - iterator.rows, err = tx.QueryContext(ctx, "SELECT address, data FROM accountbase") - } - + iterator.rows, err = tx.QueryContext(ctx, "SELECT address, data FROM accountbase ORDER BY address") if err != nil { return } @@ -1161,41 +1230,38 @@ const ( // orderedAccountsIter allows us to iterate over the accounts addresses in the order of the account hashes. type orderedAccountsIter struct { - step orderedAccountsIterStep - rows *sql.Rows - tx *sql.Tx - accountCount int - fetchAccountData bool - insertStmt *sql.Stmt + step orderedAccountsIterStep + rows *sql.Rows + tx *sql.Tx + accountCount int + insertStmt *sql.Stmt } // makeOrderedAccountsIter creates an ordered account iterator. Note that due to implementation reasons, // only a single iterator can be active at a time. -func makeOrderedAccountsIter(tx *sql.Tx, accountCount int, fetchAccountData bool) *orderedAccountsIter { +func makeOrderedAccountsIter(tx *sql.Tx, accountCount int) *orderedAccountsIter { return &orderedAccountsIter{ - tx: tx, - accountCount: accountCount, - fetchAccountData: fetchAccountData, - step: oaiStepStartup, + tx: tx, + accountCount: accountCount, + step: oaiStepStartup, } } -// accountAddressHashData is used by Next to return a single account address, hash and account data. -type accountAddressHashData struct { - address basics.Address - digest []byte - encodedAccountData []byte +// accountAddressHash is used by Next to return a single account address and the associated hash. +type accountAddressHash struct { + address basics.Address + digest []byte } -// Next returns an array containing the account address, hash and potentially data +// Next returns an array containing the account address and hash // the Next function works in multiple processing stages, where it first processs the current accounts and order them -// followed by returning the ordered accounts. In the first phase, it would return empty accountAddressHashData array +// followed by returning the ordered accounts. In the first phase, it would return empty accountAddressHash array // and sets the processedRecords to the number of accounts that were processed. On the second phase, the acct // would contain valid data ( and optionally the account data as well, if was asked in makeOrderedAccountsIter) and // the processedRecords would be zero. If err is sql.ErrNoRows it means that the iterator have completed it's work and no further // accounts exists. Otherwise, the caller is expected to keep calling "Next" to retrieve the next set of accounts // ( or let the Next function make some progress toward that goal ) -func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAddressHashData, processedRecords int, err error) { +func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAddressHash, processedRecords int, err error) { if iterator.step == oaiStepDeleteOldOrderingTable { // although we're going to delete this table anyway when completing the iterator execution, we'll try to // clean up any intermediate table. @@ -1290,11 +1356,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAd } if iterator.step == oaiStepSelectFromOrderedTable { // select the data from the ordered table - if iterator.fetchAccountData { - iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT accountsiteratorhashes.address, accountsiteratorhashes.hash, accountbase.data FROM accountsiteratorhashes JOIN accountbase ON accountbase.address=accountsiteratorhashes.address ORDER BY accountsiteratorhashes.hash") - } else { - iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT address, hash FROM accountsiteratorhashes ORDER BY hash") - } + iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT address, hash FROM accountsiteratorhashes ORDER BY hash") if err != nil { iterator.Close(ctx) @@ -1305,17 +1367,12 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAd } if iterator.step == oaiStepIterateOverOrderedTable { - acct = make([]accountAddressHashData, 0, iterator.accountCount) + acct = make([]accountAddressHash, 0, iterator.accountCount) var addr basics.Address for iterator.rows.Next() { var addrbuf []byte - var acctdata []byte var hash []byte - if iterator.fetchAccountData { - err = iterator.rows.Scan(&addrbuf, &hash, &acctdata) - } else { - err = iterator.rows.Scan(&addrbuf, &hash) - } + err = iterator.rows.Scan(&addrbuf, &hash) if err != nil { iterator.Close(ctx) return @@ -1329,7 +1386,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAd copy(addr[:], addrbuf) - acct = append(acct, accountAddressHashData{address: addr, digest: hash, encodedAccountData: acctdata}) + acct = append(acct, accountAddressHash{address: addr, digest: hash}) if len(acct) == iterator.accountCount { // we're done with this iteration. return @@ -1364,3 +1421,62 @@ func (iterator *orderedAccountsIter) Close(ctx context.Context) (err error) { _, err = iterator.tx.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes") return } + +// catchpointPendingHashesIterator allows us to iterate over the hashes in the catchpointpendinghashes table in their order. +type catchpointPendingHashesIterator struct { + hashCount int + tx *sql.Tx + rows *sql.Rows +} + +// makeCatchpointPendingHashesIterator create a pending hashes iterator that retrieves the hashes in the catchpointpendinghashes table. +func makeCatchpointPendingHashesIterator(hashCount int, tx *sql.Tx) *catchpointPendingHashesIterator { + return &catchpointPendingHashesIterator{ + hashCount: hashCount, + tx: tx, + } +} + +// Next returns an array containing the hashes, returning HashCount hashes at a time. +func (iterator *catchpointPendingHashesIterator) Next(ctx context.Context) (hashes [][]byte, err error) { + if iterator.rows == nil { + iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT data FROM catchpointpendinghashes ORDER BY data") + if err != nil { + return + } + } + + // gather up to accountCount encoded accounts. + hashes = make([][]byte, 0, iterator.hashCount) + for iterator.rows.Next() { + var hash []byte + err = iterator.rows.Scan(&hash) + if err != nil { + iterator.Close() + return + } + + hashes = append(hashes, hash) + if len(hashes) == iterator.hashCount { + // we're done with this iteration. + return + } + } + + err = iterator.rows.Err() + if err != nil { + iterator.Close() + return + } + // we just finished reading the table. + iterator.Close() + return +} + +// Close shuts down the catchpointPendingHashesIterator, releasing database resources. +func (iterator *catchpointPendingHashesIterator) Close() { + if iterator.rows != nil { + iterator.rows.Close() + iterator.rows = nil + } +} diff --git a/ledger/accountdb_test.go b/ledger/accountdb_test.go index 033bd1f9cd..063aae08d0 100644 --- a/ledger/accountdb_test.go +++ b/ledger/accountdb_test.go @@ -895,9 +895,11 @@ func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder boo balanceLoopDuration := time.Now().Sub(balancesLoopStart) last64KAccountCreationTime += balanceLoopDuration accountsGenerationDuration += balanceLoopDuration + + normalizedAccountBalances, err := prepareNormalizedBalances(balances.Balances, proto) b.StartTimer() err = l.trackerDBs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { - err = writeCatchpointStagingBalances(ctx, tx, balances.Balances, proto) + err = writeCatchpointStagingBalances(ctx, tx, normalizedAccountBalances) return }) diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 2a252351bf..1111082e85 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -1119,7 +1119,7 @@ func (au *accountUpdates) accountsInitialize(ctx context.Context, tx *sql.Tx) (b if rootHash.IsZero() { au.log.Infof("accountsInitialize rebuilding merkle trie for round %d", rnd) - accountBuilderIt := makeOrderedAccountsIter(tx, trieRebuildAccountChunkSize, false) + accountBuilderIt := makeOrderedAccountsIter(tx, trieRebuildAccountChunkSize) defer accountBuilderIt.Close(ctx) startTrieBuildTime := time.Now() accountsCount := 0 diff --git a/ledger/catchpointwriter.go b/ledger/catchpointwriter.go index 4884c26dff..86c57446f6 100644 --- a/ledger/catchpointwriter.go +++ b/ledger/catchpointwriter.go @@ -103,7 +103,6 @@ func makeCatchpointWriter(ctx context.Context, filePath string, tx *sql.Tx, bloc blocksRound: blocksRound, blockHeaderDigest: blockHeaderDigest, label: label, - accountsIterator: encodedAccountsBatchIter{orderByAddress: true}, } } diff --git a/ledger/catchupaccessor.go b/ledger/catchupaccessor.go index a3429b556b..1d612f2cb6 100644 --- a/ledger/catchupaccessor.go +++ b/ledger/catchupaccessor.go @@ -22,6 +22,7 @@ import ( "encoding/hex" "fmt" "strings" + "sync" "time" "github.com/algorand/go-algorand/config" @@ -31,6 +32,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" "github.com/algorand/go-algorand/util/metrics" ) @@ -54,6 +56,9 @@ type CatchpointCatchupAccessor interface { // ProgressStagingBalances deserialize the given bytes as a temporary staging balances ProgressStagingBalances(ctx context.Context, sectionName string, bytes []byte, progress *CatchpointCatchupAccessorProgress) (err error) + // BuildMerkleTrie inserts the account hashes into the merkle trie + BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64)) (err error) + // GetCatchupBlockRound returns the latest block round matching the current catchpoint GetCatchupBlockRound(ctx context.Context) (round basics.Round, err error) @@ -109,13 +114,6 @@ const ( // catchpointCatchupStateLast is the last entry in the CatchpointCatchupState enumeration. catchpointCatchupStateLast = CatchpointCatchupStateSwitch - - // minMerkleTrieEvictFrequency control the minimal number of accounts changes that we will attempt to update between - // two consecutive evict calls. - minMerkleTrieEvictFrequency = uint64(1024) - // maxMerkleTrieEvictionDuration is the upper bound for the time we'll let the evict call take before lowing the number - // of accounts per update. - maxMerkleTrieEvictionDuration = 2000 * time.Millisecond ) // MakeCatchpointCatchupAccessor creates a CatchpointCatchupAccessor given a ledger @@ -310,78 +308,80 @@ func (c *CatchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte return fmt.Errorf("processStagingBalances received a chunk with no accounts") } - proto := c.ledger.GenesisProto() wdb := c.ledger.trackerDB().wdb start := time.Now() ledgerProcessstagingbalancesCount.Inc(nil) - err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { - // create the merkle trie for the balances - var mc *merkleCommitter - mc, err = makeMerkleCommitter(tx, true) - if err != nil { - return - } - if progress.cachedTrie == nil { - progress.cachedTrie, err = merkletrie.MakeTrie(mc, trieMemoryConfig) - if err != nil { - return - } - } else { - progress.cachedTrie.SetCommitter(mc) - } + normalizedAccountBalances, err := prepareNormalizedBalances(balances.Balances, c.ledger.GenesisProto()) - err = writeCatchpointStagingBalances(ctx, tx, balances.Balances, proto) - if err != nil { - return - } + wg := sync.WaitGroup{} + wg.Add(3) + errChan := make(chan error, 3) - for _, balance := range balances.Balances { - var accountData basics.AccountData - err = protocol.Decode(balance.AccountData, &accountData) + // start the balances writer + go func() { + defer wg.Done() + err := wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + err = writeCatchpointStagingBalances(ctx, tx, normalizedAccountBalances) if err != nil { return } - - // if the account has any asset params, it means that it's the creator of an asset. - if len(accountData.AssetParams) > 0 { - for aidx := range accountData.AssetParams { - err = writeCatchpointStagingCreatable(ctx, tx, balance.Address, basics.CreatableIndex(aidx), basics.AssetCreatable) - if err != nil { - return - } - } + return nil + }) + if err != nil { + errChan <- err + } + }() + + // starts the creatables writer + go func() { + defer wg.Done() + hasCreatables := false + for _, accBal := range normalizedAccountBalances { + if len(accBal.accountData.AssetParams) > 0 || len(accBal.accountData.AppParams) > 0 { + hasCreatables = true + break } - - if len(accountData.AppParams) > 0 { - for aidx := range accountData.AppParams { - err = writeCatchpointStagingCreatable(ctx, tx, balance.Address, basics.CreatableIndex(aidx), basics.AppCreatable) - if err != nil { - return - } - } + } + if hasCreatables { + err := wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + err = writeCatchpointStagingCreatable(ctx, tx, normalizedAccountBalances) + return err + }) + if err != nil { + errChan <- err } + } + }() - hash := accountHashBuilder(balance.Address, accountData, balance.AccountData) - var added bool - added, err = progress.cachedTrie.Add(hash) - if !added { - return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingBalances: The provided catchpoint file contained the same account more than once. Account address %#v, account data %#v, hash '%s'", balance.Address, accountData, hex.EncodeToString(hash)) - } + // start the accounts pending hashes writer + go func() { + defer wg.Done() + err := wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + err = writeCatchpointStagingHashes(ctx, tx, normalizedAccountBalances) if err != nil { return } + return err + }) + if err != nil { + errChan <- err } + }() + + wg.Wait() + select { + case err := <-errChan: + return err + default: + } - // periodically, perform commit & evict to flush it to the disk and rebalance the cache memory utilization. - err = progress.EvictAsNeeded(uint64(len(balances.Balances))) - return - }) ledgerProcessstagingbalancesMicros.AddMicrosecondsSince(start, nil) if err == nil { progress.ProcessedAccounts += uint64(len(balances.Balances)) progress.ProcessedBytes += uint64(len(bytes)) } + // not strictly required, but clean up the pointer in case of either a failure or when we're done. if err != nil || progress.ProcessedAccounts == progress.TotalAccounts { progress.cachedTrie = nil @@ -391,34 +391,184 @@ func (c *CatchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte return err } -// EvictAsNeeded calls Evict on the cachedTrie periodically, or once we're done updating the trie. -func (progress *CatchpointCatchupAccessorProgress) EvictAsNeeded(balancesCount uint64) (err error) { - if progress.cachedTrie == nil { - return nil +// BuildMerkleTrie would process the catchpointpendinghashes and insert all the items in it into the merkle trie +func (c *CatchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64)) (err error) { + wdb := c.ledger.trackerDB().wdb + rdb := c.ledger.trackerDB().rdb + err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + // creating the index can take a while, so ensure we don't generate false alerts for no good reason. + db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(120*time.Second)) + return createCatchpointStagingHashesIndex(ctx, tx) + }) + if err != nil { + return } - if progress.evictFrequency == 0 { - progress.evictFrequency = trieRebuildCommitFrequency - } - // periodically, perform commit & evict to flush it to the disk and rebalance the cache memory utilization. - if (progress.ProcessedAccounts/progress.evictFrequency) < ((progress.ProcessedAccounts+balancesCount)/progress.evictFrequency) || - (progress.ProcessedAccounts+balancesCount) == progress.TotalAccounts { - evictStart := time.Now() - _, err = progress.cachedTrie.Evict(true) - if err == nil { - evictDelta := time.Now().Sub(evictStart) - if evictDelta > maxMerkleTrieEvictionDuration { - if progress.evictFrequency > minMerkleTrieEvictFrequency { - progress.evictFrequency /= 2 + + wg := sync.WaitGroup{} + wg.Add(2) + errChan := make(chan error, 2) + + writerQueue := make(chan [][]byte, 16) + c.ledger.setSynchronousMode(ctx, c.ledger.accountsRebuildSynchronousMode) + defer c.ledger.setSynchronousMode(ctx, c.ledger.synchronousMode) + + // starts the hashes reader + go func() { + defer wg.Done() + defer close(writerQueue) + + err := rdb.Atomic(func(transactionCtx context.Context, tx *sql.Tx) (err error) { + it := makeCatchpointPendingHashesIterator(trieRebuildAccountChunkSize, tx) + var hashes [][]byte + for { + hashes, err = it.Next(transactionCtx) + if err != nil { + break } - } else { - progress.evictFrequency *= 2 - if progress.evictFrequency > trieRebuildCommitFrequency { - progress.evictFrequency = trieRebuildCommitFrequency + if len(hashes) > 0 { + writerQueue <- hashes + } + if len(hashes) != trieRebuildAccountChunkSize { + break + } + if ctx.Err() != nil { + it.Close() + break } } + // disable the warning for over-long atomic operation execution. It's meaningless here since it's + // co-dependent on the other go-routine. + db.ResetTransactionWarnDeadline(transactionCtx, tx, time.Now().Add(5*time.Second)) + return err + }) + if err != nil { + errChan <- err + } + }() + + // starts the merkle trie writer + go func() { + defer wg.Done() + var trie *merkletrie.Trie + uncommitedHashesCount := 0 + keepWriting := true + hashesWritten := uint64(0) + var mc *merkleCommitter + if progressUpdates != nil { + progressUpdates(hashesWritten) + } + + err := wdb.Atomic(func(transactionCtx context.Context, tx *sql.Tx) (err error) { + // create the merkle trie for the balances + mc, err = makeMerkleCommitter(tx, true) + if err != nil { + return + } + + trie, err = merkletrie.MakeTrie(mc, trieMemoryConfig) + return err + }) + if err != nil { + errChan <- err + return } + + for keepWriting { + var hashesToWrite [][]byte + select { + case hashesToWrite = <-writerQueue: + if hashesToWrite == nil { + // i.e. the writerQueue is closed. + keepWriting = false + continue + } + case <-ctx.Done(): + keepWriting = false + continue + } + + err = rdb.Atomic(func(transactionCtx context.Context, tx *sql.Tx) (err error) { + mc, err = makeMerkleCommitter(tx, true) + if err != nil { + return + } + trie.SetCommitter(mc) + for _, accountHash := range hashesToWrite { + var added bool + added, err = trie.Add(accountHash) + if !added { + return fmt.Errorf("CatchpointCatchupAccessorImpl::BuildMerkleTrie: The provided catchpoint file contained the same account more than once. hash '%s'", hex.EncodeToString(accountHash)) + } + if err != nil { + return + } + } + uncommitedHashesCount += len(hashesToWrite) + hashesWritten += uint64(len(hashesToWrite)) + return nil + }) + if err != nil { + break + } + + if uncommitedHashesCount >= trieRebuildCommitFrequency { + err = wdb.Atomic(func(transactionCtx context.Context, tx *sql.Tx) (err error) { + // set a long 30-second window for the evict before warning is generated. + db.ResetTransactionWarnDeadline(transactionCtx, tx, time.Now().Add(30*time.Second)) + mc, err = makeMerkleCommitter(tx, true) + if err != nil { + return + } + trie.SetCommitter(mc) + _, err = trie.Evict(true) + if err != nil { + return + } + uncommitedHashesCount = 0 + return nil + }) + if err != nil { + keepWriting = false + continue + } + } + if progressUpdates != nil { + progressUpdates(hashesWritten) + } + } + if err != nil { + errChan <- err + return + } + if uncommitedHashesCount > 0 { + err = wdb.Atomic(func(transactionCtx context.Context, tx *sql.Tx) (err error) { + // set a long 30-second window for the evict before warning is generated. + db.ResetTransactionWarnDeadline(transactionCtx, tx, time.Now().Add(30*time.Second)) + mc, err = makeMerkleCommitter(tx, true) + if err != nil { + return + } + trie.SetCommitter(mc) + _, err = trie.Evict(true) + return + }) + } + + if err != nil { + errChan <- err + } + return + }() + + wg.Wait() + + select { + case err := <-errChan: + return err + default: } - return + + return err } // GetCatchupBlockRound returns the latest block round matching the current catchpoint diff --git a/node/node.go b/node/node.go index 580f13c682..ce8d74d0c2 100644 --- a/node/node.go +++ b/node/node.go @@ -71,6 +71,7 @@ type StatusReport struct { Catchpoint string // the catchpoint where we're currently catching up to. If the node isn't in fast catchup mode, it will be empty. CatchpointCatchupTotalAccounts uint64 CatchpointCatchupProcessedAccounts uint64 + CatchpointCatchupVerifiedAccounts uint64 CatchpointCatchupTotalBlocks uint64 CatchpointCatchupAcquiredBlocks uint64 } @@ -610,6 +611,7 @@ func (node *AlgorandFullNode) Status() (s StatusReport, err error) { s.Catchpoint = stats.CatchpointLabel s.CatchpointCatchupTotalAccounts = stats.TotalAccounts s.CatchpointCatchupProcessedAccounts = stats.ProcessedAccounts + s.CatchpointCatchupVerifiedAccounts = stats.VerifiedAccounts s.CatchpointCatchupTotalBlocks = stats.TotalBlocks s.CatchpointCatchupAcquiredBlocks = stats.AcquiredBlocks s.CatchupTime = time.Now().Sub(stats.StartTime) From 40721954f76a1de8ba7695af6157e09644f9fc1c Mon Sep 17 00:00:00 2001 From: John Jannotti Date: Tue, 24 Nov 2020 12:57:44 -0500 Subject: [PATCH 28/34] Consistently report line numbers in TEAL error messages. (#1710) Consistently report line numbers in TEAL error messages --- data/transactions/logic/assembler.go | 215 +++++++++-------- data/transactions/logic/assembler_test.go | 217 ++++++------------ .../transactions/logic/backwardCompat_test.go | 6 +- 3 files changed, 186 insertions(+), 252 deletions(-) diff --git a/data/transactions/logic/assembler.go b/data/transactions/logic/assembler.go index 1251efc345..f778827f15 100644 --- a/data/transactions/logic/assembler.go +++ b/data/transactions/logic/assembler.go @@ -90,7 +90,7 @@ func (ops *OpStream) SetLabelHere(label string) error { ops.labels = make(map[string]int) } if _, ok := ops.labels[label]; ok { - return fmt.Errorf("duplicate label %s", label) + return ops.errorf("duplicate label %s", label) } ops.labels[label] = ops.Out.Len() return nil @@ -105,8 +105,8 @@ func (ops *OpStream) RecordSourceLine() { } // ReferToLabel records an opcode label refence to resolve later -func (ops *OpStream) ReferToLabel(sourceLine, pc int, label string) { - ops.labelReferences = append(ops.labelReferences, labelReference{sourceLine, pc, label}) +func (ops *OpStream) ReferToLabel(pc int, label string) { + ops.labelReferences = append(ops.labelReferences, labelReference{ops.sourceLine, pc, label}) } func (ops *OpStream) tpush(argType StackType) { @@ -141,13 +141,13 @@ func (ops *OpStream) Intc(constIndex uint) error { ops.Out.WriteByte(0x25) // intc_3 default: if constIndex > 0xff { - return errors.New("cannot have more than 256 int constants") + return ops.error("cannot have more than 256 int constants") } ops.Out.WriteByte(0x21) // intc ops.Out.WriteByte(uint8(constIndex)) } if constIndex >= uint(len(ops.intc)) { - return fmt.Errorf("intc %d is not defined", constIndex) + return ops.errorf("intc %d is not defined", constIndex) } ops.tpush(StackUint64) return nil @@ -184,13 +184,13 @@ func (ops *OpStream) Bytec(constIndex uint) error { ops.Out.WriteByte(0x2b) // bytec_3 default: if constIndex > 0xff { - return errors.New("cannot have more than 256 byte constants") + return ops.error("cannot have more than 256 byte constants") } ops.Out.WriteByte(0x27) // bytec ops.Out.WriteByte(uint8(constIndex)) } if constIndex >= uint(len(ops.bytec)) { - return fmt.Errorf("bytec %d is not defined", constIndex) + return ops.errorf("bytec %d is not defined", constIndex) } ops.trace("bytec %d %s", constIndex, hex.EncodeToString(ops.bytec[constIndex])) ops.tpush(StackBytes) @@ -229,7 +229,7 @@ func (ops *OpStream) Arg(val uint64) error { ops.Out.WriteByte(0x30) // arg_3 default: if val > 0xff { - return errors.New("cannot have more than 256 args") + return ops.error("cannot have more than 256 args") } ops.Out.WriteByte(0x2c) ops.Out.WriteByte(uint8(val)) @@ -241,7 +241,7 @@ func (ops *OpStream) Arg(val uint64) error { // Txn writes opcodes for loading a field from the current transaction func (ops *OpStream) Txn(val uint64) error { if val >= uint64(len(TxnFieldNames)) { - return errors.New("invalid txn field") + return ops.errorf("invalid txn field: %d", val) } ops.Out.WriteByte(0x31) ops.Out.WriteByte(uint8(val)) @@ -252,10 +252,10 @@ func (ops *OpStream) Txn(val uint64) error { // Txna writes opcodes for loading array field from the current transaction func (ops *OpStream) Txna(fieldNum uint64, arrayFieldIdx uint64) error { if fieldNum >= uint64(len(TxnFieldNames)) { - return errors.New("invalid txn field") + return ops.errorf("invalid txn field: %d", fieldNum) } if arrayFieldIdx > 255 { - return errors.New("txna cannot look up beyond index 255") + return ops.errorf("txna array index beyond 255: %d", arrayFieldIdx) } ops.Out.WriteByte(0x36) ops.Out.WriteByte(uint8(fieldNum)) @@ -267,10 +267,10 @@ func (ops *OpStream) Txna(fieldNum uint64, arrayFieldIdx uint64) error { // Gtxn writes opcodes for loading a field from the current transaction func (ops *OpStream) Gtxn(gid, val uint64) error { if val >= uint64(len(TxnFieldNames)) { - return errors.New("invalid txn field") + return ops.errorf("invalid txn field: %d", val) } if gid > 255 { - return errors.New("gtxn cannot look up beyond group index 255") + return ops.errorf("gtxn transaction index beyond 255: %d", gid) } ops.Out.WriteByte(0x33) ops.Out.WriteByte(uint8(gid)) @@ -282,10 +282,13 @@ func (ops *OpStream) Gtxn(gid, val uint64) error { // Gtxna writes opcodes for loading an array field from the current transaction func (ops *OpStream) Gtxna(gid, fieldNum uint64, arrayFieldIdx uint64) error { if fieldNum >= uint64(len(TxnFieldNames)) { - return errors.New("invalid txn field") + return ops.errorf("invalid txn field: %d", fieldNum) } - if gid > 255 || arrayFieldIdx > 255 { - return errors.New("gtxna cannot look up beyond index 255") + if gid > 255 { + return ops.errorf("gtxna group index beyond 255: %d", gid) + } + if arrayFieldIdx > 255 { + return ops.errorf("gtxna array index beyond 255: %d", arrayFieldIdx) } ops.Out.WriteByte(0x37) ops.Out.WriteByte(uint8(gid)) @@ -298,7 +301,7 @@ func (ops *OpStream) Gtxna(gid, fieldNum uint64, arrayFieldIdx uint64) error { // Global writes opcodes for loading an evaluator-global field func (ops *OpStream) Global(val uint64) error { if val >= uint64(len(GlobalFieldNames)) { - return errors.New("invalid global field") + return ops.errorf("invalid global field: %d", val) } ops.Out.WriteByte(0x32) ops.Out.WriteByte(uint8(val)) @@ -310,7 +313,7 @@ func (ops *OpStream) Global(val uint64) error { // AssetHolding writes opcodes for accessing data from AssetHolding func (ops *OpStream) AssetHolding(val uint64) error { if val >= uint64(len(AssetHoldingFieldNames)) { - return errors.New("invalid asset holding field") + return ops.errorf("invalid asset holding field: %d", val) } ops.Out.WriteByte(opsByName[ops.Version]["asset_holding_get"].Opcode) ops.Out.WriteByte(uint8(val)) @@ -322,7 +325,7 @@ func (ops *OpStream) AssetHolding(val uint64) error { // AssetParams writes opcodes for accessing data from AssetParams func (ops *OpStream) AssetParams(val uint64) error { if val >= uint64(len(AssetParamsFieldNames)) { - return errors.New("invalid asset params field") + return ops.errorf("invalid asset params field: %d", val) } ops.Out.WriteByte(opsByName[ops.Version]["asset_params_get"].Opcode) ops.Out.WriteByte(uint8(val)) @@ -349,7 +352,7 @@ func assembleInt(ops *OpStream, spec *OpSpec, args []string) error { } val, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } return ops.Uint(val) } @@ -358,14 +361,14 @@ func assembleInt(ops *OpStream, spec *OpSpec, args []string) error { func assembleIntC(ops *OpStream, spec *OpSpec, args []string) error { constIndex, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } return ops.Intc(uint(constIndex)) } func assembleByteC(ops *OpStream, spec *OpSpec, args []string) error { constIndex, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } return ops.Bytec(uint(constIndex)) } @@ -521,11 +524,11 @@ func assembleByte(ops *OpStream, spec *OpSpec, args []string) error { var val []byte var err error if len(args) == 0 { - return errors.New("byte operation needs byte literal argument") + return ops.error("byte operation needs byte literal argument") } val, _, err = parseBinaryArgs(args) if err != nil { - return err + return ops.error(err) } return ops.ByteLiteral(val) } @@ -539,7 +542,7 @@ func assembleIntCBlock(ops *OpStream, spec *OpSpec, args []string) error { for i, xs := range args { cu, err := strconv.ParseUint(xs, 0, 64) if err != nil { - return err + return ops.error(err) } l = binary.PutUvarint(scratch[:], cu) ops.Out.Write(scratch[:l]) @@ -556,7 +559,7 @@ func assembleByteCBlock(ops *OpStream, spec *OpSpec, args []string) error { for len(rest) > 0 { val, consumed, err := parseBinaryArgs(rest) if err != nil { - return err + return ops.error(err) } bvals = append(bvals, val) rest = rest[consumed:] @@ -578,31 +581,31 @@ func assembleByteCBlock(ops *OpStream, spec *OpSpec, args []string) error { // parses base32-with-checksum account address strings into a byte literal func assembleAddr(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("addr operation needs one argument") + return ops.error("addr operation needs one argument") } addr, err := basics.UnmarshalChecksumAddress(args[0]) if err != nil { - return err + return ops.error(err) } return ops.ByteLiteral(addr[:]) } func assembleArg(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("arg operation needs one argument") + return ops.error("arg operation needs one argument") } val, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } return ops.Arg(val) } func assembleBranch(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("branch operation needs label argument") + return ops.error("branch operation needs label argument") } - ops.ReferToLabel(ops.sourceLine, ops.Out.Len(), args[0]) + ops.ReferToLabel(ops.Out.Len(), args[0]) err := ops.checkArgs(*spec) if err != nil { return err @@ -616,14 +619,14 @@ func assembleBranch(ops *OpStream, spec *OpSpec, args []string) error { func assembleLoad(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("load operation needs one argument") + return ops.error("load operation needs one argument") } val, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } if val > EvalMaxScratchSize { - return errors.New("load limited to 0..255") + return ops.errorf("load outside 0..255: %d", val) } ops.Out.WriteByte(0x34) ops.Out.WriteByte(byte(val)) @@ -633,14 +636,14 @@ func assembleLoad(ops *OpStream, spec *OpSpec, args []string) error { func assembleStore(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("store operation needs one argument") + return ops.error("store operation needs one argument") } val, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } if val > EvalMaxScratchSize { - return errors.New("store limited to 0..255") + return ops.errorf("store outside 0..255: %d", val) } err = ops.checkArgs(*spec) if err != nil { @@ -653,25 +656,25 @@ func assembleStore(ops *OpStream, spec *OpSpec, args []string) error { func assembleSubstring(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 2 { - return errors.New("substring expects 2 args") + return ops.error("substring expects 2 args") } start, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } if start > EvalMaxScratchSize { - return errors.New("substring limited to 0..255") + return ops.error("substring limited to 0..255") } end, err := strconv.ParseUint(args[1], 0, 64) if err != nil { - return err + return ops.error(err) } if end > EvalMaxScratchSize { - return errors.New("substring limited to 0..255") + return ops.error("substring limited to 0..255") } if end < start { - return errors.New("substring end is before start") + return ops.error("substring end is before start") } opcode := byte(0x51) err = ops.checkArgs(*spec) @@ -701,18 +704,18 @@ func disSubstring(dis *disassembleState, spec *OpSpec) { func assembleTxn(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("txn expects one argument") + return ops.error("txn expects one argument") } fs, ok := txnFieldSpecByName[args[0]] if !ok { - return fmt.Errorf("txn unknown arg %s", args[0]) + return ops.errorf("txn unknown arg: %v", args[0]) } _, ok = txnaFieldSpecByField[fs.field] if ok { - return fmt.Errorf("found txna field %s in txn op", args[0]) + return ops.errorf("found txna field %v in txn op", args[0]) } if fs.version > ops.Version { - return fmt.Errorf("txn %s available in version %d. Missed #pragma version?", args[0], fs.version) + return ops.errorf("txn %s available in version %d. Missed #pragma version?", args[0], fs.version) } val := fs.field return ops.Txn(uint64(val)) @@ -726,27 +729,27 @@ func assembleTxn2(ops *OpStream, spec *OpSpec, args []string) error { if len(args) == 2 { return assembleTxna(ops, spec, args) } - return errors.New("txn expects one or two arguments") + return ops.error("txn expects one or two arguments") } func assembleTxna(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 2 { - return errors.New("txna expects two arguments") + return ops.error("txna expects two arguments") } fs, ok := txnFieldSpecByName[args[0]] if !ok { - return fmt.Errorf("txna unknown arg %s", args[0]) + return ops.errorf("txna unknown arg: %v", args[0]) } _, ok = txnaFieldSpecByField[fs.field] if !ok { - return fmt.Errorf("txna unknown arg %s", args[0]) + return ops.errorf("txna unknown arg: %v", args[0]) } if fs.version > ops.Version { - return fmt.Errorf("txna %s available in version %d. Missed #pragma version?", args[0], fs.version) + return ops.errorf("txna %s available in version %d. Missed #pragma version?", args[0], fs.version) } arrayFieldIdx, err := strconv.ParseUint(args[1], 0, 64) if err != nil { - return err + return ops.error(err) } fieldNum := fs.field return ops.Txna(uint64(fieldNum), uint64(arrayFieldIdx)) @@ -754,22 +757,22 @@ func assembleTxna(ops *OpStream, spec *OpSpec, args []string) error { func assembleGtxn(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 2 { - return errors.New("gtxn expects two arguments") + return ops.error("gtxn expects two arguments") } gtid, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } fs, ok := txnFieldSpecByName[args[1]] if !ok { - return fmt.Errorf("gtxn unknown arg %s", args[1]) + return ops.errorf("gtxn unknown arg: %v", args[1]) } _, ok = txnaFieldSpecByField[fs.field] if ok { - return fmt.Errorf("found gtxna field %s in gtxn op", args[1]) + return ops.errorf("found gtxna field %v in gtxn op", args[1]) } if fs.version > ops.Version { - return fmt.Errorf("gtxn %s available in version %d. Missed #pragma version?", args[1], fs.version) + return ops.errorf("gtxn %s available in version %d. Missed #pragma version?", args[1], fs.version) } val := fs.field return ops.Gtxn(gtid, uint64(val)) @@ -782,31 +785,31 @@ func assembleGtxn2(ops *OpStream, spec *OpSpec, args []string) error { if len(args) == 3 { return assembleGtxna(ops, spec, args) } - return errors.New("gtxn expects two or three arguments") + return ops.error("gtxn expects two or three arguments") } func assembleGtxna(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 3 { - return errors.New("gtxna expects three arguments") + return ops.error("gtxna expects three arguments") } gtid, err := strconv.ParseUint(args[0], 0, 64) if err != nil { - return err + return ops.error(err) } fs, ok := txnFieldSpecByName[args[1]] if !ok { - return fmt.Errorf("gtxna unknown arg %s", args[1]) + return ops.errorf("gtxna unknown arg: %v", args[1]) } _, ok = txnaFieldSpecByField[fs.field] if !ok { - return fmt.Errorf("gtxna unknown arg %s", args[1]) + return ops.errorf("gtxna unknown arg: %v", args[1]) } if fs.version > ops.Version { - return fmt.Errorf("gtxna %s available in version %d. Missed #pragma version?", args[1], fs.version) + return ops.errorf("gtxna %s available in version %d. Missed #pragma version?", args[1], fs.version) } arrayFieldIdx, err := strconv.ParseUint(args[2], 0, 64) if err != nil { - return err + return ops.error(err) } fieldNum := fs.field return ops.Gtxna(gtid, uint64(fieldNum), uint64(arrayFieldIdx)) @@ -814,14 +817,14 @@ func assembleGtxna(ops *OpStream, spec *OpSpec, args []string) error { func assembleGlobal(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("global expects one argument") + return ops.error("global expects one argument") } fs, ok := globalFieldSpecByName[args[0]] if !ok { - return fmt.Errorf("global unknown arg %v", args[0]) + return ops.errorf("global unknown arg: %v", args[0]) } if fs.version > ops.Version { - return fmt.Errorf("global %s available in version %d. Missed #pragma version?", args[0], fs.version) + return ops.errorf("global %s available in version %d. Missed #pragma version?", args[0], fs.version) } val := fs.gfield return ops.Global(uint64(val)) @@ -829,22 +832,22 @@ func assembleGlobal(ops *OpStream, spec *OpSpec, args []string) error { func assembleAssetHolding(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("asset_holding_get expects one argument") + return ops.error("asset_holding_get expects one argument") } val, ok := assetHoldingFields[args[0]] if !ok { - return fmt.Errorf("asset_holding_get unknown arg %v", args[0]) + return ops.errorf("asset_holding_get unknown arg: %v", args[0]) } return ops.AssetHolding(uint64(val)) } func assembleAssetParams(ops *OpStream, spec *OpSpec, args []string) error { if len(args) != 1 { - return errors.New("asset_params_get expects one argument") + return ops.error("asset_params_get expects one argument") } val, ok := assetParamsFields[args[0]] if !ok { - return fmt.Errorf("asset_params_get unknown arg %v", args[0]) + return ops.errorf("asset_params_get unknown arg: %v", args[0]) } return ops.AssetParams(uint64(val)) } @@ -868,7 +871,7 @@ func asmDefault(ops *OpStream, spec *OpSpec, args []string) error { } err = ops.Out.WriteByte(spec.Opcode) if err != nil { - return lineErr(ops.sourceLine, err) + return ops.error(err) } return nil } @@ -885,17 +888,21 @@ func init() { // WARNING: special case op assembly by argOps functions must do their own type stack maintenance via ops.tpop() ops.tpush()/ops.tpusha() } -type lineErrorWrapper struct { +type lineError struct { Line int Err error } -func (lew *lineErrorWrapper) Error() string { - return fmt.Sprintf(":%d %s", lew.Line, lew.Err.Error()) +func fmtLineError(line int, format string, args ...interface{}) error { + return &lineError{Line: line, Err: fmt.Errorf(format, args...)} +} + +func (le *lineError) Error() string { + return fmt.Sprintf("%d: %s", le.Line, le.Err.Error()) } -func lineErr(line int, err error) error { - return &lineErrorWrapper{Line: line, Err: err} +func (le *lineError) Unwrap() error { + return le.Err } func typecheck(expected, got StackType) bool { @@ -1005,7 +1012,7 @@ func (ops *OpStream) checkArgs(spec OpSpec) error { if len(ops.labelReferences) > 0 { fmt.Fprintf(os.Stderr, "warning: %d: %s; but branches have happened and assembler does not precisely track types in this case\n", ops.sourceLine, msg) } else { - return lineErr(ops.sourceLine, errors.New(msg)) + return ops.error(msg) } } } @@ -1065,19 +1072,18 @@ func (ops *OpStream) assemble(fin io.Reader) error { // create a label err := ops.SetLabelHere(opstring[:len(opstring)-1]) if err != nil { - return lineErr(ops.sourceLine, err) + return err } continue } - err := fmt.Errorf("unknown opcode %v", opstring) - return lineErr(ops.sourceLine, err) + return ops.errorf("unknown opcode: %v", opstring) } // backward compatibility: do not allow jumps behind last instruction in TEAL v1 if ops.Version <= 1 { for label, dest := range ops.labels { if dest == ops.Out.Len() { - return fmt.Errorf(":%d label %v is too far away", ops.sourceLine, label) + return ops.errorf("label %v is too far away", label) } } } @@ -1094,16 +1100,16 @@ func (ops *OpStream) resolveLabels() (err error) { for _, lr := range ops.labelReferences { dest, ok := ops.labels[lr.label] if !ok { - return fmt.Errorf(":%d reference to undefined label %v", lr.sourceLine, lr.label) + return fmtLineError(lr.sourceLine, "reference to undefined label %v", lr.label) } // all branch instructions (currently) are opcode byte and 2 offset bytes, and the destination is relative to the next pc as if the branch was a no-op naturalPc := lr.position + 3 if dest < naturalPc { - return fmt.Errorf(":%d label %v is before reference but only forward jumps are allowed", lr.sourceLine, lr.label) + return fmtLineError(lr.sourceLine, "label %v is before reference but only forward jumps are allowed", lr.label) } jump := dest - naturalPc if jump > 0x7fff { - return fmt.Errorf(":%d label %v is too far away", lr.sourceLine, lr.label) + return fmtLineError(lr.sourceLine, "label %v is too far away", lr.label) } raw[lr.position+1] = uint8(jump >> 8) raw[lr.position+2] = uint8(jump & 0x0ff) @@ -1177,6 +1183,21 @@ func (ops *OpStream) Bytes() (program []byte, err error) { return } +func (ops *OpStream) error(problem interface{}) error { + switch p := problem.(type) { + case string: + return &lineError{Line: ops.sourceLine, Err: errors.New(p)} + case error: + return &lineError{Line: ops.sourceLine, Err: p} + default: + return &lineError{Line: ops.sourceLine, Err: fmt.Errorf("%#v", p)} + } +} + +func (ops *OpStream) errorf(format string, a ...interface{}) error { + return ops.error(fmt.Errorf(format, a...)) +} + // AssembleString takes an entire program in a string and assembles it to bytecode using AssemblerDefaultVersion func AssembleString(text string) ([]byte, error) { return AssembleStringWithVersion(text, assemblerNoVersion) @@ -1253,38 +1274,32 @@ func (ps *PragmaStream) Process(fin io.Reader) (err error) { fields := strings.Split(line, " ") if fields[0] != "#pragma" { - err = fmt.Errorf("invalid syntax: %s", fields[0]) - return + return fmtLineError(sourceLine, "invalid syntax: %s", fields[0]) } if len(fields) < 2 { - err = fmt.Errorf("empty pragma") - return + return fmtLineError(sourceLine, "empty pragma") } key := fields[1] switch key { case "version": if len(fields) < 3 { - err = fmt.Errorf("no version value") - return + return fmtLineError(sourceLine, "no version value") } value := fields[2] var ver uint64 if sourceLine != 1 { - err = fmt.Errorf("#pragma version is only allowed on 1st line") - return + return fmtLineError(sourceLine, "#pragma version is only allowed on 1st line") } ver, err = strconv.ParseUint(value, 0, 64) if err != nil { - return + return &lineError{Line: sourceLine, Err: err} } if ver < 1 || ver > AssemblerMaxVersion { - err = fmt.Errorf("unsupported version: %d", ver) - return + return fmtLineError(sourceLine, "unsupported version: %d", ver) } ps.Version = ver default: - err = fmt.Errorf("unsupported pragma directive: %s", key) - return + return fmtLineError(sourceLine, "unsupported pragma directive: %s", key) } } return diff --git a/data/transactions/logic/assembler_test.go b/data/transactions/logic/assembler_test.go index 789ccf2cf6..d8cbbbd56f 100644 --- a/data/transactions/logic/assembler_test.go +++ b/data/transactions/logic/assembler_test.go @@ -267,141 +267,54 @@ pop require.Equal(t, prog1, prog2) } +func testLine(t *testing.T, line string, ver uint64, expected string) { + // By embedding the source line between two other lines, the + // test for the correct line number in the error is more + // meaningful. + source := "int 1\n" + line + "\nint 1\n" + _, err := AssembleStringWithVersion(source, ver) + if expected == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + if strings.HasSuffix(expected, "...") { + require.Contains(t, "^"+err.Error(), "^2: "+expected[:len(expected)-3]) + } else { + require.Equal(t, "2: "+expected, err.Error()) + } + } +} func TestAssembleTxna(t *testing.T) { - source := `txna Accounts 256` - _, err := AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "txna cannot look up beyond index 255") - - source = `txna ApplicationArgs 256` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "txna cannot look up beyond index 255") - - source = `txna Sender 256` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "txna unknown arg") - - source = `gtxna 0 Accounts 256` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxna cannot look up beyond index 255") - - source = `gtxna 0 ApplicationArgs 256` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxna cannot look up beyond index 255") - - source = `gtxna 256 Accounts 0` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxna cannot look up beyond index 255") - - source = `gtxna 0 Sender 256` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxna unknown arg") - - source = `txn Accounts 0` - _, err = AssembleStringV1(source) - require.Error(t, err) - require.Contains(t, err.Error(), "txn expects one argument") - - source = `txn Accounts 0 1` - _, err = AssembleStringV2(source) - require.Error(t, err) - require.Contains(t, err.Error(), "txn expects one or two arguments") - - source = `txna Accounts 0 1` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "txna expects two arguments") - - source = `txna Accounts a` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "strconv.ParseUint") - - source = `gtxn 0 Sender 0` - _, err = AssembleStringV1(source) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxn expects two arguments") - - source = `gtxn 0 Sender 1 2` - _, err = AssembleStringV2(source) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxn expects two or three arguments") - - source = `gtxna 0 Accounts 1 2` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxna expects three arguments") - - source = `gtxna a Accounts 0` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "strconv.ParseUint") - - source = `gtxna 0 Accounts a` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "strconv.ParseUint") - - source = `txn ABC` - _, err = AssembleStringV2(source) - require.Error(t, err) - require.Contains(t, err.Error(), "txn unknown arg") - - source = `gtxn 0 ABC` - _, err = AssembleStringV2(source) - require.Error(t, err) - require.Contains(t, err.Error(), "gtxn unknown arg") - - source = `gtxn a ABC` - _, err = AssembleStringV2(source) - require.Error(t, err) - require.Contains(t, err.Error(), "strconv.ParseUint") - - source = `txn Accounts` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "found txna field Accounts in txn op") - - source = `txn Accounts` - _, err = AssembleStringV1(source) - require.Error(t, err) - require.Contains(t, err.Error(), "found txna field Accounts in txn op") - - source = `txn Accounts 0` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.NoError(t, err) - - source = `gtxn 0 Accounts` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "found gtxna field Accounts in gtxn op") - - source = `gtxn 0 Accounts` - _, err = AssembleStringV1(source) - require.Error(t, err) - require.Contains(t, err.Error(), "found gtxna field Accounts in gtxn op") - - source = `gtxn 0 Accounts 1` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.NoError(t, err) + testLine(t, "txna Accounts 256", AssemblerMaxVersion, "txna array index beyond 255: 256") + testLine(t, "txna ApplicationArgs 256", AssemblerMaxVersion, "txna array index beyond 255: 256") + testLine(t, "txna Sender 256", AssemblerMaxVersion, "txna unknown arg: Sender") + testLine(t, "gtxna 0 Accounts 256", AssemblerMaxVersion, "gtxna array index beyond 255: 256") + testLine(t, "gtxna 0 ApplicationArgs 256", AssemblerMaxVersion, "gtxna array index beyond 255: 256") + testLine(t, "gtxna 256 Accounts 0", AssemblerMaxVersion, "gtxna group index beyond 255: 256") + testLine(t, "gtxna 0 Sender 256", AssemblerMaxVersion, "gtxna unknown arg: Sender") + testLine(t, "txn Accounts 0", 1, "txn expects one argument") + testLine(t, "txn Accounts 0 1", 2, "txn expects one or two arguments") + testLine(t, "txna Accounts 0 1", AssemblerMaxVersion, "txna expects two arguments") + testLine(t, "txna Accounts a", AssemblerMaxVersion, "strconv.ParseUint...") + testLine(t, "gtxn 0 Sender 0", 1, "gtxn expects two arguments") + testLine(t, "gtxn 0 Sender 1 2", 2, "gtxn expects two or three arguments") + testLine(t, "gtxna 0 Accounts 1 2", AssemblerMaxVersion, "gtxna expects three arguments") + testLine(t, "gtxna a Accounts 0", AssemblerMaxVersion, "strconv.ParseUint...") + testLine(t, "gtxna 0 Accounts a", AssemblerMaxVersion, "strconv.ParseUint...") + testLine(t, "txn ABC", 2, "txn unknown arg: ABC") + testLine(t, "gtxn 0 ABC", 2, "gtxn unknown arg: ABC") + testLine(t, "gtxn a ABC", 2, "strconv.ParseUint...") + testLine(t, "txn Accounts", AssemblerMaxVersion, "found txna field Accounts in txn op") + testLine(t, "txn Accounts", 1, "found txna field Accounts in txn op") + testLine(t, "txn Accounts 0", AssemblerMaxVersion, "") + testLine(t, "gtxn 0 Accounts", AssemblerMaxVersion, "found gtxna field Accounts in gtxn op") + testLine(t, "gtxn 0 Accounts", 1, "found gtxna field Accounts in gtxn op") + testLine(t, "gtxn 0 Accounts 1", AssemblerMaxVersion, "") } func TestAssembleGlobal(t *testing.T) { - source := `global` - _, err := AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "global expects one argument") - - source = `global a` - _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) - require.Error(t, err) - require.Contains(t, err.Error(), "global unknown arg") + testLine(t, "global", AssemblerMaxVersion, "global expects one argument") + testLine(t, "global a", AssemblerMaxVersion, "global unknown arg: a") } func TestAssembleDefault(t *testing.T) { @@ -412,7 +325,7 @@ int 1 ` _, err := AssembleStringWithVersion(source, AssemblerMaxVersion) require.Error(t, err) - require.Contains(t, err.Error(), "wanted type uint64 got []byte") + require.Contains(t, err.Error(), "3: + arg 0 wanted type uint64 got []byte") } // mutateProgVersion replaces version (first two symbols) in hex-encoded program @@ -745,11 +658,13 @@ func TestAssembleRejectNegJump(t *testing.T) { t.Parallel() text := `wat: int 1 -bnz wat` +bnz wat +int 2` for v := uint64(1); v <= AssemblerMaxVersion; v++ { t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { program, err := AssembleStringWithVersion(text, v) require.Error(t, err) + require.Contains(t, err.Error(), "3: label wat is before reference") require.Nil(t, program) }) } @@ -782,11 +697,13 @@ byte b64 avGWRM+yy3BCavBDXO/FYTNZ6o2Jai5edsMCBdDEz//= func TestAssembleRejectUnkLabel(t *testing.T) { t.Parallel() text := `int 1 -bnz nowhere` +bnz nowhere +int 2` for v := uint64(1); v <= AssemblerMaxVersion; v++ { t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { program, err := AssembleStringWithVersion(text, v) require.Error(t, err) + require.Contains(t, err.Error(), "2: reference to undefined label nowhere") require.Nil(t, program) }) } @@ -1059,7 +976,7 @@ txna Accounts 0 _, err = AssembleStringV1(text) require.Error(t, err) - require.Contains(t, err.Error(), "unknown opcode txna") + require.Contains(t, err.Error(), "2: unknown opcode: txna") } func TestAssembleBalance(t *testing.T) { @@ -1084,7 +1001,7 @@ func TestAssembleAsset(t *testing.T) { source = "int 0\nint 0\nasset_holding_get ABC" _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) require.Error(t, err) - require.Contains(t, err.Error(), "asset_holding_get unknown arg") + require.Contains(t, err.Error(), "asset_holding_get unknown arg: ABC") source = "int 0\nasset_params_get ABC 1" _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) @@ -1094,7 +1011,7 @@ func TestAssembleAsset(t *testing.T) { source = "int 0\nasset_params_get ABC" _, err = AssembleStringWithVersion(source, AssemblerMaxVersion) require.Error(t, err) - require.Contains(t, err.Error(), "asset_params_get unknown arg") + require.Contains(t, err.Error(), "asset_params_get unknown arg: ABC") } func TestDisassembleSingleOp(t *testing.T) { @@ -1412,7 +1329,7 @@ func TestPragmaStream(t *testing.T) { ps := PragmaStream{} err := ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "unsupported version") + require.Contains(t, err.Error(), "1: unsupported version: 100") require.Equal(t, uint64(0), ps.Version) text = `#pragma version 0` @@ -1420,7 +1337,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "unsupported version") + require.Contains(t, err.Error(), "1: unsupported version: 0") require.Equal(t, uint64(0), ps.Version) text = `#pragma version a` @@ -1428,7 +1345,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "strconv.ParseUint") + require.Contains(t, err.Error(), "1: strconv.ParseUint") require.Equal(t, uint64(0), ps.Version) text = `#pragmas version 1` @@ -1436,7 +1353,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "invalid syntax") + require.Contains(t, err.Error(), "1: invalid syntax") require.Equal(t, uint64(0), ps.Version) text = ` @@ -1445,7 +1362,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "allowed on 1st line") + require.Contains(t, err.Error(), "2: #pragma version is only allowed on 1st line") require.Equal(t, uint64(0), ps.Version) text = `#pragma version 1 @@ -1454,7 +1371,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "allowed on 1st line") + require.Contains(t, err.Error(), "2: #pragma version is only allowed on 1st line") require.Equal(t, uint64(1), ps.Version) text = `#pragma version 1 @@ -1463,7 +1380,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "unsupported pragma directive: run-mode") + require.Contains(t, err.Error(), "2: unsupported pragma directive: run-mode") require.Equal(t, uint64(1), ps.Version) text = `#pragma versions` @@ -1471,7 +1388,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "unsupported pragma directive: versions") + require.Contains(t, err.Error(), "1: unsupported pragma directive: versions") require.Equal(t, uint64(0), ps.Version) text = `# pragmas version 1` @@ -1494,7 +1411,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "empty pragma") + require.Contains(t, err.Error(), "1: empty pragma") require.Equal(t, uint64(0), ps.Version) text = `#pragma version` @@ -1502,7 +1419,7 @@ func TestPragmaStream(t *testing.T) { ps = PragmaStream{} err = ps.Process(sr) require.Error(t, err) - require.Contains(t, err.Error(), "no version") + require.Contains(t, err.Error(), "1: no version") require.Equal(t, uint64(0), ps.Version) } @@ -1565,6 +1482,8 @@ len _, err = AssembleString("#pragma unk") require.Error(t, err) + require.Contains(t, err.Error(), "1: unsupported pragma directive: unk") + } func TestAssembleConstants(t *testing.T) { @@ -1573,14 +1492,14 @@ func TestAssembleConstants(t *testing.T) { t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { _, err := AssembleStringWithVersion("intc 1", v) require.Error(t, err) - require.Contains(t, err.Error(), "intc 1 is not defined") + require.Contains(t, err.Error(), "1: intc 1 is not defined") _, err = AssembleStringWithVersion("intcblock 1 2\nintc 1", v) require.NoError(t, err) _, err = AssembleStringWithVersion("bytec 1", v) require.Error(t, err) - require.Contains(t, err.Error(), "bytec 1 is not defined") + require.Contains(t, err.Error(), "1: bytec 1 is not defined") _, err = AssembleStringWithVersion("bytecblock 0x01 0x02\nbytec 1", v) require.NoError(t, err) diff --git a/data/transactions/logic/backwardCompat_test.go b/data/transactions/logic/backwardCompat_test.go index b3d56efa89..07f9eb4b4e 100644 --- a/data/transactions/logic/backwardCompat_test.go +++ b/data/transactions/logic/backwardCompat_test.go @@ -504,19 +504,19 @@ done:` t.Run("v=default", func(t *testing.T) { _, err := AssembleString(source) require.Error(t, err) - require.Contains(t, err.Error(), ":4 label done is too far away") + require.Contains(t, err.Error(), "4: label done is too far away") }) t.Run("v=0", func(t *testing.T) { _, err := AssembleStringWithVersion(source, 0) require.Error(t, err) - require.Contains(t, err.Error(), ":4 label done is too far away") + require.Contains(t, err.Error(), "4: label done is too far away") }) t.Run("v=1", func(t *testing.T) { _, err := AssembleStringWithVersion(source, 1) require.Error(t, err) - require.Contains(t, err.Error(), ":4 label done is too far away") + require.Contains(t, err.Error(), "4: label done is too far away") }) for v := uint64(2); v <= AssemblerMaxVersion; v++ { From 341395f7b0f22dd888a302c57c6ec1e46b3c69de Mon Sep 17 00:00:00 2001 From: Will Winder Date: Tue, 24 Nov 2020 12:58:20 -0500 Subject: [PATCH 29/34] Remove extra newline from version output. (#1720) Remove extra newline from version output --- config/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/version.go b/config/version.go index 9633a55723..6ac6e2b0ba 100644 --- a/config/version.go +++ b/config/version.go @@ -113,7 +113,7 @@ func GetCurrentVersion() Version { // FormatVersionAndLicense prints current version and license information func FormatVersionAndLicense() string { version := GetCurrentVersion() - return fmt.Sprintf("%d\n%s.%s [%s] (commit #%s)\n%s\n", version.AsUInt64(), version.String(), + return fmt.Sprintf("%d\n%s.%s [%s] (commit #%s)\n%s", version.AsUInt64(), version.String(), version.Channel, version.Branch, version.GetCommitHash(), GetLicenseInfo()) } From 0f3f997b417bbd7a241468ed2284ec16f9e96310 Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Tue, 24 Nov 2020 16:00:11 -0500 Subject: [PATCH 30/34] Implement opt-in in dryrun (#1721) Implement optin in dryrun --- daemon/algod/api/server/v2/dryrun.go | 15 +++++++++++++++ .../e2e-go/cli/goal/expect/goalDryrunRestTest.exp | 13 +++++++++++++ .../scripts/e2e_subs/tealprogs/app_optin_put.teal | 12 ++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 test/scripts/e2e_subs/tealprogs/app_optin_put.teal diff --git a/daemon/algod/api/server/v2/dryrun.go b/daemon/algod/api/server/v2/dryrun.go index c1b0b5f414..a815b31e0d 100644 --- a/daemon/algod/api/server/v2/dryrun.go +++ b/daemon/algod/api/server/v2/dryrun.go @@ -438,6 +438,21 @@ func doDryrunRequest(dr *DryrunRequest, proto *config.ConsensusParams, response appIdx = basics.AppIndex(dr.Apps[0].Id) } } + if stxn.Txn.OnCompletion == transactions.OptInOC { + if idx, ok := dl.accountsIn[stxn.Txn.Sender]; ok { + acct := dl.dr.Accounts[idx] + var ad basics.AccountData + if ad, err = AccountToAccountData(&acct); err != nil { + response.Error = err.Error() + return + } + if ad.AppLocalStates == nil { + ad.AppLocalStates = make(map[basics.AppIndex]basics.AppLocalState) + } + ad.AppLocalStates[appIdx] = basics.AppLocalState{KeyValue: make(basics.TealKeyValue)} + dl.accounts[stxn.Txn.Sender] = basics.BalanceRecord{Addr: stxn.Txn.Sender, AccountData: ad} + } + } l, err := makeAppLedger(&dl, &stxn.Txn, appIdx) if err != nil { diff --git a/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp b/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp index d9cff62725..e2ee04da10 100644 --- a/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp +++ b/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp @@ -52,6 +52,7 @@ if { [catch { set TEST_PRIMARY_NODE_DIR $TEST_ROOT_DIR/Primary/ set NETWORK_NAME test_net_expect_$TIME_STAMP set NETWORK_TEMPLATE "$TEST_DATA_DIR/nettemplates/TwoNodes50EachFuture.json" + set TEAL_PROGS_DIR "$TEST_DATA_DIR/../scripts/e2e_subs/tealprogs" exec cp $TEST_DATA_DIR/../../gen/devnet/genesis.json $TEST_ALGO_DIR @@ -138,6 +139,18 @@ if { [catch { TestGoalDryrunExitCode "" $TEST_PRIMARY_NODE_DIR 1 "Cannot read file : open : no such file or directory" TestGoalDryrunExitCode $INVALID_FILE_1 $TEST_PRIMARY_NODE_DIR 1 "dryrun-remote: HTTP 400 Bad Request:" + # check local state access during opt-in transaction + set GLOBAL_BYTE_SLICES 0 + set LOCAL_BYTE_SLICES 1 + set APP_ID [::AlgorandGoal::AppCreate0 $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TEAL_PROGS_DIR/app_optin_put.teal $GLOBAL_BYTE_SLICES $LOCAL_BYTE_SLICES $TEAL_PROGS_DIR/clear_program_state.teal $TEST_PRIMARY_NODE_DIR] + set DRREQ_FILE_OPTIN "$TEST_ROOT_DIR/app-optin-drreq.msgp" + spawn goal app optin --app-id $APP_ID --from $PRIMARY_ACCOUNT_ADDRESS -o $DRREQ_FILE_OPTIN --dryrun-dump --dryrun-dump-format=msgp -d $TEST_PRIMARY_NODE_DIR + expect { + timeout { ::AlgorandGoal::Abort "goal app optin timeout" } + } + + TestGoalDryrun $DRREQ_FILE_OPTIN $TEST_PRIMARY_NODE_DIR + # Shutdown the network ::AlgorandGoal::StopNetwork $NETWORK_NAME $TEST_ALGO_DIR $TEST_ROOT_DIR exit 0 diff --git a/test/scripts/e2e_subs/tealprogs/app_optin_put.teal b/test/scripts/e2e_subs/tealprogs/app_optin_put.teal new file mode 100644 index 0000000000..63cb42b9f2 --- /dev/null +++ b/test/scripts/e2e_subs/tealprogs/app_optin_put.teal @@ -0,0 +1,12 @@ +#pragma version 2 + +txn ApplicationID +bz ok + +int 0 +byte "key" +byte "value" +app_local_put + +ok: +int 1 From 1a2f3972d8674f6566216b5111d1502fb579932d Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Wed, 25 Nov 2020 13:47:32 -0500 Subject: [PATCH 31/34] TEAL: fix return type check in assembler (#1719) TEAL: fix return type check in assembler * app and asset getters return a pair * order in opcode specification was "top-left" * type check expects "bottom-left" order ## Release notes This PR addresses inconsistency in the spec where operands pushing order is different from return values order. There is no changes in TEAL evaluation so that no protocol upgrade is required. Corresponding conning to the spec is https://github.com/algorandfoundation/specs/commit/21adc34b722ff365076b7e83a797190e7f99d6a2 Please note, the consensus protocol version is pointing to the not up-to-date spec https://github.com/algorandfoundation/specs/releases/tag/August2020-TEALv2 that includes only Rekey but not applications. --- cmd/opdoc/opdoc.go | 11 +++++-- data/transactions/logic/README.md | 6 ++++ data/transactions/logic/README_in.md | 6 ++++ data/transactions/logic/TEAL_opcodes.md | 16 +++++------ data/transactions/logic/assembler.go | 5 ++-- data/transactions/logic/assembler_test.go | 35 +++++++++++++++++++++++ data/transactions/logic/opcodes.go | 8 +++--- 7 files changed, 70 insertions(+), 17 deletions(-) diff --git a/cmd/opdoc/opdoc.go b/cmd/opdoc/opdoc.go index 2b35e31650..6f9fc79aa1 100644 --- a/cmd/opdoc/opdoc.go +++ b/cmd/opdoc/opdoc.go @@ -120,12 +120,17 @@ func opToMarkdown(out io.Writer, op *logic.OpSpec) (err error) { } out.Write([]byte("\n")) } + if op.Returns == nil { fmt.Fprintf(out, "- Pushes: _None_\n") } else { - fmt.Fprintf(out, "- Pushes: %s", op.Returns[0].String()) - for _, rt := range op.Returns[1:] { - fmt.Fprintf(out, ", %s", rt.String()) + if len(op.Returns) == 1 { + fmt.Fprintf(out, "- Pushes: %s", op.Returns[0].String()) + } else { + fmt.Fprintf(out, "- Pushes: *... stack*, %s", op.Returns[0].String()) + for _, rt := range op.Returns[1:] { + fmt.Fprintf(out, ", %s", rt.String()) + } } fmt.Fprintf(out, "\n") } diff --git a/data/transactions/logic/README.md b/data/transactions/logic/README.md index 680dcf7243..f515904693 100644 --- a/data/transactions/logic/README.md +++ b/data/transactions/logic/README.md @@ -352,6 +352,12 @@ For version 1, subsequent bytes after the varuint are program opcode bytes. Futu It is important to prevent newly-introduced transaction fields from breaking assumptions made by older versions of TEAL. If one of the transactions in a group will execute a TEAL program whose version predates a given field, that field must not be set anywhere in the transaction group, or the group will be rejected. For example, executing a TEAL version 1 program on a transaction with RekeyTo set to a nonzero address will cause the program to fail, regardless of the other contents of the program itself. +This requirement is enforced as follows: + +* For every transaction, compute the earliest TEAL version that supports all the fields and and values in this transaction. For example, a transaction with a nonzero RekeyTo field will have version (at least) 2. + +* Compute the largest version number across all the transactions in a group (of size 1 or more), call it `maxVerNo`. If any transaction in this group has a TEAL program with a version smaller than `maxVerNo`, then that TEAL program will fail. + ## Varuint A '[proto-buf style variable length unsigned int](https://developers.google.com/protocol-buffers/docs/encoding#varint)' is encoded with 7 data bits per byte and the high bit is 1 if there is a following byte and 0 for the last byte. The lowest order 7 bits are in the first byte, followed by successively higher groups of 7 bits. diff --git a/data/transactions/logic/README_in.md b/data/transactions/logic/README_in.md index 51c74b8185..64bdd299cb 100644 --- a/data/transactions/logic/README_in.md +++ b/data/transactions/logic/README_in.md @@ -173,6 +173,12 @@ For version 1, subsequent bytes after the varuint are program opcode bytes. Futu It is important to prevent newly-introduced transaction fields from breaking assumptions made by older versions of TEAL. If one of the transactions in a group will execute a TEAL program whose version predates a given field, that field must not be set anywhere in the transaction group, or the group will be rejected. For example, executing a TEAL version 1 program on a transaction with RekeyTo set to a nonzero address will cause the program to fail, regardless of the other contents of the program itself. +This requirement is enforced as follows: + +* For every transaction, compute the earliest TEAL version that supports all the fields and and values in this transaction. For example, a transaction with a nonzero RekeyTo field will have version (at least) 2. + +* Compute the largest version number across all the transactions in a group (of size 1 or more), call it `maxVerNo`. If any transaction in this group has a TEAL program with a version smaller than `maxVerNo`, then that TEAL program will fail. + ## Varuint A '[proto-buf style variable length unsigned int](https://developers.google.com/protocol-buffers/docs/encoding#varint)' is encoded with 7 data bits per byte and the high bit is 1 if there is a following byte and 0 for the last byte. The lowest order 7 bits are in the first byte, followed by successively higher groups of 7 bits. diff --git a/data/transactions/logic/TEAL_opcodes.md b/data/transactions/logic/TEAL_opcodes.md index 10905cd6de..1657c107b9 100644 --- a/data/transactions/logic/TEAL_opcodes.md +++ b/data/transactions/logic/TEAL_opcodes.md @@ -208,14 +208,14 @@ Overflow is an error condition which halts execution and fails the transaction. - Opcode: 0x1d - Pops: *... stack*, {uint64 A}, {uint64 B} -- Pushes: uint64, uint64 +- Pushes: *... stack*, uint64, uint64 - A times B out to 128-bit long result as low (top) and high uint64 values on the stack ## addw - Opcode: 0x1e - Pops: *... stack*, {uint64 A}, {uint64 B} -- Pushes: uint64, uint64 +- Pushes: *... stack*, uint64, uint64 - A plus B out to 128-bit long result as sum (top) and carry-bit uint64 values on the stack - LogicSigVersion >= 2 @@ -534,14 +534,14 @@ See `bnz` for details on how branches work. `b` always jumps to the offset. - Opcode: 0x49 - Pops: *... stack*, any -- Pushes: any, any +- Pushes: *... stack*, any, any - duplicate last value on stack ## dup2 - Opcode: 0x4a - Pops: *... stack*, {any A}, {any B} -- Pushes: any, any, any, any +- Pushes: *... stack*, any, any, any, any - duplicate two last values on stack: A, B -> A, B, A, B - LogicSigVersion >= 2 @@ -606,7 +606,7 @@ params: account index, state key. Return: value. The value is zero if the key do - Opcode: 0x63 - Pops: *... stack*, {uint64 A}, {uint64 B}, {[]byte C} -- Pushes: uint64, any +- Pushes: *... stack*, any, uint64 - read from account specified by Txn.Accounts[A] from local state of the application B key C => {0 or 1 (top), value} - LogicSigVersion >= 2 - Mode: Application @@ -628,7 +628,7 @@ params: state key. Return: value. The value is zero if the key does not exist. - Opcode: 0x65 - Pops: *... stack*, {uint64 A}, {[]byte B} -- Pushes: uint64, any +- Pushes: *... stack*, any, uint64 - read from application Txn.ForeignApps[A] global state key B => {0 or 1 (top), value}. A is specified as an account index in the ForeignApps field of the ApplicationCall transaction, zero index means this app - LogicSigVersion >= 2 - Mode: Application @@ -685,7 +685,7 @@ Deleting a key which is already absent has no effect on the application global s - Opcode: 0x70 {uint8 asset holding field index} - Pops: *... stack*, {uint64 A}, {uint64 B} -- Pushes: uint64, any +- Pushes: *... stack*, any, uint64 - read from account specified by Txn.Accounts[A] and asset B holding field X (imm arg) => {0 or 1 (top), value} - LogicSigVersion >= 2 - Mode: Application @@ -704,7 +704,7 @@ params: account index, asset id. Return: did_exist flag (1 if exist and 0 otherw - Opcode: 0x71 {uint8 asset params field index} - Pops: *... stack*, uint64 -- Pushes: uint64, any +- Pushes: *... stack*, any, uint64 - read from asset Txn.ForeignAssets[A] params field X (imm arg) => {0 or 1 (top), value} - LogicSigVersion >= 2 - Mode: Application diff --git a/data/transactions/logic/assembler.go b/data/transactions/logic/assembler.go index f778827f15..21321c0861 100644 --- a/data/transactions/logic/assembler.go +++ b/data/transactions/logic/assembler.go @@ -53,6 +53,7 @@ type OpStream struct { Out bytes.Buffer Version uint64 Trace io.Writer + Stderr io.Writer vubytes [9]byte intc []uint64 @@ -1010,7 +1011,7 @@ func (ops *OpStream) checkArgs(spec OpSpec) error { if !typecheck(argType, stype) { msg := fmt.Sprintf("%s arg %d wanted type %s got %s", spec.Name, i, argType.String(), stype.String()) if len(ops.labelReferences) > 0 { - fmt.Fprintf(os.Stderr, "warning: %d: %s; but branches have happened and assembler does not precisely track types in this case\n", ops.sourceLine, msg) + fmt.Fprintf(ops.Stderr, "warning: %d: %s; but branches have happened and assembler does not precisely track types in this case\n", ops.sourceLine, msg) } else { return ops.error(msg) } @@ -1247,7 +1248,7 @@ func AssembleStringWithVersionEx(text string, version uint64) ([]byte, map[int]i } sr = strings.NewReader(text) - ops := OpStream{Version: version} + ops := OpStream{Version: version, Stderr: os.Stderr} err = ops.assemble(sr) if err != nil { return nil, nil, err diff --git a/data/transactions/logic/assembler_test.go b/data/transactions/logic/assembler_test.go index d8cbbbd56f..24bdb9f76a 100644 --- a/data/transactions/logic/assembler_test.go +++ b/data/transactions/logic/assembler_test.go @@ -1519,3 +1519,38 @@ func TestErrShortBytecblock(t *testing.T) { checkIntConstBlock(&cx) require.Equal(t, cx.err, errShortIntcblock) } + +func TestBranchAssemblyTypeCheck(t *testing.T) { + text := ` + int 0 // current app id [0] + int 1 // key [1, 0] + itob // ["\x01", 0] + app_global_get_ex // [0|1, x] + pop // [x] + btoi // [n] +` + + sr := strings.NewReader(text) + var buf bytes.Buffer + ops := OpStream{Version: AssemblerMaxVersion, Stderr: &buf} + err := ops.assemble(sr) + require.NoError(t, err) + require.Empty(t, buf, buf.String()) + + text = ` + int 0 // current app id [0] + int 1 // key [1, 0] + itob // ["\x01", 0] + app_global_get_ex // [0|1, x] + bnz flip // [x] +flip: // [x] + btoi // [n] +` + + sr = strings.NewReader(text) + buf.Reset() + ops = OpStream{Version: AssemblerMaxVersion, Stderr: &buf} + err = ops.assemble(sr) + require.NoError(t, err) + require.Empty(t, buf, buf.String()) +} diff --git a/data/transactions/logic/opcodes.go b/data/transactions/logic/opcodes.go index 6f41c0089d..15d7b12a7b 100644 --- a/data/transactions/logic/opcodes.go +++ b/data/transactions/logic/opcodes.go @@ -154,16 +154,16 @@ var OpSpecs = []OpSpec{ {0x60, "balance", opBalance, asmDefault, disDefault, oneInt, oneInt, 2, runModeApplication, opSizeDefault}, {0x61, "app_opted_in", opAppCheckOptedIn, asmDefault, disDefault, twoInts, oneInt, 2, runModeApplication, opSizeDefault}, {0x62, "app_local_get", opAppGetLocalState, asmDefault, disDefault, oneInt.plus(oneBytes), oneAny, 2, runModeApplication, opSizeDefault}, - {0x63, "app_local_get_ex", opAppGetLocalStateEx, asmDefault, disDefault, twoInts.plus(oneBytes), oneInt.plus(oneAny), 2, runModeApplication, opSizeDefault}, + {0x63, "app_local_get_ex", opAppGetLocalStateEx, asmDefault, disDefault, twoInts.plus(oneBytes), oneAny.plus(oneInt), 2, runModeApplication, opSizeDefault}, {0x64, "app_global_get", opAppGetGlobalState, asmDefault, disDefault, oneBytes, oneAny, 2, runModeApplication, opSizeDefault}, - {0x65, "app_global_get_ex", opAppGetGlobalStateEx, asmDefault, disDefault, oneInt.plus(oneBytes), oneInt.plus(oneAny), 2, runModeApplication, opSizeDefault}, + {0x65, "app_global_get_ex", opAppGetGlobalStateEx, asmDefault, disDefault, oneInt.plus(oneBytes), oneAny.plus(oneInt), 2, runModeApplication, opSizeDefault}, {0x66, "app_local_put", opAppPutLocalState, asmDefault, disDefault, oneInt.plus(oneBytes).plus(oneAny), nil, 2, runModeApplication, opSizeDefault}, {0x67, "app_global_put", opAppPutGlobalState, asmDefault, disDefault, oneBytes.plus(oneAny), nil, 2, runModeApplication, opSizeDefault}, {0x68, "app_local_del", opAppDeleteLocalState, asmDefault, disDefault, oneInt.plus(oneBytes), nil, 2, runModeApplication, opSizeDefault}, {0x69, "app_global_del", opAppDeleteGlobalState, asmDefault, disDefault, oneBytes, nil, 2, runModeApplication, opSizeDefault}, - {0x70, "asset_holding_get", opAssetHoldingGet, assembleAssetHolding, disAssetHolding, twoInts, oneInt.plus(oneAny), 2, runModeApplication, opSize{1, 2, nil}}, - {0x71, "asset_params_get", opAssetParamsGet, assembleAssetParams, disAssetParams, oneInt, oneInt.plus(oneAny), 2, runModeApplication, opSize{1, 2, nil}}, + {0x70, "asset_holding_get", opAssetHoldingGet, assembleAssetHolding, disAssetHolding, twoInts, oneAny.plus(oneInt), 2, runModeApplication, opSize{1, 2, nil}}, + {0x71, "asset_params_get", opAssetParamsGet, assembleAssetParams, disAssetParams, oneInt, oneAny.plus(oneInt), 2, runModeApplication, opSize{1, 2, nil}}, } type sortByOpcode []OpSpec From d4544c8933fdfe2eb76fcb45d08280d2a97103c6 Mon Sep 17 00:00:00 2001 From: btoll Date: Wed, 25 Nov 2020 15:42:15 -0500 Subject: [PATCH 32/34] Fix rpm upload in pipeline (#1723) This fixes the archive target that wasn't properly uploading stable rpm packages because "stable" isn't in the filename. Instead, it is sufficient to just search for the version number. --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0966d019eb..c288109dbc 100644 --- a/Makefile +++ b/Makefile @@ -305,6 +305,4 @@ install: build include ./scripts/release/mule/Makefile.mule archive: - CHANNEL=$(CHANNEL) \ - aws s3 cp tmp/node_pkgs s3://algorand-internal/channel/${CHANNEL}/$(FULLBUILDNUMBER) --recursive --exclude "*" --include "*${CHANNEL}*$(FULLBUILDNUMBER)*" - + aws s3 cp tmp/node_pkgs s3://algorand-internal/channel/$(CHANNEL)/$(FULLBUILDNUMBER) --recursive --exclude "*" --include "*$(FULLBUILDNUMBER)*" From a6f508aec81a63a2f69f50a346bed84df37b5f66 Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Wed, 25 Nov 2020 16:20:07 -0500 Subject: [PATCH 33/34] Fix opt-ins and timeouts in pingpong assets testing (#1724) Fix opt-ins and timeouts in pingpong assets testing --- shared/pingpong/accounts.go | 275 ++++++++++++++++++++++++------------ shared/pingpong/pingpong.go | 16 ++- 2 files changed, 196 insertions(+), 95 deletions(-) diff --git a/shared/pingpong/accounts.go b/shared/pingpong/accounts.go index ba1fd56bf9..94403d3fa8 100644 --- a/shared/pingpong/accounts.go +++ b/shared/pingpong/accounts.go @@ -130,12 +130,19 @@ func throttleTransactionRate(startTime time.Time, cfg PpConfig, totalSent uint64 // Step 1) Create X assets for each of the participant accounts // Step 2) For each participant account, opt-in to assets of all other participant accounts // Step 3) Evenly distribute the assets across all participant accounts -func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) (resultAssetMaps map[uint64]v1.AssetParams, err error) { +func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) (resultAssetMaps map[uint64]v1.AssetParams, optIns map[uint64][]string, err error) { + proto, err := getProto(client) + if err != nil { + return + } var startTime = time.Now() var totalSent uint64 = 0 resultAssetMaps = make(map[uint64]v1.AssetParams) + // optIns contains own and explicitly opted-in assets + optIns = make(map[uint64][]string) + numCreatedAssetsByAddr := make(map[string]int, len(accounts)) // 1) Create X assets for each of the participant accounts for addr := range accounts { addrAccount, addrErr := client.AccountInformation(addr) @@ -146,6 +153,7 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf } toCreate := int(cfg.NumAsset) - len(addrAccount.AssetParams) + numCreatedAssetsByAddr[addr] = toCreate fmt.Printf("Creating %v create asset transaction for account %v \n", toCreate, addr) fmt.Printf("cfg.NumAsset %v, addrAccount.AssetParams %v\n", cfg.NumAsset, addrAccount.AssetParams) @@ -186,135 +194,216 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf totalSent++ throttleTransactionRate(startTime, cfg, totalSent) } - account, accountErr := client.AccountInformation(addr) - if accountErr != nil { - fmt.Printf("Cannot lookup source account %v\n", addr) - err = accountErr - return - } + } + // wait until all the assets created + r, err := client.Status() + if err != nil { + fmt.Printf("Error: failed to obtain last round after assets creation") + return + } + nextRound := r.LastRound + 1 + allAssets := make(map[uint64]string, int(cfg.NumAsset)*len(accounts)) + for addr := range accounts { + var account v1.Account + deadline := time.Now().Add(3 * time.Minute) + for { + account, err = client.AccountInformation(addr) + if err != nil { + fmt.Printf("Warning: cannot lookup source account after assets creation") + time.Sleep(1 * time.Second) + continue + } + if len(account.AssetParams) >= numCreatedAssetsByAddr[addr] { + break + } + if time.Now().After(deadline) { + err = fmt.Errorf("asset creation took too long") + fmt.Printf("Error: %s\n", err.Error()) + return + } + r, err = client.WaitForRound(nextRound) + if err != nil { + fmt.Printf("Warning: failed to wait for round %d after assets creation", nextRound) + time.Sleep(1 * time.Second) + continue + } + nextRound = r.LastRound + 1 + } assetParams := account.AssetParams - fmt.Printf("Configured %d assets %+v\n", len(assetParams), assetParams) - + if !cfg.Quiet { + fmt.Printf("Configured %d assets %+v\n", len(assetParams), assetParams) + } + // add own asset to opt-ins since asset creators are auto-opted in + for k := range account.AssetParams { + optIns[k] = append(optIns[k], addr) + allAssets[k] = addr + } } - time.Sleep(time.Second * 15) + // optInsByAddr tracks only explicitly opted-in assetsA + optInsByAddr := make(map[string]map[uint64]bool) - // 2) For each participant account, opt-in to assets of all other participant accounts + // 2) For each participant account, opt-in up to proto.MaxAssetsPerAccount assets of all other participant accounts for addr := range accounts { if !cfg.Quiet { - fmt.Printf("Opting in assets from account %v\n", addr) + fmt.Printf("Opting to account %v\n", addr) } - addrAccount, addrErr := client.AccountInformation(addr) + + acct, addrErr := client.AccountInformation(addr) if addrErr != nil { - fmt.Printf("Cannot lookup source account\n") + fmt.Printf("Cannot lookup optin account\n") err = addrErr return } + numSlots := proto.MaxAssetsPerAccount - len(acct.Assets) + i := 0 + for k, creator := range allAssets { + if creator == addr { + continue + } - assetParams := addrAccount.AssetParams - if !cfg.Quiet { - fmt.Printf("Optining in %d assets %+v\n", len(assetParams), assetParams) - } - - // Opt-in Accounts for each asset - for k := range assetParams { - if !cfg.Quiet { - fmt.Printf("optin asset %+v\n", k) + // opt-in asset k for addr + tx, sendErr := client.MakeUnsignedAssetSendTx(k, 0, addr, "", "") + if sendErr != nil { + fmt.Printf("Cannot initiate asset optin %v in account %v\n", k, addr) + err = sendErr + return } - for addr2 := range accounts { - if addr != addr2 { - if !cfg.Quiet { - fmt.Printf("Opting in assets to account %v \n", addr2) - } - _, addrErr2 := client.AccountInformation(addr2) - if addrErr2 != nil { - fmt.Printf("Cannot lookup optin account\n") - err = addrErr2 - return - } + tx, err = client.FillUnsignedTxTemplate(addr, 0, 0, cfg.MaxFee, tx) + if err != nil { + fmt.Printf("Cannot fill asset optin %v in account %v\n", k, addr) + return + } - // opt-in asset k for addr - tx, sendErr := client.MakeUnsignedAssetSendTx(k, 0, addr2, "", "") - if sendErr != nil { - fmt.Printf("Cannot initiate asset optin %v in account %v\n", k, addr2) - err = sendErr - return - } + _, err = signAndBroadcastTransaction(accounts, addr, tx, client, cfg) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "signing and broadcasting asset optin failed with error %v\n", err) + return + } + optIns[k] = append(optIns[k], addr) + if _, ok := optInsByAddr[addr]; !ok { + optInsByAddr[addr] = make(map[uint64]bool) + } + optInsByAddr[addr][k] = true - tx, err = client.FillUnsignedTxTemplate(addr2, 0, 0, cfg.MaxFee, tx) - if err != nil { - fmt.Printf("Cannot fill asset optin %v in account %v\n", k, addr2) - return - } + if i >= numSlots-1 { + break + } + i++ - _, err = signAndBroadcastTransaction(accounts, addr2, tx, client, cfg) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "signing and broadcasting asset optin failed with error %v\n", err) - return - } + totalSent++ + throttleTransactionRate(startTime, cfg, totalSent) + } + } - totalSent++ - throttleTransactionRate(startTime, cfg, totalSent) - } + // wait until all opt-ins completed + r, err = client.Status() + if err != nil { + fmt.Printf("Error: failed to obtain last round after assets opt in") + return + } + nextRound = r.LastRound + 1 + for addr := range accounts { + expectedAssets := numCreatedAssetsByAddr[addr] + len(optInsByAddr[addr]) + var account v1.Account + deadline := time.Now().Add(3 * time.Minute) + for { + account, err = client.AccountInformation(addr) + if err != nil { + fmt.Printf("Warning: cannot lookup source account after assets opt in") + time.Sleep(1 * time.Second) + continue + } + if len(account.Assets) >= expectedAssets { + break + } + if time.Now().After(deadline) { + err = fmt.Errorf("asset opting in took too long") + fmt.Printf("Error: %s\n", err.Error()) + return + } + r, err = client.WaitForRound(nextRound) + if err != nil { + fmt.Printf("Warning: failed to wait for round %d after assets opt in", nextRound) + time.Sleep(1 * time.Second) + continue } + nextRound = r.LastRound + 1 } } - time.Sleep(time.Second * 15) - // Step 3) Evenly distribute the assets across all participant accounts - for addr := range accounts { + // Step 3) Evenly distribute the assets across all opted-in accounts + for k, creator := range allAssets { if !cfg.Quiet { - fmt.Printf("Distributing assets from account %v\n", addr) + fmt.Printf("Distributing asset %+v from account %v\n", k, creator) } - addrAccount, addrErr := client.AccountInformation(addr) - if addrErr != nil { + creatorAccount, creatorErr := client.AccountInformation(creator) + if creatorErr != nil { fmt.Printf("Cannot lookup source account\n") - err = addrErr + err = creatorErr return } + assetParams := creatorAccount.AssetParams - assetParams := addrAccount.AssetParams - if !cfg.Quiet { - fmt.Printf("Distributing %d assets\n", len(assetParams)) - } - - // Distribute assets to each account - for k := range assetParams { + for _, addr := range optIns[k] { + assetAmt := assetParams[k].Total / uint64(len(optIns[k])) if !cfg.Quiet { - fmt.Printf("Distributing asset %v \n", k) + fmt.Printf("Distributing assets from %v to %v \n", creator, addr) } - assetAmt := assetParams[k].Total / uint64(len(accounts)) - for addr2 := range accounts { - if addr != addr2 { - if !cfg.Quiet { - fmt.Printf("Distributing assets from %v to %v \n", addr, addr2) - } + tx, sendErr := constructTxn(creator, addr, cfg.MaxFee, assetAmt, k, CreatablesInfo{}, client, cfg) + if sendErr != nil { + fmt.Printf("Cannot transfer asset %v from account %v\n", k, creator) + err = sendErr + return + } - tx, sendErr := constructTxn(addr, addr2, cfg.MaxFee, assetAmt, k, CreatablesInfo{}, client, cfg) - if sendErr != nil { - fmt.Printf("Cannot transfer asset %v from account %v\n", k, addr) - err = sendErr - return - } + _, err = signAndBroadcastTransaction(accounts, creator, tx, client, cfg) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "signing and broadcasting asset distribution failed with error %v\n", err) + return + } - _, err = signAndBroadcastTransaction(accounts, addr, tx, client, cfg) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "signing and broadcasting asset distribution failed with error %v\n", err) - return - } + totalSent++ + throttleTransactionRate(startTime, cfg, totalSent) + } + // append the asset to the result assets + resultAssetMaps[k] = assetParams[k] + } - totalSent++ - throttleTransactionRate(startTime, cfg, totalSent) - } - } - // append the asset to the result assets - resultAssetMaps[k] = assetParams[k] + // wait for all transfers acceptance + r, err = client.Status() + if err != nil { + fmt.Printf("Error: failed to obtain last round after assets distribution") + return + } + nextRound = r.LastRound + 1 + deadline := time.Now().Add(3 * time.Minute) + var pending v1.PendingTransactions + for { + pending, err = client.GetPendingTransactions(100) + if err != nil { + fmt.Printf("Warning: cannot get pending txn") + time.Sleep(1 * time.Second) + continue + } + if pending.TotalTxns == 0 { + break + } + if time.Now().After(deadline) { + fmt.Printf("Warning: assets distribution took too long") + break + } + r, err = client.WaitForRound(nextRound) + if err != nil { + fmt.Printf("Warning: failed to wait for round %d after assets distribution", nextRound) + time.Sleep(1 * time.Second) + continue } + nextRound = r.LastRound + 1 } - time.Sleep(time.Second * 10) return } diff --git a/shared/pingpong/pingpong.go b/shared/pingpong/pingpong.go index 3249ada4a6..393a7d40bb 100644 --- a/shared/pingpong/pingpong.go +++ b/shared/pingpong/pingpong.go @@ -64,7 +64,7 @@ func PrepareAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]u return } - cinfo.AssetParams, err = prepareAssets(assetAccounts, ac, cfg) + cinfo.AssetParams, cinfo.OptIns, err = prepareAssets(assetAccounts, ac, cfg) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "prepare assets failed %v\n", err) return @@ -384,6 +384,11 @@ func sendFromTo( amt := cfg.MaxAmt fee := cfg.MaxFee + assetsByCreator := make(map[string][]*v1.AssetParams) + for _, p := range cinfo.AssetParams { + c := p.Creator + assetsByCreator[c] = append(assetsByCreator[c], &p) + } for i, from := range fromList { if cfg.RandomizeAmt { amt = rand.Uint64()%cfg.MaxAmt + 1 @@ -570,7 +575,7 @@ func constructTxn(from, to string, fee, amt, aidx uint64, cinfo CreatablesInfo, } if cfg.NumApp > 0 { // Construct app transaction - // generate random assetID or appId if we send asset/app txns + // select opted-in accounts for Txn.Accounts field var accounts []string if len(cinfo.OptIns[aidx]) > 0 { indices := rand.Perm(len(cinfo.OptIns[aidx])) @@ -594,6 +599,13 @@ func constructTxn(from, to string, fee, amt, aidx uint64, cinfo CreatablesInfo, _, _ = fmt.Fprintf(os.Stdout, "Calling app %d : %s\n", aidx, from) } } else if cfg.NumAsset > 0 { // Construct asset transaction + // select a pair of random opted-in accounts by aidx + // use them as from/to addresses + if len(cinfo.OptIns[aidx]) > 0 { + indices := rand.Perm(len(cinfo.OptIns[aidx])) + from = cinfo.OptIns[aidx][indices[0]] + to = cinfo.OptIns[aidx][indices[1]] + } txn, err = client.MakeUnsignedAssetSendTx(aidx, amt, to, "", "") if err != nil { _, _ = fmt.Fprintf(os.Stdout, "error making unsigned asset send tx %v\n", err) From 65d5a6a722f19daa7eb63cb814edbb7eda752d45 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 25 Nov 2020 16:45:00 -0500 Subject: [PATCH 34/34] Update version to 2.3.0 --- buildnumber.dat | 1 + config/version.go | 2 +- genesistimestamp.dat | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 buildnumber.dat create mode 100644 genesistimestamp.dat diff --git a/buildnumber.dat b/buildnumber.dat new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/buildnumber.dat @@ -0,0 +1 @@ +0 diff --git a/config/version.go b/config/version.go index 6ac6e2b0ba..439c9ab4dd 100644 --- a/config/version.go +++ b/config/version.go @@ -33,7 +33,7 @@ const VersionMajor = 2 // VersionMinor is the Minor semantic version number (x.#.z) - changed when backwards-compatible features are introduced. // Not enforced until after initial public release (x > 0). -const VersionMinor = 2 +const VersionMinor = 3 // Version is the type holding our full version information. type Version struct { diff --git a/genesistimestamp.dat b/genesistimestamp.dat new file mode 100644 index 0000000000..c72c6a7795 --- /dev/null +++ b/genesistimestamp.dat @@ -0,0 +1 @@ +1558657885